feat: 新增 吾爱破解、果壳、快手

This commit is contained in:
imsyy
2024-12-05 15:28:28 +08:00
parent f38d264366
commit 161b6b612b
22 changed files with 288 additions and 1050 deletions

View File

@@ -59,7 +59,7 @@ app.get("/", (c) => c.html(<Home />));
app.notFound((c) => c.html(<NotFound />, 404));
// error
app.onError((err, c) => {
logger.error(`出现致命错误:${err}`);
logger.error(`❌ [ERROR] ${err?.message}`);
return c.html(<Error error={err?.message} />, 500);
});

View File

@@ -12,15 +12,12 @@ export type Config = {
ALLOWED_HOST: string;
USE_LOG_FILE: boolean;
RSS_MODE: boolean;
USE_PUPPETEER: boolean;
};
// 验证并提取环境变量
const getEnvVariable = (key: string): string | undefined => {
const value = process.env[key];
if (value === undefined) {
return undefined;
}
if (value === undefined) return undefined;
return value;
};
@@ -28,9 +25,7 @@ const getEnvVariable = (key: string): string | undefined => {
const getNumericEnvVariable = (key: string, defaultValue: number): number => {
const value = getEnvVariable(key) ?? String(defaultValue);
const parsedValue = parseInt(value, 10);
if (isNaN(parsedValue)) {
return defaultValue;
}
if (isNaN(parsedValue)) return defaultValue;
return parsedValue;
};
@@ -45,10 +40,9 @@ export const config: Config = {
PORT: getNumericEnvVariable("PORT", 6688),
DISALLOW_ROBOT: getBooleanEnvVariable("DISALLOW_ROBOT", true),
CACHE_TTL: getNumericEnvVariable("CACHE_TTL", 3600),
REQUEST_TIMEOUT: getNumericEnvVariable("CACHE_TTL", 6000),
REQUEST_TIMEOUT: getNumericEnvVariable("REQUEST_TIMEOUT", 6000),
ALLOWED_DOMAIN: getEnvVariable("ALLOWED_DOMAIN") || "*",
ALLOWED_HOST: getEnvVariable("ALLOWED_HOST") || "imsyy.top",
USE_LOG_FILE: getBooleanEnvVariable("USE_LOG_FILE", true),
RSS_MODE: getBooleanEnvVariable("RSS_MODE", false),
USE_PUPPETEER: getBooleanEnvVariable("USE_PUPPETEER", false),
};

View File

@@ -10,8 +10,7 @@ const serveHotApi: (port?: number) => void = (port: number = config.PORT) => {
fetch: app.fetch,
port,
});
logger.info(`🔥 DailyHot API 成功在端口 ${port} 上运行`);
logger.info(`💻 Puppeteer: ${config.USE_PUPPETEER}`);
logger.info(`🔥 DailyHot API successfully runs on port ${port}`);
logger.info(`🔗 Local: 👉 http://localhost:${port}`);
return apiServer;
} catch (error) {

33
src/router.types.d.ts vendored
View File

@@ -315,14 +315,31 @@ export type RouterType = {
message: string;
replynum: number;
};
nodeseek: {
guid: {
_: string;
}[];
xueqiu: {
pic: string;
tag: string;
id: number;
hot: number;
content: string;
};
guokr: {
id: number;
title: string;
description: string | string[];
"dc:creator": string;
pubDate: string[];
link: string[];
summary: string;
author: {
nickname: string;
};
date_modified: string;
small_image: string;
};
kuaishou: {
id: string;
name: string;
hotValue: string;
iconUrl: string;
poster:string;
photoIds: {
json: string[];
};
};
};

View File

@@ -1,24 +1,27 @@
import type { RouterData, ListContext, Options, RouterResType } from "../types.js";
import { web } from "../utils/getData.js";
import { extractRss, parseRSS } from "../utils/parseRSS.js";
import { get } from "../utils/getData.js";
import { getTime } from "../utils/getTime.js";
import { parseRSS } from "../utils/parseRSS.js";
import iconv from "iconv-lite";
const typeMap: Record<string, string> = {
digest: "最新精华",
hot: "最新热门",
new: "最新回复",
newthread: "最新发表",
};
export const handleRoute = async (c: ListContext, noCache: boolean) => {
const type = c.req.query("type") || "hot";
const type = c.req.query("type") || "digest";
const listData = await getList({ type }, noCache);
const routeData: RouterData = {
name: "52pojie",
title: "吾爱破解",
type: "榜单",
type: typeMap[type],
params: {
type: {
name: "榜单分类",
type: {
tech: "新鲜出炉",
newthread: "技术分享",
hot: "人气热门",
digest: "精华采撷",
},
type: typeMap,
},
},
link: "https://www.52pojie.cn/",
@@ -31,28 +34,27 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => {
const getList = async (options: Options, noCache: boolean): Promise<RouterResType> => {
const { type } = options;
const url = `https://www.52pojie.cn/forum.php?mod=guide&view=${type}&rss=1`;
const result = await web({
const result = await get({
url,
noCache,
userAgent:
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
responseType: "arraybuffer",
headers: {
userAgent:
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
},
});
const parseData = async () => {
if (typeof result?.data !== "string") return [];
const rssContent = extractRss(result.data);
if (!rssContent) return [];
return await parseRSS(rssContent);
};
const list = await parseData();
// 转码
const utf8Data = iconv.decode(result.data, "gbk");
const list = await parseRSS(utf8Data);
return {
...result,
data: list.map((v, i) => ({
id: v.guid || i,
title: v.title || "",
desc: v.content || "",
author: v.author || "",
desc: v.content?.trim() || "",
author: v.author,
timestamp: getTime(v.pubDate || 0),
hot: 0,
hot: undefined,
url: v.link || "",
mobileUrl: v.link || "",
})),

45
src/routes/guokr.ts Normal file
View File

@@ -0,0 +1,45 @@
import type { RouterData } from "../types.js";
import type { RouterType } from "../router.types.js";
import { get } from "../utils/getData.js";
import { getTime } from "../utils/getTime.js";
export const handleRoute = async (_: undefined, noCache: boolean) => {
const listData = await getList(noCache);
const routeData: RouterData = {
name: "guokr",
title: "果壳",
type: "热门文章",
description: "科技有意思",
link: "https://www.guokr.com/",
total: listData.data?.length || 0,
...listData,
};
return routeData;
};
const getList = async (noCache: boolean) => {
const url = `https://www.guokr.com/beta/proxy/science_api/articles?limit=30`;
const result = await get({
url,
noCache,
headers: {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
},
});
const list = result.data;
return {
...result,
data: list.map((v: RouterType["guokr"]) => ({
id: v.id,
title: v.title,
desc: v.summary,
cover: v.small_image,
author: v.author?.nickname,
hot: undefined,
timestamp: getTime(v.date_modified),
url: `https://www.guokr.com/article/${v.id}`,
mobileUrl: `https://m.guokr.com/article/${v.id}`,
})),
};
};

View File

@@ -1,6 +1,6 @@
import type { RouterData, ListContext, Options } from "../types.js";
import { web } from "../utils/getData.js";
import { extractRss, parseRSS } from "../utils/parseRSS.js";
import { get } from "../utils/getData.js";
import { parseRSS } from "../utils/parseRSS.js";
import { getTime } from "../utils/getTime.js";
const typeMap: Record<string, string> = {
@@ -33,19 +33,15 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => {
const getList = async (options: Options, noCache: boolean) => {
const { type } = options;
const url = `https://hostloc.com/forum.php?mod=guide&view=${type}&rss=1`;
const result = await web({
const result = await get({
url,
noCache,
userAgent:
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
headers: {
userAgent:
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
},
});
const parseData = async () => {
if (typeof result?.data !== "string") return [];
const rssContent = extractRss(result.data);
if (!rssContent) return [];
return await parseRSS(rssContent);
};
const list = await parseData();
const list = await parseRSS(result.data);
return {
...result,
data: list.map((v, i) => ({

View File

@@ -29,8 +29,8 @@ const getList = async (noCache: boolean) => {
desc: v.post_content,
timestamp: getTime(v.created_at),
hot: v.like_count || v.comment_count,
url: v.buzz_original_url || `https://www.ifanr.com/${v.id}`,
mobileUrl: v.buzz_original_url || `https://www.ifanr.com/digest/${v.id}`,
url: v.buzz_original_url || `https://www.ifanr.com/${v.post_id}`,
mobileUrl: v.buzz_original_url || `https://www.ifanr.com/digest/${v.post_id}`,
})),
};
};

59
src/routes/kuaishou.ts Normal file
View File

@@ -0,0 +1,59 @@
import type { RouterType } from "../router.types.js";
import type { ListItem, RouterData } from "../types.js";
import { get } from "../utils/getData.js";
import { parseChineseNumber } from "../utils/getNum.js";
import UserAgent from "user-agents";
export const handleRoute = async (_: undefined, noCache: boolean) => {
const listData = await getList(noCache);
const routeData: RouterData = {
name: "kuaishou",
title: "快手",
type: "热榜",
description: "快手,拥抱每一种生活",
link: "https://www.kuaishou.com/",
total: listData.data?.length || 0,
...listData,
};
return routeData;
};
const getList = async (noCache: boolean) => {
const url = `https://www.kuaishou.com/?isHome=1`;
const userAgent = new UserAgent({
deviceCategory: "desktop",
});
const result = await get({
url,
noCache,
headers: {
"User-Agent": userAgent.toString(),
},
});
const listData: ListItem[] = [];
// 获取主要内容
const pattern = /window.__APOLLO_STATE__=(.*);\(function\(\)/s;
const matchResult = result.data?.match(pattern);
const jsonObject = JSON.parse(matchResult[1])["defaultClient"];
// 获取所有分类
const allItems = jsonObject['$ROOT_QUERY.visionHotRank({"page":"home"})']["items"];
// 获取全部热榜
allItems?.forEach((item: { id: string }) => {
// 基础数据
const hotItem: RouterType["kuaishou"] = jsonObject[item.id];
const id = hotItem.photoIds?.json?.[0];
listData.push({
id: hotItem.id,
title: hotItem.name,
cover: decodeURIComponent(hotItem.poster),
hot: parseChineseNumber(hotItem.hotValue),
timestamp: undefined,
url: `https://www.kuaishou.com/short-video/${id}`,
mobileUrl: `https://www.kuaishou.com/short-video/${id}`,
});
});
return {
...result,
data: listData,
};
};

View File

@@ -1,8 +1,7 @@
import type { RouterData } from "../types.js";
import type { RouterType } from "../router.types.js";
import { get } from "../utils/getData.js";
import { parseStringPromise } from "xml2js";
import { getTime } from "../utils/getTime.js";
import { parseRSS } from "../utils/parseRSS.js";
export const handleRoute = async (_: undefined, noCache: boolean) => {
const listData = await getList(noCache);
@@ -28,19 +27,18 @@ export const handleRoute = async (_: undefined, noCache: boolean) => {
const getList = async (noCache: boolean) => {
const url = `https://rss.nodeseek.com/`;
const result = await get({ url, noCache });
const rssData = await parseStringPromise(result.data);
const list = rssData.rss.channel[0].item;
const list = await parseRSS(result.data);
return {
...result,
data: list.map((v: RouterType["nodeseek"]) => ({
id: v.guid[0]._,
title: v.title[0],
desc: v.description ? v.description[0] : "",
author: v["dc:creator"] ? v["dc:creator"][0] : "unknown",
timestamp: getTime(v.pubDate[0]),
data: list.map((v, i) => ({
id: v.guid || i,
title: v.title || "",
desc: v.content?.trim() || "",
author: v.author,
timestamp: getTime(v.pubDate || 0),
hot: undefined,
url: v.link[0],
mobileUrl: v.link[0],
url: v.link || "",
mobileUrl: v.link || "",
})),
};
};

44
src/routes/xueqiu.ts Normal file
View File

@@ -0,0 +1,44 @@
import type { RouterData } from "../types.js";
import type { RouterType } from "../router.types.js";
import { get } from "../utils/getData.js";
export const handleRoute = async (_: undefined, noCache: boolean) => {
const listData = await getList(noCache);
const routeData: RouterData = {
name: "xueqiu",
title: "雪球",
type: "热门话题",
description: "聪明的投资者都在这里",
link: "https://xueqiu.com/",
total: listData.data?.length || 0,
...listData,
};
return routeData;
};
const getList = async (noCache: boolean) => {
const url = `https://xueqiu.com/hot_event/list.json`;
const result = await get({
url,
noCache,
headers: {
"User-Agent":
"Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/1.0 Mobile/12F69 Safari/605.1.15",
},
});
const list = result.data.list;
return {
...result,
data: list.map((v: RouterType["xueqiu"]) => ({
id: v.id,
title: v.tag?.startsWith("#") && v.tag?.endsWith("#") ? v.tag.slice(1, -1) : v.tag,
desc: v.content,
cover: v.pic,
author: undefined,
hot: v.hot,
timestamp: undefined,
url: undefined,
mobileUrl: undefined,
})),
};
};

10
src/types.d.ts vendored
View File

@@ -1,4 +1,5 @@
import type { Context } from "hono";
import type { ResponseType } from "axios";
// Context
export type ListContext = Context;
@@ -44,6 +45,7 @@ export interface Get {
noCache?: boolean;
ttl?: number;
originaInfo?: boolean;
responseType?: ResponseType;
}
export interface Post {
@@ -56,14 +58,6 @@ export interface Post {
originaInfo?: boolean;
}
export interface Web {
url: string;
timeout?: number;
noCache?: boolean;
ttl?: number;
userAgent?: string;
}
// 参数类型
export interface Options {
[key: string]: string | number | undefined;

View File

@@ -27,7 +27,7 @@ export const getCache = <T>(key: string): GetCache<T> | undefined => {
// 将数据写入缓存
export const setCache = <T>(key: string, value: T, ttl: number = config.CACHE_TTL) => {
const success = cache.set(key, value, ttl);
if (logger) logger.info("数据缓存成功", { url: key });
if (logger) logger.info(`💾 [CHCHE] ${key} has been cached`);
return success;
};

View File

@@ -1,8 +1,6 @@
import type { Get, Post, Web } from "../types.js";
import type { Page } from "puppeteer";
import type { Get, Post } from "../types.js";
import { config } from "../config.js";
import { getCache, setCache, delCache } from "./cache.js";
import { Cluster } from "puppeteer-cluster";
import logger from "./logger.js";
import axios from "axios";
@@ -13,69 +11,6 @@ const request = axios.create({
withCredentials: true,
});
// Puppeteer Cluster
let cluster: Cluster | null = null;
/**
* 创建 Puppeteer Cluster
* @returns {Promise<Cluster|null>} Puppeteer Cluster
*/
export const createCluster = async (): Promise<Cluster | null> => {
if (cluster) return cluster;
try {
cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_BROWSER,
maxConcurrency: 5,
puppeteerOptions: {
headless: true,
args: [
"--no-sandbox",
"--disable-gpu",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
],
},
});
logger.info("Puppeteer Cluster 已成功初始化");
} catch (error) {
logger.warn("启动 Puppeteer Cluster 失败,可能是没有 Chromium 安装", error);
cluster = null;
}
return cluster;
};
/**
* 处理任务
* @param page 页面
* @param data 数据
* @returns
*/
const taskHandler = async ({
page,
data: { url, userAgent },
}: {
page: Page;
data: { url: string; userAgent?: string };
}): Promise<string> => {
// 用户代理
if (userAgent) await page.setUserAgent(userAgent);
// 请求拦截
await page.setRequestInterception(true);
// 拦截非必要资源
page.on("request", (request) => {
const type = request.resourceType();
if (type === "document" || type === "script") {
request.continue();
} else {
request.abort();
}
});
// 加载页面
await page.goto(url, { waitUntil: "networkidle0", timeout: config.REQUEST_TIMEOUT });
// 返回页面内容
return page.content();
};
// 请求拦截
request.interceptors.request.use(
(request) => {
@@ -84,7 +19,7 @@ request.interceptors.request.use(
return request;
},
(error) => {
logger.error("请求失败,请稍后重试");
logger.error("❌ [ERROR] request failed");
return Promise.reject(error);
},
);
@@ -102,31 +37,38 @@ request.interceptors.response.use(
// GET
export const get = async (options: Get) => {
const { url, headers, params, noCache, ttl = config.CACHE_TTL, originaInfo = false } = options;
logger.info("发起 GET 请求", options);
const {
url,
headers,
params,
noCache,
ttl = config.CACHE_TTL,
originaInfo = false,
responseType = "json",
} = options;
logger.info(`🌐 [GET] ${url}`);
try {
// 检查缓存
if (noCache) delCache(url);
else {
const cachedData = getCache(url);
if (cachedData) {
logger.info("采用缓存", { url });
logger.info("💾 [CHCHE] The request is cached");
return { fromCache: true, data: cachedData.data, updateTime: cachedData.updateTime };
}
}
// 缓存不存在时请求接口
logger.info("请求接口", { url });
const response = await request.get(url, { headers, params });
const response = await request.get(url, { headers, params, responseType });
const responseData = response?.data || response;
// 存储新获取的数据到缓存
const updateTime = new Date().toISOString();
const data = originaInfo ? response : responseData;
setCache(url, { data, updateTime }, ttl);
// 返回数据
logger.info("接口调用成功", { status: response?.statusText });
logger.info(`✅ [${response?.statusText}] request was successful`);
return { fromCache: false, data, updateTime };
} catch (error) {
logger.error("GET 请求出错", error);
logger.error("❌ [ERROR] request failed");
throw error;
}
};
@@ -134,19 +76,18 @@ export const get = async (options: Get) => {
// POST
export const post = async (options: Post) => {
const { url, headers, body, noCache, ttl = config.CACHE_TTL, originaInfo = false } = options;
logger.info("发起 POST 请求", options);
logger.info(`🌐 [POST] ${url}`);
try {
// 检查缓存
if (noCache) delCache(url);
else {
const cachedData = getCache(url);
if (cachedData) {
logger.info("采用缓存", { url });
logger.info("💾 [CHCHE] The request is cached");
return { fromCache: true, data: cachedData.data, updateTime: cachedData.updateTime };
}
}
// 缓存不存在时请求接口
logger.info("请求接口", { url });
const response = await request.post(url, body, { headers });
const responseData = response?.data || response;
// 存储新获取的数据到缓存
@@ -156,68 +97,10 @@ export const post = async (options: Post) => {
setCache(url, { data, updateTime }, ttl);
}
// 返回数据
logger.info("接口调用成功", { status: response?.statusText });
logger.info(`✅ [${response?.statusText}] request was successful`);
return { fromCache: false, data, updateTime };
} catch (error) {
logger.error("POST 请求出错", error);
logger.error("❌ [ERROR] request failed");
throw error;
}
};
// WEB - Puppeteer
export const web = async (options: Web) => {
const { url, noCache, ttl = config.CACHE_TTL, userAgent } = options;
logger.info("使用 Puppeteer 发起页面请求", options);
if (config.USE_PUPPETEER === false) {
return {
fromCache: false,
data: [],
message: "Puppeteer is not enabled, please set USE_PUPPETEER=true",
updateTime: 0,
};
}
// 初始化 Cluster
const clusterInstance = await createCluster();
if (!clusterInstance) {
logger.error("Cluster 初始化失败,无法继续");
throw new Error("Cluster 初始化失败");
}
try {
// 检查缓存
if (noCache) {
delCache(url);
} else {
const cachedData = getCache(url);
if (cachedData) {
logger.info("采用缓存", { url });
return { fromCache: true, data: cachedData.data, updateTime: cachedData.updateTime };
}
}
// 缓存不存在时使用 Puppeteer 请求页面
logger.info("启动浏览器请求页面", { url });
const pageContent = await clusterInstance.execute({ url, userAgent }, taskHandler);
// 存储新获取的数据到缓存
const updateTime = new Date().toISOString();
setCache(url, { data: pageContent, updateTime }, ttl);
// 返回数据
logger.info("页面内容获取成功");
return { fromCache: false, data: pageContent, updateTime };
} catch (error) {
logger.error("Puppeteer 请求出错", error);
throw error;
}
};
/**
* 关闭 Cluster
*/
export const closeCluster = async () => {
if (cluster) {
try {
await cluster.close();
logger.info("Puppeteer Cluster 已成功关闭");
} catch (error) {
logger.error("关闭 Puppeteer Cluster 时出错", error);
}
}
};

View File

@@ -35,7 +35,8 @@ const getRSS = (data: RouterData) => {
const rssData = feed.rss2();
return rssData;
} catch (error) {
logger.error("RSS 生成失败:", error);
logger.error("❌ [ERROR] getRSS failed");
throw error;
}
};

View File

@@ -47,7 +47,7 @@ export const parseRSS = async (rssContent: string) => {
// 返回解析数据
return items;
} catch (error) {
logger.error("解析 RSS 内容时出错:", error);
return [];
logger.error("❌ [ERROR] An error occurred while parsing RSS content");
throw error;
}
};