diff --git a/README.md b/README.md index 4b5b084..a9b5963 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ | 稀土掘金 | 热榜 | juejin | 🟢 | | 腾讯新闻 | 热点榜 | qq-news | 🟢 | | 网易新闻 | 热点榜 | netease-news | 🟢 | +| 吾爱破解 | 榜单 | 52pojie | 🟢 | +| 全球主机交流 | 榜单 | hostloc | 🟢 | | 虎嗅 | 24小时 | huxiu | 🟢 | | 爱范儿 | 快讯 | ifanr | 🟢 | | 英雄联盟 | 更新公告 | lol | 🟢 | @@ -59,7 +61,7 @@ | 中央气象台 | 全国气象预警 | weatheralarm | 🟢 | | 中国地震台 | 地震速报 | earthquake | 🟢 | -## 使用 +## ⚙️ 使用 本项目支持 `Node.js` 调用，可在安装完成后调用 `serveHotApi` 来开启服务器 diff --git a/package.json b/package.json index 8a5c488..8b692c0 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "md5": "^2.3.0", "node-cache": "^5.1.2", "puppeteer": "^22.10.0", + "puppeteer-cluster": "^0.24.0", "rss-parser": "^3.13.0", "winston": "^3.13.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 34f006a..982b4bc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,6 +38,9 @@ importers: puppeteer: specifier: ^22.10.0 version: 22.10.0(typescript@5.4.5) + puppeteer-cluster: + specifier: ^0.24.0 + version: 0.24.0(puppeteer@22.10.0(typescript@5.4.5)) rss-parser: specifier: ^3.13.0 version: 3.13.0 @@ -1082,6 +1085,11 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + puppeteer-cluster@0.24.0: + resolution: {integrity: sha512-zHPoNsrwkFLKFtgJJv2aC13EbMASQsE048uZd7CyikEXcl+sc1Nf6PMFb9kMoZI7/zMYxvuP658o2mw079ZZyQ==} + peerDependencies: + puppeteer: '>=22.0.0' + puppeteer-core@22.10.0: resolution: {integrity: sha512-I54J4Vy4I07UHsgB1QSmuFoF7KNQjJWcvFBPhtY+ezMdBfwgGDr8dzYrJa11aPgP9kxIUHjhktcMmmfJkOAtTw==} engines: {node: '>=18'} @@ -2410,6 +2418,13 @@ snapshots: punycode@2.3.1: {} + puppeteer-cluster@0.24.0(puppeteer@22.10.0(typescript@5.4.5)): + dependencies: + debug: 4.3.5 + puppeteer: 22.10.0(typescript@5.4.5) + transitivePeerDependencies: + - supports-color + puppeteer-core@22.10.0: dependencies: '@puppeteer/browsers': 2.2.3 diff --git a/src/router.types.d.ts b/src/router.types.d.ts index f2999d1..82885d7 100644 --- a/src/router.types.d.ts +++ b/src/router.types.d.ts @@ -43,7 +43,7 @@ export type RouterType = { source_id: number; pubdate: string; }; - "52pojie": { + discuz: { title: string; link: string; guid: string; diff --git a/src/routes/52pojie.ts b/src/routes/52pojie.ts index afda037..2e35d85 100644 --- a/src/routes/52pojie.ts +++ b/src/routes/52pojie.ts @@ -52,7 +52,7 @@ const getList = async (options: Options, noCache: boolean) => { return { fromCache: result.fromCache, updateTime: result.updateTime, - data: list.map((v: RouterType["52pojie"]) => ({ + data: list.map((v: RouterType["discuz"]) => ({ id: v.guid, title: v.title, desc: v.content, diff --git a/src/routes/hostloc.ts b/src/routes/hostloc.ts new file mode 100644 index 0000000..541f559 --- /dev/null +++ b/src/routes/hostloc.ts @@ -0,0 +1,66 @@ +import type { RouterData, ListContext, Options } from "../types.js"; +import type { RouterType } from "../router.types.js"; +import { web } from "../utils/getData.js"; +import { extractRss, parseRSS } from "../utils/parseRSS.js"; +import getTime from "../utils/getTime.js"; + +export const handleRoute = async (c: ListContext, noCache: boolean) => { + const type = c.req.query("type") || "hot"; + const { fromCache, data, updateTime } = await getList({ type }, noCache); + const routeData: RouterData = { + name: "hostloc", + title: "全球主机交流", + type: "榜单", + parameData: { + type: { + name: "榜单分类", + type: { + hot: "最新热门", + digest: "最新精华", + new: "最新回复", + newthread: "最新发表", + }, + }, + }, + link: "https://hostloc.com/", + total: data?.length || 0, + updateTime, + fromCache, + data, + }; + return routeData; +}; + +const getList = async (options: Options, noCache: boolean) => { + const { type } = options; + const url = `https://hostloc.com/forum.php?mod=guide&view=${type}&rss=1`; + const result = await web({ + url, + noCache, + userAgent: + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + }); + const parseData = async () => { + if (typeof result?.data === "string") { + const rssContent = extractRss(result.data); + return await parseRSS(rssContent); + } else { + return []; + } + }; + const list = await parseData(); + return { + fromCache: result.fromCache, + updateTime: result.updateTime, + data: list.map((v: RouterType["discuz"]) => ({ + id: v.guid, + title: v.title, + desc: v.content, + author: v.author, + timestamp: getTime(v.pubDate), + hot: null, + url: v.link, + mobileUrl: v.link, + })), + }; +}; diff --git a/src/utils/getData.ts b/src/utils/getData.ts index ee939e4..e816f2a 100644 --- a/src/utils/getData.ts +++ b/src/utils/getData.ts @@ -1,7 +1,7 @@ import type { Get, Post, Web } from "../types.ts"; import { config } from "../config.js"; import { getCache, setCache, delCache } from "./cache.js"; -import puppeteer from "puppeteer"; +import { Cluster } from "puppeteer-cluster"; import logger from "./logger.js"; import axios from "axios"; @@ -12,6 +12,27 @@ const request = axios.create({ withCredentials: true, }); +// puppeteer-cluster +export const createCluster = async () => { + return await Cluster.launch({ + concurrency: Cluster.CONCURRENCY_BROWSER, + maxConcurrency: 5, + }); +}; + +// Cluster +const cluster = await createCluster(); + +// Cluster configuration +cluster.task(async ({ page, data: { url, userAgent } }) => { + if (userAgent) { + await page.setUserAgent(userAgent); + } + await page.goto(url, { waitUntil: 'networkidle0' }); + const pageContent = await page.content(); + return pageContent; +}); + // 请求拦截 request.interceptors.request.use( (request) => { @@ -117,14 +138,7 @@ export const web = async (options: Web) => { } // 缓存不存在时使用 Puppeteer 请求页面 logger.info("启动浏览器请求页面", { url }); - const browser = await puppeteer.launch(); - const page = await browser.newPage(); - // 其他 Puppeteer 操作 - if (userAgent) page.setUserAgent(userAgent); - await page.goto(url, { waitUntil: "networkidle0" }); - // 获取页面内容 - const pageContent = await page.evaluate(() => document.body.innerHTML); - await browser.close(); + const pageContent = await cluster.execute({ url, userAgent }); // 存储新获取的数据到缓存 const updateTime = new Date().toISOString(); setCache(url, { data: pageContent, updateTime }, ttl);