diff --git a/.gitignore b/.gitignore index f1c0a74..e8c5b83 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,8 @@ dist *.njsproj *.sln *.sw? + +package-lock.json +# Sentry Config File +.env.sentry-build-plugin .nvmrc diff --git a/README.md b/README.md index 1b83310..1a8890b 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,13 @@ npm run build npm run start ``` +### pm2 部署 + +```bash +npm i pm2 -g +sh ./deploy.sh +``` + 成功启动后程序会在控制台输出可访问的地址 ### Vercel 部署 diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..2945fcb --- /dev/null +++ b/deploy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# 日志文件 +LOG_FILE="deploy.log" + +# 输出时间戳的日志函数 +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a $LOG_FILE +} + +# 错误处理函数 +handle_error() { + log "错误: $1" + exit 1 +} + +# 开始拉去代码 +log "开始拉取代码..." +git pull +# 开始部署 +log "开始部署..." + +# 安装依赖 +log "正在安装依赖..." +npm install || handle_error "npm install 失败" + +# 构建项目 +log "正在构建项目..." +npm run build || handle_error "构建失败" + +# 使用 pm2 重启或启动项目 +log "正在启动/重启服务..." +pm2 restart daily-news || pm2 start ecosystem.config.cjs || handle_error "PM2 启动失败" + +log "部署完成!" diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs new file mode 100644 index 0000000..7fa756a --- /dev/null +++ b/ecosystem.config.cjs @@ -0,0 +1,15 @@ +module.exports = { + apps: [{ + name: 'daily-news', + script: 'npm', + args: 'start', + instances: 1, + autorestart: true, + watch: false, + max_memory_restart: '1G', + env: { + NODE_ENV: 'development', + PORT: 6688 + } + }] +} diff --git a/package.json b/package.json index 07d3242..d4a6980 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "dev": "cross-env NODE_ENV=development tsx watch --no-cache src/index.ts", "dev:cache": "cross-env NODE_ENV=development tsx watch src/index.ts", "build": "tsc --project tsconfig.json", - "start": "cross-env NODE_ENV=development tsx dist/index.js" + "start": "cross-env NODE_ENV=development node dist/index.js" }, "type": "module", "dependencies": { @@ -49,6 +49,7 @@ "ioredis": "^5.4.1", "md5": "^2.3.0", "node-cache": "^5.1.2", + "node-fetch": "^3.3.2", "rss-parser": "^3.13.0", "user-agents": "^1.1.379", "winston": "^3.17.0" diff --git a/src/routes/bilibili.ts b/src/routes/bilibili.ts index 5305da5..c4cbf1f 100644 --- a/src/routes/bilibili.ts +++ b/src/routes/bilibili.ts @@ -3,20 +3,19 @@ import type { RouterType } from "../router.types.js"; import { get } from "../utils/getData.js"; import getBiliWbi from "../utils/getToken/bilibili.js"; import { getTime } from "../utils/getTime.js"; - +import logger from "../utils/logger.js"; const typeMap: Record = { "0": "全站", "1": "动画", "3": "音乐", "4": "游戏", "5": "娱乐", - "36": "科技", + "188": "科技", "119": "鬼畜", "129": "舞蹈", "155": "时尚", "160": "生活", "168": "国创相关", - "188": "数码", "181": "影视", }; @@ -44,18 +43,30 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { const getList = async (options: Options, noCache: boolean): Promise => { const { type } = options; const wbiData = await getBiliWbi(); - const url = `https://api.bilibili.com/x/web-interface/ranking/v2?tid=${type}&type=all&${wbiData}`; + const url = `https://api.bilibili.com/x/web-interface/ranking/v2?rid=${type}&type=all&${wbiData}`; const result = await get({ url, headers: { - Referer: `https://www.bilibili.com/ranking/all`, - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36", + 'Referer': 'https://www.bilibili.com/ranking/all', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + 'Accept-Encoding': 'gzip, deflate, br', + 'Sec-Ch-Ua': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"', + 'Sec-Ch-Ua-Mobile': '?0', + 'Sec-Ch-Ua-Platform': '"Windows"', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'same-origin', + 'Sec-Fetch-User': '?1', + 'Upgrade-Insecure-Requests': '1', }, - noCache, + noCache: false, }); + // 是否触发风控 if (result.data?.data?.list?.length > 0) { + logger.info('bilibili 新接口') const list = result.data.data.list; return { fromCache: result.fromCache, @@ -75,7 +86,8 @@ const getList = async (options: Options, noCache: boolean): Promise { - const listData = await getList(noCache); - const routeData: RouterData = { +/** + * 定义 Trending 仓库信息的类型 + */ +type RepoInfo = { + owner: string; // 仓库所属者 + repo: string; // 仓库名称 + url: string; // 仓库链接 + description: string; // 仓库描述 + language: string; // 编程语言 + stars: string; // Stars (由于可能包含逗号或者其他符号，这里先用 string 存；实际可自行转 number) + forks: string; // Forks + todayStars?: string | number; // 今日 Star +}; + +type TrendingRepoInfo = { + data: RepoInfo[]; + updateTime: string; + fromCache: boolean; +}; + +type TrendingType = "daily" | "weekly" | "monthly"; + +const typeMap: Record = { + daily: "日榜", + weekly: "周榜", + monthly: "月榜", +}; + +function isTrendingType(value: string): value is TrendingType { + return ["daily", "weekly", "monthly"].includes(value as TrendingType); +} + +export const handleRoute = async (c: ListContext) => { + const typeParam = c.req.query("type") || "daily"; + const type = isTrendingType(typeParam) ? typeParam : "daily"; + + const listData = await getTrendingRepos(type); + + const routeData = { name: "github", - title: "GitHub", - type: "Trending", - description: "See what the GitHub community is most excited about today", - link: "https://github.com/trending", - total: listData.data?.length || 0, - ...listData, + title: "github 趋势", + type: typeMap[type], + params: { + type: { + name: '排行榜分区', + type: typeMap, + }, + }, + link: `https://github.com/trending?since=${type}`, + total: listData?.data?.length || 0, + ...{ + ...listData, + data: listData?.data?.map((v, index)=>{ + return { + id:index, + title: v.repo, + desc: v.description, + hot: v.stars, + ...v + } + }) + } }; return routeData; }; -const getList = async (noCache: boolean) => { - const baseUrl = "https://github.com"; - const result = await get({ - url: `${baseUrl}/trending?spoken_language_code=`, - noCache, - headers: { - userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" - } - }); - - try { - const $ = load(result.data); - const stories: RouterType["github"][] = []; - - $("main .Box div[data-hpc] > article").each((_, el) => { - const a = $(el).find(">h2 a"); - const title = a.text().replace(/\n+/g, "").trim(); - const path = a.attr("href"); - const star = $(el).find("[href$=stargazers]").text().replace(/\s+/g, "").trim(); - const desc = $(el).find(">p").text().replace(/\n+/g, "").trim(); - - if (path && title) { - stories.push({ - id: path.slice(1), // 移除开头的 / - title, - desc, - hot: parseInt(star.replace(/,/g, "")) || undefined, - timestamp: undefined, - url: `${baseUrl}${path}`, - mobileUrl: `${baseUrl}${path}`, - }); - } - }); - +/** + * 爬取 GitHub Trending 列表 + * @param since 可选参数: 'daily' | 'weekly' | 'monthly'，默认值为 'daily' + * @returns Promise 返回包含热门项目信息的数组 + */ +export async function getTrendingRepos( + type: TrendingType | string = "daily", + ttl = 60 * 60 * 24, +): Promise { + const url = `https://github.com/trending?since=${type}`; + // 先从缓存中取 + const cachedData = await getCache(url); + if (cachedData) { + logger.info("💾 [CHCHE] The request is cached"); return { - ...result, - data: stories, + fromCache: true, + updateTime: cachedData.updateTime, + data: (cachedData?.data as RepoInfo[]) || [], }; - } catch (error) { - throw new Error(`Failed to parse GitHub Trending HTML: ${error}`); } -}; \ No newline at end of file + logger.info(`🌐 [GET] ${url}`); + + // 更新请求头 + const headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate, br', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Cache-Control': 'max-age=0', + }; + + // 添加重试逻辑 + const maxRetries = 3; + let lastError; + + for (let i = 0; i < maxRetries; i++) { + try { + // 设置超时时间为 20 秒 + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 20000); + + const response = await fetch(url, { + headers, + signal: controller.signal + }); + clearTimeout(timeout); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const html = await response.text(); + // 1. 加载 HTML + const $ = cheerio.load(html); + // 2. 存储结果的数组 + const results: RepoInfo[] = []; + // 3. 遍历每个 article.Box-row + $("article.Box-row").each((_, el) => { + const $el = $(el); + // 仓库标题和链接 (在

> 里) + const $repoAnchor = $el.find("h2 a"); + // 可能出现 "owner / repo" 这种文本 + // eg: "owner / repo" + const fullNameText = $repoAnchor + .text() + .trim() + // 可能有多余空格，可以再做一次 split + // "owner / repo" => ["owner", "repo"] + .replace(/\r?\n/g, "") // 去掉换行 + .replace(/\s+/g, " ") // 多空格处理 + .split("/") + .map((s) => s.trim()); + + const owner = fullNameText[0] || ""; + const repoName = fullNameText[1] || ""; + + // href 即仓库链接 + const repoUrl = "https://github.com" + $repoAnchor.attr("href"); + + // 仓库描述 (

) + const description = $el.find("p.col-9.color-fg-muted").text().trim(); + + // 语言 () + const language = $el.find('[itemprop="programmingLanguage"]').text().trim(); + + const starsText = $el.find('a[href$="/stargazers"]').text().trim(); + + const forksText = $el.find(`a[href$="/forks"]`).text().trim(); + + // 整合 + results.push({ + owner, + repo: repoName, + url: repoUrl || "", + description, + language, + stars: starsText, + forks: forksText, + }); + }); + + const updateTime = new Date().toISOString(); + const data = results; + + await setCache(url, { data, updateTime }, ttl); + // 返回数据 + logger.info(`✅ [${response?.status}] 请求成功！`); + return { fromCache: false, updateTime, data }; + } catch (error: Error | unknown) { + lastError = error; + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + logger.error(`❌ [ERROR] 第 ${i + 1} 请求失败: ${errorMessage}`); + + // 如果是最后一次重试，则抛出错误 + if (i === maxRetries - 1) { + logger.error("❌ [ERROR] 所有尝试请求失败！"); + throw lastError; + } + + // 等待一段时间后重试 (1秒、2秒、4秒...) + await new Promise(resolve => setTimeout(resolve, Math.pow(2, i) * 1000)); + continue; + } + } + + throw new Error("请求失败！"); +} diff --git a/src/routes/juejin.ts b/src/routes/juejin.ts index 0d9caf1..ca15fdf 100644 --- a/src/routes/juejin.ts +++ b/src/routes/juejin.ts @@ -1,13 +1,55 @@ -import type { RouterData } from "../types.js"; +import type { ListContext, RouterData } from "../types.js"; import type { RouterType } from "../router.types.js"; import { get } from "../utils/getData.js"; -export const handleRoute = async (_: undefined, noCache: boolean) => { - const listData = await getList(noCache); + +const headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + 'Accept-Encoding': 'gzip, deflate, br', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Sec-Ch-Ua': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"', + 'Sec-Ch-Ua-Mobile': '?0', + 'Sec-Ch-Ua-Platform': '"Windows"', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'same-origin', + 'Sec-Fetch-User': '?1', + 'Upgrade-Insecure-Requests': '1', +} + +const category_url = 'https://api.juejin.cn/tag_api/v1/query_category_briefs' +const getCategory = async()=>{ + const res = await get({ + url: category_url, + headers + }) + const data = res?.data?.data || [] + const typeObj: Record = {} + typeObj['1'] = '综合' + data.forEach((c: { category_id: string; category_name: string }) => { + typeObj[c.category_id] = c.category_name + }) + + return typeObj +} + +export const handleRoute = async (c: ListContext, noCache: boolean) => { + const type = c.req.query("type") || 1; + const listData = await getList(noCache, type); + const typeMaps = await getCategory() const routeData: RouterData = { name: "juejin", title: "稀土掘金", type: "文章榜", + params: { + type: { + name: "排行榜分区", + type: typeMaps, + }, + }, link: "https://juejin.cn/hot/articles", total: listData.data?.length || 0, ...listData, @@ -15,9 +57,9 @@ export const handleRoute = async (_: undefined, noCache: boolean) => { return routeData; }; -const getList = async (noCache: boolean) => { - const url = `https://api.juejin.cn/content_api/v1/content/article_rank?category_id=1&type=hot`; - const result = await get({ url, noCache }); +const getList = async (noCache: boolean, type: number | string = 1) => { + const url = `https://api.juejin.cn/content_api/v1/content/article_rank?category_id=${type}&type=hot`; + const result = await get({ url, noCache, headers }); const list = result.data.data; return { ...result, diff --git a/src/routes/weread.ts b/src/routes/weread.ts index 0bce300..26ade23 100644 --- a/src/routes/weread.ts +++ b/src/routes/weread.ts @@ -1,15 +1,30 @@ -import type { RouterData } from "../types.js"; +import type { ListContext, RouterData } from "../types.js"; import type { RouterType } from "../router.types.js"; import { get } from "../utils/getData.js"; import getWereadID from "../utils/getToken/weread.js"; import { getTime } from "../utils/getTime.js"; -export const handleRoute = async (_: undefined, noCache: boolean) => { - const listData = await getList(noCache); +const typeMap: Record = { + rising: "飙升榜", + hot_search: "热搜榜", + newbook: "新书榜", + general_novel_rising: "小说榜", + all: "总榜", +}; + +export const handleRoute = async (c: ListContext, noCache: boolean) => { + const type = c.req.query("type") || "rising"; + const listData = await getList(noCache, type); const routeData: RouterData = { name: "weread", title: "微信读书", - type: "飙升榜", + type: `${typeMap[type]}`, + params: { + type: { + name: "排行榜分区", + type: typeMap, + }, + }, link: "https://weread.qq.com/", total: listData.data?.length || 0, ...listData, @@ -17,8 +32,8 @@ export const handleRoute = async (_: undefined, noCache: boolean) => { return routeData; }; -const getList = async (noCache: boolean) => { - const url = `https://weread.qq.com/web/bookListInCategory/rising?rank=1`; +const getList = async (noCache: boolean, type='rising') => { + const url = `https://weread.qq.com/web/bookListInCategory/${type}?rank=1`; const result = await get({ url, noCache,