406 lines
13 KiB
TypeScript
406 lines
13 KiB
TypeScript
|
// 抓取主流程：遍历订阅源 → 抓取 → 解析 → 去重 → 通知 → 写 KV
|
|||
|
|
// 迁自 check-feeds.js 的 resolveFeed + main，作为 CF Cron 的 scheduled handler。
|
|||
|
|
|
|||
|
|
import type { Env } from '../../types';
|
|||
|
|
import { fetchUrl } from './fetch';
|
|||
|
|
import { discoverFeedUrl, tryCommonFeedPaths } from './discover';
|
|||
|
|
import {
|
|||
|
|
parseFeedXml,
|
|||
|
|
parseJsonFeed,
|
|||
|
|
parseCustomJson,
|
|||
|
|
isJsonResponse,
|
|||
|
|
type Article,
|
|||
|
|
type FeedConfig,
|
|||
|
|
type ParsedFeed,
|
|||
|
|
} from './parse';
|
|||
|
|
import { sendFeishuNotification } from './notify';
|
|||
|
|
import { kvGetJson, kvPutJson } from './util';
|
|||
|
|
|
|||
|
|
const MAX_SEEN = 500;
|
|||
|
|
|
|||
|
|
/** 单源抓取超时（原来 30s：一个挂掉的源能把整批拖到 4 分钟以上） */
|
|||
|
|
const FEED_TIMEOUT = 8000;
|
|||
|
|
|
|||
|
|
/** 连续失败这么多次就进入退避期，不再每次抓（否则每小时白等 8 秒） */
|
|||
|
|
const FAIL_THRESHOLD = 3;
|
|||
|
|
/** 退避时长：6 小时（期间只在其它源都抓完后才可能被跳过） */
|
|||
|
|
const FAIL_BACKOFF_MS = 6 * 3600 * 1000;
|
|||
|
|
|
|||
|
|
interface FailRecord { n: number; at: number }
|
|||
|
|
|
|||
|
|
/** 抓取 + 解析单个订阅源（含 feed 自动发现）。失败返回 null。 */
|
|||
|
|
async function resolveFeed(env: Env, feedConfig: FeedConfig): Promise<ParsedFeed | null> {
|
|||
|
|
const url = feedConfig.url;
|
|||
|
|
const format = feedConfig.format || 'auto';
|
|||
|
|
const forceProxy = feedConfig.proxy === true;
|
|||
|
|
|
|||
|
|
let response;
|
|||
|
|
try {
|
|||
|
|
response = await fetchUrl(env, url, FEED_TIMEOUT, forceProxy);
|
|||
|
|
} catch {
|
|||
|
|
// 直连 + 代理都失败：不再重复抓同一个地址（原来会白等一个超时），
|
|||
|
|
// 直接把常见的 feed 路径猜一遍就放弃。
|
|||
|
|
response = undefined;
|
|||
|
|
}
|
|||
|
|
if (!response) {
|
|||
|
|
const guessedUrl = await tryCommonFeedPaths(env, url);
|
|||
|
|
if (!guessedUrl) return null;
|
|||
|
|
try {
|
|||
|
|
response = await fetchUrl(env, guessedUrl, FEED_TIMEOUT, forceProxy);
|
|||
|
|
} catch {
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const { text, contentType } = response;
|
|||
|
|
const isJson = isJsonResponse(text, contentType);
|
|||
|
|
|
|||
|
|
if (format === 'json' || (format === 'auto' && isJson)) {
|
|||
|
|
try {
|
|||
|
|
const json = JSON.parse(text);
|
|||
|
|
if (
|
|||
|
|
json.version?.includes('jsonfeed.org') ||
|
|||
|
|
(json.items && Array.isArray(json.items) && !feedConfig.path)
|
|||
|
|
) {
|
|||
|
|
return parseJsonFeed(json, url);
|
|||
|
|
}
|
|||
|
|
return parseCustomJson(json, feedConfig);
|
|||
|
|
} catch {
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (
|
|||
|
|
text.includes('<rss') ||
|
|||
|
|
text.includes('<feed') ||
|
|||
|
|
text.includes('<channel') ||
|
|||
|
|
text.includes('<entry')
|
|||
|
|
) {
|
|||
|
|
return parseFeedXml(text, url);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// HTML 发现
|
|||
|
|
const feedUrl = discoverFeedUrl(text, url);
|
|||
|
|
if (feedUrl) {
|
|||
|
|
const feedResp = await fetchUrl(env, feedUrl, FEED_TIMEOUT, forceProxy);
|
|||
|
|
if (feedResp) {
|
|||
|
|
if (isJsonResponse(feedResp.text, feedResp.contentType)) {
|
|||
|
|
try {
|
|||
|
|
const json = JSON.parse(feedResp.text);
|
|||
|
|
if (json.version?.includes('jsonfeed.org') || json.items) return parseJsonFeed(json, feedUrl);
|
|||
|
|
return parseCustomJson(json, { url: feedUrl });
|
|||
|
|
} catch {
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return parseFeedXml(feedResp.text, feedUrl);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const guessedUrl = await tryCommonFeedPaths(env, url);
|
|||
|
|
if (guessedUrl) {
|
|||
|
|
const guessResp = await fetchUrl(env, guessedUrl, FEED_TIMEOUT, forceProxy);
|
|||
|
|
if (guessResp) {
|
|||
|
|
if (isJsonResponse(guessResp.text, guessResp.contentType)) {
|
|||
|
|
try {
|
|||
|
|
const json = JSON.parse(guessResp.text);
|
|||
|
|
if (json.version?.includes('jsonfeed.org') || json.items) return parseJsonFeed(json, guessedUrl);
|
|||
|
|
return parseCustomJson(json, { url: guessedUrl });
|
|||
|
|
} catch {
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return parseFeedXml(guessResp.text, guessedUrl);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** 从远程 JSON / KV 加载订阅源列表 */
|
|||
|
|
async function loadFeeds(env: Env): Promise<FeedConfig[]> {
|
|||
|
|
// 优先从 KV 读（管理页维护的最新订阅源）
|
|||
|
|
const kvFeeds = await kvGetJson<{ feeds: FeedConfig[] }>(env, 'feeds_config', { feeds: [] });
|
|||
|
|
if (kvFeeds.feeds && kvFeeds.feeds.length > 0) {
|
|||
|
|
return kvFeeds.feeds.map((f) => (typeof f === 'string' ? { url: f } : f));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 回退：FEEDS_URL 远程 JSON
|
|||
|
|
if (env.FEEDS_URL) {
|
|||
|
|
try {
|
|||
|
|
const res = await fetch(env.FEEDS_URL, {
|
|||
|
|
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; RSSBot/1.0)' },
|
|||
|
|
});
|
|||
|
|
if (res.ok) {
|
|||
|
|
const json = (await res.json()) as any;
|
|||
|
|
if (Array.isArray(json)) {
|
|||
|
|
return json.map((item: string | FeedConfig) =>
|
|||
|
|
typeof item === 'string' ? { url: item } : item,
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
if (json.feeds && Array.isArray(json.feeds)) {
|
|||
|
|
return json.feeds.map((item: string | FeedConfig) =>
|
|||
|
|
typeof item === 'string' ? { url: item } : item,
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch {
|
|||
|
|
// 忽略，走空
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return [];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
interface SeenData {
|
|||
|
|
lastCheck: string | null;
|
|||
|
|
articles: Record<string, string>;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** scheduled 入口：每天抓取一次 */
|
|||
|
|
/**
|
|||
|
|
* 抓取一批订阅源。
|
|||
|
|
* 免费版 Worker 每次调用限 50 个 subrequest，66 个源全量一把抓会炸，
|
|||
|
|
* 所以按 offset/limit 分批轮转（一天两批跑完全部）；latest 按 feed 维度合并写入。
|
|||
|
|
*/
|
|||
|
|
export interface CronStats {
|
|||
|
|
/** 本批计划抓取的源数 */
|
|||
|
|
batch: number;
|
|||
|
|
/** 订阅源总数 */
|
|||
|
|
total: number;
|
|||
|
|
ok: number;
|
|||
|
|
failed: number;
|
|||
|
|
/** 本批抓到的文章条数（去重前） */
|
|||
|
|
articles: number;
|
|||
|
|
/** 判定为新文章的条数 */
|
|||
|
|
newArticles: number;
|
|||
|
|
/** 是否 dry-run（不通知、不写 KV） */
|
|||
|
|
dry: boolean;
|
|||
|
|
durationMs: number;
|
|||
|
|
/** 因连续失败处于退避期、本次跳过的源数 */
|
|||
|
|
skipped: number;
|
|||
|
|
/** 最慢的几个源，用于排查拖后腿的 feed */
|
|||
|
|
slowest: { url: string; ms: number }[];
|
|||
|
|
/** 本次失败的源（最多列 10 个，便于排查） */
|
|||
|
|
failedUrls: string[];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export async function runCron(
|
|||
|
|
env: Env,
|
|||
|
|
opts?: { offset?: number; limit?: number; dry?: boolean; rotate?: boolean },
|
|||
|
|
): Promise<CronStats> {
|
|||
|
|
const startedAt = Date.now();
|
|||
|
|
const dry = opts?.dry === true;
|
|||
|
|
const rotate = opts?.rotate === true;
|
|||
|
|
const offset = Math.max(opts?.offset ?? 0, 0);
|
|||
|
|
const batchLimit = Math.max(opts?.limit ?? 0, 0);
|
|||
|
|
const timings: { url: string; ms: number }[] = [];
|
|||
|
|
let okCount = 0;
|
|||
|
|
let failedCount = 0;
|
|||
|
|
let articleCount = 0;
|
|||
|
|
let skippedCount = 0;
|
|||
|
|
const failedUrls: string[] = [];
|
|||
|
|
const emptyStats: CronStats = {
|
|||
|
|
batch: 0, total: 0, ok: 0, failed: 0, articles: 0, newArticles: 0,
|
|||
|
|
dry, durationMs: 0, skipped: 0, slowest: [], failedUrls: [],
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
console.log('🔍 开始检查 RSS Feed...');
|
|||
|
|
|
|||
|
|
const allFeeds = await loadFeeds(env);
|
|||
|
|
if (allFeeds.length === 0) {
|
|||
|
|
console.log('⚠️ 没有可用的订阅源');
|
|||
|
|
return emptyStats;
|
|||
|
|
}
|
|||
|
|
// 失败退避表：连续失败 >= 阈值的源，在退避期内不再每次白等超时
|
|||
|
|
const failMap = await kvGetJson<Record<string, FailRecord>>(env, 'rss_fail', {});
|
|||
|
|
const nowTs = Date.now();
|
|||
|
|
const inBackoff = (u: string): boolean => {
|
|||
|
|
const r = failMap[u];
|
|||
|
|
return !!r && r.n >= FAIL_THRESHOLD && nowTs - r.at < FAIL_BACKOFF_MS;
|
|||
|
|
};
|
|||
|
|
const markFail = (u: string): void => {
|
|||
|
|
const r = failMap[u];
|
|||
|
|
failMap[u] = { n: (r?.n ?? 0) + 1, at: nowTs };
|
|||
|
|
};
|
|||
|
|
const clearFail = (u: string): void => {
|
|||
|
|
if (failMap[u]) delete failMap[u];
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
console.log(`📋 共 ${allFeeds.length} 个订阅源`);
|
|||
|
|
for (const u of Object.keys(failMap)) if (!inBackoff(u)) delete failMap[u]; // 退避期满自动重试
|
|||
|
|
|
|||
|
|
// 环形取本批：offset 开始取 limit 个（limit=0 表示全量）。
|
|||
|
|
// rotate=true 时从 KV 里的游标继续（定时任务用，保证每轮都能覆盖到所有源）。
|
|||
|
|
const feeds: FeedConfig[] = [];
|
|||
|
|
let cursorAfter = offset;
|
|||
|
|
if (batchLimit > 0 && allFeeds.length > batchLimit) {
|
|||
|
|
let start = offset % allFeeds.length;
|
|||
|
|
if (rotate && !dry) {
|
|||
|
|
const saved = await env.RSS_KV.get('rss_cursor');
|
|||
|
|
const n = parseInt(saved || '0', 10);
|
|||
|
|
if (Number.isFinite(n)) start = ((n % allFeeds.length) + allFeeds.length) % allFeeds.length;
|
|||
|
|
}
|
|||
|
|
let examined = 0;
|
|||
|
|
let i = start;
|
|||
|
|
while (feeds.length < batchLimit && examined < allFeeds.length) {
|
|||
|
|
const f = allFeeds[i % allFeeds.length];
|
|||
|
|
i += 1;
|
|||
|
|
examined += 1;
|
|||
|
|
if (inBackoff(f.url)) {
|
|||
|
|
skippedCount += 1;
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
feeds.push(f);
|
|||
|
|
}
|
|||
|
|
cursorAfter = i % allFeeds.length;
|
|||
|
|
if (rotate && !dry) await env.RSS_KV.put('rss_cursor', String(cursorAfter));
|
|||
|
|
console.log(
|
|||
|
|
`📋 本批 ${feeds.length} 个（cursor=${start}${skippedCount ? `，跳过退避中 ${skippedCount} 个` : ''}）`,
|
|||
|
|
);
|
|||
|
|
} else {
|
|||
|
|
feeds.push(...allFeeds);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const seenData = await kvGetJson<SeenData>(env, 'seen_articles', {
|
|||
|
|
lastCheck: null,
|
|||
|
|
articles: {},
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
const allNewArticles: Article[] = [];
|
|||
|
|
const allFetchedArticles: (Article & { siteUrl: string })[] = [];
|
|||
|
|
|
|||
|
|
for (const feedConfig of feeds) {
|
|||
|
|
const url = feedConfig.url;
|
|||
|
|
console.log(`🔍 检查: ${url}`);
|
|||
|
|
const t0 = Date.now();
|
|||
|
|
try {
|
|||
|
|
const result = await resolveFeed(env, feedConfig);
|
|||
|
|
timings.push({ url, ms: Date.now() - t0 });
|
|||
|
|
if (!result) {
|
|||
|
|
failedCount++;
|
|||
|
|
markFail(url);
|
|||
|
|
if (failedUrls.length < 10) failedUrls.push(url);
|
|||
|
|
console.log(' ❌ 未找到 Feed');
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
okCount++;
|
|||
|
|
articleCount += result.articles.length;
|
|||
|
|
clearFail(url);
|
|||
|
|
|
|||
|
|
const { feedTitle, articles } = result;
|
|||
|
|
console.log(` 📰 ${feedTitle || '未知'} - 共 ${articles.length} 篇`);
|
|||
|
|
|
|||
|
|
const recentArticles = articles.slice(0, 10);
|
|||
|
|
allFetchedArticles.push(
|
|||
|
|
...recentArticles.map((a) => ({ ...a, feedTitle: feedTitle || '未知', siteUrl: url })),
|
|||
|
|
);
|
|||
|
|
const newOnes = recentArticles.filter((a) => !seenData.articles[a.link]);
|
|||
|
|
|
|||
|
|
if (newOnes.length > 0) {
|
|||
|
|
console.log(` 🆕 发现 ${newOnes.length} 篇新文章`);
|
|||
|
|
allNewArticles.push(...newOnes);
|
|||
|
|
} else {
|
|||
|
|
console.log(' ✅ 无新文章');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for (const a of recentArticles) {
|
|||
|
|
seenData.articles[a.link] = a.pubDate;
|
|||
|
|
}
|
|||
|
|
} catch (err) {
|
|||
|
|
timings.push({ url, ms: Date.now() - t0 });
|
|||
|
|
failedCount++;
|
|||
|
|
markFail(url);
|
|||
|
|
if (failedUrls.length < 10) failedUrls.push(url);
|
|||
|
|
console.error(` ❌ 抓取失败: ${(err as Error).message}`);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 通知（仅最近 24h 内的）
|
|||
|
|
const oneDayAgo = Date.now() - 86400000;
|
|||
|
|
const recentNew = allNewArticles.filter(
|
|||
|
|
(a) => new Date(a.pubDate).getTime() > oneDayAgo || !seenData.lastCheck,
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
if (dry) {
|
|||
|
|
console.log(`🧪 dry-run：跳过通知（本应通知 ${recentNew.length} 篇）与 KV 写入`);
|
|||
|
|
} else if (recentNew.length > 0) {
|
|||
|
|
await sendFeishuNotification(env, recentNew);
|
|||
|
|
} else if (allNewArticles.length > 0) {
|
|||
|
|
console.log('ℹ️ 发现新文章但超过 24 小时，不推送');
|
|||
|
|
} else {
|
|||
|
|
console.log('✅ 所有博客均无新文章');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 写 latest 缓存（供 /api/results、/api/articles 读）
|
|||
|
|
// 分批模式：本批源的条目替换旧缓存里的同源条目，其他源的保留
|
|||
|
|
const byFeed: Record<string, { articles: Article[]; siteUrl: string }> = {};
|
|||
|
|
for (const a of allFetchedArticles) {
|
|||
|
|
const key = a.feedTitle || '未知';
|
|||
|
|
if (!byFeed[key]) byFeed[key] = { articles: [], siteUrl: a.siteUrl || '' };
|
|||
|
|
byFeed[key].articles.push({ title: a.title, link: a.link, pubDate: a.pubDate, author: a.author, feedTitle: key });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!dry) {
|
|||
|
|
try {
|
|||
|
|
await kvPutJson(env, 'rss_fail', failMap);
|
|||
|
|
} catch {
|
|||
|
|
/* 失败表写失败不影响主流程 */
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const durationMs = Date.now() - startedAt;
|
|||
|
|
const stats: CronStats = {
|
|||
|
|
batch: feeds.length,
|
|||
|
|
total: allFeeds.length,
|
|||
|
|
ok: okCount,
|
|||
|
|
failed: failedCount,
|
|||
|
|
articles: articleCount,
|
|||
|
|
newArticles: allNewArticles.length,
|
|||
|
|
dry,
|
|||
|
|
durationMs,
|
|||
|
|
skipped: skippedCount,
|
|||
|
|
slowest: timings.sort((a, b) => b.ms - a.ms).slice(0, 5),
|
|||
|
|
failedUrls,
|
|||
|
|
};
|
|||
|
|
if (dry) return stats;
|
|||
|
|
|
|||
|
|
const prev = await kvGetJson<{ timestamp: string | null; total: number; feeds: { name: string; siteUrl: string; favicon: string; articles: Article[] }[] }>(
|
|||
|
|
env,
|
|||
|
|
'latest',
|
|||
|
|
{ timestamp: null, total: 0, feeds: [] },
|
|||
|
|
);
|
|||
|
|
const batchUrls = new Set(feeds.map((f) => f.url));
|
|||
|
|
const kept = prev.feeds.filter((f) => !batchUrls.has(f.siteUrl || f.name));
|
|||
|
|
const mergedFeeds = [
|
|||
|
|
...kept,
|
|||
|
|
...Object.entries(byFeed).map(([name, { articles, siteUrl }]) => ({
|
|||
|
|
name,
|
|||
|
|
siteUrl,
|
|||
|
|
// 必须写绝对地址：友圈页在 usj.cc 上，相对路径 /api/* 会 404 → 退化成第三方默认头像
|
|||
|
|
favicon: siteUrl
|
|||
|
|
? `${env.PUBLIC_API_BASE || 'https://api.200181.xyz'}/api/favicon?url=${encodeURIComponent(siteUrl)}`
|
|||
|
|
: '',
|
|||
|
|
articles,
|
|||
|
|
})),
|
|||
|
|
];
|
|||
|
|
const mergedTotal = mergedFeeds.reduce((n, f) => n + f.articles.length, 0);
|
|||
|
|
|
|||
|
|
await kvPutJson(env, 'latest', {
|
|||
|
|
timestamp: new Date().toISOString(),
|
|||
|
|
total: mergedTotal,
|
|||
|
|
feeds: mergedFeeds,
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// 清理 + 保存去重记录
|
|||
|
|
const entries = Object.entries(seenData.articles);
|
|||
|
|
if (entries.length > MAX_SEEN) {
|
|||
|
|
entries.sort((a, b) => new Date(b[1]).getTime() - new Date(a[1]).getTime());
|
|||
|
|
seenData.articles = Object.fromEntries(entries.slice(0, MAX_SEEN));
|
|||
|
|
}
|
|||
|
|
seenData.lastCheck = new Date().toISOString();
|
|||
|
|
await kvPutJson(env, 'seen_articles', seenData);
|
|||
|
|
|
|||
|
|
console.log(`📝 已保存去重记录 (${Object.keys(seenData.articles).length} 条)`);
|
|||
|
|
return stats;
|
|||
|
|
}
|