From fdeb25805cc2d90bd817b6d0bbb67aec345e57d8 Mon Sep 17 00:00:00 2001 From: SepComet <202308010230@stu.csust.edu.cn> Date: Tue, 5 May 2026 09:53:51 +0800 Subject: [PATCH] feat: add gitea sync and env-aware content sync --- .env.example | 6 + package.json | 2 +- scripts/fetch-gitea.ts | 411 +++++++++++++++++++++++++++++ scripts/sync-content.ts | 108 ++++---- src/components/GiteaActivity.astro | 2 +- src/components/ProjectCard.astro | 4 +- src/pages/index.astro | 2 +- 7 files changed, 480 insertions(+), 55 deletions(-) create mode 100644 scripts/fetch-gitea.ts diff --git a/.env.example b/.env.example index ebf538f..b7676a9 100644 --- a/.env.example +++ b/.env.example @@ -38,3 +38,9 @@ STRICT_SYNC=false # ----------------------------------------------------------------------------- SEAFILE_MIRROR_DOWNLOADS=false DOWNLOADS_OUTPUT_DIR=public/downloads + +GITEA_ACTIVITY_DAYS=70 +GITEA_ACTIVITY_PER_DAY_LIMIT=50 +GITEA_RECENT_ITEM_LIMIT=8 +GITEA_REQUEST_TIMEOUT_MS=15000 +GITEA_REQUEST_CONCURRENCY=5 diff --git a/package.json b/package.json index b8eefe5..88d5c13 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ }, "scripts": { "dev": "astro dev", - "content:sync": "node --experimental-strip-types ./scripts/sync-content.ts", + "content:sync": "node --env-file-if-exists=.env --experimental-strip-types ./scripts/sync-content.ts", "build": "astro build", "rebuild": "npm run content:sync && npm run build", "preview": "astro preview", diff --git a/scripts/fetch-gitea.ts b/scripts/fetch-gitea.ts new file mode 100644 index 0000000..4998b14 --- /dev/null +++ b/scripts/fetch-gitea.ts @@ -0,0 +1,411 @@ +import process from 'node:process'; + +type ProjectSeed = { + name: string; + description: string; + gitea_repo: string; + cover_image: string; + demo_video?: string; + download_link?: string; + tags: string[]; + featured: boolean; +}; + +export type GeneratedProject = ProjectSeed & { + repo_url?: string; + updated_at: string; + source: 'seed' | 'gitea' | 'gitea+seafile'; +}; + +export type ActivityDay = { + date: string; + count: number; +}; + +export type RecentActivity = { + type: string; + repo: string; + message: string; + url: string; + time: string; +}; + +export type GeneratedGiteaActivity = { + updatedAt: string; + source: 'placeholder' | 'gitea'; + itemCount: number; + days: ActivityDay[]; + recent: RecentActivity[]; +}; + +export type GiteaSyncConfig = { + baseUrl: string; + token: string; + username: string; + activityDays?: number; + activityPerDayLimit?: number; + recentItemLimit?: number; + requestTimeoutMs?: number; + requestConcurrency?: number; + strict?: boolean; +}; + +type GiteaRepositoryResponse = { + description?: string; + full_name?: string; + html_url?: string; + updated_at?: string; + topics?: string[]; +}; + +type GiteaActivityFeedResponse = { + op_type?: string; + content?: string; + created?: string; + ref_name?: string; + repo?: { + full_name?: string; + name?: string; + html_url?: string; + }; + comment?: { + html_url?: string; + body?: string; + }; + issue?: { + html_url?: string; + title?: string; + }; + pull_request?: { + html_url?: string; + title?: string; + }; +}; + +export async function fetchGiteaSnapshot(input: { + config: GiteaSyncConfig; + projectSeeds: ProjectSeed[]; + syncedAt: string; +}) { + const { config, projectSeeds, syncedAt } = input; + const client = createGiteaClient(config); + + const [repoResults, userActivities] = await Promise.all([ + Promise.all( + projectSeeds.map((seed) => enrichProjectFromGitea({ client, config, project: seed, syncedAt })), + ), + fetchUserActivities({ client, config }), + ]); + + const projects = repoResults.map((result) => result.project); + const activity = buildActivitySnapshot({ + syncedAt, + recentItemLimit: config.recentItemLimit ?? 8, + activityDays: config.activityDays ?? 70, + feeds: userActivities, + }); + + return { projects, activity }; +} + +async function enrichProjectFromGitea(input: { + client: ReturnType; + config: GiteaSyncConfig; + project: ProjectSeed; + syncedAt: string; +}): Promise<{ project: GeneratedProject }> { + const { client, config, project, syncedAt } = input; + const repoParts = splitRepoFullName(project.gitea_repo); + const fallbackRepoUrl = buildRepoUrl(config.baseUrl, project.gitea_repo); + + if (!repoParts) { + return { project: toSeedProject(project, syncedAt, fallbackRepoUrl) }; + } + + try { + const repo = await client.getJson( + `/api/v1/repos/${repoParts.owner}/${repoParts.repo}`, + ); + + return { + project: { + ...project, + description: repo.description?.trim() || project.description, + repo_url: repo.html_url || fallbackRepoUrl, + tags: mergeTags(project.tags, repo.topics ?? []), + updated_at: repo.updated_at || syncedAt, + source: project.download_link ? 'gitea+seafile' : 'gitea', + }, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const logLine = `[fetch-gitea] failed to sync ${project.gitea_repo}: ${message}`; + + if (config.strict) { + throw new Error(logLine); + } + + console.warn(logLine); + + return { + project: toSeedProject(project, syncedAt, fallbackRepoUrl), + }; + } +} + +async function fetchUserActivities(input: { + client: ReturnType; + config: GiteaSyncConfig; +}): Promise { + const { client, config } = input; + const activityDays = config.activityDays ?? 70; + const limit = config.activityPerDayLimit ?? 50; + const concurrency = Math.max(1, config.requestConcurrency ?? 5); + const dates = Array.from({ length: activityDays }, (_, index) => { + const date = new Date(); + date.setUTCDate(date.getUTCDate() - index); + return date.toISOString().slice(0, 10); + }); + + const results = await mapWithConcurrency(dates, concurrency, async (date) => { + try { + const feeds = await client.getJson( + `/api/v1/users/${config.username}/activities/feeds`, + { + 'only-performed-by': 'true', + date, + limit: String(limit), + }, + ); + + return feeds.map((feed) => + mapActivityFeed(feed, { + fallbackRepo: config.username, + fallbackRepoUrl: `${trimTrailingSlash(config.baseUrl)}/${config.username}`, + }), + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const logLine = `[fetch-gitea] failed to sync user activity for ${config.username} on ${date}: ${message}`; + + if (config.strict) { + throw new Error(logLine); + } + + console.warn(logLine); + return []; + } + }); + + return results.flat(); +} + +function buildActivitySnapshot(input: { + syncedAt: string; + recentItemLimit: number; + activityDays: number; + feeds: RecentActivity[]; +}): GeneratedGiteaActivity { + const sortedFeeds = [...input.feeds] + .filter((item) => item.time) + .sort((a, b) => Date.parse(b.time) - Date.parse(a.time)); + + const recent = dedupeActivities(sortedFeeds).slice(0, input.recentItemLimit); + const days = buildHeatmapDays(sortedFeeds, input.activityDays); + + return { + updatedAt: input.syncedAt, + source: sortedFeeds.length > 0 ? 'gitea' : 'placeholder', + itemCount: sortedFeeds.length, + days, + recent, + }; +} + +function buildHeatmapDays(activities: RecentActivity[], dayCount: number): ActivityDay[] { + const now = new Date(); + const counts = new Map(); + + for (const item of activities) { + const dateKey = toDateKey(item.time); + if (!dateKey) { + continue; + } + counts.set(dateKey, (counts.get(dateKey) ?? 0) + 1); + } + + const days: ActivityDay[] = []; + for (let offset = dayCount - 1; offset >= 0; offset -= 1) { + const date = new Date(now); + date.setUTCDate(date.getUTCDate() - offset); + const dateKey = date.toISOString().slice(0, 10); + days.push({ + date: dateKey, + count: counts.get(dateKey) ?? 0, + }); + } + + return days; +} + +function dedupeActivities(activities: RecentActivity[]) { + const seen = new Set(); + return activities.filter((item) => { + const key = `${item.repo}|${item.time}|${item.type}|${item.url}|${item.message}`; + if (seen.has(key)) { + return false; + } + seen.add(key); + return true; + }); +} + +function mapActivityFeed( + feed: GiteaActivityFeedResponse, + fallback: { fallbackRepo: string; fallbackRepoUrl: string }, +): RecentActivity { + const repo = feed.repo?.full_name || fallback.fallbackRepo; + const type = normalizeActivityType(feed.op_type); + const message = + clean(feed.content) || + clean(feed.comment?.body) || + clean(feed.issue?.title) || + clean(feed.pull_request?.title) || + [feed.op_type, feed.ref_name].filter(Boolean).join(' ').trim() || + 'Repository activity'; + const url = + feed.comment?.html_url || + feed.issue?.html_url || + feed.pull_request?.html_url || + feed.repo?.html_url || + fallback.fallbackRepoUrl; + const time = feed.created || new Date().toISOString(); + + return { + type, + repo, + message, + url, + time, + }; +} + +async function mapWithConcurrency( + items: T[], + concurrency: number, + mapper: (item: T, index: number) => Promise, +): Promise { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const currentIndex = nextIndex; + nextIndex += 1; + results[currentIndex] = await mapper(items[currentIndex], currentIndex); + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker())); + return results; +} + +function normalizeActivityType(value?: string) { + return value?.trim().toLowerCase().replace(/\s+/g, '_') || 'activity'; +} + +function toSeedProject(project: ProjectSeed, syncedAt: string, fallbackRepoUrl: string): GeneratedProject { + return { + ...project, + repo_url: fallbackRepoUrl, + updated_at: syncedAt, + source: 'seed', + }; +} + +function mergeTags(seedTags: string[], topics: string[]) { + const merged = new Set([...seedTags, ...topics].map((tag) => tag.trim()).filter(Boolean)); + return [...merged]; +} + +function splitRepoFullName(value: string) { + const [owner, repo] = value.split('/'); + if (!owner || !repo) { + return null; + } + return { owner, repo }; +} + +function buildRepoUrl(baseUrl: string, repoFullName: string) { + return `${trimTrailingSlash(baseUrl)}/${repoFullName}`; +} + +function toDateKey(value: string) { + const time = Date.parse(value); + if (Number.isNaN(time)) { + return ''; + } + return new Date(time).toISOString().slice(0, 10); +} + +function clean(value?: string) { + return value?.replace(/\s+/g, ' ').trim() || ''; +} + +function trimTrailingSlash(value: string) { + return value.replace(/\/+$/, ''); +} + +function createGiteaClient(config: GiteaSyncConfig) { + const headers = new Headers({ + accept: 'application/json', + }); + + if (config.token) { + headers.set('Authorization', `token ${config.token}`); + } + + return { + async getJson(pathname: string, query?: Record): Promise { + const url = new URL(pathname, `${trimTrailingSlash(config.baseUrl)}/`); + if (query) { + for (const [key, value] of Object.entries(query)) { + url.searchParams.set(key, value); + } + } + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), config.requestTimeoutMs ?? 15000); + + try { + const response = await fetch(url, { + method: 'GET', + headers, + signal: controller.signal, + }); + + if (!response.ok) { + const body = await safeReadText(response); + throw new Error(`HTTP ${response.status} ${response.statusText}${body ? ` - ${body}` : ''}`); + } + + return (await response.json()) as T; + } finally { + clearTimeout(timeout); + } + }, + }; +} + +async function safeReadText(response: Response) { + try { + const text = await response.text(); + return text.slice(0, 300); + } catch { + return ''; + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + console.log('[fetch-gitea] This module is intended to be imported by scripts/sync-content.ts'); +} diff --git a/scripts/sync-content.ts b/scripts/sync-content.ts index 02fafc5..42406cb 100644 --- a/scripts/sync-content.ts +++ b/scripts/sync-content.ts @@ -1,6 +1,7 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; import process from 'node:process'; +import { fetchGiteaSnapshot, type GeneratedGiteaActivity, type GeneratedProject } from './fetch-gitea.ts'; type ProjectSeed = { name: string; @@ -20,39 +21,12 @@ type ShareSeed = { time: string; }; -type GeneratedProject = ProjectSeed & { - repo_url?: string; - updated_at: string; - source: 'seed' | 'gitea' | 'gitea+seafile'; -}; - type GeneratedShare = ShareSeed & { size?: number; source: 'seed' | 'seafile'; updated_at: string; }; -type ActivityDay = { - date: string; - count: number; -}; - -type RecentActivity = { - type: string; - repo: string; - message: string; - url: string; - time: string; -}; - -type GeneratedGiteaActivity = { - updatedAt: string; - source: 'placeholder' | 'gitea'; - itemCount: number; - days: ActivityDay[]; - recent: RecentActivity[]; -}; - type SyncConfig = { rootDir: string; outputDir: string; @@ -61,6 +35,11 @@ type SyncConfig = { baseUrl: string; token: string; username: string; + activityDays: number; + activityPerDayLimit: number; + requestConcurrency: number; + recentItemLimit: number; + requestTimeoutMs: number; }; seafile: { baseUrl: string; @@ -85,10 +64,24 @@ async function main() { ); } + const seedProjects = await readJson( + path.join(config.rootDir, 'src/content/projects/index.json'), + ); + const giteaSnapshotPromise = hasGiteaConfig(config) + ? fetchGiteaSnapshot({ + config: { + ...config.gitea, + strict: config.strictSync, + }, + projectSeeds: seedProjects, + syncedAt, + }) + : null; + const [projects, shares, giteaActivity] = await Promise.all([ - resolveProjects(config, syncedAt), + resolveProjects(config, syncedAt, seedProjects, giteaSnapshotPromise), resolveShares(config, syncedAt), - resolveGiteaActivity(config, syncedAt), + resolveGiteaActivity(config, syncedAt, giteaSnapshotPromise), ]); await Promise.all([ @@ -112,6 +105,11 @@ function loadConfig(rootDir: string): SyncConfig { baseUrl: process.env.GITEA_BASE_URL?.trim() ?? '', token: process.env.GITEA_TOKEN?.trim() ?? '', username: process.env.GITEA_USERNAME?.trim() ?? '', + activityDays: getNumberEnv('GITEA_ACTIVITY_DAYS', 70), + activityPerDayLimit: getNumberEnv('GITEA_ACTIVITY_PER_DAY_LIMIT', 50), + recentItemLimit: getNumberEnv('GITEA_RECENT_ITEM_LIMIT', 8), + requestTimeoutMs: getNumberEnv('GITEA_REQUEST_TIMEOUT_MS', 15000), + requestConcurrency: getNumberEnv('GITEA_REQUEST_CONCURRENCY', 5), }, seafile: { baseUrl: process.env.SEAFILE_BASE_URL?.trim() ?? '', @@ -128,14 +126,17 @@ function loadConfig(rootDir: string): SyncConfig { async function resolveProjects( config: SyncConfig, syncedAt: string, + seedProjects: ProjectSeed[], + giteaSnapshotPromise: Promise>> | null, ): Promise { - const seedProjects = await readJson( - path.join(config.rootDir, 'src/content/projects/index.json'), - ); + if (giteaSnapshotPromise) { + const snapshot = await giteaSnapshotPromise; + return snapshot.projects; + } - if (hasGiteaConfig(config) || hasSeafileConfig(config)) { + if (hasSeafileConfig(config)) { console.warn( - '[sync-content] remote project sync not implemented yet, falling back to local seed data.', + '[sync-content] Seafile project sync is not implemented yet, falling back to local seed data.', ); } @@ -171,26 +172,21 @@ async function resolveShares(config: SyncConfig, syncedAt: string): Promise>> | null, ): Promise { - if (hasGiteaConfig(config)) { - if (config.strictSync) { - throw new Error( - 'STRICT_SYNC=true, but remote Gitea activity sync is not implemented yet.', - ); - } - - console.warn( - '[sync-content] remote Gitea activity sync not implemented yet, writing placeholder output.', - ); + if (!giteaSnapshotPromise) { + return { + updatedAt: syncedAt, + source: 'placeholder', + itemCount: 0, + days: [], + recent: [], + }; } - return { - updatedAt: syncedAt, - source: 'placeholder', - itemCount: 0, - days: [], - recent: [], - }; + const snapshot = await giteaSnapshotPromise; + + return snapshot.activity; } function hasGiteaConfig(config: SyncConfig) { @@ -218,6 +214,16 @@ function trimTrailingSlash(value: string) { return value.replace(/\/+$/, ''); } +function getNumberEnv(name: string, fallback: number) { + const value = process.env[name]; + if (value == null || value.trim() === '') { + return fallback; + } + + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : fallback; +} + async function readJson(filePath: string): Promise { const raw = await readFile(filePath, 'utf-8'); return JSON.parse(raw) as T; diff --git a/src/components/GiteaActivity.astro b/src/components/GiteaActivity.astro index 66d804a..17c5130 100644 --- a/src/components/GiteaActivity.astro +++ b/src/components/GiteaActivity.astro @@ -43,7 +43,7 @@ const isPlaceholder = activity.source === 'placeholder'; : '这里展示构建阶段同步下来的 Gitea 热力图与最近活动摘要。'}

- + 打开 Gitea diff --git a/src/components/ProjectCard.astro b/src/components/ProjectCard.astro index 5fd3752..f958150 100644 --- a/src/components/ProjectCard.astro +++ b/src/components/ProjectCard.astro @@ -1,4 +1,6 @@ --- +import { site } from '../config'; + interface Props { name: string; description: string; @@ -23,7 +25,7 @@ const { downloadLink = '', } = Astro.props; -const fallbackRepoUrl = repo ? `https://gitea.sepcomet.xyz/${repo}` : ''; +const fallbackRepoUrl = repo ? `${site.gitea.url}/${repo}` : ''; const resolvedRepoUrl = repoUrl || fallbackRepoUrl; --- diff --git a/src/pages/index.astro b/src/pages/index.astro index 0646fee..5092024 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -19,7 +19,7 @@ const sortedLogs = logs.sort((a, b) => b.data.date.getTime() - a.data.date.getTi const latestLogs = sortedLogs.slice(0, 3); const featuredProjects = projects.filter((project) => project.featured).slice(0, 3); const latestLog = latestLogs[0]; -const activeRepo = activity.recent[0]?.repo ?? featuredProjects[0]?.gitea_repo ?? 'sepcomet/personal-homepage'; +const activeRepo = activity.recent[0]?.repo ?? featuredProjects[0]?.gitea_repo ?? `${site.gitea.username}/personal-homepage`; ---