feat: add gitea sync and env-aware content sync

.env.example

@@ -38,3 +38,9 @@ STRICT_SYNC=false
 # -----------------------------------------------------------------------------
 SEAFILE_MIRROR_DOWNLOADS=false
 DOWNLOADS_OUTPUT_DIR=public/downloads
+
+GITEA_ACTIVITY_DAYS=70
+GITEA_ACTIVITY_PER_DAY_LIMIT=50
+GITEA_RECENT_ITEM_LIMIT=8
+GITEA_REQUEST_TIMEOUT_MS=15000
+GITEA_REQUEST_CONCURRENCY=5

package.json

@@ -8,7 +8,7 @@
   },
   "scripts": {
     "dev": "astro dev",
-    "content:sync": "node --experimental-strip-types ./scripts/sync-content.ts",
+    "content:sync": "node --env-file-if-exists=.env --experimental-strip-types ./scripts/sync-content.ts",
     "build": "astro build",
     "rebuild": "npm run content:sync && npm run build",
     "preview": "astro preview",

scripts/fetch-gitea.ts

@@ -0,0 +1,411 @@
+import process from 'node:process';
+
+type ProjectSeed = {
+  name: string;
+  description: string;
+  gitea_repo: string;
+  cover_image: string;
+  demo_video?: string;
+  download_link?: string;
+  tags: string[];
+  featured: boolean;
+};
+
+export type GeneratedProject = ProjectSeed & {
+  repo_url?: string;
+  updated_at: string;
+  source: 'seed' | 'gitea' | 'gitea+seafile';
+};
+
+export type ActivityDay = {
+  date: string;
+  count: number;
+};
+
+export type RecentActivity = {
+  type: string;
+  repo: string;
+  message: string;
+  url: string;
+  time: string;
+};
+
+export type GeneratedGiteaActivity = {
+  updatedAt: string;
+  source: 'placeholder' | 'gitea';
+  itemCount: number;
+  days: ActivityDay[];
+  recent: RecentActivity[];
+};
+
+export type GiteaSyncConfig = {
+  baseUrl: string;
+  token: string;
+  username: string;
+  activityDays?: number;
+  activityPerDayLimit?: number;
+  recentItemLimit?: number;
+  requestTimeoutMs?: number;
+  requestConcurrency?: number;
+  strict?: boolean;
+};
+
+type GiteaRepositoryResponse = {
+  description?: string;
+  full_name?: string;
+  html_url?: string;
+  updated_at?: string;
+  topics?: string[];
+};
+
+type GiteaActivityFeedResponse = {
+  op_type?: string;
+  content?: string;
+  created?: string;
+  ref_name?: string;
+  repo?: {
+    full_name?: string;
+    name?: string;
+    html_url?: string;
+  };
+  comment?: {
+    html_url?: string;
+    body?: string;
+  };
+  issue?: {
+    html_url?: string;
+    title?: string;
+  };
+  pull_request?: {
+    html_url?: string;
+    title?: string;
+  };
+};
+
+export async function fetchGiteaSnapshot(input: {
+  config: GiteaSyncConfig;
+  projectSeeds: ProjectSeed[];
+  syncedAt: string;
+}) {
+  const { config, projectSeeds, syncedAt } = input;
+  const client = createGiteaClient(config);
+
+  const [repoResults, userActivities] = await Promise.all([
+    Promise.all(
+      projectSeeds.map((seed) => enrichProjectFromGitea({ client, config, project: seed, syncedAt })),
+    ),
+    fetchUserActivities({ client, config }),
+  ]);
+
+  const projects = repoResults.map((result) => result.project);
+  const activity = buildActivitySnapshot({
+    syncedAt,
+    recentItemLimit: config.recentItemLimit ?? 8,
+    activityDays: config.activityDays ?? 70,
+    feeds: userActivities,
+  });
+
+  return { projects, activity };
+}
+
+async function enrichProjectFromGitea(input: {
+  client: ReturnType<typeof createGiteaClient>;
+  config: GiteaSyncConfig;
+  project: ProjectSeed;
+  syncedAt: string;
+}): Promise<{ project: GeneratedProject }> {
+  const { client, config, project, syncedAt } = input;
+  const repoParts = splitRepoFullName(project.gitea_repo);
+  const fallbackRepoUrl = buildRepoUrl(config.baseUrl, project.gitea_repo);
+
+  if (!repoParts) {
+    return { project: toSeedProject(project, syncedAt, fallbackRepoUrl) };
+  }
+
+  try {
+    const repo = await client.getJson<GiteaRepositoryResponse>(
+      `/api/v1/repos/${repoParts.owner}/${repoParts.repo}`,
+    );
+
+    return {
+      project: {
+        ...project,
+        description: repo.description?.trim() || project.description,
+        repo_url: repo.html_url || fallbackRepoUrl,
+        tags: mergeTags(project.tags, repo.topics ?? []),
+        updated_at: repo.updated_at || syncedAt,
+        source: project.download_link ? 'gitea+seafile' : 'gitea',
+      },
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const logLine = `[fetch-gitea] failed to sync ${project.gitea_repo}: ${message}`;
+
+    if (config.strict) {
+      throw new Error(logLine);
+    }
+
+    console.warn(logLine);
+
+    return {
+      project: toSeedProject(project, syncedAt, fallbackRepoUrl),
+    };
+  }
+}
+
+async function fetchUserActivities(input: {
+  client: ReturnType<typeof createGiteaClient>;
+  config: GiteaSyncConfig;
+}): Promise<RecentActivity[]> {
+  const { client, config } = input;
+  const activityDays = config.activityDays ?? 70;
+  const limit = config.activityPerDayLimit ?? 50;
+  const concurrency = Math.max(1, config.requestConcurrency ?? 5);
+  const dates = Array.from({ length: activityDays }, (_, index) => {
+    const date = new Date();
+    date.setUTCDate(date.getUTCDate() - index);
+    return date.toISOString().slice(0, 10);
+  });
+
+  const results = await mapWithConcurrency(dates, concurrency, async (date) => {
+    try {
+      const feeds = await client.getJson<GiteaActivityFeedResponse[]>(
+        `/api/v1/users/${config.username}/activities/feeds`,
+        {
+          'only-performed-by': 'true',
+          date,
+          limit: String(limit),
+        },
+      );
+
+      return feeds.map((feed) =>
+        mapActivityFeed(feed, {
+          fallbackRepo: config.username,
+          fallbackRepoUrl: `${trimTrailingSlash(config.baseUrl)}/${config.username}`,
+        }),
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const logLine = `[fetch-gitea] failed to sync user activity for ${config.username} on ${date}: ${message}`;
+
+      if (config.strict) {
+        throw new Error(logLine);
+      }
+
+      console.warn(logLine);
+      return [];
+    }
+  });
+
+  return results.flat();
+}
+
+function buildActivitySnapshot(input: {
+  syncedAt: string;
+  recentItemLimit: number;
+  activityDays: number;
+  feeds: RecentActivity[];
+}): GeneratedGiteaActivity {
+  const sortedFeeds = [...input.feeds]
+    .filter((item) => item.time)
+    .sort((a, b) => Date.parse(b.time) - Date.parse(a.time));
+
+  const recent = dedupeActivities(sortedFeeds).slice(0, input.recentItemLimit);
+  const days = buildHeatmapDays(sortedFeeds, input.activityDays);
+
+  return {
+    updatedAt: input.syncedAt,
+    source: sortedFeeds.length > 0 ? 'gitea' : 'placeholder',
+    itemCount: sortedFeeds.length,
+    days,
+    recent,
+  };
+}
+
+function buildHeatmapDays(activities: RecentActivity[], dayCount: number): ActivityDay[] {
+  const now = new Date();
+  const counts = new Map<string, number>();
+
+  for (const item of activities) {
+    const dateKey = toDateKey(item.time);
+    if (!dateKey) {
+      continue;
+    }
+    counts.set(dateKey, (counts.get(dateKey) ?? 0) + 1);
+  }
+
+  const days: ActivityDay[] = [];
+  for (let offset = dayCount - 1; offset >= 0; offset -= 1) {
+    const date = new Date(now);
+    date.setUTCDate(date.getUTCDate() - offset);
+    const dateKey = date.toISOString().slice(0, 10);
+    days.push({
+      date: dateKey,
+      count: counts.get(dateKey) ?? 0,
+    });
+  }
+
+  return days;
+}
+
+function dedupeActivities(activities: RecentActivity[]) {
+  const seen = new Set<string>();
+  return activities.filter((item) => {
+    const key = `${item.repo}|${item.time}|${item.type}|${item.url}|${item.message}`;
+    if (seen.has(key)) {
+      return false;
+    }
+    seen.add(key);
+    return true;
+  });
+}
+
+function mapActivityFeed(
+  feed: GiteaActivityFeedResponse,
+  fallback: { fallbackRepo: string; fallbackRepoUrl: string },
+): RecentActivity {
+  const repo = feed.repo?.full_name || fallback.fallbackRepo;
+  const type = normalizeActivityType(feed.op_type);
+  const message =
+    clean(feed.content) ||
+    clean(feed.comment?.body) ||
+    clean(feed.issue?.title) ||
+    clean(feed.pull_request?.title) ||
+    [feed.op_type, feed.ref_name].filter(Boolean).join(' ').trim() ||
+    'Repository activity';
+  const url =
+    feed.comment?.html_url ||
+    feed.issue?.html_url ||
+    feed.pull_request?.html_url ||
+    feed.repo?.html_url ||
+    fallback.fallbackRepoUrl;
+  const time = feed.created || new Date().toISOString();
+
+  return {
+    type,
+    repo,
+    message,
+    url,
+    time,
+  };
+}
+
+async function mapWithConcurrency<T, R>(
+  items: T[],
+  concurrency: number,
+  mapper: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  const results = new Array<R>(items.length);
+  let nextIndex = 0;
+
+  async function worker() {
+    while (nextIndex < items.length) {
+      const currentIndex = nextIndex;
+      nextIndex += 1;
+      results[currentIndex] = await mapper(items[currentIndex], currentIndex);
+    }
+  }
+
+  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
+  return results;
+}
+
+function normalizeActivityType(value?: string) {
+  return value?.trim().toLowerCase().replace(/\s+/g, '_') || 'activity';
+}
+
+function toSeedProject(project: ProjectSeed, syncedAt: string, fallbackRepoUrl: string): GeneratedProject {
+  return {
+    ...project,
+    repo_url: fallbackRepoUrl,
+    updated_at: syncedAt,
+    source: 'seed',
+  };
+}
+
+function mergeTags(seedTags: string[], topics: string[]) {
+  const merged = new Set([...seedTags, ...topics].map((tag) => tag.trim()).filter(Boolean));
+  return [...merged];
+}
+
+function splitRepoFullName(value: string) {
+  const [owner, repo] = value.split('/');
+  if (!owner || !repo) {
+    return null;
+  }
+  return { owner, repo };
+}
+
+function buildRepoUrl(baseUrl: string, repoFullName: string) {
+  return `${trimTrailingSlash(baseUrl)}/${repoFullName}`;
+}
+
+function toDateKey(value: string) {
+  const time = Date.parse(value);
+  if (Number.isNaN(time)) {
+    return '';
+  }
+  return new Date(time).toISOString().slice(0, 10);
+}
+
+function clean(value?: string) {
+  return value?.replace(/\s+/g, ' ').trim() || '';
+}
+
+function trimTrailingSlash(value: string) {
+  return value.replace(/\/+$/, '');
+}
+
+function createGiteaClient(config: GiteaSyncConfig) {
+  const headers = new Headers({
+    accept: 'application/json',
+  });
+
+  if (config.token) {
+    headers.set('Authorization', `token ${config.token}`);
+  }
+
+  return {
+    async getJson<T>(pathname: string, query?: Record<string, string>): Promise<T> {
+      const url = new URL(pathname, `${trimTrailingSlash(config.baseUrl)}/`);
+      if (query) {
+        for (const [key, value] of Object.entries(query)) {
+          url.searchParams.set(key, value);
+        }
+      }
+
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), config.requestTimeoutMs ?? 15000);
+
+      try {
+        const response = await fetch(url, {
+          method: 'GET',
+          headers,
+          signal: controller.signal,
+        });
+
+        if (!response.ok) {
+          const body = await safeReadText(response);
+          throw new Error(`HTTP ${response.status} ${response.statusText}${body ? ` - ${body}` : ''}`);
+        }
+
+        return (await response.json()) as T;
+      } finally {
+        clearTimeout(timeout);
+      }
+    },
+  };
+}
+
+async function safeReadText(response: Response) {
+  try {
+    const text = await response.text();
+    return text.slice(0, 300);
+  } catch {
+    return '';
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  console.log('[fetch-gitea] This module is intended to be imported by scripts/sync-content.ts');
+}

scripts/sync-content.ts

@@ -1,6 +1,7 @@
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import process from 'node:process';
+import { fetchGiteaSnapshot, type GeneratedGiteaActivity, type GeneratedProject } from './fetch-gitea.ts';
 
 type ProjectSeed = {
   name: string;
@@ -20,39 +21,12 @@ type ShareSeed = {
   time: string;
 };
 
-type GeneratedProject = ProjectSeed & {
-  repo_url?: string;
-  updated_at: string;
-  source: 'seed' | 'gitea' | 'gitea+seafile';
-};
-
 type GeneratedShare = ShareSeed & {
   size?: number;
   source: 'seed' | 'seafile';
   updated_at: string;
 };
 
-type ActivityDay = {
-  date: string;
-  count: number;
-};
-
-type RecentActivity = {
-  type: string;
-  repo: string;
-  message: string;
-  url: string;
-  time: string;
-};
-
-type GeneratedGiteaActivity = {
-  updatedAt: string;
-  source: 'placeholder' | 'gitea';
-  itemCount: number;
-  days: ActivityDay[];
-  recent: RecentActivity[];
-};
-
 type SyncConfig = {
   rootDir: string;
   outputDir: string;
@@ -61,6 +35,11 @@ type SyncConfig = {
     baseUrl: string;
     token: string;
     username: string;
+    activityDays: number;
+    activityPerDayLimit: number;
+    requestConcurrency: number;
+    recentItemLimit: number;
+    requestTimeoutMs: number;
   };
   seafile: {
     baseUrl: string;
@@ -85,10 +64,24 @@ async function main() {
     );
   }
 
+  const seedProjects = await readJson<ProjectSeed[]>(
+    path.join(config.rootDir, 'src/content/projects/index.json'),
+  );
+  const giteaSnapshotPromise = hasGiteaConfig(config)
+    ? fetchGiteaSnapshot({
+        config: {
+          ...config.gitea,
+          strict: config.strictSync,
+        },
+        projectSeeds: seedProjects,
+        syncedAt,
+      })
+    : null;
+
   const [projects, shares, giteaActivity] = await Promise.all([
-    resolveProjects(config, syncedAt),
+    resolveProjects(config, syncedAt, seedProjects, giteaSnapshotPromise),
     resolveShares(config, syncedAt),
-    resolveGiteaActivity(config, syncedAt),
+    resolveGiteaActivity(config, syncedAt, giteaSnapshotPromise),
   ]);
 
   await Promise.all([
@@ -112,6 +105,11 @@ function loadConfig(rootDir: string): SyncConfig {
       baseUrl: process.env.GITEA_BASE_URL?.trim() ?? '',
       token: process.env.GITEA_TOKEN?.trim() ?? '',
       username: process.env.GITEA_USERNAME?.trim() ?? '',
+      activityDays: getNumberEnv('GITEA_ACTIVITY_DAYS', 70),
+      activityPerDayLimit: getNumberEnv('GITEA_ACTIVITY_PER_DAY_LIMIT', 50),
+      recentItemLimit: getNumberEnv('GITEA_RECENT_ITEM_LIMIT', 8),
+      requestTimeoutMs: getNumberEnv('GITEA_REQUEST_TIMEOUT_MS', 15000),
+      requestConcurrency: getNumberEnv('GITEA_REQUEST_CONCURRENCY', 5),
     },
     seafile: {
       baseUrl: process.env.SEAFILE_BASE_URL?.trim() ?? '',
@@ -128,14 +126,17 @@ function loadConfig(rootDir: string): SyncConfig {
 async function resolveProjects(
   config: SyncConfig,
   syncedAt: string,
+  seedProjects: ProjectSeed[],
+  giteaSnapshotPromise: Promise<Awaited<ReturnType<typeof fetchGiteaSnapshot>>> | null,
 ): Promise<GeneratedProject[]> {
-  const seedProjects = await readJson<ProjectSeed[]>(
+  if (giteaSnapshotPromise) {
-    path.join(config.rootDir, 'src/content/projects/index.json'),
+    const snapshot = await giteaSnapshotPromise;
-  );
+    return snapshot.projects;
+  }
 
-  if (hasGiteaConfig(config) || hasSeafileConfig(config)) {
+  if (hasSeafileConfig(config)) {
     console.warn(
-      '[sync-content] remote project sync not implemented yet, falling back to local seed data.',
+      '[sync-content] Seafile project sync is not implemented yet, falling back to local seed data.',
     );
   }
 
@@ -171,26 +172,21 @@ async function resolveShares(config: SyncConfig, syncedAt: string): Promise<Gene
 async function resolveGiteaActivity(
   config: SyncConfig,
   syncedAt: string,
+  giteaSnapshotPromise: Promise<Awaited<ReturnType<typeof fetchGiteaSnapshot>>> | null,
 ): Promise<GeneratedGiteaActivity> {
-  if (hasGiteaConfig(config)) {
+  if (!giteaSnapshotPromise) {
-    if (config.strictSync) {
+    return {
-      throw new Error(
+      updatedAt: syncedAt,
-        'STRICT_SYNC=true, but remote Gitea activity sync is not implemented yet.',
+      source: 'placeholder',
-      );
+      itemCount: 0,
-    }
+      days: [],
-
+      recent: [],
-    console.warn(
+    };
-      '[sync-content] remote Gitea activity sync not implemented yet, writing placeholder output.',
-    );
   }
 
-  return {
+  const snapshot = await giteaSnapshotPromise;
-    updatedAt: syncedAt,
+
-    source: 'placeholder',
+  return snapshot.activity;
-    itemCount: 0,
-    days: [],
-    recent: [],
-  };
 }
 
 function hasGiteaConfig(config: SyncConfig) {
@@ -218,6 +214,16 @@ function trimTrailingSlash(value: string) {
   return value.replace(/\/+$/, '');
 }
 
+function getNumberEnv(name: string, fallback: number) {
+  const value = process.env[name];
+  if (value == null || value.trim() === '') {
+    return fallback;
+  }
+
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
 async function readJson<T>(filePath: string): Promise<T> {
   const raw = await readFile(filePath, 'utf-8');
   return JSON.parse(raw) as T;

src/components/GiteaActivity.astro

@@ -43,7 +43,7 @@ const isPlaceholder = activity.source === 'placeholder';
           : '这里展示构建阶段同步下来的 Gitea 热力图与最近活动摘要。'}
       </p>
     </div>
-    <a class="section-link" href={site.gitea.baseUrl} target="_blank" rel="noreferrer">
+    <a class="section-link" href={site.gitea.url} target="_blank" rel="noreferrer">
       打开 Gitea
     </a>
   </div>

src/components/ProjectCard.astro

@@ -1,4 +1,6 @@
 ---
+import { site } from '../config';
+
 interface Props {
   name: string;
   description: string;
@@ -23,7 +25,7 @@ const {
   downloadLink = '',
 } = Astro.props;
 
-const fallbackRepoUrl = repo ? `https://gitea.sepcomet.xyz/${repo}` : '';
+const fallbackRepoUrl = repo ? `${site.gitea.url}/${repo}` : '';
 const resolvedRepoUrl = repoUrl || fallbackRepoUrl;
 ---
 

src/pages/index.astro

@@ -19,7 +19,7 @@ const sortedLogs = logs.sort((a, b) => b.data.date.getTime() - a.data.date.getTi
 const latestLogs = sortedLogs.slice(0, 3);
 const featuredProjects = projects.filter((project) => project.featured).slice(0, 3);
 const latestLog = latestLogs[0];
-const activeRepo = activity.recent[0]?.repo ?? featuredProjects[0]?.gitea_repo ?? 'sepcomet/personal-homepage';
+const activeRepo = activity.recent[0]?.repo ?? featuredProjects[0]?.gitea_repo ?? `${site.gitea.username}/personal-homepage`;
 ---
 
 <Layout title={`${site.name} | ${site.title}`} description={site.description} currentPath="/">