feat: scaffold seafile resource sync

This commit is contained in:
SepComet 2026-05-05 10:30:47 +08:00
parent fdeb25805c
commit 8265a10cff
10 changed files with 506 additions and 97 deletions

View File

@ -44,3 +44,5 @@ GITEA_ACTIVITY_PER_DAY_LIMIT=50
GITEA_RECENT_ITEM_LIMIT=8
GITEA_REQUEST_TIMEOUT_MS=15000
GITEA_REQUEST_CONCURRENCY=5
SEAFILE_REQUEST_TIMEOUT_MS=15000

View File

@ -1,4 +1,5 @@
import process from 'node:process';
import type { ProjectDownload } from './fetch-seafile.ts';
type ProjectSeed = {
name: string;
@ -13,6 +14,7 @@ type ProjectSeed = {
export type GeneratedProject = ProjectSeed & {
repo_url?: string;
downloads?: ProjectDownload[];
updated_at: string;
source: 'seed' | 'gitea' | 'gitea+seafile';
};

372
scripts/fetch-seafile.ts Normal file
View File

@ -0,0 +1,372 @@
import { readFile } from 'node:fs/promises';
import path from 'node:path';
import type { GeneratedProject } from './fetch-gitea.ts';
export type SeafileResourceType = 'build' | 'demo' | 'document' | 'asset';
export type ProjectDownload = {
name: string;
description?: string;
url?: string;
type?: SeafileResourceType;
platform?: string;
size?: number;
updated_at?: string;
source?: 'seed' | 'seafile';
};
export type GeneratedShare = {
name: string;
description?: string;
url?: string;
type?: SeafileResourceType;
project?: string;
project_repo?: string;
platform?: string;
size?: number;
updated_at?: string;
time?: string;
source: 'seed' | 'seafile';
};
export type SeafileSyncConfig = {
baseUrl: string;
token: string;
requestTimeoutMs?: number;
strict?: boolean;
};
type SeedShare = {
name: string;
description?: string;
url?: string;
time?: string;
};
type SeafileMappingFile = {
projects?: ProjectResourceMapping[];
shares?: ShareResourceMapping[];
};
type ProjectResourceMapping = {
project_repo: string;
downloads?: ResourceMapping[];
};
type ShareResourceMapping = ResourceMapping & {
project?: string;
project_repo?: string;
};
type ResourceMapping = {
name: string;
description?: string;
url?: string;
repo_id?: string;
path?: string;
type?: SeafileResourceType;
platform?: string;
size?: number;
updated_at?: string;
time?: string;
};
type SeafileFileDetailResponse = {
name?: string;
size?: number;
mtime?: number | string;
modifier_email?: string;
type?: string;
};
export async function fetchSeafileSnapshot(input: {
config: SeafileSyncConfig;
projects: GeneratedProject[];
seedShares: SeedShare[];
syncedAt: string;
mappingFilePath: string;
}) {
const { config, projects, seedShares, syncedAt, mappingFilePath } = input;
const mapping = await readMappingFile(mappingFilePath);
const client = canUseSeafileApi(config) ? createSeafileClient(config) : null;
const mappedProjects = await Promise.all(
projects.map(async (project) => {
const resourceEntry = mapping.projects?.find((item) => item.project_repo === project.gitea_repo);
const downloads = await Promise.all(
(resourceEntry?.downloads ?? []).map((download) =>
resolveResource(download, {
client,
config,
syncedAt,
}),
),
);
const normalizedDownloads = downloads.filter(hasContent);
const firstResolvedUrl = normalizedDownloads.find((item) => item.url)?.url;
return {
...project,
download_link: project.download_link || firstResolvedUrl || undefined,
downloads: normalizedDownloads.length > 0 ? normalizedDownloads : undefined,
source:
normalizedDownloads.length > 0
? project.source === 'seed'
? 'gitea+seafile'
: 'gitea+seafile'
: project.source,
} satisfies GeneratedProject;
}),
);
const projectShares = mappedProjects.flatMap((project) =>
(project.downloads ?? []).map((download) => ({
name: download.name,
description: download.description,
url: download.url,
type: download.type,
project: project.name,
project_repo: project.gitea_repo,
platform: download.platform,
size: download.size,
updated_at: download.updated_at,
time: download.updated_at,
source: 'seafile' as const,
})),
);
const extraShares = await Promise.all(
(mapping.shares ?? []).map(async (share) => {
const resolved = await resolveResource(share, {
client,
config,
syncedAt,
});
return {
name: resolved.name,
description: resolved.description,
url: resolved.url,
type: resolved.type,
project: share.project,
project_repo: share.project_repo,
platform: resolved.platform,
size: resolved.size,
updated_at: resolved.updated_at,
time: share.time || resolved.updated_at || syncedAt,
source: 'seafile' as const,
} satisfies GeneratedShare;
}),
);
const normalizedSeedShares = seedShares.map((share) => ({
name: share.name,
description: share.description,
url: share.url,
time: share.time || syncedAt,
updated_at: share.time || syncedAt,
source: 'seed' as const,
}));
const shares = dedupeShares([
...normalizedSeedShares,
...projectShares,
...extraShares.filter(hasContent),
]).sort((a, b) => Date.parse(b.updated_at || b.time || '') - Date.parse(a.updated_at || a.time || ''));
return {
projects: mappedProjects,
shares,
};
}
async function readMappingFile(filePath: string): Promise<SeafileMappingFile> {
try {
const raw = await readFile(filePath, 'utf-8');
return JSON.parse(raw) as SeafileMappingFile;
} catch (error) {
if (isMissingFileError(error)) {
return {};
}
throw error;
}
}
async function resolveResource(
resource: ResourceMapping,
input: {
client: ReturnType<typeof createSeafileClient> | null;
config: SeafileSyncConfig;
syncedAt: string;
},
): Promise<ProjectDownload> {
const fallback: ProjectDownload = {
name: resource.name,
description: resource.description,
url: resource.url,
type: resource.type,
platform: resource.platform,
size: resource.size,
updated_at: resource.updated_at || resource.time || input.syncedAt,
source: resource.url || resource.repo_id || resource.path ? 'seafile' : 'seed',
};
if (resource.url || !resource.repo_id || !resource.path || !input.client) {
return fallback;
}
try {
const [detail, downloadUrl] = await Promise.all([
input.client.getJson<SeafileFileDetailResponse>(
`/api/v2.1/repos/${resource.repo_id}/file/detail/`,
{ path: resource.path },
),
input.client.getLink(`/api/v2.1/repos/${resource.repo_id}/file/`, {
path: resource.path,
}),
]);
return {
name: resource.name || detail.name || path.basename(resource.path),
description: resource.description,
url: downloadUrl || fallback.url,
type: resource.type,
platform: resource.platform,
size: detail.size ?? resource.size,
updated_at: resource.updated_at || toIsoFromMtime(detail.mtime) || input.syncedAt,
source: 'seafile',
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const logLine = `[fetch-seafile] failed to resolve ${resource.path || resource.name}: ${message}`;
if (input.config.strict) {
throw new Error(logLine);
}
console.warn(logLine);
return fallback;
}
}
function createSeafileClient(config: SeafileSyncConfig) {
const headers = new Headers({
accept: 'application/json',
});
if (config.token) {
headers.set('Authorization', `Token ${config.token}`);
}
return {
async getJson<T>(pathname: string, query?: Record<string, string>): Promise<T> {
const response = await fetchWithTimeout(config, pathname, query, headers);
return (await response.json()) as T;
},
async getLink(pathname: string, query?: Record<string, string>) {
const response = await fetchWithTimeout(config, pathname, query, headers);
const text = (await response.text()).trim();
if (!text) {
return '';
}
try {
const parsed = JSON.parse(text) as string | { url?: string; download_link?: string };
if (typeof parsed === 'string') {
return parsed;
}
return parsed.download_link || parsed.url || text;
} catch {
return text.replace(/^"|"$/g, '');
}
},
};
}
async function fetchWithTimeout(
config: SeafileSyncConfig,
pathname: string,
query: Record<string, string> | undefined,
headers: Headers,
) {
const url = new URL(pathname, `${trimTrailingSlash(config.baseUrl)}/`);
if (query) {
for (const [key, value] of Object.entries(query)) {
url.searchParams.set(key, value);
}
}
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), config.requestTimeoutMs ?? 15000);
try {
const response = await fetch(url, {
method: 'GET',
headers,
signal: controller.signal,
});
if (!response.ok) {
const body = await safeReadText(response);
throw new Error(`HTTP ${response.status} ${response.statusText}${body ? ` - ${body}` : ''}`);
}
return response;
} finally {
clearTimeout(timeout);
}
}
function hasContent(item: { name?: string; url?: string; description?: string }) {
return Boolean(item.name || item.url || item.description);
}
function dedupeShares(items: GeneratedShare[]) {
const seen = new Set<string>();
return items.filter((item) => {
const key = [item.project_repo, item.name, item.url, item.type, item.platform].join('|');
if (seen.has(key)) {
return false;
}
seen.add(key);
return true;
});
}
function toIsoFromMtime(value?: number | string) {
if (typeof value === 'number') {
const milliseconds = value > 1_000_000_000_000 ? value : value * 1000;
return new Date(milliseconds).toISOString();
}
if (typeof value === 'string' && value.trim()) {
const parsed = Date.parse(value);
if (!Number.isNaN(parsed)) {
return new Date(parsed).toISOString();
}
}
return '';
}
function canUseSeafileApi(config: SeafileSyncConfig) {
return Boolean(config.baseUrl && config.token);
}
function trimTrailingSlash(value: string) {
return value.replace(/\/+$/, '');
}
function isMissingFileError(error: unknown) {
return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT';
}
async function safeReadText(response: Response) {
try {
return (await response.text()).slice(0, 300);
} catch {
return '';
}
}

View File

@ -2,6 +2,7 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises';
import path from 'node:path';
import process from 'node:process';
import { fetchGiteaSnapshot, type GeneratedGiteaActivity, type GeneratedProject } from './fetch-gitea.ts';
import { fetchSeafileSnapshot, type GeneratedShare } from './fetch-seafile.ts';
type ProjectSeed = {
name: string;
@ -16,15 +17,9 @@ type ProjectSeed = {
type ShareSeed = {
name: string;
description: string;
url: string;
time: string;
};
type GeneratedShare = ShareSeed & {
size?: number;
source: 'seed' | 'seafile';
updated_at: string;
description?: string;
url?: string;
time?: string;
};
type SyncConfig = {
@ -46,6 +41,7 @@ type SyncConfig = {
token: string;
mirrorDownloads: boolean;
downloadsOutputDir: string;
requestTimeoutMs: number;
};
};
@ -67,8 +63,12 @@ async function main() {
const seedProjects = await readJson<ProjectSeed[]>(
path.join(config.rootDir, 'src/content/projects/index.json'),
);
const giteaSnapshotPromise = hasGiteaConfig(config)
? fetchGiteaSnapshot({
const seedShares = await readJson<ShareSeed[]>(
path.join(config.rootDir, 'src/content/shares/index.json'),
);
const giteaSnapshot = hasGiteaConfig(config)
? await fetchGiteaSnapshot({
config: {
...config.gitea,
strict: config.strictSync,
@ -78,20 +78,47 @@ async function main() {
})
: null;
const [projects, shares, giteaActivity] = await Promise.all([
resolveProjects(config, syncedAt, seedProjects, giteaSnapshotPromise),
resolveShares(config, syncedAt),
resolveGiteaActivity(config, syncedAt, giteaSnapshotPromise),
]);
const baseProjects =
giteaSnapshot?.projects ??
seedProjects.map((project) => ({
...project,
repo_url:
project.gitea_repo && config.gitea.baseUrl
? `${trimTrailingSlash(config.gitea.baseUrl)}/${project.gitea_repo}`
: undefined,
updated_at: syncedAt,
source: 'seed' as const,
}));
const seafileSnapshot = await fetchSeafileSnapshot({
config: {
...config.seafile,
strict: config.strictSync,
},
projects: baseProjects,
seedShares,
syncedAt,
mappingFilePath: path.join(config.rootDir, 'src/content/seafile/index.json'),
});
const giteaActivity =
giteaSnapshot?.activity ??
({
updatedAt: syncedAt,
source: 'placeholder',
itemCount: 0,
days: [],
recent: [],
} satisfies GeneratedGiteaActivity);
await Promise.all([
writeJson(path.join(config.outputDir, 'projects.json'), projects),
writeJson(path.join(config.outputDir, 'shares.json'), shares),
writeJson(path.join(config.outputDir, 'projects.json'), seafileSnapshot.projects),
writeJson(path.join(config.outputDir, 'shares.json'), seafileSnapshot.shares),
writeJson(path.join(config.outputDir, 'gitea-activity.json'), giteaActivity),
]);
console.log(`[sync-content] wrote ${projects.length} projects`);
console.log(`[sync-content] wrote ${shares.length} shares`);
console.log(`[sync-content] wrote ${seafileSnapshot.projects.length} projects`);
console.log(`[sync-content] wrote ${seafileSnapshot.shares.length} shares`);
console.log(`[sync-content] wrote ${giteaActivity.itemCount} activity items`);
console.log('[sync-content] done');
}
@ -119,84 +146,15 @@ function loadConfig(rootDir: string): SyncConfig {
rootDir,
process.env.DOWNLOADS_OUTPUT_DIR ?? 'public/downloads',
),
requestTimeoutMs: getNumberEnv('SEAFILE_REQUEST_TIMEOUT_MS', 15000),
},
};
}
async function resolveProjects(
config: SyncConfig,
syncedAt: string,
seedProjects: ProjectSeed[],
giteaSnapshotPromise: Promise<Awaited<ReturnType<typeof fetchGiteaSnapshot>>> | null,
): Promise<GeneratedProject[]> {
if (giteaSnapshotPromise) {
const snapshot = await giteaSnapshotPromise;
return snapshot.projects;
}
if (hasSeafileConfig(config)) {
console.warn(
'[sync-content] Seafile project sync is not implemented yet, falling back to local seed data.',
);
}
return seedProjects.map((project) => ({
...project,
repo_url:
project.gitea_repo && config.gitea.baseUrl
? `${trimTrailingSlash(config.gitea.baseUrl)}/${project.gitea_repo}`
: undefined,
updated_at: syncedAt,
source: 'seed',
}));
}
async function resolveShares(config: SyncConfig, syncedAt: string): Promise<GeneratedShare[]> {
const seedShares = await readJson<ShareSeed[]>(
path.join(config.rootDir, 'src/content/shares/index.json'),
);
if (hasSeafileConfig(config)) {
console.warn(
'[sync-content] remote share sync not implemented yet, falling back to local seed data.',
);
}
return seedShares.map((share) => ({
...share,
source: 'seed',
updated_at: syncedAt,
}));
}
async function resolveGiteaActivity(
config: SyncConfig,
syncedAt: string,
giteaSnapshotPromise: Promise<Awaited<ReturnType<typeof fetchGiteaSnapshot>>> | null,
): Promise<GeneratedGiteaActivity> {
if (!giteaSnapshotPromise) {
return {
updatedAt: syncedAt,
source: 'placeholder',
itemCount: 0,
days: [],
recent: [],
};
}
const snapshot = await giteaSnapshotPromise;
return snapshot.activity;
}
function hasGiteaConfig(config: SyncConfig) {
return Boolean(config.gitea.baseUrl && config.gitea.token && config.gitea.username);
}
function hasSeafileConfig(config: SyncConfig) {
return Boolean(config.seafile.baseUrl && config.seafile.token);
}
function getBooleanEnv(name: string, fallback: boolean) {
const value = process.env[name];
if (value == null || value.trim() === '') {

View File

@ -1,5 +1,6 @@
---
import { site } from '../config';
import type { ProjectDownload } from '../data/loaders';
interface Props {
name: string;
@ -11,6 +12,7 @@ interface Props {
featured?: boolean;
demoVideo?: string;
downloadLink?: string;
downloads?: ProjectDownload[];
}
const {
@ -23,10 +25,12 @@ const {
featured = false,
demoVideo = '',
downloadLink = '',
downloads = [],
} = Astro.props;
const fallbackRepoUrl = repo ? `${site.gitea.url}/${repo}` : '';
const resolvedRepoUrl = repoUrl || fallbackRepoUrl;
const visibleDownloads = downloads.filter((item) => item.name || item.url);
---
<article class="card">
@ -57,10 +61,26 @@ const resolvedRepoUrl = repoUrl || fallbackRepoUrl;
Demo
</a>
) : null}
{downloadLink ? (
{downloadLink && visibleDownloads.length === 0 ? (
<a class="section-link" href={downloadLink} target="_blank" rel="noreferrer">
下载
</a>
) : null}
</div>
{
visibleDownloads.length > 0 ? (
<div class="project-links" style="margin-top: 0.75rem; flex-wrap: wrap; gap: 0.6rem 0.8rem;">
{visibleDownloads.map((item) =>
item.url ? (
<a class="section-link" href={item.url} target="_blank" rel="noreferrer">
{item.name}
</a>
) : (
<span class="tag">{item.name}</span>
),
)}
</div>
) : null
}
</article>

View File

@ -0,0 +1,28 @@
{
"projects": [
{
"project_repo": "basil/personal-homepage",
"downloads": [
{
"name": "Windows 构建包",
"description": "项目打包文件(可选)",
"url": "http://106.12.111.150:8000/f/5eb1877a7212488ca0aa/?dl=1",
"repo_id": "",
"path": "",
"type": "build",
"platform": "windows"
}
]
}
],
"shares": [
{
"name": "简历 PDF",
"description": "独立公开资源示例,可不绑定项目。",
"url": "",
"repo_id": "",
"path": "",
"type": "document"
}
]
}

View File

@ -2,6 +2,17 @@ import { readFile } from 'node:fs/promises';
import path from 'node:path';
import process from 'node:process';
export type ProjectDownload = {
name: string;
description?: string;
url?: string;
type?: 'build' | 'demo' | 'document' | 'asset';
platform?: string;
size?: number;
updated_at?: string;
source?: 'seed' | 'seafile';
};
type ProjectSeed = {
name: string;
description: string;
@ -9,6 +20,7 @@ type ProjectSeed = {
cover_image: string;
demo_video?: string;
download_link?: string;
downloads?: ProjectDownload[];
tags: string[];
featured: boolean;
};
@ -21,12 +33,16 @@ export type ProjectData = ProjectSeed & {
type ShareSeed = {
name: string;
description: string;
url: string;
time: string;
description?: string;
url?: string;
time?: string;
};
export type ShareData = ShareSeed & {
type?: 'build' | 'demo' | 'document' | 'asset';
project?: string;
project_repo?: string;
platform?: string;
size?: number;
source?: 'seed' | 'seafile';
updated_at?: string;

View File

@ -114,6 +114,7 @@ const activeRepo = activity.recent[0]?.repo ?? featuredProjects[0]?.gitea_repo ?
featured={project.featured}
demoVideo={project.demo_video}
downloadLink={project.download_link}
downloads={project.downloads}
/>
))
}

View File

@ -33,6 +33,7 @@ const projects = await loadProjects();
featured={project.featured}
demoVideo={project.demo_video}
downloadLink={project.download_link}
downloads={project.downloads}
/>
))
}

View File

@ -23,12 +23,21 @@ const shares = await loadShares();
{shares.map((share) => (
<article class="card">
<p class="log-meta">
<span class="mono">{share.time}</span>
<span class="mono">{share.updated_at || share.time || '未标注时间'}</span>
{share.project ? <span class="tag">{share.project}</span> : null}
{share.type ? <span class="tag tag--tech">{share.type}</span> : null}
</p>
<h2 class="project-title">{share.name}</h2>
<p>{share.description}</p>
{share.description ? <p>{share.description}</p> : null}
{share.project_repo ? <p class="activity-copy mono">{share.project_repo}</p> : null}
<div class="project-links" style="margin-top: 1rem;">
<a class="section-link" href={share.url}>打开链接</a>
{share.url ? (
<a class="section-link" href={share.url} target="_blank" rel="noreferrer">
打开链接
</a>
) : (
<span class="tag">链接待补充</span>
)}
</div>
</article>
))}