feat: scaffold seafile resource sync
This commit is contained in:
SepComet
@@ -1,4 +1,5 @@
|
||||
import process from 'node:process';
|
||||
import type { ProjectDownload } from './fetch-seafile.ts';
|
||||
|
||||
type ProjectSeed = {
|
||||
name: string;
|
||||
@@ -13,6 +14,7 @@ type ProjectSeed = {
|
||||
|
||||
export type GeneratedProject = ProjectSeed & {
|
||||
repo_url?: string;
|
||||
downloads?: ProjectDownload[];
|
||||
updated_at: string;
|
||||
source: 'seed' | 'gitea' | 'gitea+seafile';
|
||||
};
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { GeneratedProject } from './fetch-gitea.ts';
|
||||
|
||||
export type SeafileResourceType = 'build' | 'demo' | 'document' | 'asset';
|
||||
|
||||
export type ProjectDownload = {
|
||||
name: string;
|
||||
description?: string;
|
||||
url?: string;
|
||||
type?: SeafileResourceType;
|
||||
platform?: string;
|
||||
size?: number;
|
||||
updated_at?: string;
|
||||
source?: 'seed' | 'seafile';
|
||||
};
|
||||
|
||||
export type GeneratedShare = {
|
||||
name: string;
|
||||
description?: string;
|
||||
url?: string;
|
||||
type?: SeafileResourceType;
|
||||
project?: string;
|
||||
project_repo?: string;
|
||||
platform?: string;
|
||||
size?: number;
|
||||
updated_at?: string;
|
||||
time?: string;
|
||||
source: 'seed' | 'seafile';
|
||||
};
|
||||
|
||||
export type SeafileSyncConfig = {
|
||||
baseUrl: string;
|
||||
token: string;
|
||||
requestTimeoutMs?: number;
|
||||
strict?: boolean;
|
||||
};
|
||||
|
||||
type SeedShare = {
|
||||
name: string;
|
||||
description?: string;
|
||||
url?: string;
|
||||
time?: string;
|
||||
};
|
||||
|
||||
type SeafileMappingFile = {
|
||||
projects?: ProjectResourceMapping[];
|
||||
shares?: ShareResourceMapping[];
|
||||
};
|
||||
|
||||
type ProjectResourceMapping = {
|
||||
project_repo: string;
|
||||
downloads?: ResourceMapping[];
|
||||
};
|
||||
|
||||
type ShareResourceMapping = ResourceMapping & {
|
||||
project?: string;
|
||||
project_repo?: string;
|
||||
};
|
||||
|
||||
type ResourceMapping = {
|
||||
name: string;
|
||||
description?: string;
|
||||
url?: string;
|
||||
repo_id?: string;
|
||||
path?: string;
|
||||
type?: SeafileResourceType;
|
||||
platform?: string;
|
||||
size?: number;
|
||||
updated_at?: string;
|
||||
time?: string;
|
||||
};
|
||||
|
||||
type SeafileFileDetailResponse = {
|
||||
name?: string;
|
||||
size?: number;
|
||||
mtime?: number | string;
|
||||
modifier_email?: string;
|
||||
type?: string;
|
||||
};
|
||||
|
||||
export async function fetchSeafileSnapshot(input: {
|
||||
config: SeafileSyncConfig;
|
||||
projects: GeneratedProject[];
|
||||
seedShares: SeedShare[];
|
||||
syncedAt: string;
|
||||
mappingFilePath: string;
|
||||
}) {
|
||||
const { config, projects, seedShares, syncedAt, mappingFilePath } = input;
|
||||
const mapping = await readMappingFile(mappingFilePath);
|
||||
const client = canUseSeafileApi(config) ? createSeafileClient(config) : null;
|
||||
|
||||
const mappedProjects = await Promise.all(
|
||||
projects.map(async (project) => {
|
||||
const resourceEntry = mapping.projects?.find((item) => item.project_repo === project.gitea_repo);
|
||||
const downloads = await Promise.all(
|
||||
(resourceEntry?.downloads ?? []).map((download) =>
|
||||
resolveResource(download, {
|
||||
client,
|
||||
config,
|
||||
syncedAt,
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const normalizedDownloads = downloads.filter(hasContent);
|
||||
const firstResolvedUrl = normalizedDownloads.find((item) => item.url)?.url;
|
||||
|
||||
return {
|
||||
...project,
|
||||
download_link: project.download_link || firstResolvedUrl || undefined,
|
||||
downloads: normalizedDownloads.length > 0 ? normalizedDownloads : undefined,
|
||||
source:
|
||||
normalizedDownloads.length > 0
|
||||
? project.source === 'seed'
|
||||
? 'gitea+seafile'
|
||||
: 'gitea+seafile'
|
||||
: project.source,
|
||||
} satisfies GeneratedProject;
|
||||
}),
|
||||
);
|
||||
|
||||
const projectShares = mappedProjects.flatMap((project) =>
|
||||
(project.downloads ?? []).map((download) => ({
|
||||
name: download.name,
|
||||
description: download.description,
|
||||
url: download.url,
|
||||
type: download.type,
|
||||
project: project.name,
|
||||
project_repo: project.gitea_repo,
|
||||
platform: download.platform,
|
||||
size: download.size,
|
||||
updated_at: download.updated_at,
|
||||
time: download.updated_at,
|
||||
source: 'seafile' as const,
|
||||
})),
|
||||
);
|
||||
|
||||
const extraShares = await Promise.all(
|
||||
(mapping.shares ?? []).map(async (share) => {
|
||||
const resolved = await resolveResource(share, {
|
||||
client,
|
||||
config,
|
||||
syncedAt,
|
||||
});
|
||||
|
||||
return {
|
||||
name: resolved.name,
|
||||
description: resolved.description,
|
||||
url: resolved.url,
|
||||
type: resolved.type,
|
||||
project: share.project,
|
||||
project_repo: share.project_repo,
|
||||
platform: resolved.platform,
|
||||
size: resolved.size,
|
||||
updated_at: resolved.updated_at,
|
||||
time: share.time || resolved.updated_at || syncedAt,
|
||||
source: 'seafile' as const,
|
||||
} satisfies GeneratedShare;
|
||||
}),
|
||||
);
|
||||
|
||||
const normalizedSeedShares = seedShares.map((share) => ({
|
||||
name: share.name,
|
||||
description: share.description,
|
||||
url: share.url,
|
||||
time: share.time || syncedAt,
|
||||
updated_at: share.time || syncedAt,
|
||||
source: 'seed' as const,
|
||||
}));
|
||||
|
||||
const shares = dedupeShares([
|
||||
...normalizedSeedShares,
|
||||
...projectShares,
|
||||
...extraShares.filter(hasContent),
|
||||
]).sort((a, b) => Date.parse(b.updated_at || b.time || '') - Date.parse(a.updated_at || a.time || ''));
|
||||
|
||||
return {
|
||||
projects: mappedProjects,
|
||||
shares,
|
||||
};
|
||||
}
|
||||
|
||||
async function readMappingFile(filePath: string): Promise<SeafileMappingFile> {
|
||||
try {
|
||||
const raw = await readFile(filePath, 'utf-8');
|
||||
return JSON.parse(raw) as SeafileMappingFile;
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error)) {
|
||||
return {};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveResource(
|
||||
resource: ResourceMapping,
|
||||
input: {
|
||||
client: ReturnType<typeof createSeafileClient> | null;
|
||||
config: SeafileSyncConfig;
|
||||
syncedAt: string;
|
||||
},
|
||||
): Promise<ProjectDownload> {
|
||||
const fallback: ProjectDownload = {
|
||||
name: resource.name,
|
||||
description: resource.description,
|
||||
url: resource.url,
|
||||
type: resource.type,
|
||||
platform: resource.platform,
|
||||
size: resource.size,
|
||||
updated_at: resource.updated_at || resource.time || input.syncedAt,
|
||||
source: resource.url || resource.repo_id || resource.path ? 'seafile' : 'seed',
|
||||
};
|
||||
|
||||
if (resource.url || !resource.repo_id || !resource.path || !input.client) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
try {
|
||||
const [detail, downloadUrl] = await Promise.all([
|
||||
input.client.getJson<SeafileFileDetailResponse>(
|
||||
`/api/v2.1/repos/${resource.repo_id}/file/detail/`,
|
||||
{ path: resource.path },
|
||||
),
|
||||
input.client.getLink(`/api/v2.1/repos/${resource.repo_id}/file/`, {
|
||||
path: resource.path,
|
||||
}),
|
||||
]);
|
||||
|
||||
return {
|
||||
name: resource.name || detail.name || path.basename(resource.path),
|
||||
description: resource.description,
|
||||
url: downloadUrl || fallback.url,
|
||||
type: resource.type,
|
||||
platform: resource.platform,
|
||||
size: detail.size ?? resource.size,
|
||||
updated_at: resource.updated_at || toIsoFromMtime(detail.mtime) || input.syncedAt,
|
||||
source: 'seafile',
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const logLine = `[fetch-seafile] failed to resolve ${resource.path || resource.name}: ${message}`;
|
||||
|
||||
if (input.config.strict) {
|
||||
throw new Error(logLine);
|
||||
}
|
||||
|
||||
console.warn(logLine);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function createSeafileClient(config: SeafileSyncConfig) {
|
||||
const headers = new Headers({
|
||||
accept: 'application/json',
|
||||
});
|
||||
|
||||
if (config.token) {
|
||||
headers.set('Authorization', `Token ${config.token}`);
|
||||
}
|
||||
|
||||
return {
|
||||
async getJson<T>(pathname: string, query?: Record<string, string>): Promise<T> {
|
||||
const response = await fetchWithTimeout(config, pathname, query, headers);
|
||||
return (await response.json()) as T;
|
||||
},
|
||||
async getLink(pathname: string, query?: Record<string, string>) {
|
||||
const response = await fetchWithTimeout(config, pathname, query, headers);
|
||||
const text = (await response.text()).trim();
|
||||
|
||||
if (!text) {
|
||||
return '';
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(text) as string | { url?: string; download_link?: string };
|
||||
if (typeof parsed === 'string') {
|
||||
return parsed;
|
||||
}
|
||||
return parsed.download_link || parsed.url || text;
|
||||
} catch {
|
||||
return text.replace(/^"|"$/g, '');
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(
|
||||
config: SeafileSyncConfig,
|
||||
pathname: string,
|
||||
query: Record<string, string> | undefined,
|
||||
headers: Headers,
|
||||
) {
|
||||
const url = new URL(pathname, `${trimTrailingSlash(config.baseUrl)}/`);
|
||||
if (query) {
|
||||
for (const [key, value] of Object.entries(query)) {
|
||||
url.searchParams.set(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), config.requestTimeoutMs ?? 15000);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await safeReadText(response);
|
||||
throw new Error(`HTTP ${response.status} ${response.statusText}${body ? ` - ${body}` : ''}`);
|
||||
}
|
||||
|
||||
return response;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
function hasContent(item: { name?: string; url?: string; description?: string }) {
|
||||
return Boolean(item.name || item.url || item.description);
|
||||
}
|
||||
|
||||
function dedupeShares(items: GeneratedShare[]) {
|
||||
const seen = new Set<string>();
|
||||
return items.filter((item) => {
|
||||
const key = [item.project_repo, item.name, item.url, item.type, item.platform].join('|');
|
||||
if (seen.has(key)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function toIsoFromMtime(value?: number | string) {
|
||||
if (typeof value === 'number') {
|
||||
const milliseconds = value > 1_000_000_000_000 ? value : value * 1000;
|
||||
return new Date(milliseconds).toISOString();
|
||||
}
|
||||
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
const parsed = Date.parse(value);
|
||||
if (!Number.isNaN(parsed)) {
|
||||
return new Date(parsed).toISOString();
|
||||
}
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
function canUseSeafileApi(config: SeafileSyncConfig) {
|
||||
return Boolean(config.baseUrl && config.token);
|
||||
}
|
||||
|
||||
function trimTrailingSlash(value: string) {
|
||||
return value.replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
function isMissingFileError(error: unknown) {
|
||||
return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT';
|
||||
}
|
||||
|
||||
async function safeReadText(response: Response) {
|
||||
try {
|
||||
return (await response.text()).slice(0, 300);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
+48
-90
@@ -2,6 +2,7 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
import { fetchGiteaSnapshot, type GeneratedGiteaActivity, type GeneratedProject } from './fetch-gitea.ts';
|
||||
import { fetchSeafileSnapshot, type GeneratedShare } from './fetch-seafile.ts';
|
||||
|
||||
type ProjectSeed = {
|
||||
name: string;
|
||||
@@ -16,15 +17,9 @@ type ProjectSeed = {
|
||||
|
||||
type ShareSeed = {
|
||||
name: string;
|
||||
description: string;
|
||||
url: string;
|
||||
time: string;
|
||||
};
|
||||
|
||||
type GeneratedShare = ShareSeed & {
|
||||
size?: number;
|
||||
source: 'seed' | 'seafile';
|
||||
updated_at: string;
|
||||
description?: string;
|
||||
url?: string;
|
||||
time?: string;
|
||||
};
|
||||
|
||||
type SyncConfig = {
|
||||
@@ -46,6 +41,7 @@ type SyncConfig = {
|
||||
token: string;
|
||||
mirrorDownloads: boolean;
|
||||
downloadsOutputDir: string;
|
||||
requestTimeoutMs: number;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -67,8 +63,12 @@ async function main() {
|
||||
const seedProjects = await readJson<ProjectSeed[]>(
|
||||
path.join(config.rootDir, 'src/content/projects/index.json'),
|
||||
);
|
||||
const giteaSnapshotPromise = hasGiteaConfig(config)
|
||||
? fetchGiteaSnapshot({
|
||||
const seedShares = await readJson<ShareSeed[]>(
|
||||
path.join(config.rootDir, 'src/content/shares/index.json'),
|
||||
);
|
||||
|
||||
const giteaSnapshot = hasGiteaConfig(config)
|
||||
? await fetchGiteaSnapshot({
|
||||
config: {
|
||||
...config.gitea,
|
||||
strict: config.strictSync,
|
||||
@@ -78,20 +78,47 @@ async function main() {
|
||||
})
|
||||
: null;
|
||||
|
||||
const [projects, shares, giteaActivity] = await Promise.all([
|
||||
resolveProjects(config, syncedAt, seedProjects, giteaSnapshotPromise),
|
||||
resolveShares(config, syncedAt),
|
||||
resolveGiteaActivity(config, syncedAt, giteaSnapshotPromise),
|
||||
]);
|
||||
const baseProjects =
|
||||
giteaSnapshot?.projects ??
|
||||
seedProjects.map((project) => ({
|
||||
...project,
|
||||
repo_url:
|
||||
project.gitea_repo && config.gitea.baseUrl
|
||||
? `${trimTrailingSlash(config.gitea.baseUrl)}/${project.gitea_repo}`
|
||||
: undefined,
|
||||
updated_at: syncedAt,
|
||||
source: 'seed' as const,
|
||||
}));
|
||||
|
||||
const seafileSnapshot = await fetchSeafileSnapshot({
|
||||
config: {
|
||||
...config.seafile,
|
||||
strict: config.strictSync,
|
||||
},
|
||||
projects: baseProjects,
|
||||
seedShares,
|
||||
syncedAt,
|
||||
mappingFilePath: path.join(config.rootDir, 'src/content/seafile/index.json'),
|
||||
});
|
||||
|
||||
const giteaActivity =
|
||||
giteaSnapshot?.activity ??
|
||||
({
|
||||
updatedAt: syncedAt,
|
||||
source: 'placeholder',
|
||||
itemCount: 0,
|
||||
days: [],
|
||||
recent: [],
|
||||
} satisfies GeneratedGiteaActivity);
|
||||
|
||||
await Promise.all([
|
||||
writeJson(path.join(config.outputDir, 'projects.json'), projects),
|
||||
writeJson(path.join(config.outputDir, 'shares.json'), shares),
|
||||
writeJson(path.join(config.outputDir, 'projects.json'), seafileSnapshot.projects),
|
||||
writeJson(path.join(config.outputDir, 'shares.json'), seafileSnapshot.shares),
|
||||
writeJson(path.join(config.outputDir, 'gitea-activity.json'), giteaActivity),
|
||||
]);
|
||||
|
||||
console.log(`[sync-content] wrote ${projects.length} projects`);
|
||||
console.log(`[sync-content] wrote ${shares.length} shares`);
|
||||
console.log(`[sync-content] wrote ${seafileSnapshot.projects.length} projects`);
|
||||
console.log(`[sync-content] wrote ${seafileSnapshot.shares.length} shares`);
|
||||
console.log(`[sync-content] wrote ${giteaActivity.itemCount} activity items`);
|
||||
console.log('[sync-content] done');
|
||||
}
|
||||
@@ -119,84 +146,15 @@ function loadConfig(rootDir: string): SyncConfig {
|
||||
rootDir,
|
||||
process.env.DOWNLOADS_OUTPUT_DIR ?? 'public/downloads',
|
||||
),
|
||||
requestTimeoutMs: getNumberEnv('SEAFILE_REQUEST_TIMEOUT_MS', 15000),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveProjects(
|
||||
config: SyncConfig,
|
||||
syncedAt: string,
|
||||
seedProjects: ProjectSeed[],
|
||||
giteaSnapshotPromise: Promise<Awaited<ReturnType<typeof fetchGiteaSnapshot>>> | null,
|
||||
): Promise<GeneratedProject[]> {
|
||||
if (giteaSnapshotPromise) {
|
||||
const snapshot = await giteaSnapshotPromise;
|
||||
return snapshot.projects;
|
||||
}
|
||||
|
||||
if (hasSeafileConfig(config)) {
|
||||
console.warn(
|
||||
'[sync-content] Seafile project sync is not implemented yet, falling back to local seed data.',
|
||||
);
|
||||
}
|
||||
|
||||
return seedProjects.map((project) => ({
|
||||
...project,
|
||||
repo_url:
|
||||
project.gitea_repo && config.gitea.baseUrl
|
||||
? `${trimTrailingSlash(config.gitea.baseUrl)}/${project.gitea_repo}`
|
||||
: undefined,
|
||||
updated_at: syncedAt,
|
||||
source: 'seed',
|
||||
}));
|
||||
}
|
||||
|
||||
async function resolveShares(config: SyncConfig, syncedAt: string): Promise<GeneratedShare[]> {
|
||||
const seedShares = await readJson<ShareSeed[]>(
|
||||
path.join(config.rootDir, 'src/content/shares/index.json'),
|
||||
);
|
||||
|
||||
if (hasSeafileConfig(config)) {
|
||||
console.warn(
|
||||
'[sync-content] remote share sync not implemented yet, falling back to local seed data.',
|
||||
);
|
||||
}
|
||||
|
||||
return seedShares.map((share) => ({
|
||||
...share,
|
||||
source: 'seed',
|
||||
updated_at: syncedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
async function resolveGiteaActivity(
|
||||
config: SyncConfig,
|
||||
syncedAt: string,
|
||||
giteaSnapshotPromise: Promise<Awaited<ReturnType<typeof fetchGiteaSnapshot>>> | null,
|
||||
): Promise<GeneratedGiteaActivity> {
|
||||
if (!giteaSnapshotPromise) {
|
||||
return {
|
||||
updatedAt: syncedAt,
|
||||
source: 'placeholder',
|
||||
itemCount: 0,
|
||||
days: [],
|
||||
recent: [],
|
||||
};
|
||||
}
|
||||
|
||||
const snapshot = await giteaSnapshotPromise;
|
||||
|
||||
return snapshot.activity;
|
||||
}
|
||||
|
||||
function hasGiteaConfig(config: SyncConfig) {
|
||||
return Boolean(config.gitea.baseUrl && config.gitea.token && config.gitea.username);
|
||||
}
|
||||
|
||||
function hasSeafileConfig(config: SyncConfig) {
|
||||
return Boolean(config.seafile.baseUrl && config.seafile.token);
|
||||
}
|
||||
|
||||
function getBooleanEnv(name: string, fallback: boolean) {
|
||||
const value = process.env[name];
|
||||
if (value == null || value.trim() === '') {
|
||||
|
||||
Reference in New Issue
Block a user