From 9ad73276e4bbfc9e9fd7a5fc6c3720e6954ecae9 Mon Sep 17 00:00:00 2001 From: SepComet <202308010230@stu.csust.edu.cn> Date: Thu, 7 May 2026 14:38:44 +0800 Subject: [PATCH] Prepare build-time static asset syncing without touching page templates Constraint: Static images need to come from remote sources before log sync and build Rejected: Per-file source mapping | directory-grouped manifests are easier to maintain Confidence: high Scope-risk: narrow Directive: Prefer grouped target_dir manifests and keep page-facing paths stable Tested: npm run assets:sync; npm run build; bash -n scripts/deploy-homepage.sh; npx tsc --noEmit --pretty false --project tsconfig.json Not-tested: Real Seafile download against production URLs --- .env.example | 3 + README.md | 53 ++++++ docs/linux-cron-deploy.md | 43 ++++- package.json | 1 + scripts/deploy-homepage.sh | 28 +++ scripts/sync-static-assets.ts | 267 +++++++++++++++++++++++++++ src/content/static-assets/index.json | 24 +++ 7 files changed, 416 insertions(+), 3 deletions(-) create mode 100644 scripts/sync-static-assets.ts create mode 100644 src/content/static-assets/index.json diff --git a/.env.example b/.env.example index 6a61c75..8598f83 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,7 @@ SEAFILE_TOKEN= # Relative paths are resolved from the repo root. # ----------------------------------------------------------------------------- SYNC_OUTPUT_DIR=src/data/generated +STATIC_ASSET_MANIFEST=src/content/static-assets/index.json # ----------------------------------------------------------------------------- # Optional behavior @@ -41,6 +42,7 @@ SYNC_OUTPUT_DIR=src/data/generated # falling back to cached/seed data. # ----------------------------------------------------------------------------- STRICT_SYNC=false +STATIC_ASSET_SYNC_STRICT=false # ----------------------------------------------------------------------------- # Optional future behavior @@ -57,3 +59,4 @@ GITEA_REQUEST_TIMEOUT_MS=15000 GITEA_REQUEST_CONCURRENCY=5 SEAFILE_REQUEST_TIMEOUT_MS=15000 +STATIC_ASSET_REQUEST_TIMEOUT_MS=15000 diff --git a/README.md b/README.md index 4523e51..b17ea70 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ - Markdown 日志内容 - 项目 / 分享 / Gitea 活动页 - 构建时同步 Gitea / Seafile 数据 +- 部署前按清单同步静态图片资源 - generated JSON schema 校验 - 统一重建入口 `npm run rebuild` - 结构化错误码与 `REBUILD_RESULT` 输出，便于 AstrBot / cron 调用 @@ -76,6 +77,7 @@ npm run rebuild | 命令 | 用途 | |------|------| | `npm run dev` | 本地开发 | +| `npm run assets:sync` | 按清单下载静态资源到本地目标目录 | | `npm run content:sync` | 仅执行内容同步 | | `npm run build` | 仅执行 Astro 构建 | | `npm run rebuild` | 同步 + 构建，适合作为 AstrBot / cron 统一入口 | @@ -109,6 +111,11 @@ npm run rebuild - 文件：`src/content/seafile/index.json` +### 静态资源同步清单 + +- 文件：`src/content/static-assets/index.json` +- 作用：在部署脚本里按“目录 -> 多个文件 URL”把远端图片下载到本地，例如 `public/images/projects/...` + ## 推荐的内容维护边界 当前更推荐把下面这些文件视为 **AstrBot 可维护输入文件**： @@ -125,6 +132,52 @@ npm run rebuild 这样比手工直接改 JSON 更适合日常维护，也不需要额外引入数据库或在线后台。 +静态资源（例如项目封面、图库图片）如果不想直接提交到仓库，可以维护： + +- `src/content/static-assets/index.json` + +推荐格式示例： + +```json +[ + { + "target_dir": "public/images/projects", + "files": [ + { + "url": "https://example.com/seafile-cover.png", + "filename": "personal-homepage.png" + }, + { + "url": "https://example.com/seafile-cover-2.png", + "filename": "devlog-pipeline.png" + } + ] + }, + { + "target_dir": "public/images/gallery", + "files": [ + { + "url": "https://example.com/seafile-gallery-cat.jpg", + "filename": "cat.jpg" + } + ] + } +] +``` + +然后执行： + +```bash +npm run assets:sync +``` + +或直接走宿主机统一部署脚本；它会在同步 log 之前先跑一轮静态资源同步。 + +兼容说明： + +- 现在脚本仍兼容旧格式 `{"url":"...","target":"..."}`； +- 但后续更推荐使用按目录分组的格式，维护起来更清楚。 + ## 统一重建入口 适合 AstrBot / cron 调用的命令： diff --git a/docs/linux-cron-deploy.md b/docs/linux-cron-deploy.md index 9fbfdbe..672f8b3 100644 --- a/docs/linux-cron-deploy.md +++ b/docs/linux-cron-deploy.md @@ -2,8 +2,10 @@ 适用于当前这套流程： +- 静态图片可由远端 URL 拉取 - log 由另一个服务生成 - log 落到宿主机目录 +- 宿主机先按清单同步静态资源到仓库目标目录 - 宿主机定时任务复制 log 到 homepage 仓库 - 宿主机执行 `npm run rebuild` - 宿主机执行 `docker compose up -d --build homepage` @@ -23,11 +25,13 @@ scripts/deploy-homepage.sh | 变量 | 默认值 | 说明 | |---|---|---| | `LOG_SOURCE_DIR` | `/home/basil/bot/data/git-summary` | 日志生成服务写入的源目录 | +| `STATIC_ASSET_MANIFEST` | `src/content/static-assets/index.json` | 静态资源下载清单 | | `SITE_DIR` | `/home/basil/source/personal-homepage` | 个人主页仓库根目录 | | `LOG_TARGET_DIR` | `$SITE_DIR/src/content/logs` | 复制后的日志目标目录 | | `DOCKER_COMPOSE_FILE` | `$SITE_DIR/docker-compose.yml` | Compose 文件 | | `DOCKER_SERVICE_NAME` | `homepage` | 要重建/重启的服务名 | | `RSYNC_DELETE` | `false` | 同步时默认不删除旧日志 | +| `RUN_STATIC_ASSET_SYNC` | `true` | 默认先按清单同步静态资源 | | `RUN_LOCAL_REBUILD` | `true` | 默认先在宿主机执行 `npm run rebuild` | | `LOCK_FILE` | `/tmp/personal-homepage-deploy.lock` | 防止重复执行的锁文件 | @@ -41,9 +45,10 @@ scripts/deploy-homepage.sh 1. 检查目录与命令是否存在 2. 获取锁文件，避免重复执行 -3. 把 `LOG_SOURCE_DIR` 同步到 `LOG_TARGET_DIR` -4. 进入 `SITE_DIR` 执行 `npm run rebuild` -5. 进入 `SITE_DIR` 执行： +3. 按 `STATIC_ASSET_MANIFEST` 下载静态资源到目标目录 +4. 把 `LOG_SOURCE_DIR` 同步到 `LOG_TARGET_DIR` +5. 进入 `SITE_DIR` 执行 `npm run rebuild` +6. 进入 `SITE_DIR` 执行： ```bash docker compose -f "$DOCKER_COMPOSE_FILE" up -d --build "$DOCKER_SERVICE_NAME" @@ -68,6 +73,12 @@ RSYNC_DELETE=true \ ./scripts/deploy-homepage.sh ``` +如果这次不想拉远端图片： + +```bash +RUN_STATIC_ASSET_SYNC=false ./scripts/deploy-homepage.sh +``` + 如果你只想做 log 同步 + Docker 重建，不想先在宿主机跑一次 `npm run rebuild`： ```bash @@ -114,6 +125,7 @@ RUN_LOCAL_REBUILD=false ./scripts/deploy-homepage.sh 当前建议： +- 静态图片清单维护在 `src/content/static-assets/index.json` - `LOG_SOURCE_DIR` 作为日志服务的宿主机挂载目录 - `SITE_DIR` 直接指向 homepage 仓库根目录 - 优先先手动跑通一次： @@ -123,3 +135,28 @@ RUN_LOCAL_REBUILD=false ./scripts/deploy-homepage.sh ``` 确认没问题后再挂 cron。 + +静态资源清单推荐写成按目录分组的格式： + +```json +[ + { + "target_dir": "public/images/projects", + "files": [ + { + "url": "https://example.com/seafile-cover.png", + "filename": "personal-homepage.png" + } + ] + }, + { + "target_dir": "public/images/gallery", + "files": [ + { + "url": "https://example.com/seafile-gallery-1.jpg", + "filename": "gallery-1.jpg" + } + ] + } +] +``` diff --git a/package.json b/package.json index acd74fa..4505f50 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ }, "scripts": { "dev": "astro dev", + "assets:sync": "node --env-file-if-exists=.env --experimental-strip-types ./scripts/sync-static-assets.ts", "content:sync": "node --env-file-if-exists=.env --experimental-strip-types ./scripts/sync-content.ts", "build": "astro build", "rebuild": "node --env-file-if-exists=.env --experimental-strip-types ./scripts/rebuild.ts", diff --git a/scripts/deploy-homepage.sh b/scripts/deploy-homepage.sh index 221d8bd..0639469 100755 --- a/scripts/deploy-homepage.sh +++ b/scripts/deploy-homepage.sh @@ -6,6 +6,10 @@ set -Eeuo pipefail # 默认值：/home/basil/bot/data/git-summary LOG_SOURCE_DIR="${LOG_SOURCE_DIR:-/home/basil/bot/data/git-summary}" +# 静态资源同步清单 +# 默认值：$SITE_DIR/src/content/static-assets/index.json +STATIC_ASSET_MANIFEST="${STATIC_ASSET_MANIFEST:-src/content/static-assets/index.json}" + # 个人主页仓库目录 # 默认值：/home/basil/source/personal-homepage SITE_DIR="${SITE_DIR:-/home/basil/source/personal-homepage}" @@ -28,6 +32,10 @@ RSYNC_DELETE="${RSYNC_DELETE:-false}" # 默认值：true RUN_LOCAL_REBUILD="${RUN_LOCAL_REBUILD:-true}" +# 是否在 log 同步前先同步静态资源 +# 默认值：true +RUN_STATIC_ASSET_SYNC="${RUN_STATIC_ASSET_SYNC:-true}" + # 锁文件路径，避免定时任务重入 # 默认值：/tmp/personal-homepage-deploy.lock LOCK_FILE="${LOCK_FILE:-/tmp/personal-homepage-deploy.lock}" @@ -79,6 +87,15 @@ sync_logs() { cp -a "${LOG_SOURCE_DIR}/." "$LOG_TARGET_DIR/" } +sync_static_assets() { + log "同步静态资源清单: $STATIC_ASSET_MANIFEST" + ( + cd "$SITE_DIR" + STATIC_ASSET_MANIFEST="$STATIC_ASSET_MANIFEST" \ + node --env-file-if-exists=.env --experimental-strip-types ./scripts/sync-static-assets.ts + ) +} + run_local_rebuild() { log "开始宿主机构建校验: npm run rebuild" ( @@ -97,6 +114,9 @@ refresh_docker_service() { main_impl() { require_command docker + if is_true "$RUN_STATIC_ASSET_SYNC"; then + require_command node + fi if is_true "$RUN_LOCAL_REBUILD"; then require_command npm fi @@ -111,12 +131,20 @@ main_impl() { log "LOG_SOURCE_DIR=$LOG_SOURCE_DIR" log "LOG_TARGET_DIR=$LOG_TARGET_DIR" log "SITE_DIR=$SITE_DIR" + log "STATIC_ASSET_MANIFEST=$STATIC_ASSET_MANIFEST" log "DOCKER_COMPOSE_FILE=$DOCKER_COMPOSE_FILE" log "DOCKER_SERVICE_NAME=$DOCKER_SERVICE_NAME" log "RSYNC_DELETE=$RSYNC_DELETE" + log "RUN_STATIC_ASSET_SYNC=$RUN_STATIC_ASSET_SYNC" log "RUN_LOCAL_REBUILD=$RUN_LOCAL_REBUILD" log "LOCK_FILE=$LOCK_FILE" + if is_true "$RUN_STATIC_ASSET_SYNC"; then + sync_static_assets + else + log "跳过静态资源同步（RUN_STATIC_ASSET_SYNC=false）" + fi + sync_logs if is_true "$RUN_LOCAL_REBUILD"; then diff --git a/scripts/sync-static-assets.ts b/scripts/sync-static-assets.ts new file mode 100644 index 0000000..ce88f5d --- /dev/null +++ b/scripts/sync-static-assets.ts @@ -0,0 +1,267 @@ +import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; + +type StaticAssetManifestItem = { + url: string; + target: string; +}; + +type StaticAssetManifestGroup = { + target_dir: string; + files: Array<{ + url: string; + filename: string; + }>; +}; + +type StaticAssetConfig = { + rootDir: string; + manifestPath: string; + strict: boolean; + requestTimeoutMs: number; +}; + +async function main() { + const config = loadConfig(process.cwd()); + const items = await readManifest(config.manifestPath); + + console.log('[sync-static-assets] start'); + console.log(`[sync-static-assets] manifest: ${config.manifestPath}`); + + if (items.length === 0) { + console.log('[sync-static-assets] no assets configured, skip'); + return; + } + + let failureCount = 0; + + for (const item of items) { + try { + await syncAsset(item, config); + } catch (error) { + failureCount += 1; + const message = error instanceof Error ? error.message : String(error); + + if (config.strict) { + throw new Error(`[sync-static-assets] failed: ${message}`); + } + + console.warn(`[sync-static-assets] skip failed asset: ${message}`); + } + } + + console.log( + `[sync-static-assets] done: total=${items.length} failed=${failureCount} strict=${config.strict}`, + ); +} + +function loadConfig(rootDir: string): StaticAssetConfig { + return { + rootDir, + manifestPath: resolveFromRoot( + rootDir, + process.env.STATIC_ASSET_MANIFEST ?? 'src/content/static-assets/index.json', + ), + strict: getBooleanEnv('STATIC_ASSET_SYNC_STRICT', false), + requestTimeoutMs: getNumberEnv('STATIC_ASSET_REQUEST_TIMEOUT_MS', 15000), + }; +} + +async function readManifest(filePath: string): Promise { + try { + const raw = await readFile(filePath, 'utf-8'); + const parsed = JSON.parse(raw) as unknown; + + if (!Array.isArray(parsed)) { + throw new Error('manifest must be a JSON array'); + } + + return parsed.flatMap((item, index) => normalizeManifestEntry(item, index)); + } catch (error) { + if (isMissingFileError(error)) { + return []; + } + + throw error; + } +} + +function normalizeManifestEntry(item: unknown, index: number): StaticAssetManifestItem[] { + if (!item || typeof item !== 'object') { + throw new Error(`manifest item #${index + 1} must be an object`); + } + + const record = item as Record; + const targetDir = typeof record.target_dir === 'string' ? record.target_dir.trim() : ''; + const files = Array.isArray(record.files) ? record.files : null; + + if (targetDir || files) { + return normalizeManifestGroup({ target_dir: targetDir, files: files ?? [] }, index); + } + + return [normalizeLegacyManifestItem(record, index)]; +} + +function normalizeLegacyManifestItem( + record: Record, + index: number, +): StaticAssetManifestItem { + const url = typeof record.url === 'string' ? record.url.trim() : ''; + const target = typeof record.target === 'string' ? record.target.trim() : ''; + + if (!url) { + throw new Error(`manifest item #${index + 1} is missing url`); + } + + if (!target) { + throw new Error(`manifest item #${index + 1} is missing target`); + } + + return { url, target }; +} + +function normalizeManifestGroup( + group: StaticAssetManifestGroup, + index: number, +): StaticAssetManifestItem[] { + const targetDir = group.target_dir.trim(); + + if (!targetDir) { + throw new Error(`manifest group #${index + 1} is missing target_dir`); + } + + if (!Array.isArray(group.files)) { + throw new Error(`manifest group #${index + 1} files must be an array`); + } + + return group.files.map((item, fileIndex) => { + if (!item || typeof item !== 'object') { + throw new Error(`manifest group #${index + 1} file #${fileIndex + 1} must be an object`); + } + + const record = item as Record; + const url = typeof record.url === 'string' ? record.url.trim() : ''; + const filename = typeof record.filename === 'string' ? record.filename.trim() : ''; + + if (!url) { + throw new Error(`manifest group #${index + 1} file #${fileIndex + 1} is missing url`); + } + + if (!filename) { + throw new Error( + `manifest group #${index + 1} file #${fileIndex + 1} is missing filename`, + ); + } + + return { + url, + target: path.posix.join(targetDir.replace(/\\/g, '/'), filename), + }; + }); +} + +async function syncAsset(item: StaticAssetManifestItem, config: StaticAssetConfig) { + const targetPath = ensurePathInsideRoot(config.rootDir, item.target); + await mkdir(path.dirname(targetPath), { recursive: true }); + + const tempPath = `${targetPath}.tmp`; + const response = await fetchWithTimeout(item.url, config.requestTimeoutMs); + + try { + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + + await writeFile(tempPath, buffer); + await rename(tempPath, targetPath); + + console.log( + `[sync-static-assets] wrote ${path.relative(config.rootDir, targetPath)} <- ${item.url}`, + ); + } catch (error) { + await safeRemove(tempPath); + throw error; + } +} + +function ensurePathInsideRoot(rootDir: string, target: string) { + const resolvedRoot = path.resolve(rootDir); + const resolvedTarget = path.resolve(rootDir, target); + const relative = path.relative(resolvedRoot, resolvedTarget); + + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error(`target path escapes repo root: ${target}`); + } + + return resolvedTarget; +} + +async function fetchWithTimeout(url: string, timeoutMs: number) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + try { + const response = await fetch(url, { + method: 'GET', + signal: controller.signal, + }); + + if (!response.ok) { + const body = await safeReadText(response); + throw new Error(`HTTP ${response.status} ${response.statusText}${body ? ` - ${body}` : ''}`); + } + + return response; + } finally { + clearTimeout(timeout); + } +} + +function getBooleanEnv(name: string, fallback: boolean) { + const value = process.env[name]; + if (value == null || value.trim() === '') { + return fallback; + } + + return ['1', 'true', 'yes', 'on'].includes(value.trim().toLowerCase()); +} + +function getNumberEnv(name: string, fallback: number) { + const value = process.env[name]; + if (value == null || value.trim() === '') { + return fallback; + } + + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : fallback; +} + +function resolveFromRoot(rootDir: string, targetPath: string) { + return path.isAbsolute(targetPath) ? targetPath : path.join(rootDir, targetPath); +} + +function isMissingFileError(error: unknown) { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT'; +} + +async function safeReadText(response: Response) { + try { + return (await response.text()).slice(0, 300); + } catch { + return ''; + } +} + +async function safeRemove(filePath: string) { + try { + await unlink(filePath); + } catch { + // Ignore temp cleanup errors. + } +} + +main().catch((error) => { + console.error('[sync-static-assets] failed'); + console.error(error); + process.exitCode = 1; +}); diff --git a/src/content/static-assets/index.json b/src/content/static-assets/index.json new file mode 100644 index 0000000..d40cc66 --- /dev/null +++ b/src/content/static-assets/index.json @@ -0,0 +1,24 @@ +[ + { + "target_dir": "public/images/projects", + "files": [] + }, + { + "target_dir": "public/images/gallery", + "files": + [ + { + "url": "http://106.12.111.150:8000/f/0d357852716f492eb92f/?dl=1", + "filename": "Alphys.png" + }, + { + "url": "http://106.12.111.150:8000/f/ec2e2e282ed7478c8fa0/?dl=1", + "filename": "Asgore.png" + }, + { + "url": "http://106.12.111.150:8000/f/7b7fb8414ab2485ba05a/?dl=1", + "filename": "Asriel.png" + } + ] + } +]