diff --git a/.gitea/workflows/deploy-staging-live.yml b/.gitea/workflows/deploy-staging-live.yml index 0d2b475..7401691 100644 --- a/.gitea/workflows/deploy-staging-live.yml +++ b/.gitea/workflows/deploy-staging-live.yml @@ -118,14 +118,14 @@ jobs: echo "ℹ️ no annotations/media change -> skip deploy" fi - - name: Install docker client + docker compose plugin (v2) + - name: Install docker client + docker compose plugin (v2) + python yaml run: | set -euo pipefail source /tmp/deploy.env [[ "${GO:-0}" == "1" ]] || { echo "ℹ️ skipped"; exit 0; } apt-get -o Acquire::Retries=5 -o Acquire::ForceIPv4=true update - apt-get install -y --no-install-recommends ca-certificates curl docker.io + apt-get install -y --no-install-recommends ca-certificates curl docker.io python3 python3-yaml rm -rf /var/lib/apt/lists/* mkdir -p /usr/local/lib/docker/cli-plugins @@ -136,6 +136,7 @@ jobs: docker version docker compose version + python3 --version # 🔥 KEY FIX: reuse existing compose project name if containers already exist PROJ="$(docker inspect archicratie-web-blue --format '{{ index .Config.Labels "com.docker.compose.project" }}' 2>/dev/null || true)" @@ -244,4 +245,171 @@ jobs: } echo "✅ live OK" - set -e \ No newline at end of file + set -e + + - name: Hotpatch annotations-index.json (deep merge shards) into blue+green + run: | + set -euo pipefail + source /tmp/deploy.env + [[ "${GO:-0}" == "1" ]] || { echo "ℹ️ skipped"; exit 0; } + + python3 - <<'PY' + import os, re, json, glob, datetime + import yaml + + ROOT = os.getcwd() + ANNO_ROOT = os.path.join(ROOT, "src", "annotations") + + def is_obj(x): return isinstance(x, dict) + def is_arr(x): return isinstance(x, list) + + def key_media(it): + return str((it or {}).get("src","")) + + def key_ref(it): + it = it or {} + return "||".join([ + str(it.get("url","")), + str(it.get("label","")), + str(it.get("kind","")), + str(it.get("citation","")), + ]) + + def key_comment(it): + return str((it or {}).get("text","")).strip() + + def dedup_extend(dst_list, src_list, key_fn): + seen = set() + out = [] + for x in (dst_list or []): + k = key_fn(x) + if k and k not in seen: + seen.add(k); out.append(x) + for x in (src_list or []): + k = key_fn(x) + if k and k not in seen: + seen.add(k); out.append(x) + return out + + def deep_merge(dst, src): + # non destructif : ne supprime rien, n'écrase pas les scalaires existants + for k, v in (src or {}).items(): + if k in ("media", "refs", "comments_editorial") and is_arr(v): + if k == "media": + dst[k] = dedup_extend(dst.get(k, []), v, key_media) + elif k == "refs": + dst[k] = dedup_extend(dst.get(k, []), v, key_ref) + else: + dst[k] = dedup_extend(dst.get(k, []), v, key_comment) + continue + + if is_obj(v): + if not is_obj(dst.get(k)): + dst[k] = dst.get(k) if is_obj(dst.get(k)) else {} + deep_merge(dst[k], v) + continue + + if is_arr(v): + # fallback: union by json string + cur = dst.get(k, []) + if not is_arr(cur): cur = [] + seen = set() + out = [] + for x in cur: + s = json.dumps(x, sort_keys=True, ensure_ascii=False) + if s not in seen: + seen.add(s); out.append(x) + for x in v: + s = json.dumps(x, sort_keys=True, ensure_ascii=False) + if s not in seen: + seen.add(s); out.append(x) + dst[k] = out + continue + + # scalaires: set seulement si absent / vide + if k not in dst or dst.get(k) in (None, ""): + dst[k] = v + + def para_num(pid): + m = re.match(r"^p-(\d+)-", str(pid)) + return int(m.group(1)) if m else 10**9 + + def sort_lists(entry): + for k in ("media","refs","comments_editorial"): + arr = entry.get(k) + if not is_arr(arr): continue + def ts(x): + try: + return datetime.datetime.fromisoformat(str((x or {}).get("ts","")).replace("Z","+00:00")).timestamp() + except Exception: + return 0 + arr.sort(key=lambda x: (ts(x), json.dumps(x, sort_keys=True, ensure_ascii=False))) + entry[k] = arr + + pages = {} + errors = [] + + if not os.path.isdir(ANNO_ROOT): + raise SystemExit(f"Missing annotations root: {ANNO_ROOT}") + + files = sorted(glob.glob(os.path.join(ANNO_ROOT, "**", "*.yml"), recursive=True)) + for fp in files: + try: + with open(fp, "r", encoding="utf-8") as f: + doc = yaml.safe_load(f) or {} + if not isinstance(doc, dict) or doc.get("schema") != 1: + continue + page = str(doc.get("page","")).strip().strip("/") + paras = doc.get("paras") or {} + if not page or not isinstance(paras, dict): + continue + + pg = pages.setdefault(page, {"paras": {}}) + for pid, entry in paras.items(): + pid = str(pid) + if pid not in pg["paras"] or not isinstance(pg["paras"].get(pid), dict): + pg["paras"][pid] = {} + if isinstance(entry, dict): + deep_merge(pg["paras"][pid], entry) + sort_lists(pg["paras"][pid]) + + except Exception as e: + errors.append({"file": os.path.relpath(fp, ROOT), "error": str(e)}) + + # tri paras + for page, obj in pages.items(): + keys = list((obj.get("paras") or {}).keys()) + keys.sort(key=lambda k: (para_num(k), k)) + obj["paras"] = {k: obj["paras"][k] for k in keys} + + out = { + "schema": 1, + "generatedAt": datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat().replace("+00:00","Z"), + "pages": pages, + "stats": { + "pages": len(pages), + "paras": sum(len(v.get("paras") or {}) for v in pages.values()), + "errors": len(errors), + }, + "errors": errors, + } + + with open("/tmp/annotations-index.json", "w", encoding="utf-8") as f: + json.dump(out, f, ensure_ascii=False) + + print("OK: wrote /tmp/annotations-index.json pages=", out["stats"]["pages"], "paras=", out["stats"]["paras"], "errors=", out["stats"]["errors"]) + PY + + # inject into running containers + for c in archicratie-web-blue archicratie-web-green; do + echo "== patch $c ==" + docker cp /tmp/annotations-index.json "${c}:/usr/share/nginx/html/annotations-index.json" + done + + # quick smoke: check new file exists and is readable + for p in 8081 8082; do + echo "== smoke annotations-index on $p ==" + curl -fsS "http://127.0.0.1:${p}/annotations-index.json" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("generatedAt:", j.get("generatedAt")); print("pages:", len(j.get("pages") or {}))' + done + + echo "✅ hotpatch annotations-index done" \ No newline at end of file diff --git a/scripts/apply-annotation-ticket.mjs b/scripts/apply-annotation-ticket.mjs index 9fb0477..4424e5e 100644 --- a/scripts/apply-annotation-ticket.mjs +++ b/scripts/apply-annotation-ticket.mjs @@ -1,9 +1,13 @@ #!/usr/bin/env node // scripts/apply-annotation-ticket.mjs -// Applique un ticket Gitea "type/media | type/reference | type/comment" vers src/annotations + public/media -// Robuste, idempotent, non destructif +// Applique un ticket Gitea "type/media | type/reference | type/comment" vers: // -// DRY RUN par défaut si --dry-run +// ✅ src/annotations///.yml (sharding par paragraphe) +// ✅ public/media//// +// +// Robuste, idempotent, non destructif. +// +// DRY RUN si --dry-run // Options: --dry-run --no-download --verify --strict --commit --close // // Env requis: @@ -36,7 +40,7 @@ import YAML from "yaml"; function usage(exitCode = 0) { console.log(` -apply-annotation-ticket — applique un ticket SidePanel (media/ref/comment) vers src/annotations/ +apply-annotation-ticket — applique un ticket SidePanel (media/ref/comment) vers src/annotations/ (shard par paragraphe) Usage: node scripts/apply-annotation-ticket.mjs [--dry-run] [--no-download] [--verify] [--strict] [--commit] [--close] @@ -44,7 +48,7 @@ Usage: Flags: --dry-run : n'écrit rien (affiche un aperçu) --no-download : n'essaie pas de télécharger les pièces jointes (media) - --verify : tente de vérifier que (page, ancre) existent (baseline/dist si dispo) + --verify : vérifie que (page, ancre) existent (dist/para-index.json si dispo, sinon baseline) --strict : refuse si URL ref invalide (http/https) OU caption media vide --commit : git add + git commit (le script commit dans la branche courante) --close : ferme le ticket (nécessite --commit) @@ -57,7 +61,7 @@ Env optionnel: GITEA_OWNER / GITEA_REPO (sinon auto-détecté via git remote) ANNO_DIR (défaut: src/annotations) PUBLIC_DIR (défaut: public) - MEDIA_ROOT (défaut URL: /media) -> écrit dans public/media/... + MEDIA_ROOT (défaut URL: /media) Exit codes: 0 ok @@ -102,6 +106,8 @@ const ANNO_DIR = path.join(CWD, process.env.ANNO_DIR || "src", "annotations"); const PUBLIC_DIR = path.join(CWD, process.env.PUBLIC_DIR || "public"); const MEDIA_URL_ROOT = String(process.env.MEDIA_ROOT || "/media").replace(/\/+$/, ""); +/* --------------------------------- helpers -------------------------------- */ + function getEnv(name, fallback = "") { return (process.env[name] ?? fallback).trim(); } @@ -123,7 +129,12 @@ function runQuiet(cmd, args, opts = {}) { } async function exists(p) { - try { await fs.access(p); return true; } catch { return false; } + try { + await fs.access(p); + return true; + } catch { + return false; + } } function inferOwnerRepoFromGit() { @@ -140,6 +151,317 @@ function gitHasStagedChanges() { return r.status === 1; } +function escapeRegExp(s) { + return String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function pickLine(body, key) { + const re = new RegExp(`^\\s*${escapeRegExp(key)}\\s*:\\s*([^\\n\\r]+)`, "mi"); + const m = String(body || "").match(re); + return m ? m[1].trim() : ""; +} + +function pickSection(body, markers) { + const text = String(body || "").replace(/\r\n/g, "\n"); + const idx = markers + .map((m) => ({ m, i: text.toLowerCase().indexOf(m.toLowerCase()) })) + .filter((x) => x.i >= 0) + .sort((a, b) => a.i - b.i)[0]; + if (!idx) return ""; + + const start = idx.i + idx.m.length; + const tail = text.slice(start); + + const stops = ["\n## ", "\n---", "\nJustification", "\nProposition", "\nSources"]; + let end = tail.length; + for (const s of stops) { + const j = tail.toLowerCase().indexOf(s.toLowerCase()); + if (j >= 0 && j < end) end = j; + } + return tail.slice(0, end).trim(); +} + +function normalizeChemin(chemin) { + let c = String(chemin || "").trim(); + if (!c) return ""; + if (!c.startsWith("/")) c = "/" + c; + if (!c.endsWith("/")) c = c + "/"; + c = c.replace(/\/{2,}/g, "/"); + return c; +} + +function normalizePageKeyFromChemin(chemin) { + return normalizeChemin(chemin).replace(/^\/+|\/+$/g, ""); +} + +function normalizeAnchorId(s) { + let a = String(s || "").trim(); + if (a.startsWith("#")) a = a.slice(1); + return a; +} + +function assert(cond, msg, code = 1) { + if (!cond) { + const e = new Error(msg); + e.__exitCode = code; + throw e; + } +} + +function isPlainObject(x) { + return !!x && typeof x === "object" && !Array.isArray(x); +} + +function paraIndexFromId(id) { + const m = String(id).match(/^p-(\d+)-/i); + return m ? Number(m[1]) : Number.NaN; +} + +function isHttpUrl(u) { + try { + const x = new URL(String(u)); + return x.protocol === "http:" || x.protocol === "https:"; + } catch { + return false; + } +} + +/* ------------------------------ para-index (verify + sort) ------------------------------ */ + +async function loadParaOrderFromDist(pageKey) { + const distIdx = path.join(CWD, "dist", "para-index.json"); + if (!(await exists(distIdx))) return null; + + let j; + try { + j = JSON.parse(await fs.readFile(distIdx, "utf8")); + } catch { + return null; + } + + // Support several shapes: + // A) { items:[{id,page,...}, ...] } + if (Array.isArray(j?.items)) { + const ids = []; + for (const it of j.items) { + const p = String(it?.page || it?.pageKey || ""); + const id = String(it?.id || it?.paraId || ""); + if (p === pageKey && id) ids.push(id); + } + if (ids.length) return ids; + } + + // B) { byId: { "p-...": { page:"archicrat-ia/chapitre-4", ... }, ... } } + if (j?.byId && typeof j.byId === "object") { + // cannot rebuild full order; but can verify existence + // return a pseudo-order map from known ids sorted by p-- then alpha + const ids = Object.keys(j.byId).filter((id) => String(j.byId[id]?.page || "") === pageKey); + if (ids.length) { + ids.sort((a, b) => { + const ia = paraIndexFromId(a); + const ib = paraIndexFromId(b); + if (Number.isFinite(ia) && Number.isFinite(ib) && ia !== ib) return ia - ib; + return String(a).localeCompare(String(b)); + }); + return ids; + } + } + + // C) { pages: { "archicrat-ia/chapitre-4": { ids:[...]} } } + if (j?.pages && typeof j.pages === "object") { + const pg = j.pages[pageKey]; + if (Array.isArray(pg?.ids)) return pg.ids.map(String); + if (Array.isArray(pg?.paras)) return pg.paras.map(String); + } + + return null; +} + +async function tryVerifyAnchor(pageKey, anchorId) { + // 1) dist/para-index.json + const order = await loadParaOrderFromDist(pageKey); + if (order) return order.includes(anchorId); + + // 2) tests/anchors-baseline.json (fallback) + const base = path.join(CWD, "tests", "anchors-baseline.json"); + if (await exists(base)) { + try { + const j = JSON.parse(await fs.readFile(base, "utf8")); + const candidates = []; + if (j?.pages && typeof j.pages === "object") { + for (const [k, v] of Object.entries(j.pages)) { + if (!Array.isArray(v)) continue; + if (String(k).includes(pageKey)) candidates.push(...v); + } + } + if (Array.isArray(j?.entries)) { + for (const it of j.entries) { + const p = String(it?.page || ""); + const ids = it?.ids; + if (Array.isArray(ids) && p.includes(pageKey)) candidates.push(...ids); + } + } + if (candidates.length) return candidates.some((x) => String(x) === anchorId); + } catch { + // ignore + } + } + + return null; // cannot verify +} + +/* ----------------------------- deep merge helpers (non destructive) ----------------------------- */ + +function keyMedia(x) { + return String(x?.src || ""); +} +function keyRef(x) { + return `${x?.url || ""}||${x?.label || ""}||${x?.kind || ""}||${x?.citation || ""}`; +} +function keyComment(x) { + return String(x?.text || "").trim(); +} + +function uniqUnion(dstArr, srcArr, keyFn) { + const out = Array.isArray(dstArr) ? [...dstArr] : []; + const seen = new Set(out.map((x) => keyFn(x))); + for (const it of (Array.isArray(srcArr) ? srcArr : [])) { + const k = keyFn(it); + if (!k) continue; + if (!seen.has(k)) { + seen.add(k); + out.push(it); + } + } + return out; +} + +function deepMergeEntry(dst, src) { + if (!isPlainObject(dst) || !isPlainObject(src)) return; + + for (const [k, v] of Object.entries(src)) { + if (k === "media" && Array.isArray(v)) { + dst.media = uniqUnion(dst.media, v, keyMedia); + continue; + } + if (k === "refs" && Array.isArray(v)) { + dst.refs = uniqUnion(dst.refs, v, keyRef); + continue; + } + if (k === "comments_editorial" && Array.isArray(v)) { + dst.comments_editorial = uniqUnion(dst.comments_editorial, v, keyComment); + continue; + } + + if (isPlainObject(v)) { + if (!isPlainObject(dst[k])) dst[k] = {}; + deepMergeEntry(dst[k], v); + continue; + } + + if (Array.isArray(v)) { + // fallback: union by JSON string + const cur = Array.isArray(dst[k]) ? dst[k] : []; + const seen = new Set(cur.map((x) => JSON.stringify(x))); + const out = [...cur]; + for (const it of v) { + const s = JSON.stringify(it); + if (!seen.has(s)) { + seen.add(s); + out.push(it); + } + } + dst[k] = out; + continue; + } + + // scalar: set only if missing/empty + if (!(k in dst) || dst[k] == null || dst[k] === "") { + dst[k] = v; + } + } +} + +function stableSortByTs(arr) { + if (!Array.isArray(arr)) return; + arr.sort((a, b) => { + const ta = Date.parse(a?.ts || "") || 0; + const tb = Date.parse(b?.ts || "") || 0; + if (ta !== tb) return ta - tb; + return JSON.stringify(a).localeCompare(JSON.stringify(b)); + }); +} + +/* ----------------------------- annotations I/O ----------------------------- */ + +async function loadAnnoDocYaml(fileAbs, pageKey) { + if (!(await exists(fileAbs))) { + return { schema: 1, page: pageKey, paras: {} }; + } + + const raw = await fs.readFile(fileAbs, "utf8"); + let doc; + try { + doc = YAML.parse(raw); + } catch (e) { + throw new Error(`${path.relative(CWD, fileAbs)}: parse failed: ${String(e?.message ?? e)}`); + } + + assert(isPlainObject(doc), `${path.relative(CWD, fileAbs)}: doc must be an object`, 2); + assert(doc.schema === 1, `${path.relative(CWD, fileAbs)}: schema must be 1`, 2); + assert(isPlainObject(doc.paras), `${path.relative(CWD, fileAbs)}: missing object key "paras"`, 2); + + if (doc.page != null) { + const got = String(doc.page).replace(/^\/+/, "").replace(/\/+$/, ""); + assert(got === pageKey, `${path.relative(CWD, fileAbs)}: page mismatch (page="${doc.page}" vs path="${pageKey}")`, 2); + } else { + doc.page = pageKey; + } + + return doc; +} + +function sortParasObject(paras, order) { + const keys = Object.keys(paras || {}); + const idx = new Map(); + if (Array.isArray(order)) { + order.forEach((id, i) => idx.set(String(id), i)); + } + + keys.sort((a, b) => { + const ha = idx.has(a); + const hb = idx.has(b); + if (ha && hb) return idx.get(a) - idx.get(b); + if (ha && !hb) return -1; + if (!ha && hb) return 1; + + const ia = paraIndexFromId(a); + const ib = paraIndexFromId(b); + if (Number.isFinite(ia) && Number.isFinite(ib) && ia !== ib) return ia - ib; + return String(a).localeCompare(String(b)); + }); + + const out = {}; + for (const k of keys) out[k] = paras[k]; + return out; +} + +async function saveAnnoDocYaml(fileAbs, doc, order = null) { + await fs.mkdir(path.dirname(fileAbs), { recursive: true }); + doc.paras = sortParasObject(doc.paras, order); + + // also sort known lists inside each para for stable diffs + for (const e of Object.values(doc.paras || {})) { + if (!isPlainObject(e)) continue; + stableSortByTs(e.media); + stableSortByTs(e.refs); + stableSortByTs(e.comments_editorial); + } + + const out = YAML.stringify(doc); + await fs.writeFile(fileAbs, out, "utf8"); +} + /* ------------------------------ gitea helpers ------------------------------ */ function apiBaseNorm(forgeApiBase) { @@ -167,7 +489,7 @@ async function fetchIssue({ forgeApiBase, owner, repo, token, issueNum }) { } async function fetchIssueAssets({ forgeApiBase, owner, repo, token, issueNum }) { - // ✅ Gitea: /issues/{index}/assets + // Gitea: /issues/{index}/assets const url = `${apiBaseNorm(forgeApiBase)}/api/v1/repos/${owner}/${repo}/issues/${issueNum}/assets`; try { const json = await giteaGET(url, token); @@ -215,200 +537,43 @@ async function closeIssue({ forgeApiBase, owner, repo, token, issueNum, comment } } -/* ------------------------------ parsing helpers ---------------------------- */ +/* ------------------------------ media helpers ------------------------------ */ -function escapeRegExp(s) { - return String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +function inferMediaTypeFromFilename(name) { + const n = String(name || "").toLowerCase(); + if (/\.(png|jpe?g|webp|gif|svg)$/.test(n)) return "image"; + if (/\.(mp4|webm|mov|m4v)$/.test(n)) return "video"; + if (/\.(mp3|wav|ogg|m4a)$/.test(n)) return "audio"; + return "link"; } -function pickLine(body, key) { - const re = new RegExp(`^\\s*${escapeRegExp(key)}\\s*:\\s*([^\\n\\r]+)`, "mi"); - const m = String(body || "").match(re); - return m ? m[1].trim() : ""; +function sanitizeFilename(name) { + return String(name || "file") + .replace(/[\/\\]/g, "_") + .replace(/[^\w.\-]+/g, "_") + .replace(/_+/g, "_") + .slice(0, 180); } -function pickSection(body, markers) { - const text = String(body || "").replace(/\r\n/g, "\n"); - const idx = markers - .map((m) => ({ m, i: text.toLowerCase().indexOf(m.toLowerCase()) })) - .filter((x) => x.i >= 0) - .sort((a, b) => a.i - b.i)[0]; - if (!idx) return ""; - - const start = idx.i + idx.m.length; - const tail = text.slice(start); - - const stops = [ - "\n## ", - "\n---", - "\nJustification", - "\nProposition", - "\nSources", - ]; - let end = tail.length; - for (const s of stops) { - const j = tail.toLowerCase().indexOf(s.toLowerCase()); - if (j >= 0 && j < end) end = j; - } - return tail.slice(0, end).trim(); -} - -function normalizeChemin(chemin) { - let c = String(chemin || "").trim(); - if (!c) return ""; - if (!c.startsWith("/")) c = "/" + c; - if (!c.endsWith("/")) c = c + "/"; - c = c.replace(/\/{2,}/g, "/"); - return c; -} - -function normalizePageKeyFromChemin(chemin) { - return normalizeChemin(chemin).replace(/^\/+|\/+$/g, ""); -} - -function normalizeAnchorId(s) { - let a = String(s || "").trim(); - if (a.startsWith("#")) a = a.slice(1); - return a; -} - -function assert(cond, msg, code = 1) { - if (!cond) { - const e = new Error(msg); - e.__exitCode = code; - throw e; - } -} - -function isPlainObject(x) { - return !!x && typeof x === "object" && !Array.isArray(x); -} - -/* ----------------------------- verify helpers ------------------------------ */ - -function paraIndexFromId(id) { - const m = String(id).match(/^p-(\d+)-/i); - return m ? Number(m[1]) : Number.NaN; -} - -async function tryVerifyAnchor(pageKey, anchorId) { - // 1) dist/para-index.json (si build déjà faite) - const distIdx = path.join(CWD, "dist", "para-index.json"); - if (await exists(distIdx)) { - const raw = await fs.readFile(distIdx, "utf8"); - const idx = JSON.parse(raw); - const byId = idx?.byId; - if (byId && typeof byId === "object" && byId[anchorId] != null) return true; - } - - // 2) tests/anchors-baseline.json (si dispo) - const base = path.join(CWD, "tests", "anchors-baseline.json"); - if (await exists(base)) { - const raw = await fs.readFile(base, "utf8"); - const j = JSON.parse(raw); - - // tolérant: cherche un array d'ids associé à la page - const candidates = []; - - // cas 1: j.pages[...] - if (j?.pages && typeof j.pages === "object") { - for (const [k, v] of Object.entries(j.pages)) { - if (!Array.isArray(v)) continue; - // on matche large: pageKey inclus dans le path - if (String(k).includes(pageKey)) candidates.push(...v); - } - } - - // cas 2: j.entries = [{page, ids}] - if (Array.isArray(j?.entries)) { - for (const it of j.entries) { - const p = String(it?.page || ""); - const ids = it?.ids; - if (Array.isArray(ids) && p.includes(pageKey)) candidates.push(...ids); - } - } - - if (candidates.length) { - return candidates.some((x) => String(x) === anchorId); - } - } - - // impossible à vérifier - return null; -} - -/* ----------------------------- annotations I/O ----------------------------- */ - -async function loadAnnoDoc(fileAbs, pageKey) { - if (!(await exists(fileAbs))) { - return { schema: 1, page: pageKey, paras: {} }; - } - - const raw = await fs.readFile(fileAbs, "utf8"); - let doc; - try { - doc = YAML.parse(raw); - } catch (e) { - throw new Error(`${path.relative(CWD, fileAbs)}: parse failed: ${String(e?.message ?? e)}`); - } - - assert(isPlainObject(doc), `${path.relative(CWD, fileAbs)}: doc must be an object`); - assert(doc.schema === 1, `${path.relative(CWD, fileAbs)}: schema must be 1`); - assert(isPlainObject(doc.paras), `${path.relative(CWD, fileAbs)}: missing object key "paras"`); - - if (doc.page != null) { - const got = String(doc.page).replace(/^\/+/, "").replace(/\/+$/, ""); - assert(got === pageKey, `${path.relative(CWD, fileAbs)}: page mismatch (page="${doc.page}" vs path="${pageKey}")`); - } else { - doc.page = pageKey; - } - - return doc; -} - -function sortParasObject(paras) { - const keys = Object.keys(paras || {}); - keys.sort((a, b) => { - const ia = paraIndexFromId(a); - const ib = paraIndexFromId(b); - if (Number.isFinite(ia) && Number.isFinite(ib) && ia !== ib) return ia - ib; - return String(a).localeCompare(String(b)); +async function downloadToFile(url, token, destAbs) { + const res = await fetch(url, { + headers: { + Authorization: `token ${token}`, + "User-Agent": "archicratie-apply-annotation/1.0", + }, + redirect: "follow", }); - const out = {}; - for (const k of keys) out[k] = paras[k]; - return out; + if (!res.ok) { + const t = await res.text().catch(() => ""); + throw new Error(`download failed HTTP ${res.status}: ${url}\n${t}`); + } + const buf = Buffer.from(await res.arrayBuffer()); + await fs.mkdir(path.dirname(destAbs), { recursive: true }); + await fs.writeFile(destAbs, buf); + return buf.length; } -async function saveAnnoDocYaml(fileAbs, doc) { - await fs.mkdir(path.dirname(fileAbs), { recursive: true }); - doc.paras = sortParasObject(doc.paras); - const out = YAML.stringify(doc); - await fs.writeFile(fileAbs, out, "utf8"); -} - -/* ------------------------------ apply per type ----------------------------- */ - -function ensureEntry(doc, paraId) { - if (!doc.paras[paraId] || !isPlainObject(doc.paras[paraId])) doc.paras[paraId] = {}; - return doc.paras[paraId]; -} - -function uniqPush(arr, item, keyFn) { - const k = keyFn(item); - const exists = arr.some((x) => keyFn(x) === k); - if (!exists) arr.push(item); - return !exists; -} - -function stableSortByTs(arr) { - if (!Array.isArray(arr)) return; - arr.sort((a, b) => { - const ta = Date.parse(a?.ts || "") || 0; - const tb = Date.parse(b?.ts || "") || 0; - if (ta !== tb) return ta - tb; - return JSON.stringify(a).localeCompare(JSON.stringify(b)); - }); -} +/* ------------------------------ type parsers ------------------------------ */ function parseReferenceBlock(body) { const block = @@ -431,50 +596,6 @@ function parseReferenceBlock(body) { }; } -function inferMediaTypeFromFilename(name) { - const n = String(name || "").toLowerCase(); - if (/\.(png|jpe?g|webp|gif|svg)$/.test(n)) return "image"; - if (/\.(mp4|webm|mov|m4v)$/.test(n)) return "video"; - if (/\.(mp3|wav|ogg|m4a)$/.test(n)) return "audio"; - return "link"; -} - -function sanitizeFilename(name) { - return String(name || "file") - .replace(/[\/\\]/g, "_") - .replace(/[^\w.\-]+/g, "_") - .replace(/_+/g, "_") - .slice(0, 180); -} - -function isHttpUrl(u) { - try { - const x = new URL(String(u)); - return x.protocol === "http:" || x.protocol === "https:"; - } catch { - return false; - } -} - -async function downloadToFile(url, token, destAbs) { - const res = await fetch(url, { - headers: { - // la plupart des /attachments sont publics, mais on garde le token “au cas où” - Authorization: `token ${token}`, - "User-Agent": "archicratie-apply-annotation/1.0", - }, - redirect: "follow", - }); - if (!res.ok) { - const t = await res.text().catch(() => ""); - throw new Error(`download failed HTTP ${res.status}: ${url}\n${t}`); - } - const buf = Buffer.from(await res.arrayBuffer()); - await fs.mkdir(path.dirname(destAbs), { recursive: true }); - await fs.writeFile(destAbs, buf); - return buf.length; -} - /* ----------------------------------- main ---------------------------------- */ async function main() { @@ -511,29 +632,57 @@ async function main() { const pageKey = normalizePageKeyFromChemin(chemin); assert(pageKey, "Ticket: impossible de dériver pageKey.", 2); + // para order (used for verify + sorting) + const paraOrder = DO_VERIFY ? await loadParaOrderFromDist(pageKey) : null; + if (DO_VERIFY) { const ok = await tryVerifyAnchor(pageKey, ancre); if (ok === false) { throw Object.assign(new Error(`Ticket verify: ancre introuvable pour page "${pageKey}" => ${ancre}`), { __exitCode: 2 }); } if (ok === null) { - // pas de source de vérité dispo - if (STRICT) throw Object.assign(new Error(`Ticket verify (strict): impossible de vérifier (pas de baseline/dist)`), { __exitCode: 2 }); - console.warn("⚠️ verify: impossible de vérifier (pas de baseline/dist) — on continue."); + if (STRICT) throw Object.assign(new Error(`Ticket verify (strict): impossible de vérifier (pas de dist/para-index.json ou baseline)`), { __exitCode: 2 }); + console.warn("⚠️ verify: impossible de vérifier (pas de dist/para-index.json ou baseline) — on continue."); } } - const annoFileAbs = path.join(ANNO_DIR, `${pageKey}.yml`); - const annoFileRel = path.relative(CWD, annoFileAbs).replace(/\\/g, "/"); + // ✅ SHARD FILE: src/annotations//.yml + const annoShardFileAbs = path.join(ANNO_DIR, pageKey, `${ancre}.yml`); + const annoShardFileRel = path.relative(CWD, annoShardFileAbs).replace(/\\/g, "/"); - console.log("✅ Parsed:", { type, chemin, ancre: `#${ancre}`, pageKey, annoFile: annoFileRel }); + // legacy (read-only, used as base to avoid losing previously stored data) + const annoLegacyFileAbs = path.join(ANNO_DIR, `${pageKey}.yml`); - const doc = await loadAnnoDoc(annoFileAbs, pageKey); - const entry = ensureEntry(doc, ancre); + console.log("✅ Parsed:", { + type, + chemin, + ancre: `#${ancre}`, + pageKey, + annoFile: annoShardFileRel, + }); + + // load shard doc + const doc = await loadAnnoDocYaml(annoShardFileAbs, pageKey); + + // merge legacy para into shard as base (non destructive) + if (await exists(annoLegacyFileAbs)) { + try { + const legacy = await loadAnnoDocYaml(annoLegacyFileAbs, pageKey); + const legacyEntry = legacy?.paras?.[ancre]; + if (isPlainObject(legacyEntry)) { + if (!isPlainObject(doc.paras[ancre])) doc.paras[ancre] = {}; + deepMergeEntry(doc.paras[ancre], legacyEntry); + } + } catch { + // ignore legacy parse issues (shard still works) + } + } + + if (!isPlainObject(doc.paras[ancre])) doc.paras[ancre] = {}; + const entry = doc.paras[ancre]; const touchedFiles = []; const notes = []; - let changed = false; const nowIso = new Date().toISOString(); @@ -545,16 +694,16 @@ async function main() { if (!Array.isArray(entry.comments_editorial)) entry.comments_editorial = []; const item = { text, status: "new", ts: nowIso, fromIssue: issueNum }; - const added = uniqPush(entry.comments_editorial, item, (x) => `${(x?.text || "").trim()}`); - if (added) { changed = true; notes.push(`+ comment added (len=${text.length})`); } - else notes.push(`~ comment already present (dedup)`); + const before = entry.comments_editorial.length; + entry.comments_editorial = uniqUnion(entry.comments_editorial, [item], keyComment); + changed = changed || entry.comments_editorial.length !== before; stableSortByTs(entry.comments_editorial); + notes.push(changed ? `+ comment added (len=${text.length})` : `~ comment already present (dedup)`); } else if (type === "type/reference") { const ref = parseReferenceBlock(body); - assert(ref.url || ref.label, "Ticket reference: renseigne au moins - URL: ou - Label: dans le ticket.", 2); if (STRICT && ref.url && !isHttpUrl(ref.url)) { @@ -571,32 +720,30 @@ async function main() { }; if (ref.citation) item.citation = ref.citation; - const added = uniqPush(entry.refs, item, (x) => `${x?.url || ""}||${x?.label || ""}||${x?.kind || ""}||${x?.citation || ""}`); - if (added) { changed = true; notes.push(`+ reference added (${item.url ? "url" : "label"})`); } - else notes.push(`~ reference already present (dedup)`); + const before = entry.refs.length; + entry.refs = uniqUnion(entry.refs, [item], keyRef); + changed = changed || entry.refs.length !== before; stableSortByTs(entry.refs); + notes.push(changed ? `+ reference added (${item.url ? "url" : "label"})` : `~ reference already present (dedup)`); } else if (type === "type/media") { if (!Array.isArray(entry.media)) entry.media = []; const atts = NO_DOWNLOAD ? [] : await fetchIssueAssets({ forgeApiBase, owner, repo, token, issueNum }); - - if (!atts.length) { - notes.push("! no assets found (nothing to download)."); - } + if (!atts.length) notes.push("! no assets found (nothing to download)."); for (const a of atts) { const name = sanitizeFilename(a?.name || `asset-${a?.id || "x"}`); const dl = a?.browser_download_url || a?.download_url || ""; if (!dl) { notes.push(`! asset missing download url: ${name}`); continue; } - // caption = title du ticket (fallback ".") - const caption = (title || "").trim() || "."; - if (STRICT && !caption.trim()) { - throw Object.assign(new Error("Ticket media (strict): caption vide."), { __exitCode: 2 }); + const caption = (title || "").trim(); + if (STRICT && !caption) { + throw Object.assign(new Error("Ticket media (strict): caption vide (titre de ticket requis)."), { __exitCode: 2 }); } + const captionFinal = caption || "."; const mediaDirAbs = path.join(PUBLIC_DIR, "media", pageKey, ancre); const destAbs = path.join(mediaDirAbs, name); @@ -608,21 +755,24 @@ async function main() { const bytes = await downloadToFile(dl, token, destAbs); notes.push(`+ downloaded ${name} (${bytes} bytes) -> ${urlPath}`); touchedFiles.push(path.relative(CWD, destAbs).replace(/\\/g, "/")); + changed = true; } else { notes.push(`(dry) would download ${name} -> ${urlPath}`); + changed = true; } const item = { type: inferMediaTypeFromFilename(name), src: urlPath, - caption, + caption: captionFinal, credit: "", ts: nowIso, fromIssue: issueNum, }; - const added = uniqPush(entry.media, item, (x) => String(x?.src || "")); - if (added) changed = true; + const before = entry.media.length; + entry.media = uniqUnion(entry.media, [item], keyMedia); + if (entry.media.length !== before) changed = true; } stableSortByTs(entry.media); @@ -640,7 +790,7 @@ async function main() { if (DRY_RUN) { console.log("\n--- DRY RUN (no write) ---"); - console.log(`Would update: ${annoFileRel}`); + console.log(`Would update: ${annoShardFileRel}`); for (const n of notes) console.log(" ", n); console.log("\nExcerpt (resulting entry):"); console.log(YAML.stringify({ [ancre]: doc.paras[ancre] }).trimEnd()); @@ -648,10 +798,10 @@ async function main() { return; } - await saveAnnoDocYaml(annoFileAbs, doc); - touchedFiles.unshift(annoFileRel); + await saveAnnoDocYaml(annoShardFileAbs, doc, paraOrder); + touchedFiles.unshift(annoShardFileRel); - console.log(`✅ Updated: ${annoFileRel}`); + console.log(`✅ Updated: ${annoShardFileRel}`); for (const n of notes) console.log(" ", n); if (DO_COMMIT) { @@ -685,4 +835,4 @@ main().catch((e) => { const code = e?.__exitCode || 1; console.error("💥", e?.message || e); process.exit(code); -}); +}); \ No newline at end of file