ci: shard annotations by para + deploy deep-merge hotpatch

.gitea/workflows/deploy-staging-live.yml

@@ -118,14 +118,14 @@ jobs:
             echo "ℹ️ no annotations/media change -> skip deploy"
           fi
 
-      - name: Install docker client + docker compose plugin (v2)
+      - name: Install docker client + docker compose plugin (v2) + python yaml
         run: |
           set -euo pipefail
           source /tmp/deploy.env
           [[ "${GO:-0}" == "1" ]] || { echo "ℹ️ skipped"; exit 0; }
 
           apt-get -o Acquire::Retries=5 -o Acquire::ForceIPv4=true update
-          apt-get install -y --no-install-recommends ca-certificates curl docker.io
+          apt-get install -y --no-install-recommends ca-certificates curl docker.io python3 python3-yaml
           rm -rf /var/lib/apt/lists/*
 
           mkdir -p /usr/local/lib/docker/cli-plugins
@@ -136,6 +136,7 @@ jobs:
 
           docker version
           docker compose version
+          python3 --version
 
           # 🔥 KEY FIX: reuse existing compose project name if containers already exist
           PROJ="$(docker inspect archicratie-web-blue --format '{{ index .Config.Labels "com.docker.compose.project" }}' 2>/dev/null || true)"
@@ -244,4 +245,171 @@ jobs:
           }
 
           echo "✅ live OK"
-          set -e
+          set -e
+
+      - name: Hotpatch annotations-index.json (deep merge shards) into blue+green
+        run: |
+          set -euo pipefail
+          source /tmp/deploy.env
+          [[ "${GO:-0}" == "1" ]] || { echo "ℹ️ skipped"; exit 0; }
+
+          python3 - <<'PY'
+          import os, re, json, glob, datetime
+          import yaml
+
+          ROOT = os.getcwd()
+          ANNO_ROOT = os.path.join(ROOT, "src", "annotations")
+
+          def is_obj(x): return isinstance(x, dict)
+          def is_arr(x): return isinstance(x, list)
+
+          def key_media(it):
+            return str((it or {}).get("src",""))
+
+          def key_ref(it):
+            it = it or {}
+            return "||".join([
+              str(it.get("url","")),
+              str(it.get("label","")),
+              str(it.get("kind","")),
+              str(it.get("citation","")),
+            ])
+
+          def key_comment(it):
+            return str((it or {}).get("text","")).strip()
+
+          def dedup_extend(dst_list, src_list, key_fn):
+            seen = set()
+            out = []
+            for x in (dst_list or []):
+              k = key_fn(x)
+              if k and k not in seen:
+                seen.add(k); out.append(x)
+            for x in (src_list or []):
+              k = key_fn(x)
+              if k and k not in seen:
+                seen.add(k); out.append(x)
+            return out
+
+          def deep_merge(dst, src):
+            # non destructif : ne supprime rien, n'écrase pas les scalaires existants
+            for k, v in (src or {}).items():
+              if k in ("media", "refs", "comments_editorial") and is_arr(v):
+                if k == "media":
+                  dst[k] = dedup_extend(dst.get(k, []), v, key_media)
+                elif k == "refs":
+                  dst[k] = dedup_extend(dst.get(k, []), v, key_ref)
+                else:
+                  dst[k] = dedup_extend(dst.get(k, []), v, key_comment)
+                continue
+
+              if is_obj(v):
+                if not is_obj(dst.get(k)):
+                  dst[k] = dst.get(k) if is_obj(dst.get(k)) else {}
+                deep_merge(dst[k], v)
+                continue
+
+              if is_arr(v):
+                # fallback: union by json string
+                cur = dst.get(k, [])
+                if not is_arr(cur): cur = []
+                seen = set()
+                out = []
+                for x in cur:
+                  s = json.dumps(x, sort_keys=True, ensure_ascii=False)
+                  if s not in seen:
+                    seen.add(s); out.append(x)
+                for x in v:
+                  s = json.dumps(x, sort_keys=True, ensure_ascii=False)
+                  if s not in seen:
+                    seen.add(s); out.append(x)
+                dst[k] = out
+                continue
+
+              # scalaires: set seulement si absent / vide
+              if k not in dst or dst.get(k) in (None, ""):
+                dst[k] = v
+
+          def para_num(pid):
+            m = re.match(r"^p-(\d+)-", str(pid))
+            return int(m.group(1)) if m else 10**9
+
+          def sort_lists(entry):
+            for k in ("media","refs","comments_editorial"):
+              arr = entry.get(k)
+              if not is_arr(arr): continue
+              def ts(x):
+                try:
+                  return datetime.datetime.fromisoformat(str((x or {}).get("ts","")).replace("Z","+00:00")).timestamp()
+                except Exception:
+                  return 0
+              arr.sort(key=lambda x: (ts(x), json.dumps(x, sort_keys=True, ensure_ascii=False)))
+              entry[k] = arr
+
+          pages = {}
+          errors = []
+
+          if not os.path.isdir(ANNO_ROOT):
+            raise SystemExit(f"Missing annotations root: {ANNO_ROOT}")
+
+          files = sorted(glob.glob(os.path.join(ANNO_ROOT, "**", "*.yml"), recursive=True))
+          for fp in files:
+            try:
+              with open(fp, "r", encoding="utf-8") as f:
+                doc = yaml.safe_load(f) or {}
+              if not isinstance(doc, dict) or doc.get("schema") != 1:
+                continue
+              page = str(doc.get("page","")).strip().strip("/")
+              paras = doc.get("paras") or {}
+              if not page or not isinstance(paras, dict):
+                continue
+
+              pg = pages.setdefault(page, {"paras": {}})
+              for pid, entry in paras.items():
+                pid = str(pid)
+                if pid not in pg["paras"] or not isinstance(pg["paras"].get(pid), dict):
+                  pg["paras"][pid] = {}
+                if isinstance(entry, dict):
+                  deep_merge(pg["paras"][pid], entry)
+                sort_lists(pg["paras"][pid])
+
+            except Exception as e:
+              errors.append({"file": os.path.relpath(fp, ROOT), "error": str(e)})
+
+          # tri paras
+          for page, obj in pages.items():
+            keys = list((obj.get("paras") or {}).keys())
+            keys.sort(key=lambda k: (para_num(k), k))
+            obj["paras"] = {k: obj["paras"][k] for k in keys}
+
+          out = {
+            "schema": 1,
+            "generatedAt": datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat().replace("+00:00","Z"),
+            "pages": pages,
+            "stats": {
+              "pages": len(pages),
+              "paras": sum(len(v.get("paras") or {}) for v in pages.values()),
+              "errors": len(errors),
+            },
+            "errors": errors,
+          }
+
+          with open("/tmp/annotations-index.json", "w", encoding="utf-8") as f:
+            json.dump(out, f, ensure_ascii=False)
+
+          print("OK: wrote /tmp/annotations-index.json pages=", out["stats"]["pages"], "paras=", out["stats"]["paras"], "errors=", out["stats"]["errors"])
+          PY
+
+          # inject into running containers
+          for c in archicratie-web-blue archicratie-web-green; do
+            echo "== patch $c =="
+            docker cp /tmp/annotations-index.json "${c}:/usr/share/nginx/html/annotations-index.json"
+          done
+
+          # quick smoke: check new file exists and is readable
+          for p in 8081 8082; do
+            echo "== smoke annotations-index on $p =="
+            curl -fsS "http://127.0.0.1:${p}/annotations-index.json" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("generatedAt:", j.get("generatedAt")); print("pages:", len(j.get("pages") or {}))'
+          done
+
+          echo "✅ hotpatch annotations-index done"