arozantsev commited on
Commit
c2bbd0e
·
verified ·
1 Parent(s): f53996d

Deploy simready-assets run 20261113-698f0b6 (GPU variant) (part 2)

Browse files
pipeline/categories.py CHANGED
@@ -226,6 +226,52 @@ def augment_hierarchy(doc: Dict, values: Iterable[str], vendors: Dict, pins: Dic
226
  return out, unplaced
227
 
228
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
229
  # ── install list: package folder -> category folder ──────────────────────────
230
 
231
 
 
226
  return out, unplaced
227
 
228
 
229
+ def relabel_vendors(doc: Dict, vendors: Dict) -> List[str]:
230
+ """Give a stored run's tree the partner labels vendors.yaml has now, in place: every `oem:<owner>` node, the
231
+ path steps naming it and `central_category.vendors` (which the Explorer's Provider names come from, ahead of
232
+ makers.json). The Space runs this on the restored tree at start (hf_space_seed), so a renamed partner shows on
233
+ the run it already serves, without a re-index (2026-11-18: M-XR -> Marso by M-XR). Only labels change: node
234
+ ids, `folder:` ids and install_categories.json keep the run's. Returns the owners whose label changed."""
235
+ labels = vendors.get("labels") or {}
236
+ changed = set()
237
+
238
+ def new_label(qcode: str, old: str) -> str:
239
+ owner = qcode[len(OEM_PREFIX):]
240
+ label = labels.get(owner)
241
+ if label and label != old:
242
+ changed.add(owner)
243
+ return label
244
+ return old
245
+
246
+ for key, node in (doc.get("qcodes") or {}).items():
247
+ if not isinstance(node, dict):
248
+ continue
249
+ if key.startswith(OEM_PREFIX) and isinstance(node.get("label"), str):
250
+ node["label"] = new_label(key, node["label"])
251
+ for step in node.get("path") or []:
252
+ if isinstance(step, dict) and str(step.get("qcode") or "").startswith(OEM_PREFIX):
253
+ step["label"] = new_label(step["qcode"], str(step.get("label") or ""))
254
+ cc = doc.get("central_category")
255
+ if isinstance(cc, dict) and isinstance(cc.get("vendors"), dict):
256
+ for owner, old in cc["vendors"].items():
257
+ cc["vendors"][owner] = new_label(OEM_PREFIX + owner, str(old))
258
+ return sorted(changed)
259
+
260
+
261
+ def relabel_vendors_file(path: Path, vendors_file: Path = VENDORS_FILE) -> List[str]:
262
+ """relabel_vendors on a tree file, rewritten atomically only when a label changed. A missing tree or
263
+ vendors.yaml changes nothing (a run without a tree, an image without the file)."""
264
+ if not path.exists() or not vendors_file.exists():
265
+ return []
266
+ doc = json.loads(path.read_text())
267
+ changed = relabel_vendors(doc, load_vendors(vendors_file))
268
+ if changed:
269
+ tmp = path.with_name(path.name + ".tmp")
270
+ tmp.write_text(json.dumps(doc, indent=1, ensure_ascii=False, sort_keys=True) + "\n")
271
+ os.replace(tmp, path)
272
+ return changed
273
+
274
+
275
  # ── install list: package folder -> category folder ──────────────────────────
276
 
277
 
pipeline/hf_endpoint.py CHANGED
@@ -55,8 +55,9 @@ ROOT = Path(__file__).resolve().parents[1]
55
  PROXY_SRC = ROOT / "docker" / "hf-space-proxy"
56
  PROXY_STAGE = ROOT / ".build" / "hf-space-proxy-stage"
57
  # The thin Space's repository: these files from docker/hf-space-proxy, and README.md (PROXY_README).
58
- PROXY_FILES = ("Dockerfile", "nginx.conf.template", "start.sh", "LICENSE", "CONTRIBUTING.md", "THIRD_PARTY_NOTICE.md",
59
- "licenses") # licenses/: the notice's license texts, one per package
 
60
  # Files a thin Space may hold besides those: its README and the .gitattributes HF writes into every new repo.
61
  PROXY_REPO_FILES = frozenset(PROXY_FILES) | {"README.md", ".gitattributes"}
62
  PROXY_DIRS = frozenset({"licenses"}) # the PROXY_FILES that are directories: their files are the Space's too
 
55
  PROXY_SRC = ROOT / "docker" / "hf-space-proxy"
56
  PROXY_STAGE = ROOT / ".build" / "hf-space-proxy-stage"
57
  # The thin Space's repository: these files from docker/hf-space-proxy, and README.md (PROXY_README).
58
+ # third-party-sources.sh: the build step that puts the added packages' source code in the image (/licenses/sources).
59
+ PROXY_FILES = ("Dockerfile", "nginx.conf.template", "start.sh", "third-party-sources.sh", "LICENSE", "CONTRIBUTING.md",
60
+ "THIRD_PARTY_NOTICE.md", "licenses") # licenses/: the notice's license texts, one per package
61
  # Files a thin Space may hold besides those: its README and the .gitattributes HF writes into every new repo.
62
  PROXY_REPO_FILES = frozenset(PROXY_FILES) | {"README.md", ".gitattributes"}
63
  PROXY_DIRS = frozenset({"licenses"}) # the PROXY_FILES that are directories: their files are the Space's too
pipeline/hf_space.py CHANGED
@@ -224,6 +224,9 @@ def stage_repo(dest: Path, marker: str, readme: str, dockerfile_name: str = "Doc
224
  REPO_ROOT / "pipeline", dest / "pipeline",
225
  ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
226
  )
 
 
 
227
  # The Dockerfile COPYs this in directly now (see its own comment) —
228
  # must ship alongside it or HF's build fails with a missing path.
229
  # A dist built by simready-central's slim/scripts/build-explorer-public.sh
 
224
  REPO_ROOT / "pipeline", dest / "pipeline",
225
  ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
226
  )
227
+ # The partner labels the Space puts on the restored Category tree at start (categories.relabel_vendors).
228
+ (dest / "config" / "hierarchy").mkdir(parents=True)
229
+ shutil.copy2(REPO_ROOT / "config" / "hierarchy" / "vendors.yaml", dest / "config" / "hierarchy" / "vendors.yaml")
230
  # The Dockerfile COPYs this in directly now (see its own comment) —
231
  # must ship alongside it or HF's build fails with a missing path.
232
  # A dist built by simready-central's slim/scripts/build-explorer-public.sh
pipeline/hf_space_seed.py CHANGED
@@ -57,6 +57,7 @@ from pathlib import Path
57
  import httpx
58
 
59
  from .artifact_store import download_prefix, read_text
 
60
  from .config import BUILD_DIR, Config
61
  from .restore import OPENSEARCH_TIMEOUT, RESTORE_DIR, download, restore_opensearch
62
 
@@ -124,6 +125,14 @@ def main_opensearch(cfg: Config, base: str, corpus_name: str) -> int:
124
  prefix = resolve_prefix(cfg, base, corpus_name)
125
  print(f"==> restoring from {prefix}")
126
  download(cfg, prefix)
 
 
 
 
 
 
 
 
127
  # The artifact set is complete on disk: hf-storage-api (supervisord) waits
128
  # for its catalog OR this marker, so a run without a catalog is reported
129
  # instead of waited on forever.
 
57
  import httpx
58
 
59
  from .artifact_store import download_prefix, read_text
60
+ from .categories import HIERARCHY_FILE, relabel_vendors_file
61
  from .config import BUILD_DIR, Config
62
  from .restore import OPENSEARCH_TIMEOUT, RESTORE_DIR, download, restore_opensearch
63
 
 
125
  prefix = resolve_prefix(cfg, base, corpus_name)
126
  print(f"==> restoring from {prefix}")
127
  download(cfg, prefix)
128
+ # The tree's partner names as vendors.yaml has them now, not as the run baked them (a renamed partner shows
129
+ # without a re-index). Before the marker: nginx serves the tree from this directory.
130
+ try:
131
+ changed = relabel_vendors_file(RESTORE_DIR / "hierarchy" / HIERARCHY_FILE)
132
+ if changed:
133
+ print(f"==> tree: partner labels from vendors.yaml for {', '.join(changed)}")
134
+ except Exception as e: # a label is cosmetic: never fail the seed over it
135
+ print(f"==> tree: partner relabel skipped ({e})")
136
  # The artifact set is complete on disk: hf-storage-api (supervisord) waits
137
  # for its catalog OR this marker, so a run without a catalog is reported
138
  # instead of waited on forever.
pipeline/media/__main__.py CHANGED
@@ -16,6 +16,9 @@
16
  fix-batch --from DIR --batch DIR --asset ID a one-asset batch to fix an asset by hand (its row,
17
  package and files from the service's batch)
18
  replay the nightly check: snapshot + overlays == manifest
 
 
 
19
 
20
  Each prints one JSON line (its result) and exits 0, or 1 when the step itself failed (an asset held or failed is a
21
  result, not a failure: see the batch's state.json). With RESULT_FILE set (an Argo step: its output parameter
@@ -76,6 +79,12 @@ def main(argv=None) -> int:
76
  fx.add_argument("--batch", type=Path, required=True)
77
  fx.add_argument("--asset", required=True)
78
  sub.add_parser("replay")
 
 
 
 
 
 
79
  a = ap.parse_args(argv)
80
  try:
81
  out = run(a)
@@ -106,7 +115,8 @@ def run(a) -> dict:
106
  res = gaps.compute(rows, store.load_manifest(store.client()), a.backfill_clips)
107
  bmod.write_json(a.out, {"commit": commit, **res})
108
  out = {"commit": commit, "rows": res["rows"], "queue": len(res["queue"]), "complete": res["complete"],
109
- "clip_backlog": len(res["clip_backlog"])}
 
110
  if a.emit_rows:
111
  import base64
112
  out["rows_b64"] = base64.b64encode(json.dumps(res["queue"]).encode()).decode()
@@ -157,6 +167,9 @@ def run(a) -> dict:
157
  return stages.fix_batch(bmod.Batch(a.src), bmod.Batch(a.batch), a.asset)
158
  if a.cmd == "replay":
159
  return store.replay()
 
 
 
160
  raise SystemExit(f"unknown command {a.cmd}")
161
 
162
 
 
16
  fix-batch --from DIR --batch DIR --asset ID a one-asset batch to fix an asset by hand (its row,
17
  package and files from the service's batch)
18
  replay the nightly check: snapshot + overlays == manifest
19
+ drop-blocks --asset ID --kinds rtxclip --batch-id ID --reason TEXT [--dry-run]
20
+ take blocks off one served entry by hand (an overlay
21
+ that re-adds it without them; skill simready-media)
22
 
23
  Each prints one JSON line (its result) and exits 0, or 1 when the step itself failed (an asset held or failed is a
24
  result, not a failure: see the batch's state.json). With RESULT_FILE set (an Argo step: its output parameter
 
79
  fx.add_argument("--batch", type=Path, required=True)
80
  fx.add_argument("--asset", required=True)
81
  sub.add_parser("replay")
82
+ dr = sub.add_parser("drop-blocks")
83
+ dr.add_argument("--asset", required=True)
84
+ dr.add_argument("--kinds", required=True, help="comma-separated: still, spin, rtxclip, usd")
85
+ dr.add_argument("--batch-id", required=True)
86
+ dr.add_argument("--reason", required=True)
87
+ dr.add_argument("--dry-run", action="store_true")
88
  a = ap.parse_args(argv)
89
  try:
90
  out = run(a)
 
115
  res = gaps.compute(rows, store.load_manifest(store.client()), a.backfill_clips)
116
  bmod.write_json(a.out, {"commit": commit, **res})
117
  out = {"commit": commit, "rows": res["rows"], "queue": len(res["queue"]), "complete": res["complete"],
118
+ "clip_backlog": len(res["clip_backlog"]), "no_clip": len(res["no_clip"]),
119
+ "no_clip_served": res["no_clip_served"]}
120
  if a.emit_rows:
121
  import base64
122
  out["rows_b64"] = base64.b64encode(json.dumps(res["queue"]).encode()).decode()
 
167
  return stages.fix_batch(bmod.Batch(a.src), bmod.Batch(a.batch), a.asset)
168
  if a.cmd == "replay":
169
  return store.replay()
170
+ if a.cmd == "drop-blocks":
171
+ return store.drop_blocks(a.asset, [k for k in a.kinds.split(",") if k], a.batch_id, a.reason,
172
+ dry_run=a.dry_run)
173
  raise SystemExit(f"unknown command {a.cmd}")
174
 
175
 
pipeline/media/config.py CHANGED
@@ -12,7 +12,7 @@ from dataclasses import dataclass
12
  from pathlib import Path
13
 
14
  INDEX = "usdsearch-dev"
15
- RENDER_SCRIPT_ID = os.environ.get("RENDER_SCRIPT_ID", "06ea2c6e3978")
16
  LOOK = "drop-agx-hc-v1-ev-1"
17
  SPP = 64
18
  # finalize's still check: the still tile's saturated-hue histogram (fraction of its pixels per hue bin) against the
@@ -44,6 +44,36 @@ DROP_KEYS = {
44
  "https://huggingface.co/datasets/ALLSIDES/simready-scanned-objects/blob/7ad563ba3fa904c1e5fbd99c9e065f4db7d18ae6"
45
  "/SLXRSTZD87/simready_usd/SLXRSTZD87.usd": "unserved leftover: only a Web USD, the index serves dartboard_SLXRSTZD87",
46
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  # Packages of these dataset repos are converted as robots (the kit's robot checks, the clip planner's robot reader).
48
  ROBOT_REPOS = frozenset(os.environ.get("ROBOT_REPOS", ",".join([
49
  "AGIBOT-Yuanzheng/A3_Ultra_usd", "Comau/N220-27", "Comau/S18-17", "FANUC-CORPORATION/SimReady-Assets",
 
12
  from pathlib import Path
13
 
14
  INDEX = "usdsearch-dev"
15
+ RENDER_SCRIPT_ID = os.environ.get("RENDER_SCRIPT_ID", "ce84f6440742") # render-patches/genlab-0001 on 06ea2c6e3978
16
  LOOK = "drop-agx-hc-v1-ev-1"
17
  SPP = 64
18
  # finalize's still check: the still tile's saturated-hue histogram (fraction of its pixels per hue bin) against the
 
44
  "https://huggingface.co/datasets/ALLSIDES/simready-scanned-objects/blob/7ad563ba3fa904c1e5fbd99c9e065f4db7d18ae6"
45
  "/SLXRSTZD87/simready_usd/SLXRSTZD87.usd": "unserved leftover: only a Web USD, the index serves dartboard_SLXRSTZD87",
46
  }
47
+ # Packages that get no RTX Motion clip ON PURPOSE (a person turned the clip down: skill simready-media, references/
48
+ # motion-known-issues.md): "<dataset repo>:<entry path in the repo>", any revision, so a re-pin keeps the decision.
49
+ # The gap scan neither queues them for a clip nor counts them in the clip backlog (it lists them as `no_clip`),
50
+ # clip_plan never plans one, and `drop-blocks --kinds rtxclip` takes a published clip off only for these.
51
+ NO_CLIP = {
52
+ "M-XR/marso-simready:folding_step_ladder/usd/folding_step_ladder.usda":
53
+ "1-DOF fold: the tread's ribs pass 6.4-6.7 mm through the rear crossbar in the first 5 degrees on every path "
54
+ "(owner, 2026-11-16)",
55
+ "M-XR/marso-simready:folding_chair/usd/folding_chair.usda":
56
+ "seat and rear legs are linked in the real chair: every scripted fold brushes 6.5-9.6 mm (owner, 2026-11-11)",
57
+ "nvidia/simready-assets:tireTool_tire_changer_a01/simready_usd/sm_tireTool_tire_changer_a01_01.usd":
58
+ "the wheel cart swing leaves the tile crop even on a 960 refit (2026-10-17)",
59
+ }
60
+
61
+
62
+ def package_of(key: str) -> str:
63
+ """"<repo>:<path>" of a Space key (https://<host>/datasets/<repo>/blob/<rev>/<path>), the form NO_CLIP lists;
64
+ "" for a key of another shape."""
65
+ head, sep, rest = key.partition("/datasets/")
66
+ repo, sep2, tail = rest.partition("/blob/")
67
+ if not (sep and sep2 and "/" in tail):
68
+ return ""
69
+ return f"{repo}:{tail.split('/', 1)[1]}"
70
+
71
+
72
+ def no_clip(key: str) -> str:
73
+ """Why the package of ``key`` gets no clip on purpose, or ""."""
74
+ return NO_CLIP.get(package_of(key), "")
75
+
76
+
77
  # Packages of these dataset repos are converted as robots (the kit's robot checks, the clip planner's robot reader).
78
  ROBOT_REPOS = frozenset(os.environ.get("ROBOT_REPOS", ",".join([
79
  "AGIBOT-Yuanzheng/A3_Ultra_usd", "Comau/N220-27", "Comau/S18-17", "FANUC-CORPORATION/SimReady-Assets",
pipeline/media/controller.py CHANGED
@@ -49,7 +49,7 @@ import threading
49
  import time
50
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
51
 
52
- from pipeline.media import jobs, workflows
53
 
54
  log = logging.getLogger("simready-media")
55
  E = os.environ.get
@@ -187,10 +187,17 @@ class Kube:
187
  return line
188
  return {}
189
 
 
 
 
 
 
 
 
190
  def pod_line(self, pod: str, container: str) -> dict:
191
  """The last JSON line of a pod's log ({} when none or unreadable)."""
192
  try:
193
- text = self.core.read_namespaced_pod_log(pod, NS, container=container, tail_lines=5)
194
  except self.ApiException:
195
  return {}
196
  for line in reversed(text.strip().splitlines()):
@@ -310,8 +317,7 @@ class Kube:
310
 
311
  def _tail(self, pod, container: str) -> str:
312
  try:
313
- text = self.core.read_namespaced_pod_log(pod.metadata.name, NS, container=container, tail_lines=1,
314
- limit_bytes=2000)
315
  return text.strip().splitlines()[-1][:300] if text.strip() else ""
316
  except self.ApiException:
317
  return ""
@@ -462,7 +468,14 @@ class Controller:
462
  self.pending_gaps = None # done, failed, or gone (deleted): never waited on forever
463
  res = self.kube.last_line(name) if j is not None and j.status.succeeded else {}
464
  if not res.get("rows_b64"):
465
- self.db.q("INSERT INTO polls (commit, error) VALUES (%s, %s)", (commit, res.get("error") or "job failed"))
 
 
 
 
 
 
 
466
  return
467
  rows = json.loads(base64.b64decode(res["rows_b64"]))
468
  added = 0
@@ -757,17 +770,24 @@ class Controller:
757
  # by hand ----------------------------------------------------------------------------------------
758
  def by_hand(self, aid: str, act: str, who: str, note: str = "", overlay: str = "") -> tuple[bool, str]:
759
  """claim: held / failed -> fixing (someone is fixing it by hand); done: held / failed / fixing -> manual (its
760
- media was made and published by hand: never queued again, off Needs attention)."""
 
761
  if act == "claim":
762
  n = self.db.q("UPDATE assets SET state='fixing', reason=%s, updated=now() WHERE id=%s AND state IN "
763
  "('held', 'failed') RETURNING id, reason",
764
  (f"being fixed by hand by {who}" + (f": {note}" if note else ""), aid))
 
 
 
 
 
765
  else:
766
  why = f"fixed by hand by {who}" + (f": {note}" if note else "") + (f" (overlay {overlay})" if overlay else "")
767
  n = self.db.q("UPDATE assets SET state='manual', reason=%s, updated=now() WHERE id=%s AND state IN "
768
  + ATTENTION_SQL + " RETURNING id", (why[:500], aid))
769
  if not n:
770
- return False, f"no {'held or failed' if act == 'claim' else 'held, failed or fixing'} asset {aid}"
 
771
  self.db.event(f"by_hand_{act}", key=aid, by=who, note=note, overlay=overlay)
772
  return True, act
773
 
@@ -1005,7 +1025,7 @@ details.agent pre{{white-space:pre-wrap;max-width:420px;margin:6px 0 0;padding:6
1005
  </div></header>
1006
  <main>
1007
  <div class=meta><span>mode <b>{mode}</b></span><span>submissions commit <code>{commit}</code></span>
1008
- <span>last poll <b>{poll}</b></span><span><b>{clip_backlog}</b> clips in backlog</span></div>
1009
  <div class=tiles>{tiles}</div>
1010
  <h2>Running now</h2>{running}
1011
  <h2>Queue <span class=mute>({queued} assets; the next batch takes the first {batch_max})</span></h2>{queue}
@@ -1261,7 +1281,7 @@ def page(c: Controller, base: str = "") -> str:
1261
  for j in [j for j in jl if j["state"] in ("done", "failed")][:20]]
1262
  return PAGE.format(base=e(base), kui=KUI_PATH, themes=THEMES,
1263
  mode=e(MODE), commit=e(str(c.state.get("last_commit") or poll.get("commit") or "")[:12]), poll=_ago(poll.get("at")) + " ago"
1264
- if poll.get("at") else "never", clip_backlog=poll.get("clip_backlog") or 0,
1265
  tiles="".join(f'<div class="nv-card-root tile{" t-" + t if t else ""}"><div class=nv-card-content>'
1266
  f"<b>{v}</b><span>{e(k)}</span></div></div>" for k, v, t in tiles),
1267
  running=running, queued=counts.get("queued", 0), batch_max=BATCH_MAX,
@@ -1391,7 +1411,7 @@ def make_handler(c: Controller):
1391
  if not ok and not self.headers.get("Accept", "").startswith("text/html"):
1392
  return self._send(409, json.dumps({"ok": False, "detail": why}).encode())
1393
  note = str(body.get("note") or "")[:400]
1394
- if act in ("claim", "done"):
1395
  ok, why = c.by_hand(aid, act, who, note, str(body.get("overlay") or "")[:200])
1396
  if not ok and not self.headers.get("Accept", "").startswith("text/html"):
1397
  return self._send(409, json.dumps({"ok": False, "detail": why}).encode())
 
49
  import time
50
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
51
 
52
+ from pipeline.media import config, jobs, workflows
53
 
54
  log = logging.getLogger("simready-media")
55
  E = os.environ.get
 
187
  return line
188
  return {}
189
 
190
+ def pod_log(self, pod: str, container: str, **kw) -> str:
191
+ """A pod's log as raw text. Read without `_preload_content`, the client json-decodes a body that parses
192
+ as JSON and returns `str(dict)` (a Python repr): that is the whole tail once one result line is over
193
+ 16 KiB, because the runtime splits it into partial records that `tail_lines` counts (issue #225)."""
194
+ resp = self.core.read_namespaced_pod_log(pod, NS, container=container, _preload_content=False, **kw)
195
+ return resp.data.decode("utf-8", "replace")
196
+
197
  def pod_line(self, pod: str, container: str) -> dict:
198
  """The last JSON line of a pod's log ({} when none or unreadable)."""
199
  try:
200
+ text = self.pod_log(pod, container, tail_lines=5)
201
  except self.ApiException:
202
  return {}
203
  for line in reversed(text.strip().splitlines()):
 
317
 
318
  def _tail(self, pod, container: str) -> str:
319
  try:
320
+ text = self.pod_log(pod.metadata.name, container, tail_lines=1, limit_bytes=2000)
 
321
  return text.strip().splitlines()[-1][:300] if text.strip() else ""
322
  except self.ApiException:
323
  return ""
 
468
  self.pending_gaps = None # done, failed, or gone (deleted): never waited on forever
469
  res = self.kube.last_line(name) if j is not None and j.status.succeeded else {}
470
  if not res.get("rows_b64"):
471
+ if res.get("error"):
472
+ err = res["error"]
473
+ elif j is not None and j.status.succeeded:
474
+ err = "job succeeded but no result line" # the log could not be read or parsed
475
+ else:
476
+ err = "job failed" if j is not None else "job gone"
477
+ log.warning("gaps %s for %s: %s", name, commit[:8], err)
478
+ self.db.q("INSERT INTO polls (commit, error) VALUES (%s, %s)", (commit, err))
479
  return
480
  rows = json.loads(base64.b64decode(res["rows_b64"]))
481
  added = 0
 
770
  # by hand ----------------------------------------------------------------------------------------
771
  def by_hand(self, aid: str, act: str, who: str, note: str = "", overlay: str = "") -> tuple[bool, str]:
772
  """claim: held / failed -> fixing (someone is fixing it by hand); done: held / failed / fixing -> manual (its
773
+ media was made and published by hand: never queued again, off Needs attention); dropped: a published /
774
+ manual asset keeps its state, its reason records the blocks taken off it by hand (`drop-blocks`, overlay)."""
775
  if act == "claim":
776
  n = self.db.q("UPDATE assets SET state='fixing', reason=%s, updated=now() WHERE id=%s AND state IN "
777
  "('held', 'failed') RETURNING id, reason",
778
  (f"being fixed by hand by {who}" + (f": {note}" if note else ""), aid))
779
+ elif act == "dropped":
780
+ why = f"blocks dropped by hand by {who}" + (f": {note}" if note else "") + \
781
+ (f" (overlay {overlay})" if overlay else "")
782
+ n = self.db.q("UPDATE assets SET reason=%s, updated=now() WHERE id=%s AND state IN ('published', 'manual') "
783
+ "RETURNING id", (why[:500], aid))
784
  else:
785
  why = f"fixed by hand by {who}" + (f": {note}" if note else "") + (f" (overlay {overlay})" if overlay else "")
786
  n = self.db.q("UPDATE assets SET state='manual', reason=%s, updated=now() WHERE id=%s AND state IN "
787
  + ATTENTION_SQL + " RETURNING id", (why[:500], aid))
788
  if not n:
789
+ want = {"claim": "held or failed", "dropped": "published or manual"}.get(act, "held, failed or fixing")
790
+ return False, f"no {want} asset {aid}"
791
  self.db.event(f"by_hand_{act}", key=aid, by=who, note=note, overlay=overlay)
792
  return True, act
793
 
 
1025
  </div></header>
1026
  <main>
1027
  <div class=meta><span>mode <b>{mode}</b></span><span>submissions commit <code>{commit}</code></span>
1028
+ <span>last poll <b>{poll}</b></span><span><b>{clip_backlog}</b> clips in backlog</span><span title="config.NO_CLIP: clips turned down on purpose"><b>{no_clip}</b> without a clip on purpose</span></div>
1029
  <div class=tiles>{tiles}</div>
1030
  <h2>Running now</h2>{running}
1031
  <h2>Queue <span class=mute>({queued} assets; the next batch takes the first {batch_max})</span></h2>{queue}
 
1281
  for j in [j for j in jl if j["state"] in ("done", "failed")][:20]]
1282
  return PAGE.format(base=e(base), kui=KUI_PATH, themes=THEMES,
1283
  mode=e(MODE), commit=e(str(c.state.get("last_commit") or poll.get("commit") or "")[:12]), poll=_ago(poll.get("at")) + " ago"
1284
+ if poll.get("at") else "never", clip_backlog=poll.get("clip_backlog") or 0, no_clip=len(config.NO_CLIP),
1285
  tiles="".join(f'<div class="nv-card-root tile{" t-" + t if t else ""}"><div class=nv-card-content>'
1286
  f"<b>{v}</b><span>{e(k)}</span></div></div>" for k, v, t in tiles),
1287
  running=running, queued=counts.get("queued", 0), batch_max=BATCH_MAX,
 
1411
  if not ok and not self.headers.get("Accept", "").startswith("text/html"):
1412
  return self._send(409, json.dumps({"ok": False, "detail": why}).encode())
1413
  note = str(body.get("note") or "")[:400]
1414
+ if act in ("claim", "done", "dropped"):
1415
  ok, why = c.by_hand(aid, act, who, note, str(body.get("overlay") or "")[:200])
1416
  if not ok and not self.headers.get("Accept", "").startswith("text/html"):
1417
  return self._send(409, json.dumps({"ok": False, "detail": why}).encode())
pipeline/media/gaps.py CHANGED
@@ -3,10 +3,14 @@
3
  A row needs work when its key has no entry (`new`: every block), or its entry lacks `usd`, `still` or `spin`, or its
4
  spin lacks `hd` (`partial`: only the missing blocks). An entry with user joints and no `rtxclip` is reported as
5
  `clip_backlog` and queued only with backfill_clips (82 such served entries on 2026-11-02, some of them clips the
6
- motion review turned down: re-rendering them is a decision, not a gap).
 
 
7
  """
8
  from __future__ import annotations
9
 
 
 
10
  BLOCKS = ("usd", "still", "spin")
11
 
12
 
@@ -21,13 +25,18 @@ def needs(entry: dict | None) -> list[str]:
21
 
22
 
23
  def compute(rows: list[dict], manifest: dict, backfill_clips: bool = False) -> dict:
24
- """{queue: [row + {need, status}], clip_backlog: [row ids], complete: n}."""
25
  assets = manifest["assets"]
26
- queue, backlog, complete = [], [], 0
27
  for r in rows:
28
  e = assets.get(r["base_key"])
29
  need = needs(e)
30
- if e is not None and e.get("moving") and "rtxclip" not in e:
 
 
 
 
 
31
  backlog.append(r["id"])
32
  if backfill_clips:
33
  need.append("rtxclip")
@@ -37,4 +46,5 @@ def compute(rows: list[dict], manifest: dict, backfill_clips: bool = False) -> d
37
  "status": "new" if e is None else "partial"})
38
  else:
39
  complete += 1
40
- return {"queue": queue, "clip_backlog": backlog, "complete": complete, "rows": len(rows)}
 
 
3
  A row needs work when its key has no entry (`new`: every block), or its entry lacks `usd`, `still` or `spin`, or its
4
  spin lacks `hd` (`partial`: only the missing blocks). An entry with user joints and no `rtxclip` is reported as
5
  `clip_backlog` and queued only with backfill_clips (82 such served entries on 2026-11-02, some of them clips the
6
+ motion review turned down: re-rendering them is a decision, not a gap). A package in config.NO_CLIP (its clip turned
7
+ down on purpose) is never queued for a clip, new or served, nor counted in the backlog: it is listed as `no_clip`, and
8
+ as `no_clip_served` too while the store still serves a clip for it (a `drop-blocks` not run yet).
9
  """
10
  from __future__ import annotations
11
 
12
+ from pipeline.media import config
13
+
14
  BLOCKS = ("usd", "still", "spin")
15
 
16
 
 
25
 
26
 
27
  def compute(rows: list[dict], manifest: dict, backfill_clips: bool = False) -> dict:
28
+ """{queue: [row + {need, status}], clip_backlog: [row ids], no_clip / no_clip_served: [row ids], complete: n}."""
29
  assets = manifest["assets"]
30
+ queue, backlog, no_clip, no_clip_served, complete = [], [], [], [], 0
31
  for r in rows:
32
  e = assets.get(r["base_key"])
33
  need = needs(e)
34
+ if config.no_clip(r["base_key"]):
35
+ no_clip.append(r["id"])
36
+ need = [n for n in need if n != "rtxclip?"]
37
+ if e is not None and "rtxclip" in e:
38
+ no_clip_served.append(r["id"])
39
+ elif e is not None and e.get("moving") and "rtxclip" not in e:
40
  backlog.append(r["id"])
41
  if backfill_clips:
42
  need.append("rtxclip")
 
46
  "status": "new" if e is None else "partial"})
47
  else:
48
  complete += 1
49
+ return {"queue": queue, "clip_backlog": backlog, "no_clip": no_clip, "no_clip_served": no_clip_served,
50
+ "complete": complete, "rows": len(rows)}
pipeline/media/stages.py CHANGED
@@ -560,9 +560,10 @@ def clip_plan(b: Batch) -> dict:
560
  """Movers only (user joints > 0): the library's moving-parts plan rendered from the spin's start frame
561
  (rtx/joints/showcase_w2.py: check v2 over every pose, the floor under every pose), then its 1440 HD input
562
  (rtx/hd/hdclip_build.py rekey). Held: a refused plan (a collision, no pair answer) or predicted cuts past the
563
- frame (the 960 motion-fit refit is a person's call: skill simready-media, stage refit)."""
 
564
  p = P()
565
- movers = [r for r in b.rows() if r.get("moving")]
566
  if not movers:
567
  return summary(b, "clip_plan", [], movers=0)
568
  sw = b.root / "sw"
 
560
  """Movers only (user joints > 0): the library's moving-parts plan rendered from the spin's start frame
561
  (rtx/joints/showcase_w2.py: check v2 over every pose, the floor under every pose), then its 1440 HD input
562
  (rtx/hd/hdclip_build.py rekey). Held: a refused plan (a collision, no pair answer) or predicted cuts past the
563
+ frame (the 960 motion-fit refit is a person's call: skill simready-media, stage refit). A package in
564
+ config.NO_CLIP (its clip turned down on purpose) is never planned: it packs with its still, spin and Web USD."""
565
  p = P()
566
+ movers = [r for r in b.rows() if r.get("moving") and not config.no_clip(r["base_key"])]
567
  if not movers:
568
  return summary(b, "clip_plan", [], movers=0)
569
  sw = b.root / "sw"
pipeline/media/store.py CHANGED
@@ -242,6 +242,74 @@ def publish(packs: list[Path], batch_id: str, c=None) -> dict:
242
  "overlay": okey, "manifest_entries": len(manifest["assets"]), "manifest_tries": tries}
243
 
244
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
  # ------------------------------------------------------------------------------------------- concurrent publishes
246
  # Two publishes at once (seven approved batches, or a batch and a hand fix) must not lose each other's entries. The
247
  # overlay number is claimed with a create-only write, and manifest.json is replaced only if it is still the version
 
242
  "overlay": okey, "manifest_entries": len(manifest["assets"]), "manifest_tries": tries}
243
 
244
 
245
+ ENTRY_FIELDS = ("id", "moving", "robot")
246
+
247
+
248
+ def drop_overlay(manifest: dict, aid: str, kinds: list[str]) -> tuple[str, dict]:
249
+ """(key, overlay item) that takes ``kinds`` off the entry with id ``aid`` and keeps everything else: the overlay
250
+ removes the key and re-adds it ({id, entry: {moving, robot?}} plus each kept block as published, its ``base``
251
+ included), so apply_overlay and replay need nothing new. Raises ValueError for an unknown kind, an id the manifest
252
+ doesn't list (or lists twice), an entry without a block it is asked to drop or without any block left, and an
253
+ entry with a field the re-add would lose."""
254
+ bad = [k for k in kinds if k not in KINDS]
255
+ if bad or not kinds:
256
+ raise ValueError(f"drop: kinds {kinds} are not some of {list(KINDS)}")
257
+ keys = [k for k, e in manifest["assets"].items() if e.get("id") == aid]
258
+ if len(keys) != 1:
259
+ raise ValueError(f"drop: {aid} is the id of {len(keys)} manifest entries, not 1")
260
+ key, e = keys[0], manifest["assets"][keys[0]]
261
+ if "rtxclip" in kinds and not config.no_clip(key):
262
+ raise ValueError(f"drop: {config.package_of(key) or key} is not in config.NO_CLIP: record the decision there "
263
+ "first (else the service queues its clip again)")
264
+ missing = [k for k in kinds if k not in e]
265
+ if missing:
266
+ raise ValueError(f"drop: {aid} has no {missing}")
267
+ extra = sorted(set(e) - set(ENTRY_FIELDS) - set(KINDS))
268
+ if extra:
269
+ raise ValueError(f"drop: {aid} has fields {extra} the re-added entry would lose")
270
+ item = {"id": aid, "entry": {"moving": e.get("moving", 0), **({"robot": True} if e.get("robot") else {})},
271
+ **{k: copy.deepcopy(e[k]) for k in KINDS if k in e and k not in kinds}}
272
+ if not any(k in item for k in KINDS):
273
+ raise ValueError(f"drop: {aid} would be left with no block (remove the entry instead)")
274
+ want = {k: v for k, v in e.items() if k not in kinds}
275
+ got = apply_overlay({"assets": {key: e}}, {key: item}, [key])["assets"][key]
276
+ if got != want:
277
+ raise ValueError(f"drop: {aid}: the re-added entry would differ from the kept one")
278
+ return key, item
279
+
280
+
281
+ def drop_blocks(aid: str, kinds: list[str], batch_id: str, reason: str, c=None, dry_run: bool = False) -> dict:
282
+ """Take ``kinds`` (e.g. ["rtxclip"]) off one served entry, by hand (skill simready-media: a clip turned down
283
+ after it was published): an overlay record as a publish writes one (``remove`` [key] + the entry re-added
284
+ without them, ``dropped`` {key: kinds} and ``reason`` for the reader), then the manifest. No file is written or
285
+ deleted: the dropped blocks' files stay under v1/assets/ (write-once), unlisted. Unlike publish, the re-added
286
+ entry is built from the manifest as update_manifest applies it (a publish in between can't be reverted by a
287
+ stale copy), and the overlay record, holding exactly what was applied, is written after the manifest. The
288
+ Spaces' tar loader (artifact bucket) can't drop a block yet: this changes only what the store's manifest serves."""
289
+ if not reason.strip():
290
+ raise ValueError("drop: a reason is required (it is recorded in the overlay)")
291
+ c = c or client()
292
+ key, item = drop_overlay(load_manifest(c), aid, kinds) # refusals before any write
293
+ out = {"key": key, "id": aid, "dropped": kinds}
294
+ if dry_run:
295
+ return {**out, "kept": [k for k in KINDS if k in item], "dry_run": True}
296
+ seq = claim_seq(c, batch_id)
297
+ okey = f"{config.OVERLAYS_PREFIX}{seq:06d}-{batch_id}.json"
298
+ at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
299
+ applied = {}
300
+
301
+ def change(m: dict) -> dict:
302
+ applied["key"], applied["item"] = drop_overlay(m, aid, kinds)
303
+ return apply_overlay(m, {applied["key"]: applied["item"]}, [applied["key"]])
304
+ manifest, tries = update_manifest(c, change, at)
305
+ k, it = applied["key"], applied["item"]
306
+ record = {"batch": batch_id, "seq": seq, "at": at, "assets": {k: it}, "remove": [k], "dropped": {k: kinds},
307
+ "reason": reason}
308
+ _put(c, okey, json.dumps(record, sort_keys=True).encode(), ContentType="application/json", IfNoneMatch="*")
309
+ return {**out, "kept": [x for x in KINDS if x in it], "overlay": okey, "manifest_entries": len(manifest["assets"]),
310
+ "manifest_tries": tries}
311
+
312
+
313
  # ------------------------------------------------------------------------------------------- concurrent publishes
314
  # Two publishes at once (seven approved batches, or a batch and a hand fix) must not lose each other's entries. The
315
  # overlay number is claimed with a create-only write, and manifest.json is replaced only if it is still the version
pipeline/mirror.py CHANGED
@@ -7,6 +7,9 @@
7
  Staged for scripts/rsync-simready-central.sh, which syncs it into the mirror's checkout:
8
 
9
  README.md LICENSE CONTRIBUTING.md .gitattributes
 
 
 
10
  endpoint/ the search services image the Inference Endpoint runs (docker build endpoint/): what CI builds from
11
  docker/Dockerfile.hf-space-gpu, staged by hf_space.stage_repo() exactly as deploy-space-gpu does, with
12
  its LICENSE, THIRD_PARTY_NOTICE.md and licenses/
@@ -20,6 +23,7 @@ Needs no credentials.
20
  from __future__ import annotations
21
 
22
  import argparse
 
23
  import shutil
24
  from pathlib import Path
25
  from typing import List, Optional
@@ -30,6 +34,10 @@ from . import hf_endpoint, hf_space
30
  ROOT = Path(__file__).resolve().parents[1]
31
  ENDPOINT_DIR = "endpoint"
32
  SPACE_DIR = "space"
 
 
 
 
33
 
34
  # The mirror's LFS rules (rsync-simready-central.sh appends those the mirror lacks). The encoder weights never leave
35
  # this repo (stage_repo), so the tokenizer and the service wheels are the only large files.
@@ -79,6 +87,29 @@ Third-party notices: [THIRD_PARTY_NOTICE.md](THIRD_PARTY_NOTICE.md) and [license
79
  """
80
 
81
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  def stage(cfg: Config, dest: Path, marker: str) -> Path:
83
  """Stage the mirror's tree at `dest`. `marker` (hf_space.source_marker()) is the endpoint Dockerfile's first
84
  line, tying a mirror commit to this repository's commit."""
@@ -95,13 +126,19 @@ def stage(cfg: Config, dest: Path, marker: str) -> Path:
95
  shutil.copy2(ROOT / "LICENSE", dest / "LICENSE")
96
  shutil.copy2(hf_endpoint.PROXY_SRC / "CONTRIBUTING.md", dest / "CONTRIBUTING.md")
97
  (dest / ".gitattributes").write_text(GITATTRIBUTES)
 
 
 
 
 
 
98
  return dest
99
 
100
 
101
  def source_path(rel: str) -> Optional[str]:
102
  """The tracked file of this repository a staged file is a copy of; None for a generated one."""
103
  if rel in {"README.md", ".gitattributes", f"{ENDPOINT_DIR}/Dockerfile", f"{ENDPOINT_DIR}/README.md",
104
- f"{SPACE_DIR}/README.md"}:
105
  return None
106
  if rel == "CONTRIBUTING.md":
107
  return "docker/hf-space-proxy/CONTRIBUTING.md"
 
7
  Staged for scripts/rsync-simready-central.sh, which syncs it into the mirror's checkout:
8
 
9
  README.md LICENSE CONTRIBUTING.md .gitattributes
10
+ docs/SRD.md, docs/thin-space-runtime.svg
11
+ the SRD and its diagram (DOCS), so the mirror's copy is always this repository's: links to files the
12
+ mirror doesn't hold become their paths here, under a note naming the commit
13
  endpoint/ the search services image the Inference Endpoint runs (docker build endpoint/): what CI builds from
14
  docker/Dockerfile.hf-space-gpu, staged by hf_space.stage_repo() exactly as deploy-space-gpu does, with
15
  its LICENSE, THIRD_PARTY_NOTICE.md and licenses/
 
23
  from __future__ import annotations
24
 
25
  import argparse
26
+ import re
27
  import shutil
28
  from pathlib import Path
29
  from typing import List, Optional
 
34
  ROOT = Path(__file__).resolve().parents[1]
35
  ENDPOINT_DIR = "endpoint"
36
  SPACE_DIR = "space"
37
+ # This repository's docs the mirror holds at the same paths: the sync owns these; the rest of the mirror's docs/ is
38
+ # its own (rsync-simready-central.sh). SRD.md is rewritten (srd_for_mirror); every other entry is a plain copy.
39
+ DOCS = ("docs/SRD.md", "docs/thin-space-runtime.svg")
40
+ SRD = "docs/SRD.md"
41
 
42
  # The mirror's LFS rules (rsync-simready-central.sh appends those the mirror lacks). The encoder weights never leave
43
  # this repo (stage_repo), so the tokenizer and the service wheels are the only large files.
 
87
  """
88
 
89
 
90
+ MD_LINK = re.compile(r"(?<!!)\[([^\]]+)\]\(([^)\s]+)\)")
91
+
92
+
93
+ def srd_for_mirror(text: str, marker: str) -> str:
94
+ """The SRD as the mirror holds it. A relative link to a file the mirror doesn't hold becomes that file's path in
95
+ this repository (`docs/thin-space.md`); links to DOCS, images and URLs stay. A note under the title says where it
96
+ comes from and that it is edited here."""
97
+ staged = {Path(d).name for d in DOCS}
98
+
99
+ def link(m: re.Match) -> str:
100
+ label, target = m.group(1), m.group(2)
101
+ if "://" in target or target.startswith(("#", "mailto:")) or target.split("#")[0] in staged:
102
+ return m.group(0)
103
+ path = f"`{(Path('docs') / target).as_posix()}`"
104
+ return path if label == target else f"{label} ({path})"
105
+
106
+ sha = marker.split(":", 1)[1].strip()
107
+ title, _, rest = text.partition("\n")
108
+ note = (f"> Synced from `simready-central-indexing-internal` (`{SRD}` at `{sha}`) by "
109
+ "`scripts/rsync-simready-central.sh`. Edit it there, not here; the paths it names are in that repository.")
110
+ return f"{title}\n\n{note}\n{MD_LINK.sub(link, rest)}"
111
+
112
+
113
  def stage(cfg: Config, dest: Path, marker: str) -> Path:
114
  """Stage the mirror's tree at `dest`. `marker` (hf_space.source_marker()) is the endpoint Dockerfile's first
115
  line, tying a mirror commit to this repository's commit."""
 
126
  shutil.copy2(ROOT / "LICENSE", dest / "LICENSE")
127
  shutil.copy2(hf_endpoint.PROXY_SRC / "CONTRIBUTING.md", dest / "CONTRIBUTING.md")
128
  (dest / ".gitattributes").write_text(GITATTRIBUTES)
129
+ for rel in DOCS:
130
+ (dest / rel).parent.mkdir(parents=True, exist_ok=True)
131
+ if rel == SRD:
132
+ (dest / rel).write_text(srd_for_mirror((ROOT / rel).read_text(), marker))
133
+ else:
134
+ shutil.copy2(ROOT / rel, dest / rel)
135
  return dest
136
 
137
 
138
  def source_path(rel: str) -> Optional[str]:
139
  """The tracked file of this repository a staged file is a copy of; None for a generated one."""
140
  if rel in {"README.md", ".gitattributes", f"{ENDPOINT_DIR}/Dockerfile", f"{ENDPOINT_DIR}/README.md",
141
+ f"{SPACE_DIR}/README.md", SRD}:
142
  return None
143
  if rel == "CONTRIBUTING.md":
144
  return "docker/hf-space-proxy/CONTRIBUTING.md"
services/explorer/dist/makers.json CHANGED
@@ -249,7 +249,7 @@
249
  },
250
  {
251
  "code": "oem:projectsim",
252
- "name": "Kaedim - projectsim",
253
  "line": "Articulated props",
254
  "key": "https://huggingface.co/datasets/projectsim/simready-assets/blob/9326854525a6c941c1d46c82a3262deea79bd20a/garlic_press/simready_usd/garlic_press.usd",
255
  "count": 7,
@@ -259,7 +259,7 @@
259
  },
260
  {
261
  "code": "oem:M-XR",
262
- "name": "M-XR",
263
  "line": "Scanned everyday products",
264
  "key": "https://huggingface.co/datasets/M-XR/marso-simready/blob/0bf9664adf0f7fdaace26c3e36a518eaac16fc38/jug_kettle/usd/igenix_jug_kettle.usda",
265
  "count": 84,
 
249
  },
250
  {
251
  "code": "oem:projectsim",
252
+ "name": "Projectsim",
253
  "line": "Articulated props",
254
  "key": "https://huggingface.co/datasets/projectsim/simready-assets/blob/9326854525a6c941c1d46c82a3262deea79bd20a/garlic_press/simready_usd/garlic_press.usd",
255
  "count": 7,
 
259
  },
260
  {
261
  "code": "oem:M-XR",
262
+ "name": "Marso by M-XR",
263
  "line": "Scanned everyday products",
264
  "key": "https://huggingface.co/datasets/M-XR/marso-simready/blob/0bf9664adf0f7fdaace26c3e36a518eaac16fc38/jug_kettle/usd/igenix_jug_kettle.usda",
265
  "count": 84,