Spaces:
Running on RTX PRO 6000
Running on RTX PRO 6000
Deploy simready-assets run 20261113-698f0b6 (GPU variant) (part 2)
Browse files- pipeline/categories.py +46 -0
- pipeline/hf_endpoint.py +3 -2
- pipeline/hf_space.py +3 -0
- pipeline/hf_space_seed.py +9 -0
- pipeline/media/__main__.py +14 -1
- pipeline/media/config.py +31 -1
- pipeline/media/controller.py +30 -10
- pipeline/media/gaps.py +15 -5
- pipeline/media/stages.py +3 -2
- pipeline/media/store.py +68 -0
- pipeline/mirror.py +38 -1
- services/explorer/dist/makers.json +2 -2
pipeline/categories.py
CHANGED
|
@@ -226,6 +226,52 @@ def augment_hierarchy(doc: Dict, values: Iterable[str], vendors: Dict, pins: Dic
|
|
| 226 |
return out, unplaced
|
| 227 |
|
| 228 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 229 |
# ── install list: package folder -> category folder ──────────────────────────
|
| 230 |
|
| 231 |
|
|
|
|
| 226 |
return out, unplaced
|
| 227 |
|
| 228 |
|
| 229 |
+
def relabel_vendors(doc: Dict, vendors: Dict) -> List[str]:
|
| 230 |
+
"""Give a stored run's tree the partner labels vendors.yaml has now, in place: every `oem:<owner>` node, the
|
| 231 |
+
path steps naming it and `central_category.vendors` (which the Explorer's Provider names come from, ahead of
|
| 232 |
+
makers.json). The Space runs this on the restored tree at start (hf_space_seed), so a renamed partner shows on
|
| 233 |
+
the run it already serves, without a re-index (2026-11-18: M-XR -> Marso by M-XR). Only labels change: node
|
| 234 |
+
ids, `folder:` ids and install_categories.json keep the run's. Returns the owners whose label changed."""
|
| 235 |
+
labels = vendors.get("labels") or {}
|
| 236 |
+
changed = set()
|
| 237 |
+
|
| 238 |
+
def new_label(qcode: str, old: str) -> str:
|
| 239 |
+
owner = qcode[len(OEM_PREFIX):]
|
| 240 |
+
label = labels.get(owner)
|
| 241 |
+
if label and label != old:
|
| 242 |
+
changed.add(owner)
|
| 243 |
+
return label
|
| 244 |
+
return old
|
| 245 |
+
|
| 246 |
+
for key, node in (doc.get("qcodes") or {}).items():
|
| 247 |
+
if not isinstance(node, dict):
|
| 248 |
+
continue
|
| 249 |
+
if key.startswith(OEM_PREFIX) and isinstance(node.get("label"), str):
|
| 250 |
+
node["label"] = new_label(key, node["label"])
|
| 251 |
+
for step in node.get("path") or []:
|
| 252 |
+
if isinstance(step, dict) and str(step.get("qcode") or "").startswith(OEM_PREFIX):
|
| 253 |
+
step["label"] = new_label(step["qcode"], str(step.get("label") or ""))
|
| 254 |
+
cc = doc.get("central_category")
|
| 255 |
+
if isinstance(cc, dict) and isinstance(cc.get("vendors"), dict):
|
| 256 |
+
for owner, old in cc["vendors"].items():
|
| 257 |
+
cc["vendors"][owner] = new_label(OEM_PREFIX + owner, str(old))
|
| 258 |
+
return sorted(changed)
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def relabel_vendors_file(path: Path, vendors_file: Path = VENDORS_FILE) -> List[str]:
|
| 262 |
+
"""relabel_vendors on a tree file, rewritten atomically only when a label changed. A missing tree or
|
| 263 |
+
vendors.yaml changes nothing (a run without a tree, an image without the file)."""
|
| 264 |
+
if not path.exists() or not vendors_file.exists():
|
| 265 |
+
return []
|
| 266 |
+
doc = json.loads(path.read_text())
|
| 267 |
+
changed = relabel_vendors(doc, load_vendors(vendors_file))
|
| 268 |
+
if changed:
|
| 269 |
+
tmp = path.with_name(path.name + ".tmp")
|
| 270 |
+
tmp.write_text(json.dumps(doc, indent=1, ensure_ascii=False, sort_keys=True) + "\n")
|
| 271 |
+
os.replace(tmp, path)
|
| 272 |
+
return changed
|
| 273 |
+
|
| 274 |
+
|
| 275 |
# ── install list: package folder -> category folder ──────────────────────────
|
| 276 |
|
| 277 |
|
pipeline/hf_endpoint.py
CHANGED
|
@@ -55,8 +55,9 @@ ROOT = Path(__file__).resolve().parents[1]
|
|
| 55 |
PROXY_SRC = ROOT / "docker" / "hf-space-proxy"
|
| 56 |
PROXY_STAGE = ROOT / ".build" / "hf-space-proxy-stage"
|
| 57 |
# The thin Space's repository: these files from docker/hf-space-proxy, and README.md (PROXY_README).
|
| 58 |
-
|
| 59 |
-
|
|
|
|
| 60 |
# Files a thin Space may hold besides those: its README and the .gitattributes HF writes into every new repo.
|
| 61 |
PROXY_REPO_FILES = frozenset(PROXY_FILES) | {"README.md", ".gitattributes"}
|
| 62 |
PROXY_DIRS = frozenset({"licenses"}) # the PROXY_FILES that are directories: their files are the Space's too
|
|
|
|
| 55 |
PROXY_SRC = ROOT / "docker" / "hf-space-proxy"
|
| 56 |
PROXY_STAGE = ROOT / ".build" / "hf-space-proxy-stage"
|
| 57 |
# The thin Space's repository: these files from docker/hf-space-proxy, and README.md (PROXY_README).
|
| 58 |
+
# third-party-sources.sh: the build step that puts the added packages' source code in the image (/licenses/sources).
|
| 59 |
+
PROXY_FILES = ("Dockerfile", "nginx.conf.template", "start.sh", "third-party-sources.sh", "LICENSE", "CONTRIBUTING.md",
|
| 60 |
+
"THIRD_PARTY_NOTICE.md", "licenses") # licenses/: the notice's license texts, one per package
|
| 61 |
# Files a thin Space may hold besides those: its README and the .gitattributes HF writes into every new repo.
|
| 62 |
PROXY_REPO_FILES = frozenset(PROXY_FILES) | {"README.md", ".gitattributes"}
|
| 63 |
PROXY_DIRS = frozenset({"licenses"}) # the PROXY_FILES that are directories: their files are the Space's too
|
pipeline/hf_space.py
CHANGED
|
@@ -224,6 +224,9 @@ def stage_repo(dest: Path, marker: str, readme: str, dockerfile_name: str = "Doc
|
|
| 224 |
REPO_ROOT / "pipeline", dest / "pipeline",
|
| 225 |
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
|
| 226 |
)
|
|
|
|
|
|
|
|
|
|
| 227 |
# The Dockerfile COPYs this in directly now (see its own comment) —
|
| 228 |
# must ship alongside it or HF's build fails with a missing path.
|
| 229 |
# A dist built by simready-central's slim/scripts/build-explorer-public.sh
|
|
|
|
| 224 |
REPO_ROOT / "pipeline", dest / "pipeline",
|
| 225 |
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
|
| 226 |
)
|
| 227 |
+
# The partner labels the Space puts on the restored Category tree at start (categories.relabel_vendors).
|
| 228 |
+
(dest / "config" / "hierarchy").mkdir(parents=True)
|
| 229 |
+
shutil.copy2(REPO_ROOT / "config" / "hierarchy" / "vendors.yaml", dest / "config" / "hierarchy" / "vendors.yaml")
|
| 230 |
# The Dockerfile COPYs this in directly now (see its own comment) —
|
| 231 |
# must ship alongside it or HF's build fails with a missing path.
|
| 232 |
# A dist built by simready-central's slim/scripts/build-explorer-public.sh
|
pipeline/hf_space_seed.py
CHANGED
|
@@ -57,6 +57,7 @@ from pathlib import Path
|
|
| 57 |
import httpx
|
| 58 |
|
| 59 |
from .artifact_store import download_prefix, read_text
|
|
|
|
| 60 |
from .config import BUILD_DIR, Config
|
| 61 |
from .restore import OPENSEARCH_TIMEOUT, RESTORE_DIR, download, restore_opensearch
|
| 62 |
|
|
@@ -124,6 +125,14 @@ def main_opensearch(cfg: Config, base: str, corpus_name: str) -> int:
|
|
| 124 |
prefix = resolve_prefix(cfg, base, corpus_name)
|
| 125 |
print(f"==> restoring from {prefix}")
|
| 126 |
download(cfg, prefix)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
# The artifact set is complete on disk: hf-storage-api (supervisord) waits
|
| 128 |
# for its catalog OR this marker, so a run without a catalog is reported
|
| 129 |
# instead of waited on forever.
|
|
|
|
| 57 |
import httpx
|
| 58 |
|
| 59 |
from .artifact_store import download_prefix, read_text
|
| 60 |
+
from .categories import HIERARCHY_FILE, relabel_vendors_file
|
| 61 |
from .config import BUILD_DIR, Config
|
| 62 |
from .restore import OPENSEARCH_TIMEOUT, RESTORE_DIR, download, restore_opensearch
|
| 63 |
|
|
|
|
| 125 |
prefix = resolve_prefix(cfg, base, corpus_name)
|
| 126 |
print(f"==> restoring from {prefix}")
|
| 127 |
download(cfg, prefix)
|
| 128 |
+
# The tree's partner names as vendors.yaml has them now, not as the run baked them (a renamed partner shows
|
| 129 |
+
# without a re-index). Before the marker: nginx serves the tree from this directory.
|
| 130 |
+
try:
|
| 131 |
+
changed = relabel_vendors_file(RESTORE_DIR / "hierarchy" / HIERARCHY_FILE)
|
| 132 |
+
if changed:
|
| 133 |
+
print(f"==> tree: partner labels from vendors.yaml for {', '.join(changed)}")
|
| 134 |
+
except Exception as e: # a label is cosmetic: never fail the seed over it
|
| 135 |
+
print(f"==> tree: partner relabel skipped ({e})")
|
| 136 |
# The artifact set is complete on disk: hf-storage-api (supervisord) waits
|
| 137 |
# for its catalog OR this marker, so a run without a catalog is reported
|
| 138 |
# instead of waited on forever.
|
pipeline/media/__main__.py
CHANGED
|
@@ -16,6 +16,9 @@
|
|
| 16 |
fix-batch --from DIR --batch DIR --asset ID a one-asset batch to fix an asset by hand (its row,
|
| 17 |
package and files from the service's batch)
|
| 18 |
replay the nightly check: snapshot + overlays == manifest
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
Each prints one JSON line (its result) and exits 0, or 1 when the step itself failed (an asset held or failed is a
|
| 21 |
result, not a failure: see the batch's state.json). With RESULT_FILE set (an Argo step: its output parameter
|
|
@@ -76,6 +79,12 @@ def main(argv=None) -> int:
|
|
| 76 |
fx.add_argument("--batch", type=Path, required=True)
|
| 77 |
fx.add_argument("--asset", required=True)
|
| 78 |
sub.add_parser("replay")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
a = ap.parse_args(argv)
|
| 80 |
try:
|
| 81 |
out = run(a)
|
|
@@ -106,7 +115,8 @@ def run(a) -> dict:
|
|
| 106 |
res = gaps.compute(rows, store.load_manifest(store.client()), a.backfill_clips)
|
| 107 |
bmod.write_json(a.out, {"commit": commit, **res})
|
| 108 |
out = {"commit": commit, "rows": res["rows"], "queue": len(res["queue"]), "complete": res["complete"],
|
| 109 |
-
"clip_backlog": len(res["clip_backlog"])
|
|
|
|
| 110 |
if a.emit_rows:
|
| 111 |
import base64
|
| 112 |
out["rows_b64"] = base64.b64encode(json.dumps(res["queue"]).encode()).decode()
|
|
@@ -157,6 +167,9 @@ def run(a) -> dict:
|
|
| 157 |
return stages.fix_batch(bmod.Batch(a.src), bmod.Batch(a.batch), a.asset)
|
| 158 |
if a.cmd == "replay":
|
| 159 |
return store.replay()
|
|
|
|
|
|
|
|
|
|
| 160 |
raise SystemExit(f"unknown command {a.cmd}")
|
| 161 |
|
| 162 |
|
|
|
|
| 16 |
fix-batch --from DIR --batch DIR --asset ID a one-asset batch to fix an asset by hand (its row,
|
| 17 |
package and files from the service's batch)
|
| 18 |
replay the nightly check: snapshot + overlays == manifest
|
| 19 |
+
drop-blocks --asset ID --kinds rtxclip --batch-id ID --reason TEXT [--dry-run]
|
| 20 |
+
take blocks off one served entry by hand (an overlay
|
| 21 |
+
that re-adds it without them; skill simready-media)
|
| 22 |
|
| 23 |
Each prints one JSON line (its result) and exits 0, or 1 when the step itself failed (an asset held or failed is a
|
| 24 |
result, not a failure: see the batch's state.json). With RESULT_FILE set (an Argo step: its output parameter
|
|
|
|
| 79 |
fx.add_argument("--batch", type=Path, required=True)
|
| 80 |
fx.add_argument("--asset", required=True)
|
| 81 |
sub.add_parser("replay")
|
| 82 |
+
dr = sub.add_parser("drop-blocks")
|
| 83 |
+
dr.add_argument("--asset", required=True)
|
| 84 |
+
dr.add_argument("--kinds", required=True, help="comma-separated: still, spin, rtxclip, usd")
|
| 85 |
+
dr.add_argument("--batch-id", required=True)
|
| 86 |
+
dr.add_argument("--reason", required=True)
|
| 87 |
+
dr.add_argument("--dry-run", action="store_true")
|
| 88 |
a = ap.parse_args(argv)
|
| 89 |
try:
|
| 90 |
out = run(a)
|
|
|
|
| 115 |
res = gaps.compute(rows, store.load_manifest(store.client()), a.backfill_clips)
|
| 116 |
bmod.write_json(a.out, {"commit": commit, **res})
|
| 117 |
out = {"commit": commit, "rows": res["rows"], "queue": len(res["queue"]), "complete": res["complete"],
|
| 118 |
+
"clip_backlog": len(res["clip_backlog"]), "no_clip": len(res["no_clip"]),
|
| 119 |
+
"no_clip_served": res["no_clip_served"]}
|
| 120 |
if a.emit_rows:
|
| 121 |
import base64
|
| 122 |
out["rows_b64"] = base64.b64encode(json.dumps(res["queue"]).encode()).decode()
|
|
|
|
| 167 |
return stages.fix_batch(bmod.Batch(a.src), bmod.Batch(a.batch), a.asset)
|
| 168 |
if a.cmd == "replay":
|
| 169 |
return store.replay()
|
| 170 |
+
if a.cmd == "drop-blocks":
|
| 171 |
+
return store.drop_blocks(a.asset, [k for k in a.kinds.split(",") if k], a.batch_id, a.reason,
|
| 172 |
+
dry_run=a.dry_run)
|
| 173 |
raise SystemExit(f"unknown command {a.cmd}")
|
| 174 |
|
| 175 |
|
pipeline/media/config.py
CHANGED
|
@@ -12,7 +12,7 @@ from dataclasses import dataclass
|
|
| 12 |
from pathlib import Path
|
| 13 |
|
| 14 |
INDEX = "usdsearch-dev"
|
| 15 |
-
RENDER_SCRIPT_ID = os.environ.get("RENDER_SCRIPT_ID", "
|
| 16 |
LOOK = "drop-agx-hc-v1-ev-1"
|
| 17 |
SPP = 64
|
| 18 |
# finalize's still check: the still tile's saturated-hue histogram (fraction of its pixels per hue bin) against the
|
|
@@ -44,6 +44,36 @@ DROP_KEYS = {
|
|
| 44 |
"https://huggingface.co/datasets/ALLSIDES/simready-scanned-objects/blob/7ad563ba3fa904c1e5fbd99c9e065f4db7d18ae6"
|
| 45 |
"/SLXRSTZD87/simready_usd/SLXRSTZD87.usd": "unserved leftover: only a Web USD, the index serves dartboard_SLXRSTZD87",
|
| 46 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
# Packages of these dataset repos are converted as robots (the kit's robot checks, the clip planner's robot reader).
|
| 48 |
ROBOT_REPOS = frozenset(os.environ.get("ROBOT_REPOS", ",".join([
|
| 49 |
"AGIBOT-Yuanzheng/A3_Ultra_usd", "Comau/N220-27", "Comau/S18-17", "FANUC-CORPORATION/SimReady-Assets",
|
|
|
|
| 12 |
from pathlib import Path
|
| 13 |
|
| 14 |
INDEX = "usdsearch-dev"
|
| 15 |
+
RENDER_SCRIPT_ID = os.environ.get("RENDER_SCRIPT_ID", "ce84f6440742") # render-patches/genlab-0001 on 06ea2c6e3978
|
| 16 |
LOOK = "drop-agx-hc-v1-ev-1"
|
| 17 |
SPP = 64
|
| 18 |
# finalize's still check: the still tile's saturated-hue histogram (fraction of its pixels per hue bin) against the
|
|
|
|
| 44 |
"https://huggingface.co/datasets/ALLSIDES/simready-scanned-objects/blob/7ad563ba3fa904c1e5fbd99c9e065f4db7d18ae6"
|
| 45 |
"/SLXRSTZD87/simready_usd/SLXRSTZD87.usd": "unserved leftover: only a Web USD, the index serves dartboard_SLXRSTZD87",
|
| 46 |
}
|
| 47 |
+
# Packages that get no RTX Motion clip ON PURPOSE (a person turned the clip down: skill simready-media, references/
|
| 48 |
+
# motion-known-issues.md): "<dataset repo>:<entry path in the repo>", any revision, so a re-pin keeps the decision.
|
| 49 |
+
# The gap scan neither queues them for a clip nor counts them in the clip backlog (it lists them as `no_clip`),
|
| 50 |
+
# clip_plan never plans one, and `drop-blocks --kinds rtxclip` takes a published clip off only for these.
|
| 51 |
+
NO_CLIP = {
|
| 52 |
+
"M-XR/marso-simready:folding_step_ladder/usd/folding_step_ladder.usda":
|
| 53 |
+
"1-DOF fold: the tread's ribs pass 6.4-6.7 mm through the rear crossbar in the first 5 degrees on every path "
|
| 54 |
+
"(owner, 2026-11-16)",
|
| 55 |
+
"M-XR/marso-simready:folding_chair/usd/folding_chair.usda":
|
| 56 |
+
"seat and rear legs are linked in the real chair: every scripted fold brushes 6.5-9.6 mm (owner, 2026-11-11)",
|
| 57 |
+
"nvidia/simready-assets:tireTool_tire_changer_a01/simready_usd/sm_tireTool_tire_changer_a01_01.usd":
|
| 58 |
+
"the wheel cart swing leaves the tile crop even on a 960 refit (2026-10-17)",
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def package_of(key: str) -> str:
|
| 63 |
+
""""<repo>:<path>" of a Space key (https://<host>/datasets/<repo>/blob/<rev>/<path>), the form NO_CLIP lists;
|
| 64 |
+
"" for a key of another shape."""
|
| 65 |
+
head, sep, rest = key.partition("/datasets/")
|
| 66 |
+
repo, sep2, tail = rest.partition("/blob/")
|
| 67 |
+
if not (sep and sep2 and "/" in tail):
|
| 68 |
+
return ""
|
| 69 |
+
return f"{repo}:{tail.split('/', 1)[1]}"
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def no_clip(key: str) -> str:
|
| 73 |
+
"""Why the package of ``key`` gets no clip on purpose, or ""."""
|
| 74 |
+
return NO_CLIP.get(package_of(key), "")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
# Packages of these dataset repos are converted as robots (the kit's robot checks, the clip planner's robot reader).
|
| 78 |
ROBOT_REPOS = frozenset(os.environ.get("ROBOT_REPOS", ",".join([
|
| 79 |
"AGIBOT-Yuanzheng/A3_Ultra_usd", "Comau/N220-27", "Comau/S18-17", "FANUC-CORPORATION/SimReady-Assets",
|
pipeline/media/controller.py
CHANGED
|
@@ -49,7 +49,7 @@ import threading
|
|
| 49 |
import time
|
| 50 |
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
| 51 |
|
| 52 |
-
from pipeline.media import jobs, workflows
|
| 53 |
|
| 54 |
log = logging.getLogger("simready-media")
|
| 55 |
E = os.environ.get
|
|
@@ -187,10 +187,17 @@ class Kube:
|
|
| 187 |
return line
|
| 188 |
return {}
|
| 189 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
def pod_line(self, pod: str, container: str) -> dict:
|
| 191 |
"""The last JSON line of a pod's log ({} when none or unreadable)."""
|
| 192 |
try:
|
| 193 |
-
text = self.
|
| 194 |
except self.ApiException:
|
| 195 |
return {}
|
| 196 |
for line in reversed(text.strip().splitlines()):
|
|
@@ -310,8 +317,7 @@ class Kube:
|
|
| 310 |
|
| 311 |
def _tail(self, pod, container: str) -> str:
|
| 312 |
try:
|
| 313 |
-
text = self.
|
| 314 |
-
limit_bytes=2000)
|
| 315 |
return text.strip().splitlines()[-1][:300] if text.strip() else ""
|
| 316 |
except self.ApiException:
|
| 317 |
return ""
|
|
@@ -462,7 +468,14 @@ class Controller:
|
|
| 462 |
self.pending_gaps = None # done, failed, or gone (deleted): never waited on forever
|
| 463 |
res = self.kube.last_line(name) if j is not None and j.status.succeeded else {}
|
| 464 |
if not res.get("rows_b64"):
|
| 465 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 466 |
return
|
| 467 |
rows = json.loads(base64.b64decode(res["rows_b64"]))
|
| 468 |
added = 0
|
|
@@ -757,17 +770,24 @@ class Controller:
|
|
| 757 |
# by hand ----------------------------------------------------------------------------------------
|
| 758 |
def by_hand(self, aid: str, act: str, who: str, note: str = "", overlay: str = "") -> tuple[bool, str]:
|
| 759 |
"""claim: held / failed -> fixing (someone is fixing it by hand); done: held / failed / fixing -> manual (its
|
| 760 |
-
media was made and published by hand: never queued again, off Needs attention)
|
|
|
|
| 761 |
if act == "claim":
|
| 762 |
n = self.db.q("UPDATE assets SET state='fixing', reason=%s, updated=now() WHERE id=%s AND state IN "
|
| 763 |
"('held', 'failed') RETURNING id, reason",
|
| 764 |
(f"being fixed by hand by {who}" + (f": {note}" if note else ""), aid))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 765 |
else:
|
| 766 |
why = f"fixed by hand by {who}" + (f": {note}" if note else "") + (f" (overlay {overlay})" if overlay else "")
|
| 767 |
n = self.db.q("UPDATE assets SET state='manual', reason=%s, updated=now() WHERE id=%s AND state IN "
|
| 768 |
+ ATTENTION_SQL + " RETURNING id", (why[:500], aid))
|
| 769 |
if not n:
|
| 770 |
-
|
|
|
|
| 771 |
self.db.event(f"by_hand_{act}", key=aid, by=who, note=note, overlay=overlay)
|
| 772 |
return True, act
|
| 773 |
|
|
@@ -1005,7 +1025,7 @@ details.agent pre{{white-space:pre-wrap;max-width:420px;margin:6px 0 0;padding:6
|
|
| 1005 |
</div></header>
|
| 1006 |
<main>
|
| 1007 |
<div class=meta><span>mode <b>{mode}</b></span><span>submissions commit <code>{commit}</code></span>
|
| 1008 |
-
<span>last poll <b>{poll}</b></span><span><b>{clip_backlog}</b> clips in backlog</span></div>
|
| 1009 |
<div class=tiles>{tiles}</div>
|
| 1010 |
<h2>Running now</h2>{running}
|
| 1011 |
<h2>Queue <span class=mute>({queued} assets; the next batch takes the first {batch_max})</span></h2>{queue}
|
|
@@ -1261,7 +1281,7 @@ def page(c: Controller, base: str = "") -> str:
|
|
| 1261 |
for j in [j for j in jl if j["state"] in ("done", "failed")][:20]]
|
| 1262 |
return PAGE.format(base=e(base), kui=KUI_PATH, themes=THEMES,
|
| 1263 |
mode=e(MODE), commit=e(str(c.state.get("last_commit") or poll.get("commit") or "")[:12]), poll=_ago(poll.get("at")) + " ago"
|
| 1264 |
-
if poll.get("at") else "never", clip_backlog=poll.get("clip_backlog") or 0,
|
| 1265 |
tiles="".join(f'<div class="nv-card-root tile{" t-" + t if t else ""}"><div class=nv-card-content>'
|
| 1266 |
f"<b>{v}</b><span>{e(k)}</span></div></div>" for k, v, t in tiles),
|
| 1267 |
running=running, queued=counts.get("queued", 0), batch_max=BATCH_MAX,
|
|
@@ -1391,7 +1411,7 @@ def make_handler(c: Controller):
|
|
| 1391 |
if not ok and not self.headers.get("Accept", "").startswith("text/html"):
|
| 1392 |
return self._send(409, json.dumps({"ok": False, "detail": why}).encode())
|
| 1393 |
note = str(body.get("note") or "")[:400]
|
| 1394 |
-
if act in ("claim", "done"):
|
| 1395 |
ok, why = c.by_hand(aid, act, who, note, str(body.get("overlay") or "")[:200])
|
| 1396 |
if not ok and not self.headers.get("Accept", "").startswith("text/html"):
|
| 1397 |
return self._send(409, json.dumps({"ok": False, "detail": why}).encode())
|
|
|
|
| 49 |
import time
|
| 50 |
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
| 51 |
|
| 52 |
+
from pipeline.media import config, jobs, workflows
|
| 53 |
|
| 54 |
log = logging.getLogger("simready-media")
|
| 55 |
E = os.environ.get
|
|
|
|
| 187 |
return line
|
| 188 |
return {}
|
| 189 |
|
| 190 |
+
def pod_log(self, pod: str, container: str, **kw) -> str:
|
| 191 |
+
"""A pod's log as raw text. Read without `_preload_content`, the client json-decodes a body that parses
|
| 192 |
+
as JSON and returns `str(dict)` (a Python repr): that is the whole tail once one result line is over
|
| 193 |
+
16 KiB, because the runtime splits it into partial records that `tail_lines` counts (issue #225)."""
|
| 194 |
+
resp = self.core.read_namespaced_pod_log(pod, NS, container=container, _preload_content=False, **kw)
|
| 195 |
+
return resp.data.decode("utf-8", "replace")
|
| 196 |
+
|
| 197 |
def pod_line(self, pod: str, container: str) -> dict:
|
| 198 |
"""The last JSON line of a pod's log ({} when none or unreadable)."""
|
| 199 |
try:
|
| 200 |
+
text = self.pod_log(pod, container, tail_lines=5)
|
| 201 |
except self.ApiException:
|
| 202 |
return {}
|
| 203 |
for line in reversed(text.strip().splitlines()):
|
|
|
|
| 317 |
|
| 318 |
def _tail(self, pod, container: str) -> str:
|
| 319 |
try:
|
| 320 |
+
text = self.pod_log(pod.metadata.name, container, tail_lines=1, limit_bytes=2000)
|
|
|
|
| 321 |
return text.strip().splitlines()[-1][:300] if text.strip() else ""
|
| 322 |
except self.ApiException:
|
| 323 |
return ""
|
|
|
|
| 468 |
self.pending_gaps = None # done, failed, or gone (deleted): never waited on forever
|
| 469 |
res = self.kube.last_line(name) if j is not None and j.status.succeeded else {}
|
| 470 |
if not res.get("rows_b64"):
|
| 471 |
+
if res.get("error"):
|
| 472 |
+
err = res["error"]
|
| 473 |
+
elif j is not None and j.status.succeeded:
|
| 474 |
+
err = "job succeeded but no result line" # the log could not be read or parsed
|
| 475 |
+
else:
|
| 476 |
+
err = "job failed" if j is not None else "job gone"
|
| 477 |
+
log.warning("gaps %s for %s: %s", name, commit[:8], err)
|
| 478 |
+
self.db.q("INSERT INTO polls (commit, error) VALUES (%s, %s)", (commit, err))
|
| 479 |
return
|
| 480 |
rows = json.loads(base64.b64decode(res["rows_b64"]))
|
| 481 |
added = 0
|
|
|
|
| 770 |
# by hand ----------------------------------------------------------------------------------------
|
| 771 |
def by_hand(self, aid: str, act: str, who: str, note: str = "", overlay: str = "") -> tuple[bool, str]:
|
| 772 |
"""claim: held / failed -> fixing (someone is fixing it by hand); done: held / failed / fixing -> manual (its
|
| 773 |
+
media was made and published by hand: never queued again, off Needs attention); dropped: a published /
|
| 774 |
+
manual asset keeps its state, its reason records the blocks taken off it by hand (`drop-blocks`, overlay)."""
|
| 775 |
if act == "claim":
|
| 776 |
n = self.db.q("UPDATE assets SET state='fixing', reason=%s, updated=now() WHERE id=%s AND state IN "
|
| 777 |
"('held', 'failed') RETURNING id, reason",
|
| 778 |
(f"being fixed by hand by {who}" + (f": {note}" if note else ""), aid))
|
| 779 |
+
elif act == "dropped":
|
| 780 |
+
why = f"blocks dropped by hand by {who}" + (f": {note}" if note else "") + \
|
| 781 |
+
(f" (overlay {overlay})" if overlay else "")
|
| 782 |
+
n = self.db.q("UPDATE assets SET reason=%s, updated=now() WHERE id=%s AND state IN ('published', 'manual') "
|
| 783 |
+
"RETURNING id", (why[:500], aid))
|
| 784 |
else:
|
| 785 |
why = f"fixed by hand by {who}" + (f": {note}" if note else "") + (f" (overlay {overlay})" if overlay else "")
|
| 786 |
n = self.db.q("UPDATE assets SET state='manual', reason=%s, updated=now() WHERE id=%s AND state IN "
|
| 787 |
+ ATTENTION_SQL + " RETURNING id", (why[:500], aid))
|
| 788 |
if not n:
|
| 789 |
+
want = {"claim": "held or failed", "dropped": "published or manual"}.get(act, "held, failed or fixing")
|
| 790 |
+
return False, f"no {want} asset {aid}"
|
| 791 |
self.db.event(f"by_hand_{act}", key=aid, by=who, note=note, overlay=overlay)
|
| 792 |
return True, act
|
| 793 |
|
|
|
|
| 1025 |
</div></header>
|
| 1026 |
<main>
|
| 1027 |
<div class=meta><span>mode <b>{mode}</b></span><span>submissions commit <code>{commit}</code></span>
|
| 1028 |
+
<span>last poll <b>{poll}</b></span><span><b>{clip_backlog}</b> clips in backlog</span><span title="config.NO_CLIP: clips turned down on purpose"><b>{no_clip}</b> without a clip on purpose</span></div>
|
| 1029 |
<div class=tiles>{tiles}</div>
|
| 1030 |
<h2>Running now</h2>{running}
|
| 1031 |
<h2>Queue <span class=mute>({queued} assets; the next batch takes the first {batch_max})</span></h2>{queue}
|
|
|
|
| 1281 |
for j in [j for j in jl if j["state"] in ("done", "failed")][:20]]
|
| 1282 |
return PAGE.format(base=e(base), kui=KUI_PATH, themes=THEMES,
|
| 1283 |
mode=e(MODE), commit=e(str(c.state.get("last_commit") or poll.get("commit") or "")[:12]), poll=_ago(poll.get("at")) + " ago"
|
| 1284 |
+
if poll.get("at") else "never", clip_backlog=poll.get("clip_backlog") or 0, no_clip=len(config.NO_CLIP),
|
| 1285 |
tiles="".join(f'<div class="nv-card-root tile{" t-" + t if t else ""}"><div class=nv-card-content>'
|
| 1286 |
f"<b>{v}</b><span>{e(k)}</span></div></div>" for k, v, t in tiles),
|
| 1287 |
running=running, queued=counts.get("queued", 0), batch_max=BATCH_MAX,
|
|
|
|
| 1411 |
if not ok and not self.headers.get("Accept", "").startswith("text/html"):
|
| 1412 |
return self._send(409, json.dumps({"ok": False, "detail": why}).encode())
|
| 1413 |
note = str(body.get("note") or "")[:400]
|
| 1414 |
+
if act in ("claim", "done", "dropped"):
|
| 1415 |
ok, why = c.by_hand(aid, act, who, note, str(body.get("overlay") or "")[:200])
|
| 1416 |
if not ok and not self.headers.get("Accept", "").startswith("text/html"):
|
| 1417 |
return self._send(409, json.dumps({"ok": False, "detail": why}).encode())
|
pipeline/media/gaps.py
CHANGED
|
@@ -3,10 +3,14 @@
|
|
| 3 |
A row needs work when its key has no entry (`new`: every block), or its entry lacks `usd`, `still` or `spin`, or its
|
| 4 |
spin lacks `hd` (`partial`: only the missing blocks). An entry with user joints and no `rtxclip` is reported as
|
| 5 |
`clip_backlog` and queued only with backfill_clips (82 such served entries on 2026-11-02, some of them clips the
|
| 6 |
-
motion review turned down: re-rendering them is a decision, not a gap).
|
|
|
|
|
|
|
| 7 |
"""
|
| 8 |
from __future__ import annotations
|
| 9 |
|
|
|
|
|
|
|
| 10 |
BLOCKS = ("usd", "still", "spin")
|
| 11 |
|
| 12 |
|
|
@@ -21,13 +25,18 @@ def needs(entry: dict | None) -> list[str]:
|
|
| 21 |
|
| 22 |
|
| 23 |
def compute(rows: list[dict], manifest: dict, backfill_clips: bool = False) -> dict:
|
| 24 |
-
"""{queue: [row + {need, status}], clip_backlog: [row ids], complete: n}."""
|
| 25 |
assets = manifest["assets"]
|
| 26 |
-
queue, backlog, complete = [], [], 0
|
| 27 |
for r in rows:
|
| 28 |
e = assets.get(r["base_key"])
|
| 29 |
need = needs(e)
|
| 30 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
backlog.append(r["id"])
|
| 32 |
if backfill_clips:
|
| 33 |
need.append("rtxclip")
|
|
@@ -37,4 +46,5 @@ def compute(rows: list[dict], manifest: dict, backfill_clips: bool = False) -> d
|
|
| 37 |
"status": "new" if e is None else "partial"})
|
| 38 |
else:
|
| 39 |
complete += 1
|
| 40 |
-
return {"queue": queue, "clip_backlog": backlog, "
|
|
|
|
|
|
| 3 |
A row needs work when its key has no entry (`new`: every block), or its entry lacks `usd`, `still` or `spin`, or its
|
| 4 |
spin lacks `hd` (`partial`: only the missing blocks). An entry with user joints and no `rtxclip` is reported as
|
| 5 |
`clip_backlog` and queued only with backfill_clips (82 such served entries on 2026-11-02, some of them clips the
|
| 6 |
+
motion review turned down: re-rendering them is a decision, not a gap). A package in config.NO_CLIP (its clip turned
|
| 7 |
+
down on purpose) is never queued for a clip, new or served, nor counted in the backlog: it is listed as `no_clip`, and
|
| 8 |
+
as `no_clip_served` too while the store still serves a clip for it (a `drop-blocks` not run yet).
|
| 9 |
"""
|
| 10 |
from __future__ import annotations
|
| 11 |
|
| 12 |
+
from pipeline.media import config
|
| 13 |
+
|
| 14 |
BLOCKS = ("usd", "still", "spin")
|
| 15 |
|
| 16 |
|
|
|
|
| 25 |
|
| 26 |
|
| 27 |
def compute(rows: list[dict], manifest: dict, backfill_clips: bool = False) -> dict:
|
| 28 |
+
"""{queue: [row + {need, status}], clip_backlog: [row ids], no_clip / no_clip_served: [row ids], complete: n}."""
|
| 29 |
assets = manifest["assets"]
|
| 30 |
+
queue, backlog, no_clip, no_clip_served, complete = [], [], [], [], 0
|
| 31 |
for r in rows:
|
| 32 |
e = assets.get(r["base_key"])
|
| 33 |
need = needs(e)
|
| 34 |
+
if config.no_clip(r["base_key"]):
|
| 35 |
+
no_clip.append(r["id"])
|
| 36 |
+
need = [n for n in need if n != "rtxclip?"]
|
| 37 |
+
if e is not None and "rtxclip" in e:
|
| 38 |
+
no_clip_served.append(r["id"])
|
| 39 |
+
elif e is not None and e.get("moving") and "rtxclip" not in e:
|
| 40 |
backlog.append(r["id"])
|
| 41 |
if backfill_clips:
|
| 42 |
need.append("rtxclip")
|
|
|
|
| 46 |
"status": "new" if e is None else "partial"})
|
| 47 |
else:
|
| 48 |
complete += 1
|
| 49 |
+
return {"queue": queue, "clip_backlog": backlog, "no_clip": no_clip, "no_clip_served": no_clip_served,
|
| 50 |
+
"complete": complete, "rows": len(rows)}
|
pipeline/media/stages.py
CHANGED
|
@@ -560,9 +560,10 @@ def clip_plan(b: Batch) -> dict:
|
|
| 560 |
"""Movers only (user joints > 0): the library's moving-parts plan rendered from the spin's start frame
|
| 561 |
(rtx/joints/showcase_w2.py: check v2 over every pose, the floor under every pose), then its 1440 HD input
|
| 562 |
(rtx/hd/hdclip_build.py rekey). Held: a refused plan (a collision, no pair answer) or predicted cuts past the
|
| 563 |
-
frame (the 960 motion-fit refit is a person's call: skill simready-media, stage refit).
|
|
|
|
| 564 |
p = P()
|
| 565 |
-
movers = [r for r in b.rows() if r.get("moving")]
|
| 566 |
if not movers:
|
| 567 |
return summary(b, "clip_plan", [], movers=0)
|
| 568 |
sw = b.root / "sw"
|
|
|
|
| 560 |
"""Movers only (user joints > 0): the library's moving-parts plan rendered from the spin's start frame
|
| 561 |
(rtx/joints/showcase_w2.py: check v2 over every pose, the floor under every pose), then its 1440 HD input
|
| 562 |
(rtx/hd/hdclip_build.py rekey). Held: a refused plan (a collision, no pair answer) or predicted cuts past the
|
| 563 |
+
frame (the 960 motion-fit refit is a person's call: skill simready-media, stage refit). A package in
|
| 564 |
+
config.NO_CLIP (its clip turned down on purpose) is never planned: it packs with its still, spin and Web USD."""
|
| 565 |
p = P()
|
| 566 |
+
movers = [r for r in b.rows() if r.get("moving") and not config.no_clip(r["base_key"])]
|
| 567 |
if not movers:
|
| 568 |
return summary(b, "clip_plan", [], movers=0)
|
| 569 |
sw = b.root / "sw"
|
pipeline/media/store.py
CHANGED
|
@@ -242,6 +242,74 @@ def publish(packs: list[Path], batch_id: str, c=None) -> dict:
|
|
| 242 |
"overlay": okey, "manifest_entries": len(manifest["assets"]), "manifest_tries": tries}
|
| 243 |
|
| 244 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
# ------------------------------------------------------------------------------------------- concurrent publishes
|
| 246 |
# Two publishes at once (seven approved batches, or a batch and a hand fix) must not lose each other's entries. The
|
| 247 |
# overlay number is claimed with a create-only write, and manifest.json is replaced only if it is still the version
|
|
|
|
| 242 |
"overlay": okey, "manifest_entries": len(manifest["assets"]), "manifest_tries": tries}
|
| 243 |
|
| 244 |
|
| 245 |
+
ENTRY_FIELDS = ("id", "moving", "robot")
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def drop_overlay(manifest: dict, aid: str, kinds: list[str]) -> tuple[str, dict]:
|
| 249 |
+
"""(key, overlay item) that takes ``kinds`` off the entry with id ``aid`` and keeps everything else: the overlay
|
| 250 |
+
removes the key and re-adds it ({id, entry: {moving, robot?}} plus each kept block as published, its ``base``
|
| 251 |
+
included), so apply_overlay and replay need nothing new. Raises ValueError for an unknown kind, an id the manifest
|
| 252 |
+
doesn't list (or lists twice), an entry without a block it is asked to drop or without any block left, and an
|
| 253 |
+
entry with a field the re-add would lose."""
|
| 254 |
+
bad = [k for k in kinds if k not in KINDS]
|
| 255 |
+
if bad or not kinds:
|
| 256 |
+
raise ValueError(f"drop: kinds {kinds} are not some of {list(KINDS)}")
|
| 257 |
+
keys = [k for k, e in manifest["assets"].items() if e.get("id") == aid]
|
| 258 |
+
if len(keys) != 1:
|
| 259 |
+
raise ValueError(f"drop: {aid} is the id of {len(keys)} manifest entries, not 1")
|
| 260 |
+
key, e = keys[0], manifest["assets"][keys[0]]
|
| 261 |
+
if "rtxclip" in kinds and not config.no_clip(key):
|
| 262 |
+
raise ValueError(f"drop: {config.package_of(key) or key} is not in config.NO_CLIP: record the decision there "
|
| 263 |
+
"first (else the service queues its clip again)")
|
| 264 |
+
missing = [k for k in kinds if k not in e]
|
| 265 |
+
if missing:
|
| 266 |
+
raise ValueError(f"drop: {aid} has no {missing}")
|
| 267 |
+
extra = sorted(set(e) - set(ENTRY_FIELDS) - set(KINDS))
|
| 268 |
+
if extra:
|
| 269 |
+
raise ValueError(f"drop: {aid} has fields {extra} the re-added entry would lose")
|
| 270 |
+
item = {"id": aid, "entry": {"moving": e.get("moving", 0), **({"robot": True} if e.get("robot") else {})},
|
| 271 |
+
**{k: copy.deepcopy(e[k]) for k in KINDS if k in e and k not in kinds}}
|
| 272 |
+
if not any(k in item for k in KINDS):
|
| 273 |
+
raise ValueError(f"drop: {aid} would be left with no block (remove the entry instead)")
|
| 274 |
+
want = {k: v for k, v in e.items() if k not in kinds}
|
| 275 |
+
got = apply_overlay({"assets": {key: e}}, {key: item}, [key])["assets"][key]
|
| 276 |
+
if got != want:
|
| 277 |
+
raise ValueError(f"drop: {aid}: the re-added entry would differ from the kept one")
|
| 278 |
+
return key, item
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def drop_blocks(aid: str, kinds: list[str], batch_id: str, reason: str, c=None, dry_run: bool = False) -> dict:
|
| 282 |
+
"""Take ``kinds`` (e.g. ["rtxclip"]) off one served entry, by hand (skill simready-media: a clip turned down
|
| 283 |
+
after it was published): an overlay record as a publish writes one (``remove`` [key] + the entry re-added
|
| 284 |
+
without them, ``dropped`` {key: kinds} and ``reason`` for the reader), then the manifest. No file is written or
|
| 285 |
+
deleted: the dropped blocks' files stay under v1/assets/ (write-once), unlisted. Unlike publish, the re-added
|
| 286 |
+
entry is built from the manifest as update_manifest applies it (a publish in between can't be reverted by a
|
| 287 |
+
stale copy), and the overlay record, holding exactly what was applied, is written after the manifest. The
|
| 288 |
+
Spaces' tar loader (artifact bucket) can't drop a block yet: this changes only what the store's manifest serves."""
|
| 289 |
+
if not reason.strip():
|
| 290 |
+
raise ValueError("drop: a reason is required (it is recorded in the overlay)")
|
| 291 |
+
c = c or client()
|
| 292 |
+
key, item = drop_overlay(load_manifest(c), aid, kinds) # refusals before any write
|
| 293 |
+
out = {"key": key, "id": aid, "dropped": kinds}
|
| 294 |
+
if dry_run:
|
| 295 |
+
return {**out, "kept": [k for k in KINDS if k in item], "dry_run": True}
|
| 296 |
+
seq = claim_seq(c, batch_id)
|
| 297 |
+
okey = f"{config.OVERLAYS_PREFIX}{seq:06d}-{batch_id}.json"
|
| 298 |
+
at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
| 299 |
+
applied = {}
|
| 300 |
+
|
| 301 |
+
def change(m: dict) -> dict:
|
| 302 |
+
applied["key"], applied["item"] = drop_overlay(m, aid, kinds)
|
| 303 |
+
return apply_overlay(m, {applied["key"]: applied["item"]}, [applied["key"]])
|
| 304 |
+
manifest, tries = update_manifest(c, change, at)
|
| 305 |
+
k, it = applied["key"], applied["item"]
|
| 306 |
+
record = {"batch": batch_id, "seq": seq, "at": at, "assets": {k: it}, "remove": [k], "dropped": {k: kinds},
|
| 307 |
+
"reason": reason}
|
| 308 |
+
_put(c, okey, json.dumps(record, sort_keys=True).encode(), ContentType="application/json", IfNoneMatch="*")
|
| 309 |
+
return {**out, "kept": [x for x in KINDS if x in it], "overlay": okey, "manifest_entries": len(manifest["assets"]),
|
| 310 |
+
"manifest_tries": tries}
|
| 311 |
+
|
| 312 |
+
|
| 313 |
# ------------------------------------------------------------------------------------------- concurrent publishes
|
| 314 |
# Two publishes at once (seven approved batches, or a batch and a hand fix) must not lose each other's entries. The
|
| 315 |
# overlay number is claimed with a create-only write, and manifest.json is replaced only if it is still the version
|
pipeline/mirror.py
CHANGED
|
@@ -7,6 +7,9 @@
|
|
| 7 |
Staged for scripts/rsync-simready-central.sh, which syncs it into the mirror's checkout:
|
| 8 |
|
| 9 |
README.md LICENSE CONTRIBUTING.md .gitattributes
|
|
|
|
|
|
|
|
|
|
| 10 |
endpoint/ the search services image the Inference Endpoint runs (docker build endpoint/): what CI builds from
|
| 11 |
docker/Dockerfile.hf-space-gpu, staged by hf_space.stage_repo() exactly as deploy-space-gpu does, with
|
| 12 |
its LICENSE, THIRD_PARTY_NOTICE.md and licenses/
|
|
@@ -20,6 +23,7 @@ Needs no credentials.
|
|
| 20 |
from __future__ import annotations
|
| 21 |
|
| 22 |
import argparse
|
|
|
|
| 23 |
import shutil
|
| 24 |
from pathlib import Path
|
| 25 |
from typing import List, Optional
|
|
@@ -30,6 +34,10 @@ from . import hf_endpoint, hf_space
|
|
| 30 |
ROOT = Path(__file__).resolve().parents[1]
|
| 31 |
ENDPOINT_DIR = "endpoint"
|
| 32 |
SPACE_DIR = "space"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
# The mirror's LFS rules (rsync-simready-central.sh appends those the mirror lacks). The encoder weights never leave
|
| 35 |
# this repo (stage_repo), so the tokenizer and the service wheels are the only large files.
|
|
@@ -79,6 +87,29 @@ Third-party notices: [THIRD_PARTY_NOTICE.md](THIRD_PARTY_NOTICE.md) and [license
|
|
| 79 |
"""
|
| 80 |
|
| 81 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
def stage(cfg: Config, dest: Path, marker: str) -> Path:
|
| 83 |
"""Stage the mirror's tree at `dest`. `marker` (hf_space.source_marker()) is the endpoint Dockerfile's first
|
| 84 |
line, tying a mirror commit to this repository's commit."""
|
|
@@ -95,13 +126,19 @@ def stage(cfg: Config, dest: Path, marker: str) -> Path:
|
|
| 95 |
shutil.copy2(ROOT / "LICENSE", dest / "LICENSE")
|
| 96 |
shutil.copy2(hf_endpoint.PROXY_SRC / "CONTRIBUTING.md", dest / "CONTRIBUTING.md")
|
| 97 |
(dest / ".gitattributes").write_text(GITATTRIBUTES)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
return dest
|
| 99 |
|
| 100 |
|
| 101 |
def source_path(rel: str) -> Optional[str]:
|
| 102 |
"""The tracked file of this repository a staged file is a copy of; None for a generated one."""
|
| 103 |
if rel in {"README.md", ".gitattributes", f"{ENDPOINT_DIR}/Dockerfile", f"{ENDPOINT_DIR}/README.md",
|
| 104 |
-
f"{SPACE_DIR}/README.md"}:
|
| 105 |
return None
|
| 106 |
if rel == "CONTRIBUTING.md":
|
| 107 |
return "docker/hf-space-proxy/CONTRIBUTING.md"
|
|
|
|
| 7 |
Staged for scripts/rsync-simready-central.sh, which syncs it into the mirror's checkout:
|
| 8 |
|
| 9 |
README.md LICENSE CONTRIBUTING.md .gitattributes
|
| 10 |
+
docs/SRD.md, docs/thin-space-runtime.svg
|
| 11 |
+
the SRD and its diagram (DOCS), so the mirror's copy is always this repository's: links to files the
|
| 12 |
+
mirror doesn't hold become their paths here, under a note naming the commit
|
| 13 |
endpoint/ the search services image the Inference Endpoint runs (docker build endpoint/): what CI builds from
|
| 14 |
docker/Dockerfile.hf-space-gpu, staged by hf_space.stage_repo() exactly as deploy-space-gpu does, with
|
| 15 |
its LICENSE, THIRD_PARTY_NOTICE.md and licenses/
|
|
|
|
| 23 |
from __future__ import annotations
|
| 24 |
|
| 25 |
import argparse
|
| 26 |
+
import re
|
| 27 |
import shutil
|
| 28 |
from pathlib import Path
|
| 29 |
from typing import List, Optional
|
|
|
|
| 34 |
ROOT = Path(__file__).resolve().parents[1]
|
| 35 |
ENDPOINT_DIR = "endpoint"
|
| 36 |
SPACE_DIR = "space"
|
| 37 |
+
# This repository's docs the mirror holds at the same paths: the sync owns these; the rest of the mirror's docs/ is
|
| 38 |
+
# its own (rsync-simready-central.sh). SRD.md is rewritten (srd_for_mirror); every other entry is a plain copy.
|
| 39 |
+
DOCS = ("docs/SRD.md", "docs/thin-space-runtime.svg")
|
| 40 |
+
SRD = "docs/SRD.md"
|
| 41 |
|
| 42 |
# The mirror's LFS rules (rsync-simready-central.sh appends those the mirror lacks). The encoder weights never leave
|
| 43 |
# this repo (stage_repo), so the tokenizer and the service wheels are the only large files.
|
|
|
|
| 87 |
"""
|
| 88 |
|
| 89 |
|
| 90 |
+
MD_LINK = re.compile(r"(?<!!)\[([^\]]+)\]\(([^)\s]+)\)")
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def srd_for_mirror(text: str, marker: str) -> str:
|
| 94 |
+
"""The SRD as the mirror holds it. A relative link to a file the mirror doesn't hold becomes that file's path in
|
| 95 |
+
this repository (`docs/thin-space.md`); links to DOCS, images and URLs stay. A note under the title says where it
|
| 96 |
+
comes from and that it is edited here."""
|
| 97 |
+
staged = {Path(d).name for d in DOCS}
|
| 98 |
+
|
| 99 |
+
def link(m: re.Match) -> str:
|
| 100 |
+
label, target = m.group(1), m.group(2)
|
| 101 |
+
if "://" in target or target.startswith(("#", "mailto:")) or target.split("#")[0] in staged:
|
| 102 |
+
return m.group(0)
|
| 103 |
+
path = f"`{(Path('docs') / target).as_posix()}`"
|
| 104 |
+
return path if label == target else f"{label} ({path})"
|
| 105 |
+
|
| 106 |
+
sha = marker.split(":", 1)[1].strip()
|
| 107 |
+
title, _, rest = text.partition("\n")
|
| 108 |
+
note = (f"> Synced from `simready-central-indexing-internal` (`{SRD}` at `{sha}`) by "
|
| 109 |
+
"`scripts/rsync-simready-central.sh`. Edit it there, not here; the paths it names are in that repository.")
|
| 110 |
+
return f"{title}\n\n{note}\n{MD_LINK.sub(link, rest)}"
|
| 111 |
+
|
| 112 |
+
|
| 113 |
def stage(cfg: Config, dest: Path, marker: str) -> Path:
|
| 114 |
"""Stage the mirror's tree at `dest`. `marker` (hf_space.source_marker()) is the endpoint Dockerfile's first
|
| 115 |
line, tying a mirror commit to this repository's commit."""
|
|
|
|
| 126 |
shutil.copy2(ROOT / "LICENSE", dest / "LICENSE")
|
| 127 |
shutil.copy2(hf_endpoint.PROXY_SRC / "CONTRIBUTING.md", dest / "CONTRIBUTING.md")
|
| 128 |
(dest / ".gitattributes").write_text(GITATTRIBUTES)
|
| 129 |
+
for rel in DOCS:
|
| 130 |
+
(dest / rel).parent.mkdir(parents=True, exist_ok=True)
|
| 131 |
+
if rel == SRD:
|
| 132 |
+
(dest / rel).write_text(srd_for_mirror((ROOT / rel).read_text(), marker))
|
| 133 |
+
else:
|
| 134 |
+
shutil.copy2(ROOT / rel, dest / rel)
|
| 135 |
return dest
|
| 136 |
|
| 137 |
|
| 138 |
def source_path(rel: str) -> Optional[str]:
|
| 139 |
"""The tracked file of this repository a staged file is a copy of; None for a generated one."""
|
| 140 |
if rel in {"README.md", ".gitattributes", f"{ENDPOINT_DIR}/Dockerfile", f"{ENDPOINT_DIR}/README.md",
|
| 141 |
+
f"{SPACE_DIR}/README.md", SRD}:
|
| 142 |
return None
|
| 143 |
if rel == "CONTRIBUTING.md":
|
| 144 |
return "docker/hf-space-proxy/CONTRIBUTING.md"
|
services/explorer/dist/makers.json
CHANGED
|
@@ -249,7 +249,7 @@
|
|
| 249 |
},
|
| 250 |
{
|
| 251 |
"code": "oem:projectsim",
|
| 252 |
-
"name": "
|
| 253 |
"line": "Articulated props",
|
| 254 |
"key": "https://huggingface.co/datasets/projectsim/simready-assets/blob/9326854525a6c941c1d46c82a3262deea79bd20a/garlic_press/simready_usd/garlic_press.usd",
|
| 255 |
"count": 7,
|
|
@@ -259,7 +259,7 @@
|
|
| 259 |
},
|
| 260 |
{
|
| 261 |
"code": "oem:M-XR",
|
| 262 |
-
"name": "M-XR",
|
| 263 |
"line": "Scanned everyday products",
|
| 264 |
"key": "https://huggingface.co/datasets/M-XR/marso-simready/blob/0bf9664adf0f7fdaace26c3e36a518eaac16fc38/jug_kettle/usd/igenix_jug_kettle.usda",
|
| 265 |
"count": 84,
|
|
|
|
| 249 |
},
|
| 250 |
{
|
| 251 |
"code": "oem:projectsim",
|
| 252 |
+
"name": "Projectsim",
|
| 253 |
"line": "Articulated props",
|
| 254 |
"key": "https://huggingface.co/datasets/projectsim/simready-assets/blob/9326854525a6c941c1d46c82a3262deea79bd20a/garlic_press/simready_usd/garlic_press.usd",
|
| 255 |
"count": 7,
|
|
|
|
| 259 |
},
|
| 260 |
{
|
| 261 |
"code": "oem:M-XR",
|
| 262 |
+
"name": "Marso by M-XR",
|
| 263 |
"line": "Scanned everyday products",
|
| 264 |
"key": "https://huggingface.co/datasets/M-XR/marso-simready/blob/0bf9664adf0f7fdaace26c3e36a518eaac16fc38/jug_kettle/usd/igenix_jug_kettle.usda",
|
| 265 |
"count": 84,
|