| # /// script | |
| # requires-python = ">=3.11" | |
| # dependencies = ["matplotlib>=3.8"] | |
| # /// | |
| """Regenerate the README result figures from the submitted scores. | |
| Run from the repository root: uv run scripts/make_figures.py | |
| Reads output/*/result.json and writes artifacts/{repair,evidence,specs}.png, under | |
| output/bucket/ in the code repository or at the top level of the bucket, and refreshes | |
| the copies in docs/assets/figures/ when that folder exists. | |
| Figure 1 (environments.svg) is edited by hand and is not produced here. | |
| """ | |
| import json | |
| import shutil | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| from matplotlib.patches import FancyArrowPatch, FancyBboxPatch, Patch | |
| ROOT = Path(__file__).resolve().parents[1] | |
| # In the code repository the bucket is mirrored under output/bucket; in the bucket itself | |
| # the results and artifacts sit at the top level. | |
| BUCKET = ROOT / "output/bucket" if (ROOT / "output/bucket").is_dir() else ROOT | |
| RESULTS = BUCKET / "output" | |
| ARTIFACTS = BUCKET / "artifacts" | |
| DOCS = ROOT / "docs/assets/figures" | |
| INK = "#213546" | |
| MUTED = "#596a76" | |
| TEAL = "#187a81" | |
| TEAL_DARK = "#17696e" | |
| TEAL_LIGHT = "#7ab5ba" | |
| SAND = "#ddd6cb" | |
| ORANGE = "#d68a35" | |
| RUST = "#b55b35" | |
| RUST_TEXT = "#a55b23" | |
| BOX = "#eaf3f3" | |
| BOX_EDGE = "#b8cccc" | |
| PANEL = "#f4f8f9" | |
| WARN_BOX = "#fbf0e6" | |
| WARN_EDGE = "#d9a679" | |
| plt.rcParams.update({"font.family": "DejaVu Sans", "text.color": INK, "axes.labelcolor": INK, | |
| "xtick.color": INK, "ytick.color": INK}) | |
| def load_results(): | |
| rows = [] | |
| for path in sorted(RESULTS.glob("*/result.json")): | |
| record = json.loads(path.read_text()) | |
| rows.append((record["metrics"], record["result"])) | |
| if len(rows) != 120: | |
| raise SystemExit(f"expected 120 attempts, found {len(rows)}") | |
| return rows | |
| def mismatched_groups(result): | |
| """Check groups with at least one mismatch, across observed and unobserved cases.""" | |
| bad = set() | |
| for part in ("consistency_with_observations", "accuracy_beyond_observations"): | |
| for name, group in ((result.get(part) or {}).get("groups") or {}).items(): | |
| if group.get("accuracy", 1.0) < 1.0: | |
| bad.add(name) | |
| return bad | |
| def category(group): | |
| if group.startswith("action/"): | |
| return "operations" | |
| if group in ("initial_observation", "initial_hidden_state"): | |
| return "initial state" | |
| if group == "query/observe": | |
| return "observations" | |
| if group.startswith("query/"): | |
| return "calculations" | |
| return None # record_values: correct in every specification | |
| # ---------------------------------------------------------------- Figure 2 | |
| def box(ax, x, y, w, h, face=BOX, edge=BOX_EDGE, lw=1.2): | |
| ax.add_patch(FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0,rounding_size=0.6", | |
| facecolor=face, edgecolor=edge, linewidth=lw)) | |
| def arrow(ax, start, end, style="-|>", ls="-"): | |
| ax.add_patch(FancyArrowPatch(start, end, arrowstyle=style, mutation_scale=11, color=MUTED, | |
| linewidth=1.3, linestyle=ls, shrinkA=0, shrinkB=0)) | |
| def step(ax, x, y, w, h, number, title, detail): | |
| box(ax, x, y, w, h, face="white") | |
| ax.add_patch(plt.Circle((x + 2.0, y + h - 2.1), 1.05, color=TEAL)) | |
| ax.text(x + 2.0, y + h - 2.15, str(number), color="white", fontsize=9.5, fontweight="bold", | |
| ha="center", va="center") | |
| ax.text(x + 3.7, y + h - 2.15, title, fontsize=10.5, fontweight="bold", va="center") | |
| ax.text(x + w / 2, y + 1.6, detail, fontsize=9, color=MUTED, ha="center", va="center") | |
| def outcome(ax, x, y, w, h, reading, meaning, warn=False): | |
| box(ax, x, y, w, h, face=WARN_BOX if warn else BOX, edge=WARN_EDGE if warn else BOX_EDGE) | |
| ax.text(x + w / 2, y + h - 2.1, reading, fontsize=10.5, fontweight="bold", | |
| color=RUST_TEXT if warn else TEAL, ha="center", va="center") | |
| ax.text(x + w / 2, y + 2.0, meaning, fontsize=11, fontweight="bold", ha="center", va="center") | |
| def figure_repair(path): | |
| fig = plt.figure(figsize=(16.5, 11.6), dpi=200) | |
| ax = fig.add_axes([0, 0, 1, 1]) | |
| ax.set_xlim(0, 100) | |
| ax.set_ylim(0, 70) | |
| ax.axis("off") | |
| ax.text(2.5, 67.2, "Repair: identifying the fault and finding the effect of replacement", | |
| fontsize=17, fontweight="bold", va="center") | |
| ax.text(2.5, 64.6, "One possible investigation in configuration 1, where the valve is faulty. " | |
| "The device starts with one fault: valve, sensor or regulator.", | |
| fontsize=10.5, color=MUTED, va="center") | |
| # Records the model can read. | |
| box(ax, 2.5, 9.5, 23.5, 52.5, face=PANEL, edge="#d5e2e4") | |
| ax.text(4.0, 59.3, "Read the records", fontsize=13, fontweight="bold", color=TEAL, va="center") | |
| left = [ | |
| (55.5, "Service record", True, INK), | |
| (53.4, "Measurements require isolation, an open\ncover and settling. Isolation starts a\ntwo-unit settling period.", False, INK), | |
| (46.0, "Gauge record", True, INK), | |
| (43.9, "Independent pressure", True, TEAL), | |
| (41.8, "100 if the valve and regulator work;\n130 otherwise.", False, INK), | |
| (37.7, "Independent flow", True, TEAL), | |
| (35.6, "10 if the valve works; 20 otherwise.", False, INK), | |
| (32.9, "A faulty sensor displays 130, so the\nstarting display alone does not\nidentify the fault.", False, INK), | |
| (24.6, "Parts record", True, INK), | |
| (22.5, "Exactly one component is faulty.\nReplacements repair their named\ncomponent. Effects on calibration must\nbe established from observations.", False, INK), | |
| ] | |
| for y, text, bold, colour in left: | |
| ax.text(4.0, y, text, fontsize=10.2 if not bold else 10.8, fontweight="bold" if bold else "normal", | |
| color=colour, va="top", linespacing=1.45) | |
| for y in (47.6, 26.3): | |
| ax.plot([4.0, 24.5], [y, y], color=BOX_EDGE, linewidth=1) | |
| # Starting observation. | |
| box(ax, 30, 56.5, 67.5, 5.0) | |
| ax.text(31.5, 59.0, "Starting observation", fontsize=11, fontweight="bold", va="center") | |
| ax.text(53, 59.0, "Display 130 · Time 0 · Energy 12", fontsize=11, va="center") | |
| # Steps 1 to 4. | |
| xs = [30, 47.5, 65, 82.5] | |
| steps = [("Isolate", "Time 1 · Energy 12"), ("Open cover", "Time 2 · Energy 12"), | |
| ("Wait", "Time 3 · Energy 12"), ("Measure pressure", "Time 4 · Energy 11")] | |
| for i, (x, (title, detail)) in enumerate(zip(xs, steps), start=1): | |
| step(ax, x, 46.5, 15, 5.5, i, title, detail) | |
| if i < 4: | |
| arrow(ax, (x + 15.3, 49.25), (xs[i] - 0.3, 49.25)) | |
| arrow(ax, (37.5, 56.5), (37.5, 52.1)) | |
| ax.plot([72.5, 72.5, 90.0, 90.0], [52.6, 53.6, 53.6, 52.1], color=BOX_EDGE, linewidth=1) | |
| ax.text(73.0, 54.4, "Settled at time 3", fontsize=9, color=TEAL, va="center") | |
| # Pressure outcomes. | |
| ax.plot([90, 90, 40, 40], [46.4, 44.6, 44.6, 43.2], color=MUTED, linewidth=1.3) | |
| arrow(ax, (40, 43.6), (40, 42.3)) | |
| ax.plot([71.5, 71.5], [44.6, 43.2], color=MUTED, linewidth=1.3) | |
| arrow(ax, (71.5, 43.6), (71.5, 42.3)) | |
| outcome(ax, 31, 36.0, 18, 6.2, "Pressure = 100", "Sensor fault") | |
| outcome(ax, 62.5, 36.0, 18, 6.2, "Pressure = 130", "Valve or regulator fault") | |
| # Step 5 and flow outcomes. | |
| arrow(ax, (71.5, 35.9), (71.5, 32.7)) | |
| step(ax, 64.5, 27.0, 14, 5.6, 5, "Measure flow", "Time 5 · Energy 10") | |
| arrow(ax, (68.0, 26.9), (63.0, 23.4)) | |
| arrow(ax, (75.0, 26.9), (82.5, 23.4)) | |
| outcome(ax, 54.5, 17.0, 14, 6.2, "Flow = 20", "Valve fault") | |
| outcome(ax, 78.5, 17.0, 14, 6.2, "Flow = 10", "Regulator fault") | |
| # Step 6: the effect of replacement can only be observed. | |
| arrow(ax, (61.5, 16.9), (61.5, 13.9)) | |
| step(ax, 54.0, 8.3, 15, 5.6, 6, "Replace valve", "Time 6 · Energy 6") | |
| arrow(ax, (53.8, 11.1), (49.6, 11.1)) | |
| box(ax, 30.5, 7.6, 19.0, 7.0, face=WARN_BOX, edge=WARN_EDGE) | |
| ax.text(40.0, 12.6, "Calibrated: false", fontsize=10.5, fontweight="bold", color=RUST_TEXT, | |
| ha="center", va="center") | |
| ax.text(40.0, 9.8, "Calibration must be restored\nbefore release", fontsize=9.8, ha="center", | |
| va="center", linespacing=1.35) | |
| ax.plot([2.5, 97.5], [5.4, 5.4], color=BOX_EDGE, linewidth=1) | |
| ax.text(2.5, 3.7, "Diagnosis is only part of the task.", fontsize=11, fontweight="bold", | |
| color=RUST_TEXT, va="center") | |
| ax.text(2.5, 1.7, "The records do not say whether a replacement invalidates calibration. " | |
| "The model has to replace the part and observe the result.", fontsize=10.5, va="center") | |
| fig.savefig(path, facecolor="white") | |
| plt.close(fig) | |
| # ---------------------------------------------------------------- Figure 3 | |
| READING = ["workshop-1", "workshop-2", "workshop-3", "workshop-4", "allocation-2"] | |
| ACTING = ["allocation-1", "allocation-3", "allocation-4", "repair-1", "repair-2", "repair-3", "repair-4"] | |
| SEGMENTS = [ | |
| ("Sufficient evidence · correct answer", TEAL_DARK, "white"), | |
| ("Sufficient evidence · incorrect answer", ORANGE, INK), | |
| ("Insufficient evidence · correct answer", TEAL_LIGHT, INK), | |
| ("Insufficient evidence · incorrect answer", SAND, INK), | |
| ] | |
| def figure_evidence(rows, path): | |
| counts = defaultdict(lambda: [0, 0, 0, 0]) | |
| for metrics, _ in rows: | |
| sufficient = metrics["evidence_sufficiency"] == 1.0 | |
| correct = metrics["solution_correctness"] == 1.0 | |
| counts[metrics["configuration_name"]][(0 if sufficient else 2) + (0 if correct else 1)] += 1 | |
| order = READING + ACTING | |
| ys, y = {}, 0.0 | |
| for name in reversed(ACTING): | |
| ys[name] = y | |
| y += 1 | |
| y += 1.2 | |
| for name in reversed(READING): | |
| ys[name] = y | |
| y += 1 | |
| fig, ax = plt.subplots(figsize=(11, 7.2), dpi=200) | |
| for name in order: | |
| left = 0 | |
| for value, (_, colour, text_colour) in zip(counts[name], SEGMENTS): | |
| if value: | |
| ax.barh(ys[name], value, left=left, height=0.72, color=colour, edgecolor="white") | |
| ax.text(left + value / 2, ys[name], str(value), ha="center", va="center", | |
| fontsize=10.5, fontweight="bold", color=text_colour) | |
| left += value | |
| ax.set_yticks([ys[n] for n in order]) | |
| ax.set_yticklabels([n.replace("-", " ").capitalize() for n in order], fontsize=11) | |
| ax.set_xlim(0, 10) | |
| ax.set_xticks(range(0, 11, 2)) | |
| ax.set_xlabel("Attempts", fontsize=11) | |
| top = max(ys.values()) | |
| ax.text(-0.02, top + 0.9, "Solvable from inspected records", transform=ax.get_yaxis_transform(), | |
| ha="right", fontsize=11, fontweight="bold", color=TEAL) | |
| ax.text(-0.02, ys[ACTING[0]] + 0.9, "Require operations", transform=ax.get_yaxis_transform(), | |
| ha="right", fontsize=11, fontweight="bold", color=RUST_TEXT) | |
| ax.axhline(ys[ACTING[0]] + 0.85, color=BOX_EDGE, linewidth=1) | |
| ax.set_ylim(-0.7, top + 1.4) | |
| for side in ("top", "right"): | |
| ax.spines[side].set_visible(False) | |
| ax.grid(axis="x", color="#eeeeee") | |
| ax.set_axisbelow(True) | |
| ax.set_title("Evidence gathered and final-answer correctness by configuration", fontsize=14, | |
| fontweight="bold", loc="left", pad=26) | |
| ax.legend(handles=[Patch(color=c, label=l) for l, c, _ in SEGMENTS], loc="upper center", | |
| bbox_to_anchor=(0.45, -0.1), ncol=2, frameon=False, fontsize=10.5) | |
| fig.tight_layout() | |
| fig.savefig(path, facecolor="white") | |
| plt.close(fig) | |
| # ---------------------------------------------------------------- Figure 4 | |
| OPERATION_LABELS = { | |
| "workshop": {"return_kit": "Return equipment", "check_kit": "Inspect equipment", "wait": "Wait", | |
| "close": "Close out", "pay_venue": "Pay venue", "pay_deposits": "Refund deposits", | |
| "pay_materials": "Pay materials", "return_grant": "Repay award"}, | |
| "allocation": {"reserve_a": "Assign Alpha to A", "release_alpha": "Release Alpha", | |
| "approve": "Approve Alpha", "dispatch": "Dispatch", "fit_adapter": "Fit adapter", | |
| "reserve_b": "Assign Beta to B", "release_beta": "Release Beta", | |
| "reserve_c": "Assign Alpha to C", "unknown": "Unknown operation names"}, | |
| "repair": {"calibrate": "Calibrate", "isolate": "Isolate", "measure_flow": "Measure flow", | |
| "measure_pressure": "Measure pressure", "replace_regulator": "Replace regulator", | |
| "replace_sensor": "Replace sensor", "replace_valve": "Replace valve", | |
| "energize": "Restore power", "test": "Test", "open": "Open cover", | |
| "close": "Close cover", "wait": "Wait", "release": "Release device"}, | |
| } | |
| CATEGORIES = [("initial state", "Initial state"), ("observations", "Observations"), | |
| ("calculations", "Calculations")] | |
| def figure_specs(rows, path): | |
| n = Counter() | |
| operations = defaultdict(Counter) | |
| categories = defaultdict(Counter) | |
| for metrics, result in rows: | |
| if metrics.get("consistency_with_observations") is None: | |
| continue | |
| problem = metrics["problem"] | |
| n[problem] += 1 | |
| bad = mismatched_groups(result) | |
| for group in bad: | |
| if group.startswith("action/"): | |
| operations[problem][group[len("action/"):]] += 1 | |
| for key, _ in CATEGORIES: | |
| categories[problem][key] += any(category(g) == key for g in bad) | |
| problems = ["workshop", "allocation", "repair"] | |
| fig, axes = plt.subplots(1, 3, figsize=(16.5, 9.4), dpi=200) | |
| for ax, problem in zip(axes, problems): | |
| total = n[problem] | |
| ops = sorted(((OPERATION_LABELS[problem][k], v) for k, v in operations[problem].items() if v), | |
| key=lambda item: (-item[1], item[0])) | |
| labels, values, colours = [], [], [] | |
| for label, value in ops: | |
| labels.append(label); values.append(value); colours.append(RUST) | |
| labels.append(""); values.append(None); colours.append(None) | |
| for key, label in CATEGORIES: | |
| labels.append(label); values.append(categories[problem][key]); colours.append(TEAL) | |
| ypos = list(range(len(labels)))[::-1] | |
| for y, value, colour in zip(ypos, values, colours): | |
| if value is None: | |
| continue | |
| ax.barh(y, 100 * value / total, height=0.68, color=colour) | |
| ax.text(100 * value / total + 1.5, y, f"{value}/{total}", va="center", fontsize=9.5) | |
| ax.set_yticks(ypos) | |
| ax.set_yticklabels(labels, fontsize=10.5) | |
| ax.set_xlim(0, 118) | |
| ax.set_xticks([0, 25, 50, 75, 100]) | |
| ax.set_xticklabels(["0%", "25%", "50%", "75%", "100%"], fontsize=10) | |
| ax.tick_params(axis="y", length=0) | |
| for side in ("top", "right", "left"): | |
| ax.spines[side].set_visible(False) | |
| ax.grid(axis="x", color="#eeeeee") | |
| ax.set_axisbelow(True) | |
| ax.set_title(f"{problem.capitalize()} ({total} specifications)", fontsize=12.5, | |
| fontweight="bold", pad=12) | |
| fig.suptitle("Which parts of the environment did the specifications describe incorrectly?", | |
| fontsize=15, fontweight="bold", y=0.985) | |
| fig.supxlabel("Specifications with at least one mismatch", fontsize=11.5, y=0.075) | |
| fig.legend(handles=[Patch(color=RUST, label="Operation behavior"), | |
| Patch(color=TEAL, label="Initial state, observations and calculations")], | |
| loc="lower center", ncol=2, frameon=False, fontsize=11, bbox_to_anchor=(0.5, 0.0)) | |
| fig.tight_layout(rect=(0, 0.1, 1, 0.95)) | |
| fig.savefig(path, facecolor="white") | |
| plt.close(fig) | |
| def main(): | |
| rows = load_results() | |
| figure_repair(ARTIFACTS / "repair.png") | |
| figure_evidence(rows, ARTIFACTS / "evidence.png") | |
| figure_specs(rows, ARTIFACTS / "specs.png") | |
| if DOCS.is_dir(): | |
| shutil.copyfile(ARTIFACTS / "repair.png", DOCS / "repair-investigation.png") | |
| shutil.copyfile(ARTIFACTS / "evidence.png", DOCS / "results/evidence-and-answers.png") | |
| shutil.copyfile(ARTIFACTS / "specs.png", DOCS / "results/specification-errors.png") | |
| print("wrote repair.png, evidence.png and specs.png") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 16.2 kB
- Xet hash:
- 46f56a0ead4391e949cc5623e345ff0206693862c6e386459d500aa4267ab726
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.