abdufelsayed's picture
download
raw
16.2 kB
# /// script
# requires-python = ">=3.11"
# dependencies = ["matplotlib>=3.8"]
# ///
"""Regenerate the README result figures from the submitted scores.
Run from the repository root: uv run scripts/make_figures.py
Reads output/*/result.json and writes artifacts/{repair,evidence,specs}.png, under
output/bucket/ in the code repository or at the top level of the bucket, and refreshes
the copies in docs/assets/figures/ when that folder exists.
Figure 1 (environments.svg) is edited by hand and is not produced here.
"""
import json
import shutil
from collections import Counter, defaultdict
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.patches import FancyArrowPatch, FancyBboxPatch, Patch
ROOT = Path(__file__).resolve().parents[1]
# In the code repository the bucket is mirrored under output/bucket; in the bucket itself
# the results and artifacts sit at the top level.
BUCKET = ROOT / "output/bucket" if (ROOT / "output/bucket").is_dir() else ROOT
RESULTS = BUCKET / "output"
ARTIFACTS = BUCKET / "artifacts"
DOCS = ROOT / "docs/assets/figures"
INK = "#213546"
MUTED = "#596a76"
TEAL = "#187a81"
TEAL_DARK = "#17696e"
TEAL_LIGHT = "#7ab5ba"
SAND = "#ddd6cb"
ORANGE = "#d68a35"
RUST = "#b55b35"
RUST_TEXT = "#a55b23"
BOX = "#eaf3f3"
BOX_EDGE = "#b8cccc"
PANEL = "#f4f8f9"
WARN_BOX = "#fbf0e6"
WARN_EDGE = "#d9a679"
plt.rcParams.update({"font.family": "DejaVu Sans", "text.color": INK, "axes.labelcolor": INK,
"xtick.color": INK, "ytick.color": INK})
def load_results():
rows = []
for path in sorted(RESULTS.glob("*/result.json")):
record = json.loads(path.read_text())
rows.append((record["metrics"], record["result"]))
if len(rows) != 120:
raise SystemExit(f"expected 120 attempts, found {len(rows)}")
return rows
def mismatched_groups(result):
"""Check groups with at least one mismatch, across observed and unobserved cases."""
bad = set()
for part in ("consistency_with_observations", "accuracy_beyond_observations"):
for name, group in ((result.get(part) or {}).get("groups") or {}).items():
if group.get("accuracy", 1.0) < 1.0:
bad.add(name)
return bad
def category(group):
if group.startswith("action/"):
return "operations"
if group in ("initial_observation", "initial_hidden_state"):
return "initial state"
if group == "query/observe":
return "observations"
if group.startswith("query/"):
return "calculations"
return None # record_values: correct in every specification
# ---------------------------------------------------------------- Figure 2
def box(ax, x, y, w, h, face=BOX, edge=BOX_EDGE, lw=1.2):
ax.add_patch(FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0,rounding_size=0.6",
facecolor=face, edgecolor=edge, linewidth=lw))
def arrow(ax, start, end, style="-|>", ls="-"):
ax.add_patch(FancyArrowPatch(start, end, arrowstyle=style, mutation_scale=11, color=MUTED,
linewidth=1.3, linestyle=ls, shrinkA=0, shrinkB=0))
def step(ax, x, y, w, h, number, title, detail):
box(ax, x, y, w, h, face="white")
ax.add_patch(plt.Circle((x + 2.0, y + h - 2.1), 1.05, color=TEAL))
ax.text(x + 2.0, y + h - 2.15, str(number), color="white", fontsize=9.5, fontweight="bold",
ha="center", va="center")
ax.text(x + 3.7, y + h - 2.15, title, fontsize=10.5, fontweight="bold", va="center")
ax.text(x + w / 2, y + 1.6, detail, fontsize=9, color=MUTED, ha="center", va="center")
def outcome(ax, x, y, w, h, reading, meaning, warn=False):
box(ax, x, y, w, h, face=WARN_BOX if warn else BOX, edge=WARN_EDGE if warn else BOX_EDGE)
ax.text(x + w / 2, y + h - 2.1, reading, fontsize=10.5, fontweight="bold",
color=RUST_TEXT if warn else TEAL, ha="center", va="center")
ax.text(x + w / 2, y + 2.0, meaning, fontsize=11, fontweight="bold", ha="center", va="center")
def figure_repair(path):
fig = plt.figure(figsize=(16.5, 11.6), dpi=200)
ax = fig.add_axes([0, 0, 1, 1])
ax.set_xlim(0, 100)
ax.set_ylim(0, 70)
ax.axis("off")
ax.text(2.5, 67.2, "Repair: identifying the fault and finding the effect of replacement",
fontsize=17, fontweight="bold", va="center")
ax.text(2.5, 64.6, "One possible investigation in configuration 1, where the valve is faulty. "
"The device starts with one fault: valve, sensor or regulator.",
fontsize=10.5, color=MUTED, va="center")
# Records the model can read.
box(ax, 2.5, 9.5, 23.5, 52.5, face=PANEL, edge="#d5e2e4")
ax.text(4.0, 59.3, "Read the records", fontsize=13, fontweight="bold", color=TEAL, va="center")
left = [
(55.5, "Service record", True, INK),
(53.4, "Measurements require isolation, an open\ncover and settling. Isolation starts a\ntwo-unit settling period.", False, INK),
(46.0, "Gauge record", True, INK),
(43.9, "Independent pressure", True, TEAL),
(41.8, "100 if the valve and regulator work;\n130 otherwise.", False, INK),
(37.7, "Independent flow", True, TEAL),
(35.6, "10 if the valve works; 20 otherwise.", False, INK),
(32.9, "A faulty sensor displays 130, so the\nstarting display alone does not\nidentify the fault.", False, INK),
(24.6, "Parts record", True, INK),
(22.5, "Exactly one component is faulty.\nReplacements repair their named\ncomponent. Effects on calibration must\nbe established from observations.", False, INK),
]
for y, text, bold, colour in left:
ax.text(4.0, y, text, fontsize=10.2 if not bold else 10.8, fontweight="bold" if bold else "normal",
color=colour, va="top", linespacing=1.45)
for y in (47.6, 26.3):
ax.plot([4.0, 24.5], [y, y], color=BOX_EDGE, linewidth=1)
# Starting observation.
box(ax, 30, 56.5, 67.5, 5.0)
ax.text(31.5, 59.0, "Starting observation", fontsize=11, fontweight="bold", va="center")
ax.text(53, 59.0, "Display 130 · Time 0 · Energy 12", fontsize=11, va="center")
# Steps 1 to 4.
xs = [30, 47.5, 65, 82.5]
steps = [("Isolate", "Time 1 · Energy 12"), ("Open cover", "Time 2 · Energy 12"),
("Wait", "Time 3 · Energy 12"), ("Measure pressure", "Time 4 · Energy 11")]
for i, (x, (title, detail)) in enumerate(zip(xs, steps), start=1):
step(ax, x, 46.5, 15, 5.5, i, title, detail)
if i < 4:
arrow(ax, (x + 15.3, 49.25), (xs[i] - 0.3, 49.25))
arrow(ax, (37.5, 56.5), (37.5, 52.1))
ax.plot([72.5, 72.5, 90.0, 90.0], [52.6, 53.6, 53.6, 52.1], color=BOX_EDGE, linewidth=1)
ax.text(73.0, 54.4, "Settled at time 3", fontsize=9, color=TEAL, va="center")
# Pressure outcomes.
ax.plot([90, 90, 40, 40], [46.4, 44.6, 44.6, 43.2], color=MUTED, linewidth=1.3)
arrow(ax, (40, 43.6), (40, 42.3))
ax.plot([71.5, 71.5], [44.6, 43.2], color=MUTED, linewidth=1.3)
arrow(ax, (71.5, 43.6), (71.5, 42.3))
outcome(ax, 31, 36.0, 18, 6.2, "Pressure = 100", "Sensor fault")
outcome(ax, 62.5, 36.0, 18, 6.2, "Pressure = 130", "Valve or regulator fault")
# Step 5 and flow outcomes.
arrow(ax, (71.5, 35.9), (71.5, 32.7))
step(ax, 64.5, 27.0, 14, 5.6, 5, "Measure flow", "Time 5 · Energy 10")
arrow(ax, (68.0, 26.9), (63.0, 23.4))
arrow(ax, (75.0, 26.9), (82.5, 23.4))
outcome(ax, 54.5, 17.0, 14, 6.2, "Flow = 20", "Valve fault")
outcome(ax, 78.5, 17.0, 14, 6.2, "Flow = 10", "Regulator fault")
# Step 6: the effect of replacement can only be observed.
arrow(ax, (61.5, 16.9), (61.5, 13.9))
step(ax, 54.0, 8.3, 15, 5.6, 6, "Replace valve", "Time 6 · Energy 6")
arrow(ax, (53.8, 11.1), (49.6, 11.1))
box(ax, 30.5, 7.6, 19.0, 7.0, face=WARN_BOX, edge=WARN_EDGE)
ax.text(40.0, 12.6, "Calibrated: false", fontsize=10.5, fontweight="bold", color=RUST_TEXT,
ha="center", va="center")
ax.text(40.0, 9.8, "Calibration must be restored\nbefore release", fontsize=9.8, ha="center",
va="center", linespacing=1.35)
ax.plot([2.5, 97.5], [5.4, 5.4], color=BOX_EDGE, linewidth=1)
ax.text(2.5, 3.7, "Diagnosis is only part of the task.", fontsize=11, fontweight="bold",
color=RUST_TEXT, va="center")
ax.text(2.5, 1.7, "The records do not say whether a replacement invalidates calibration. "
"The model has to replace the part and observe the result.", fontsize=10.5, va="center")
fig.savefig(path, facecolor="white")
plt.close(fig)
# ---------------------------------------------------------------- Figure 3
READING = ["workshop-1", "workshop-2", "workshop-3", "workshop-4", "allocation-2"]
ACTING = ["allocation-1", "allocation-3", "allocation-4", "repair-1", "repair-2", "repair-3", "repair-4"]
SEGMENTS = [
("Sufficient evidence · correct answer", TEAL_DARK, "white"),
("Sufficient evidence · incorrect answer", ORANGE, INK),
("Insufficient evidence · correct answer", TEAL_LIGHT, INK),
("Insufficient evidence · incorrect answer", SAND, INK),
]
def figure_evidence(rows, path):
counts = defaultdict(lambda: [0, 0, 0, 0])
for metrics, _ in rows:
sufficient = metrics["evidence_sufficiency"] == 1.0
correct = metrics["solution_correctness"] == 1.0
counts[metrics["configuration_name"]][(0 if sufficient else 2) + (0 if correct else 1)] += 1
order = READING + ACTING
ys, y = {}, 0.0
for name in reversed(ACTING):
ys[name] = y
y += 1
y += 1.2
for name in reversed(READING):
ys[name] = y
y += 1
fig, ax = plt.subplots(figsize=(11, 7.2), dpi=200)
for name in order:
left = 0
for value, (_, colour, text_colour) in zip(counts[name], SEGMENTS):
if value:
ax.barh(ys[name], value, left=left, height=0.72, color=colour, edgecolor="white")
ax.text(left + value / 2, ys[name], str(value), ha="center", va="center",
fontsize=10.5, fontweight="bold", color=text_colour)
left += value
ax.set_yticks([ys[n] for n in order])
ax.set_yticklabels([n.replace("-", " ").capitalize() for n in order], fontsize=11)
ax.set_xlim(0, 10)
ax.set_xticks(range(0, 11, 2))
ax.set_xlabel("Attempts", fontsize=11)
top = max(ys.values())
ax.text(-0.02, top + 0.9, "Solvable from inspected records", transform=ax.get_yaxis_transform(),
ha="right", fontsize=11, fontweight="bold", color=TEAL)
ax.text(-0.02, ys[ACTING[0]] + 0.9, "Require operations", transform=ax.get_yaxis_transform(),
ha="right", fontsize=11, fontweight="bold", color=RUST_TEXT)
ax.axhline(ys[ACTING[0]] + 0.85, color=BOX_EDGE, linewidth=1)
ax.set_ylim(-0.7, top + 1.4)
for side in ("top", "right"):
ax.spines[side].set_visible(False)
ax.grid(axis="x", color="#eeeeee")
ax.set_axisbelow(True)
ax.set_title("Evidence gathered and final-answer correctness by configuration", fontsize=14,
fontweight="bold", loc="left", pad=26)
ax.legend(handles=[Patch(color=c, label=l) for l, c, _ in SEGMENTS], loc="upper center",
bbox_to_anchor=(0.45, -0.1), ncol=2, frameon=False, fontsize=10.5)
fig.tight_layout()
fig.savefig(path, facecolor="white")
plt.close(fig)
# ---------------------------------------------------------------- Figure 4
OPERATION_LABELS = {
"workshop": {"return_kit": "Return equipment", "check_kit": "Inspect equipment", "wait": "Wait",
"close": "Close out", "pay_venue": "Pay venue", "pay_deposits": "Refund deposits",
"pay_materials": "Pay materials", "return_grant": "Repay award"},
"allocation": {"reserve_a": "Assign Alpha to A", "release_alpha": "Release Alpha",
"approve": "Approve Alpha", "dispatch": "Dispatch", "fit_adapter": "Fit adapter",
"reserve_b": "Assign Beta to B", "release_beta": "Release Beta",
"reserve_c": "Assign Alpha to C", "unknown": "Unknown operation names"},
"repair": {"calibrate": "Calibrate", "isolate": "Isolate", "measure_flow": "Measure flow",
"measure_pressure": "Measure pressure", "replace_regulator": "Replace regulator",
"replace_sensor": "Replace sensor", "replace_valve": "Replace valve",
"energize": "Restore power", "test": "Test", "open": "Open cover",
"close": "Close cover", "wait": "Wait", "release": "Release device"},
}
CATEGORIES = [("initial state", "Initial state"), ("observations", "Observations"),
("calculations", "Calculations")]
def figure_specs(rows, path):
n = Counter()
operations = defaultdict(Counter)
categories = defaultdict(Counter)
for metrics, result in rows:
if metrics.get("consistency_with_observations") is None:
continue
problem = metrics["problem"]
n[problem] += 1
bad = mismatched_groups(result)
for group in bad:
if group.startswith("action/"):
operations[problem][group[len("action/"):]] += 1
for key, _ in CATEGORIES:
categories[problem][key] += any(category(g) == key for g in bad)
problems = ["workshop", "allocation", "repair"]
fig, axes = plt.subplots(1, 3, figsize=(16.5, 9.4), dpi=200)
for ax, problem in zip(axes, problems):
total = n[problem]
ops = sorted(((OPERATION_LABELS[problem][k], v) for k, v in operations[problem].items() if v),
key=lambda item: (-item[1], item[0]))
labels, values, colours = [], [], []
for label, value in ops:
labels.append(label); values.append(value); colours.append(RUST)
labels.append(""); values.append(None); colours.append(None)
for key, label in CATEGORIES:
labels.append(label); values.append(categories[problem][key]); colours.append(TEAL)
ypos = list(range(len(labels)))[::-1]
for y, value, colour in zip(ypos, values, colours):
if value is None:
continue
ax.barh(y, 100 * value / total, height=0.68, color=colour)
ax.text(100 * value / total + 1.5, y, f"{value}/{total}", va="center", fontsize=9.5)
ax.set_yticks(ypos)
ax.set_yticklabels(labels, fontsize=10.5)
ax.set_xlim(0, 118)
ax.set_xticks([0, 25, 50, 75, 100])
ax.set_xticklabels(["0%", "25%", "50%", "75%", "100%"], fontsize=10)
ax.tick_params(axis="y", length=0)
for side in ("top", "right", "left"):
ax.spines[side].set_visible(False)
ax.grid(axis="x", color="#eeeeee")
ax.set_axisbelow(True)
ax.set_title(f"{problem.capitalize()} ({total} specifications)", fontsize=12.5,
fontweight="bold", pad=12)
fig.suptitle("Which parts of the environment did the specifications describe incorrectly?",
fontsize=15, fontweight="bold", y=0.985)
fig.supxlabel("Specifications with at least one mismatch", fontsize=11.5, y=0.075)
fig.legend(handles=[Patch(color=RUST, label="Operation behavior"),
Patch(color=TEAL, label="Initial state, observations and calculations")],
loc="lower center", ncol=2, frameon=False, fontsize=11, bbox_to_anchor=(0.5, 0.0))
fig.tight_layout(rect=(0, 0.1, 1, 0.95))
fig.savefig(path, facecolor="white")
plt.close(fig)
def main():
rows = load_results()
figure_repair(ARTIFACTS / "repair.png")
figure_evidence(rows, ARTIFACTS / "evidence.png")
figure_specs(rows, ARTIFACTS / "specs.png")
if DOCS.is_dir():
shutil.copyfile(ARTIFACTS / "repair.png", DOCS / "repair-investigation.png")
shutil.copyfile(ARTIFACTS / "evidence.png", DOCS / "results/evidence-and-answers.png")
shutil.copyfile(ARTIFACTS / "specs.png", DOCS / "results/specification-errors.png")
print("wrote repair.png, evidence.png and specs.png")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
16.2 kB
·
Xet hash:
46f56a0ead4391e949cc5623e345ff0206693862c6e386459d500aa4267ab726

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.