Overlay measured Python source lines on progress chart

Count source lines from Git history on a separate right axis, leaving the unavailable initial seed unestimated.

Co-authored-by: Codex <noreply@openai.com>

README.md

@@ -33,7 +33,7 @@ Keep local logs and temporary files in `scratch/`, which is ignored by Git.
 Generate the chart snapshot from the leaderboard's v3 cache before starting the app:
 
 ```sh
-python build_progress.py /path/to/swarmchat-leaderboard-v3.json
+python build_progress.py /path/to/swarmchat-leaderboard-v3.json --source-repo scratch/v8-history.git
 ```
 
 The generated data stays in `scratch/progress.json` and is never committed. Supply
@@ -43,3 +43,11 @@ It uses the leaderboard's length-penalised percentage metric and latest successf
 1,000-prompt run per revision, with all three SmolLM baselines. The contiguous outage
 plateau from November 7, 2026, 12:12:47 through 13:15:11 UTC (commit times) is omitted;
 other score fluctuations and the original revision positions are preserved.
+
+The right axis overlays physical lines in `bot/**/*.py` at each retained evaluated
+commit, including blank lines and comments but excluding tests. Counts come from
+the source repository's first-parent Git diffs, including deletions and merges.
+Fetch the source history into `scratch/v8-history.git` before building the snapshot;
+unavailable commits get a null code-size value, rather than an estimate. The first
+evaluated seed (`541b8def5107`) predates the current main history and is no longer
+available from the Hub; its win-rate point remains, but its code-size point is absent.

build_progress.py

@@ -3,6 +3,7 @@
 import argparse
 import json
 import math
+import subprocess
 from pathlib import Path
 
 # The abrupt outage plateau, bounded by the last/first affected commit timestamps.
@@ -12,6 +13,57 @@ OUTAGE_END = "20261107T131511Z"
 MODEL = "HuggingFaceSwarms/SwarmChat-v8"
 
 
+def count_source_tree(repository, commit):
+    """Count a source tree directly for evaluated commits outside the main history."""
+    tree = subprocess.check_output(
+        ["git", "-C", str(repository), "ls-tree", "-r", commit, "--", "bot"], text=True)
+    objects = [line.split()[2] for line in tree.splitlines() if line.endswith(".py")]
+    total = 0
+    with subprocess.Popen(["git", "-C", str(repository), "cat-file", "--batch"],
+                          stdin=subprocess.PIPE, stdout=subprocess.PIPE) as process:
+        for oid in objects:
+            process.stdin.write((oid + "\n").encode())
+            process.stdin.flush()
+            size = int(process.stdout.readline().split()[2])
+            blob = process.stdout.read(size)
+            process.stdout.read(1)
+            total += blob.count(b"\n") + bool(blob and not blob.endswith(b"\n"))
+        process.stdin.close()
+        if process.wait() != 0:
+            raise RuntimeError(f"Could not read source tree for {commit}")
+    return total
+
+
+def source_line_counts(repository):
+    """Physical lines in bot/**/*.py, including blanks/comments, excluding tests.
+
+    Accumulate first-parent diffs, including deletions and merge changes. Disabling
+    rename detection makes moves across the bot/ boundary count correctly.
+    """
+    command = ["git", "-C", str(repository), "log", "--first-parent", "--reverse",
+               "--format=commit:%H", "--numstat", "--no-renames",
+               "--diff-merges=first-parent", "HEAD"]
+    counts, commit, total = {}, None, 0
+    with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as process:
+        for line in process.stdout:
+            line = line.rstrip("\n")
+            if line.startswith("commit:"):
+                if commit is not None:
+                    counts[commit[:12]] = total
+                commit = line.removeprefix("commit:")
+            elif "\t" in line:
+                added, removed, path = line.split("\t", 2)
+                if path.startswith("bot/") and path.endswith(".py"):
+                    if added == "-" or removed == "-":
+                        raise ValueError(f"Cannot count binary Python file: {path}")
+                    total += int(added) - int(removed)
+        if process.wait() != 0:
+            raise RuntimeError("Could not read source history")
+    if commit is not None:
+        counts[commit[:12]] = total
+    return counts
+
+
 def build_progress(cache):
     latest = {}
     for path, summary in cache["summaries"].items():
@@ -51,8 +103,24 @@ def build_progress(cache):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("cache", type=Path)
+    parser.add_argument("--source-repo", type=Path, required=True,
+                        help="Git repository containing the evaluated source history")
     args = parser.parse_args()
     result = build_progress(json.loads(args.cache.read_text()))
+    counts = source_line_counts(args.source_repo)
+    missing = []
+    for point in result["points"]:
+        if point["commit"] not in counts:
+            exists = subprocess.run(
+                ["git", "-C", str(args.source_repo), "cat-file", "-e", point["commit"]],
+                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
+            counts[point["commit"]] = count_source_tree(args.source_repo, point["commit"]) if exists else None
+        if counts[point["commit"]] is None:
+            missing.append(point["commit"])
+        point["lines_of_code"] = counts[point["commit"]]
+    result["missing_source_commits"] = missing
+    if missing:
+        print(f"No source available for {len(missing)} commits; leaving code-size gaps: {', '.join(missing)}")
     output = Path(__file__).parent / "scratch" / "progress.json"
     output.parent.mkdir(exist_ok=True)
     output.write_text(json.dumps(result, separators=(",", ":")) + "\n")

site/index.html

@@ -84,7 +84,7 @@
   <p class="foot">PyBot at commit <a id="commit" target="_blank" rel="noopener"></a>. Click a file to open it on the Hub.</p>
   <section class="progress" aria-labelledby="progress-title">
     <h2 id="progress-title">Learning to converse, one revision at a time</h2>
-    <div id="progress-chart" role="img" aria-label="Line chart of PyBot win rate over evaluated revisions, with three SmolLM baselines"></div>
+    <div id="progress-chart" role="img" aria-label="PyBot win rate in orange on the left axis and Python lines of code in blue on the right axis, over evaluated revisions, with three SmolLM baselines"></div>
     <p class="foot" id="progress-status" aria-live="polite">Loading evaluation history…</p>
   </section>
 </main>

site/progress.js

@@ -12,6 +12,12 @@ async function drawProgress() {
       customdata: points.map(p => p.commit),
       line: {color: "#eb6834", width: 2},
       hovertemplate: "PyBot · revision %{x:,}<br>Win rate: %{y:.2f}%<br>Commit %{customdata}<extra></extra>"
+    }, {
+      type: "scatter", mode: "lines", name: "Lines of code", yaxis: "y2",
+      x: points.map(p => p.revision), y: points.map(p => p.lines_of_code),
+      customdata: points.map(p => p.commit),
+      line: {color: "#287a9b", width: 2},
+      hovertemplate: "Revision %{x:,}<br>Lines of code: %{y:,}<br>Python source in bot/ (including blanks and comments)<br>Commit %{customdata}<extra></extra>"
     }, ...data.baselines.map((baseline, index) => ({
       type: "scatter", mode: "lines",
       name: baseline.name,
@@ -21,13 +27,17 @@ async function drawProgress() {
       hovertemplate: `${baseline.name}<br>Win rate: %{y:.2f}%<extra></extra>`
     }))];
     await Plotly.newPlot("progress-chart", traces, {
-      margin: {l: 60, r: 12, t: 30, b: 60},
+      margin: {l: 60, r: 72, t: 30, b: 60},
       paper_bgcolor: "#ffffff", plot_bgcolor: "#ffffff",
       font: {family: "'Letter Gothic', monospace", size: 12, color: "#000000"},
       xaxis: {title: {text: "Evaluated revisions", standoff: 12},
               range: extent, gridcolor: "#eeeeee", zeroline: false, automargin: true},
       yaxis: {title: {text: "Win rate (%)", standoff: 10}, rangemode: "tozero",
+              color: "#eb6834",
               gridcolor: "#eeeeee", zerolinecolor: "#dddddd", automargin: true},
+      yaxis2: {title: {text: "Lines of code", standoff: 10}, overlaying: "y", side: "right",
+               rangemode: "tozero", color: "#287a9b", tickformat: "~s",
+               showgrid: false, zeroline: false, automargin: true},
       showlegend: false,
       annotations: data.baselines.map(baseline => ({
         text: baseline.name, x: 1, xref: "paper", y: baseline.score, yref: "y",