submitting the same digest twice is a no-op (#14)

- submitting the same digest twice is a no-op (7c28fb0c8619dcc32d19a13aa817b948e1ea0dbf)

submit_test.py

@@ -147,6 +147,55 @@ def _split_image_ref(ref: str) -> tuple[str, str, str]:
     return ref, "", ""
 
 
+def find_existing_submission(
+    api: HfApi, hf_user: str, track: str, division: str, image_digest: str
+) -> dict | None:
+    """This team's existing submission of this exact image, if there is one.
+
+    Makes submitting idempotent. The same digest is the same image by
+    definition, so registering it twice cannot produce a different score; it
+    would only spend a second slot out of three. A double-click or a re-run of
+    the instructions did exactly that in the rehearsal, producing two identical
+    PENDING rows.
+
+    Keyed on the digest, not the tag or the model name, because the digest is
+    what is actually scored. Same reasoning as everywhere else in the pipeline.
+
+    Returns the request record, or None. An unreadable request is skipped: the
+    lenient direction, since the cost of missing one is a duplicate rather than
+    a refusal.
+    """
+    digest = (image_digest or "").strip()
+    if not digest:
+        return None
+    prefix = f"requests/{config.TEST_PHASE}/{track}/{division}/"
+    try:
+        files = api.list_repo_files(config.REQUESTS_REPO, repo_type="dataset")
+    except Exception:
+        return None
+    for f in files:
+        if not (f.startswith(prefix) and f.endswith("/request.json")):
+            continue
+        try:
+            local = hf_hub_download(
+                config.REQUESTS_REPO, f, repo_type="dataset", token=api.token
+            )
+            with open(local, encoding="utf-8") as fh:
+                obj = json.load(fh)
+        except Exception:
+            continue
+        if obj.get("hf_user") != hf_user:
+            continue
+        # The digest is NESTED under "image", which is the contract with the
+        # worker's parse_request(). A flat obj["image_digest"] reads None on
+        # every record, so the comparison would never match and this whole
+        # function would silently do nothing.
+        recorded = ((obj.get("image") or {}).get("digest") or "").strip()
+        if recorded == digest:
+            return obj
+    return None
+
+
 def count_test_submissions(
     api: HfApi, hf_user: str, track: str, division: str
 ) -> int:
@@ -280,6 +329,25 @@ def validate_and_submit(
 
     api = HfApi(token=token)
 
+    # Idempotent: the same digest is the same image, so registering it twice
+    # cannot produce a different score. Checked BEFORE the cap, because a
+    # re-submission is not a new submission and a team already at the limit
+    # must not be told it has run out for something it already sent.
+    existing = find_existing_submission(api, hf_user, track, division, image_digest)
+    if existing is not None:
+        status = existing.get("status", "PENDING")
+        when = (existing.get("created_at") or "")[:19].replace("T", " ")
+        return True, (
+            f"✅ Already submitted. This exact image is registered for "
+            f"{config.TRACK_LABELS[track]} / {division} as "
+            f"`{existing.get('submission_id', '')}`"
+            + (f", submitted {when} UTC" if when else "")
+            + f", currently **{status}**. Nothing was registered again and no "
+            "submission was spent, because the digest is the same image. Track "
+            "it in **My Submissions**; push a new image and submit that digest "
+            "if you want a different one scored."
+        )
+
     held = count_test_submissions(api, hf_user, track, division)
     if held >= config.MAX_TEST_SUBMISSIONS_PER_TRACK:
         return False, (

tests/test_submit_test.py

@@ -196,6 +196,141 @@ class ValidateOnlyTest(unittest.TestCase):
         self.assertFalse(self._call(image_ref="docker.io/x/y")[0])
 
 
+class SubmitIsIdempotentTest(unittest.TestCase):
+    """The end-to-end property, across the seam that matters.
+
+    Testing the lookup alone would not have caught the real bug: the lookup read
+    a flat field while the writer nests it. This drives validate_and_submit and
+    asserts nothing is written.
+    """
+
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        p = mock.patch.object(config, "TEST_PHASE_OPEN", True); p.start()
+        self.addCleanup(p.stop)
+        self.records = {}
+        d = mock.patch.object(st, "hf_hub_download", self._download); d.start()
+        self.addCleanup(d.stop)
+
+    def _download(self, repo_id, path, repo_type=None, token=None):
+        local = os.path.join(self._tmp.name, path.replace("/", "_"))
+        with open(local, "w", encoding="utf-8") as fh:
+            json.dump(self.records[path], fh)
+        return local
+
+    def _submit(self, api, digest=DIGEST):
+        with mock.patch.object(st, "HfApi", return_value=api):
+            return st.validate_and_submit(
+                hf_user="team_a", track="convqa", division="large",
+                team_name="Team A", model_name="m", license_str="mit",
+                open_weight=True, total_params=8_000_000_000,
+                active_params=8_000_000_000, image_ref=REF,
+                image_digest=digest, token="t",
+            )
+
+    def _existing(self, sid, digest, status="PENDING"):
+        path = f"requests/test/convqa/large/{sid}/request.json"
+        self.records[path] = {
+            "submission_id": sid, "hf_user": "team_a", "status": status,
+            "created_at": "2026-08-08T19:56:00Z",
+            "image": {"ref": REF, "digest": digest},
+        }
+        return path
+
+    def test_resubmitting_the_same_image_writes_nothing(self):
+        self._existing("s1", DIGEST)
+        api = FakeApi(list(self.records))
+        ok, msg = self._submit(api)
+        self.assertTrue(ok, msg)
+        self.assertEqual([], api.uploads, "a duplicate must not be written")
+        self.assertIn("Already submitted", msg)
+        self.assertIn("s1", msg)
+
+    def test_a_team_at_the_cap_can_still_resubmit_what_it_already_sent(self):
+        # The idempotency check must run BEFORE the cap, or a team at 3/3 is
+        # told it is out of submissions for an image it already sent.
+        for i in range(config.MAX_TEST_SUBMISSIONS_PER_TRACK):
+            self._existing(f"s{i}", f"sha256:{i}" + "a" * 63)
+        self._existing("mine", DIGEST)
+        api = FakeApi(list(self.records))
+        ok, msg = self._submit(api)
+        self.assertTrue(ok, msg)
+        self.assertIn("Already submitted", msg)
+        self.assertNotIn("limit reached", msg)
+        self.assertEqual([], api.uploads)
+
+    def test_a_genuinely_new_image_is_still_written(self):
+        # The other half: idempotency must not swallow a real submission.
+        self._existing("s1", DIGEST)
+        api = FakeApi(list(self.records))
+        ok, msg = self._submit(api, digest="sha256:" + "c" * 64)
+        self.assertTrue(ok, msg)
+        self.assertTrue(api.uploads, "a new digest must still be registered")
+
+
+class FindExistingSubmissionTest(unittest.TestCase):
+    """Submitting the same digest twice must not spend a second slot.
+
+    The digest is the image, so registering it again cannot change the score.
+    Two identical PENDING rows appeared in the rehearsal from one repeated
+    submit, each holding a slot out of three.
+    """
+
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self.records = {}
+        patch = mock.patch.object(st, "hf_hub_download", self._download)
+        patch.start(); self.addCleanup(patch.stop)
+
+    def _download(self, repo_id, path, repo_type=None, token=None):
+        local = os.path.join(self._tmp.name, path.replace("/", "_"))
+        with open(local, "w", encoding="utf-8") as fh:
+            json.dump(self.records[path], fh)
+        return local
+
+    def _add(self, sid, digest, hf_user="team_a", track="convqa", division="large"):
+        path = f"requests/test/{track}/{division}/{sid}/request.json"
+        # Shaped like the real record: the digest is NESTED under "image".
+        self.records[path] = {
+            "submission_id": sid, "hf_user": hf_user, "status": "PENDING",
+            "created_at": "2026-08-08T19:56:00Z",
+            "image": {"ref": REF, "digest": digest},
+        }
+
+    def _find(self, digest, hf_user="team_a"):
+        return st.find_existing_submission(
+            FakeApi(list(self.records)), hf_user, "convqa", "large", digest
+        )
+
+    def test_the_same_digest_is_found(self):
+        self._add("s1", DIGEST)
+        found = self._find(DIGEST)
+        self.assertIsNotNone(found, "an identical digest must be recognised")
+        self.assertEqual("s1", found["submission_id"])
+
+    def test_it_reads_the_nested_digest_not_a_flat_field(self):
+        # The record stores image.digest. A flat obj["image_digest"] reads None
+        # on every record, so the lookup would match nothing and silently do
+        # nothing. That bug was written and caught here.
+        self._add("s1", DIGEST)
+        self.assertIn("image", self.records[list(self.records)[0]])
+        self.assertIsNotNone(self._find(DIGEST))
+
+    def test_a_different_digest_is_not_a_duplicate(self):
+        self._add("s1", DIGEST)
+        self.assertIsNone(self._find("sha256:" + "b" * 64))
+
+    def test_another_teams_identical_image_is_not_mine(self):
+        self._add("s1", DIGEST, hf_user="someone_else")
+        self.assertIsNone(self._find(DIGEST))
+
+    def test_an_empty_digest_matches_nothing(self):
+        self._add("s1", DIGEST)
+        self.assertIsNone(self._find(""))
+
+
 class CountTestSubmissionsTest(unittest.TestCase):
     """Which request states hold a slot, and whose."""