Accept mesh candidates in leaderboard submissions

README.md

@@ -30,6 +30,6 @@ textual or visual description of a mechanical part into a valid,
 geometrically correct 3D model?
 
 Browse the **Leaderboard** tab for ranked submissions, or upload your
-own STEP files on the **Submit** tab: the Space runs the eval and
+own STEP/BREP or mesh candidate files on the **Submit** tab: the Space runs the eval and
 publishes a row with a per-submission report. The **About** tab covers
 the scoring axes and links to the benchmark code and datasets.

app.py

@@ -107,8 +107,9 @@ ABOUT_MD = f"""## About
 turn a description of a mechanical part into a valid, geometrically
 correct 3D model.
 
-- **Reference baseline**: an iterative AI agent that writes build123d Python.
-- **Submission flow**: upload a zip of per-fixture STEP files; the Space
+- **Reference baseline**: an iterative AI agent that writes CAD code.
+- **Submission flow**: upload a zip of per-fixture candidate files (STEP/BREP
+  or mesh); the Space
   runs the eval and appends a row to the submissions dataset.
 - **Datasets**: fixture inputs in
   [`{HF_DATA_REPO}`](https://huggingface.co/datasets/{HF_DATA_REPO});
@@ -1151,9 +1152,11 @@ with gr.Blocks(
             f"""
 **Submission format.** A single zip with:
 
-- one folder per sample in `{HF_DATA_REPO}`; include `output.step` for
-  samples where your system produced a candidate. Missing `output.step`
-  scores zero for that sample;
+- one folder per sample in `{HF_DATA_REPO}`; include one accepted candidate
+  file for samples where your system produced a result. Accepted names:
+  `output.step`, `output.stp`, `output.stl`, `output.obj`, `output.off`,
+  `output.3mf`, or `output.ply`. Missing `output.*` scores zero for that
+  sample;
 - a top-level `meta.json`:
 
 ```json

metrics_page.py

@@ -15,7 +15,7 @@
 """Self-contained "Metrics" explainer page for the Space.
 
 Builds one static, dependency-free HTML document explaining how a
-candidate STEP is scored: the validity gate, the three orthogonal
+candidate file is scored: the validity gate, the three orthogonal
 axes (shape / topology / interface), and the editing renormalization.
 
 It is curated (a Space-tailored summary, deliberately a little
@@ -147,8 +147,8 @@ def build_metrics_page() -> str:
         tag="CAD Score",
         title="How one part is scored",
         body=(
-            "<p>CADGenBench scores a generated part (a STEP file) against one "
-            "ground-truth STEP. First a hard <b>validity gate</b>; if it "
+            "<p>CADGenBench scores a generated part (STEP/BREP or mesh) "
+            "against one ground-truth STEP. First a hard <b>validity gate</b>; if it "
             "passes, the <b>CAD Score</b> is a weighted mean of three "
             "independent metrics, each in [0, 1].</p>"
             '<pre class="formula">'
@@ -190,10 +190,11 @@ def build_metrics_page() -> str:
         tag="Gate",
         title="CAD Validity",
         body=(
-            "<p>Runs before every other metric on the raw candidate STEP. Any "
+            "<p>Runs before every other metric on the raw candidate. Any "
             "failure sets <code>is_valid = False</code> and forces "
             "<code>cad_score = 0</code>, so an invalid solid never beats a worse "
-            "but valid one. Passing requires all of:</p>"
+            "but valid one.</p>"
+            "<p><b>STEP/BREP candidates</b> must pass all of:</p>"
             "<ol>"
             "<li><b>Well-formed BREP</b>: no per-face / edge / vertex errors "
             "(self-intersecting wires, edges off their surface, etc.).</li>"
@@ -203,6 +204,11 @@ def build_metrics_page() -> str:
             "to a manifold, closed (3F = 2E), orientation-consistent triangle "
             "mesh.</li>"
             "</ol>"
+            "<p><b>Mesh candidates</b> (<code>output.stl</code>, "
+            "<code>output.obj</code>, <code>output.off</code>, "
+            "<code>output.3mf</code>, or <code>output.ply</code>) skip BREP "
+            "checks and must directly satisfy the mesh gate: manifold, closed, "
+            "and orientation-consistent.</p>"
         ),
     )
 

submit.py

@@ -44,11 +44,12 @@ Validation gates, in order:
 4. Fixture-set match: the set of folders inside the zip equals the
    set of fixture directories in :func:`cadgenbench.common.paths.data_inputs_dir`
    (no missing, no extras).
-5. STEP parseability: any present ``<fixture>/output.step`` loads as STEP
-   geometry. A missing ``output.step`` is allowed and scores zero via the
-   evaluator's ``status="missing"`` path. Per-fixture validity (watertight,
-   manifold, etc) is *not* checked here; this gate only rejects files that are
-   present but not actually STEP.
+5. Candidate parseability: any present ``<fixture>/output.*`` candidate
+   (STEP/BREP or accepted mesh format) can be cheaply loaded. A missing
+   candidate is allowed and scores zero via the evaluator's
+   ``status="missing"`` path. Per-fixture validity (watertight, manifold,
+   etc.) is *not* checked here; this gate only rejects files that are present
+   but not actually loadable as their declared candidate kind.
 
 Hub-write ordering (after validation passes):
 
@@ -103,6 +104,7 @@ from typing import Any
 import cadgenbench
 import gradio as gr
 from cadgenbench.common.paths import data_inputs_dir
+from cadgenbench.common.mesh import MESH_FILE_SUFFIXES, mesh_from_file
 from cadgenbench.common.validity import parse_step
 from huggingface_hub import (
     CommitOperationAdd,
@@ -443,13 +445,13 @@ def handle_submit(
     yield _submit_status(
         "working",
         "Validating submission (unpacking the zip, checking the sample set "
-        "and STEP files)…",
+        "and candidate files)…",
     )
 
     zip_path = Path(zip_file.name)
 
     # The tempdir lives only for the cheap-sync validation pass
-    # (unpack zip, validate meta + fixture set + STEP parseability).
+    # (unpack zip, validate meta + fixture set + candidate parseability).
     # The Job downloads the zip itself from the Hub, so the
     # Space-local unpack is throwaway and the tempdir gets cleaned
     # up unconditionally in the outer finally.
@@ -461,7 +463,7 @@ def handle_submit(
             _extract_zip(zip_path, run_dir)
             meta = _load_and_validate_meta(run_dir)
             fixture_names = _validate_fixture_set(run_dir)
-            _validate_steps_parseable(run_dir, fixture_names)
+            _validate_candidates_parseable(run_dir, fixture_names)
         except _ValidationError as e:
             msg = f"Submission rejected: {e}"
             yield _submit_status("error", msg)
@@ -691,9 +693,10 @@ def _validate_fixture_set(unpacked: Path) -> set[str]:
     return expected
 
 
-def _validate_steps_parseable(unpacked: Path, fixture_names: set[str]) -> None:
+def _validate_candidates_parseable(unpacked: Path, fixture_names: set[str]) -> None:
     # Threads (not processes): OCC's parse_step releases the GIL during
-    # the C++ STEP read, and this gate doesn't touch the VTK renderer
+    # the C++ STEP read, and mesh loading is I/O + numpy/trimesh work. This
+    # gate doesn't touch the VTK renderer
     # (which is the only piece in the eval pipeline that needs the
     # ProcessPoolExecutor + spawn dance). Per-fixture I/O + OCC load is
     # 1-5s, so fanning out a 5+ fixture set across cpu-upgrade vCPUs
@@ -701,36 +704,58 @@ def _validate_steps_parseable(unpacked: Path, fixture_names: set[str]) -> None:
     # exception when its iterator is consumed, so wrapping in list()
     # preserves the same `Sample <name>` rejection text as the
     # sequential loop did.
-    def _check_one_step(name: str) -> None:
-        step = _candidate_step_path(unpacked / name)
-        if step is None:
+    def _check_one_candidate(name: str) -> None:
+        candidate = _candidate_path(unpacked / name)
+        if candidate is None:
             # Missing output is a valid benchmark outcome: the evaluator writes
             # status="missing" and the fixture contributes cad_score=0.
             return
-        if step.stat().st_size == 0:
+        if candidate.stat().st_size == 0:
             raise _ValidationError(
-                f"Sample `{name}` has an empty `{step.name}`."
+                f"Sample `{name}` has an empty `{candidate.name}`."
             )
-        try:
-            parse_step(step)
-        except RuntimeError as e:
+        suffix = candidate.suffix.lower()
+        if suffix in {".step", ".stp"}:
+            try:
+                parse_step(candidate)
+            except RuntimeError as e:
+                raise _ValidationError(
+                    f"Sample `{name}` has an `{candidate.name}` that is not "
+                    f"loadable as STEP geometry: {e}"
+                ) from e
+        elif suffix in MESH_FILE_SUFFIXES:
+            try:
+                mesh_from_file(candidate)
+            except Exception as e:  # noqa: BLE001 - normalize user-facing error
+                raise _ValidationError(
+                    f"Sample `{name}` has an `{candidate.name}` that is not "
+                    f"loadable as a triangle mesh: {e}"
+                ) from e
+        else:  # pragma: no cover - _candidate_path constrains suffixes
             raise _ValidationError(
-                f"Sample `{name}` has an `{step.name}` that is not loadable "
-                f"as STEP geometry: {e}"
-            ) from e
+                f"Sample `{name}` uses unsupported candidate file `{candidate.name}`."
+            )
 
     with ThreadPoolExecutor(
         max_workers=min(8, os.cpu_count() or 1),
     ) as ex:
-        list(ex.map(_check_one_step, sorted(fixture_names)))
+        list(ex.map(_check_one_candidate, sorted(fixture_names)))
+
 
+def _candidate_path(fixture_dir: Path) -> Path | None:
+    """Return the submitted candidate for *fixture_dir*, if present.
 
-def _candidate_step_path(fixture_dir: Path) -> Path | None:
-    """Return the submitted candidate STEP for *fixture_dir*, if present."""
+    STEP/BREP wins when both a STEP and a mesh are present, matching the
+    evaluator's candidate discovery policy.
+    """
     for name in ("output.step", "output.stp"):
-        step = fixture_dir / name
-        if step.is_file():
-            return step
+        candidate = fixture_dir / name
+        if candidate.is_file():
+            return candidate
+    for suffix in sorted(MESH_FILE_SUFFIXES):
+        candidate = fixture_dir / f"output{suffix}"
+        if candidate.is_file():
+            return candidate
     return None
 
 

tests/test_submit.py

@@ -392,14 +392,14 @@ def test_pending_row_preserves_existing_metadata(monkeypatch):
         assert row[k] is None
 
 
-def test_validate_steps_allows_missing_output_step(tmp_path: Path, monkeypatch):
+def test_validate_steps_allows_missing_output_candidate(tmp_path: Path, monkeypatch):
     """Missing fixture outputs are accepted; evaluator scores them as missing."""
     (tmp_path / "101").mkdir()
     (tmp_path / "102").mkdir()
     calls: list[Path] = []
     monkeypatch.setattr(submit, "parse_step", lambda p: calls.append(p))
 
-    submit._validate_steps_parseable(tmp_path, {"101", "102"})
+    submit._validate_candidates_parseable(tmp_path, {"101", "102"})
 
     assert calls == []
 
@@ -413,6 +413,23 @@ def test_validate_steps_checks_present_output_stp(tmp_path: Path, monkeypatch):
     calls: list[Path] = []
     monkeypatch.setattr(submit, "parse_step", lambda p: calls.append(p))
 
-    submit._validate_steps_parseable(tmp_path, {"101"})
+    submit._validate_candidates_parseable(tmp_path, {"101"})
 
     assert calls == [candidate]
+
+
+def test_validate_steps_checks_present_output_mesh(tmp_path: Path, monkeypatch):
+    """Mesh candidates are cheap-load checked instead of STEP-parsed."""
+    fixture = tmp_path / "101"
+    fixture.mkdir()
+    candidate = fixture / "output.stl"
+    candidate.write_text("solid empty\nendsolid empty\n")
+    step_calls: list[Path] = []
+    mesh_calls: list[Path] = []
+    monkeypatch.setattr(submit, "parse_step", lambda p: step_calls.append(p))
+    monkeypatch.setattr(submit, "mesh_from_file", lambda p: mesh_calls.append(p))
+
+    submit._validate_candidates_parseable(tmp_path, {"101"})
+
+    assert step_calls == []
+    assert mesh_calls == [candidate]