{
  "schema_version": 2,
  "reference_set": "nv100-v1",
  "reference_count": 100,
  "evaluator_version": "2.2.0",
  "source_revision": "5ecafa8612a5d16d83869587a1018bf5e3d75d00",
  "job_ids": [
    "0f2b2409-7945-44b8-abb4-c4faef268035",
    "bb865915-d3cc-400f-bda1-0eaf4b3e4b4a",
    "dc30f963-ebba-47f8-a052-535ce8a706f4",
    "e03b16cc-bf75-4247-9b96-467a3685c942",
    "e6465830-0d2a-4663-8811-76a32076b78e",
    "ed242be1-ed4f-4caa-815d-9ce76615afe4",
    "f5f94694-fb90-4593-9d21-8c5b1cb75e42",
    "03d80d61-938f-40cb-9ea7-31cd51e63b3b"
  ],
  "formal_fidelity_primary": true,
  "runs": [
    {
      "id": "harbor-evaluator-2.2.0-gpt-6.1-sol-unspecified",
      "label": "ChatGPT 6.1 Sol \u00b7 low \u00b7 Codex",
      "model": "ChatGPT 6.1 Sol",
      "model_family": "Sol",
      "model_display_name": "ChatGPT 6.1 Sol",
      "reasoning_display": "low",
      "requested_model": "openai/gpt-6.1-sol",
      "reasoning": null,
      "harness": "Codex",
      "authoring_label": "Codex",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "dc30f963-ebba-47f8-a052-535ce8a706f4",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 303,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/e586f232e0b1939f84c59956e88eed38c167bd2f/results/harbor/dc30f963-ebba-47f8-a052-535ce8a706f4",
      "scores": {
        "formal": {
          "value": 97.69239130434782,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known"
        },
        "visual": {
          "value": 51.16534437171281,
          "measured_attempts": 297,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        },
        "kinematic": {
          "value": 2.438710345248361,
          "measured_attempts": 298,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.206476566,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 61.9429698,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 97.73144040224786,
              "x_value": 0.206476566,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 51.056117170760956,
              "x_value": 0.206476566,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 2.6265986852291427,
              "x_value": 0.206476566,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_unverified_basis",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_unverified_basis"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 351.35501172333335,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 303,
          "total_attempts": 303,
          "unknown_execution_count": 0,
          "sum_recorded": 105406.503517,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 97.73144040224786,
              "x_value": 351.35501172333335,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 51.056117170760956,
              "x_value": 351.35501172333335,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 2.6265986852291427,
              "x_value": 351.35501172333335,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 478511.38333333336,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 302,
          "total_attempts": 303,
          "unknown_execution_count": 1,
          "sum_recorded": 143553415.0,
          "complete": false,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 97.73144040224786,
              "x_value": 478511.38333333336,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": true
            },
            "visual": {
              "value": 51.056117170760956,
              "x_value": 478511.38333333336,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": true
            },
            "kinematic": {
              "value": 2.6265986852291427,
              "x_value": 478511.38333333336,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": true
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 478511.38333333336,
          "sum_recorded": 143553415.0,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "input": {
          "value": 470783.68333333335,
          "sum_recorded": 141235105.0,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "output": {
          "value": 7727.7,
          "sum_recorded": 2318310.0,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "cached_input": {
          "value": 436238.10333333333,
          "sum_recorded": 130871431.0,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.206476566,
        "sum_recorded": 61.9429698,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 351.35501172333335,
        "sum_recorded": 105406.503517,
        "denominator": 300,
        "known_executions": 303,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.0021135378430521714,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.004035476913825919,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.08466629356057133,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 71.33333333333333,
          "successes": 214,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 71.33333333333333,
          "upper_bound": 71.33333333333333
        },
        "pass_at_3": {
          "value": 96.0,
          "successes": 96,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 96.0,
          "upper_bound": 96.0
        },
        "pass_all_3": {
          "value": 39.0,
          "successes": 39,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 39.0,
          "upper_bound": 39.0
        },
        "successes_per_asset": {
          "0": 4,
          "1": 17,
          "2": 40,
          "3": 39
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          }
        ]
      },
      "optional_criteria": {
        "value": null,
        "unit": "percent",
        "status": "partial",
        "lower_bound_percent": 73.57516339869281,
        "upper_bound_percent": 73.61437908496733,
        "passed_checks": 11257,
        "failed_checks": 4037,
        "unknown_checks": 6,
        "total_checks": 15300,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "fully_scored_attempts": 298,
        "unknown_attempts": 2,
        "coverage_fraction": 0.9996078431372549,
        "mean_over_fully_scored_attempts": 73.59520989603895,
        "source_field": "scores.formal.evidence.optional_score",
        "scope": "all_optional_profile",
        "profile_inventory_sha256": "e47935d0d76c63b5e5dbe8b79a53408a8234544f6bbd5246686a65691a3231a9",
        "retries_excluded": true
      },
      "repetitions": 3,
      "inference_spend": {
        "final": {
          "value": 0.206476566,
          "sum_recorded": 61.9429698,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 0.0007616943333333333,
          "sum_recorded": 0.2285083,
          "denominator": 300,
          "known_executions": 2,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.20723826033333334,
          "sum_recorded": 62.1714781,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_unverified_basis"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "harbor-evaluator-2.2.0-gpt-6.1-sol-medium",
      "label": "ChatGPT 6.1 Sol \u00b7 medium \u00b7 Codex",
      "model": "ChatGPT 6.1 Sol",
      "model_family": "Sol",
      "model_display_name": "ChatGPT 6.1 Sol",
      "reasoning_display": "medium",
      "requested_model": "openai/gpt-6.1-sol",
      "reasoning": "medium",
      "harness": "Codex",
      "authoring_label": "Codex",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "dc30f963-ebba-47f8-a052-535ce8a706f4",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 302,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/e586f232e0b1939f84c59956e88eed38c167bd2f/results/harbor/dc30f963-ebba-47f8-a052-535ce8a706f4",
      "scores": {
        "formal": {
          "value": 99.97644927536231,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known"
        },
        "visual": {
          "value": 53.65528916857613,
          "measured_attempts": 295,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        },
        "kinematic": {
          "value": 3.142825677790428,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known"
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.4125363053333333,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 123.7608916,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 99.9759686483289,
              "x_value": 0.4125363053333333,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 53.81366071899747,
              "x_value": 0.4125363053333333,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 3.4180296057006734,
              "x_value": 0.4125363053333333,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_unverified_basis",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_unverified_basis"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 559.2767091,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 302,
          "total_attempts": 302,
          "unknown_execution_count": 0,
          "sum_recorded": 167783.01273,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 99.9759686483289,
              "x_value": 559.2767091,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 53.81366071899747,
              "x_value": 559.2767091,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 3.4180296057006734,
              "x_value": 559.2767091,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 1064313.18,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 302,
          "total_attempts": 302,
          "unknown_execution_count": 0,
          "sum_recorded": 319293954.0,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 99.9759686483289,
              "x_value": 1064313.18,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 53.81366071899747,
              "x_value": 1064313.18,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 3.4180296057006734,
              "x_value": 1064313.18,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 1064313.18,
          "sum_recorded": 319293954.0,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "input": {
          "value": 1045908.9566666667,
          "sum_recorded": 313772687.0,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "output": {
          "value": 18404.22333333333,
          "sum_recorded": 5521267.0,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cached_input": {
          "value": 989904.26,
          "sum_recorded": 296971278.0,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.4125363053333333,
        "sum_recorded": 123.7608916,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 559.2767091,
        "sum_recorded": 167783.01273,
        "denominator": 300,
        "known_executions": 302,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.004126334835087973,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.0076886419163115826,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.1312628658498705,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 99.66666666666667,
          "successes": 299,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 99.66666666666667,
          "upper_bound": 99.66666666666667
        },
        "pass_at_3": {
          "value": 100.0,
          "successes": 100,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_3": {
          "value": 99.0,
          "successes": 99,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 99.0,
          "upper_bound": 99.0
        },
        "successes_per_asset": {
          "0": 0,
          "1": 0,
          "2": 1,
          "3": 99
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          }
        ]
      },
      "optional_criteria": {
        "value": null,
        "unit": "percent",
        "status": "partial",
        "lower_bound_percent": 73.2156862745098,
        "upper_bound_percent": 73.23529411764706,
        "passed_checks": 11202,
        "failed_checks": 4095,
        "unknown_checks": 3,
        "total_checks": 15300,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "fully_scored_attempts": 299,
        "unknown_attempts": 1,
        "coverage_fraction": 0.9998039215686274,
        "mean_over_fully_scored_attempts": 73.231031543052,
        "source_field": "scores.formal.evidence.optional_score",
        "scope": "all_optional_profile",
        "profile_inventory_sha256": "e47935d0d76c63b5e5dbe8b79a53408a8234544f6bbd5246686a65691a3231a9",
        "retries_excluded": true
      },
      "repetitions": 3,
      "inference_spend": {
        "final": {
          "value": 0.4125363053333333,
          "sum_recorded": 123.7608916,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 0.0019417876666666667,
          "sum_recorded": 0.5825363,
          "denominator": 300,
          "known_executions": 2,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.414478093,
          "sum_recorded": 124.3434279,
          "denominator": 300,
          "known_executions": 302,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_unverified_basis"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "e6465830-0d2a-4663-8811-76a32076b78e:codex-text2cad-build123d:gpt-6.1-sol:medium",
      "label": "ChatGPT 6.1 Sol \u00b7 medium \u00b7 Codex + CAD/build123d",
      "model": "ChatGPT 6.1 Sol",
      "model_family": "Sol",
      "model_display_name": "ChatGPT 6.1 Sol",
      "reasoning_display": "medium",
      "requested_model": "openai/gpt-6.1-sol",
      "reasoning": "medium",
      "harness": "Codex + CAD/build123d",
      "authoring_label": "Codex + CAD/build123d",
      "agent_revision": "cb68fc11a27a5c6d046946ec6de9ed876915d8a2",
      "evaluator_version": "2.2.0",
      "job_id": "e6465830-0d2a-4663-8811-76a32076b78e",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 400,
      "execution_count": 401,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/3f791f561dd935c8f1d4318b477f2908b7ba22bb/results/harbor/e6465830-0d2a-4663-8811-76a32076b78e",
      "scores": {
        "formal": {
          "value": 100.0,
          "measured_attempts": 400,
          "eligible_attempts": 400,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 100.0,
          "upper_bound_percent": 100.0
        },
        "visual": {
          "value": 55.5640022459195,
          "measured_attempts": 388,
          "eligible_attempts": 400,
          "complete": false,
          "status": "partial",
          "lower_bound_percent": 53.897082178541915,
          "upper_bound_percent": 56.54283364956439
        },
        "kinematic": {
          "value": 3.004327094889817,
          "measured_attempts": 400,
          "eligible_attempts": 400,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 3.004327094889817,
          "upper_bound_percent": 3.004327094889817
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.7382938889999999,
          "unit": "USD/asset",
          "denominator": 400,
          "measured_attempts": 400,
          "total_attempts": 400,
          "unknown_execution_count": 0,
          "sum_recorded": 295.3175556,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 0.7382938889999999,
              "measured_attempts": 294,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 55.23798927636517,
              "x_value": 0.7382938889999999,
              "measured_attempts": 184,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 2.8987035040224516,
              "x_value": 0.7382938889999999,
              "measured_attempts": 271,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "kind": "estimated",
          "pricing_policy": "openai-standard-global-30m",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "estimated"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 567.08734337,
          "unit": "seconds/asset",
          "denominator": 400,
          "measured_attempts": 401,
          "total_attempts": 401,
          "unknown_execution_count": 0,
          "sum_recorded": 226834.937348,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 567.08734337,
              "measured_attempts": 294,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 55.23798927636517,
              "x_value": 567.08734337,
              "measured_attempts": 184,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 2.8987035040224516,
              "x_value": 567.08734337,
              "measured_attempts": 271,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 3369249.315,
          "unit": "tokens/asset",
          "denominator": 400,
          "measured_attempts": 400,
          "total_attempts": 401,
          "unknown_execution_count": 1,
          "sum_recorded": 1347699726,
          "complete": false,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 3369249.315,
              "measured_attempts": 294,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": true
            },
            "visual": {
              "value": 55.23798927636517,
              "x_value": 3369249.315,
              "measured_attempts": 184,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": true
            },
            "kinematic": {
              "value": 2.8987035040224516,
              "x_value": 3369249.315,
              "measured_attempts": 271,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": true
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 3369249.315,
          "sum_recorded": 1347699726.0,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "input": {
          "value": 3351437.9375,
          "sum_recorded": 1340575175.0,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "output": {
          "value": 17811.3775,
          "sum_recorded": 7124551.0,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "cached_input": {
          "value": 3257605.5275,
          "sum_recorded": 1303042211.0,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.7382938889999999,
        "sum_recorded": 295.3175556,
        "denominator": 400,
        "known_executions": 400,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 567.08734337,
        "sum_recorded": 226834.937348,
        "denominator": 400,
        "known_executions": 401,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.007382938889999999,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.013287269799831934,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.24574351116953752,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "n": 4,
        "assets": 100,
        "final_attempts": 400,
        "pass_at_1": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_1": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_at_3": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_3": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_at_4": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_4": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "successes_per_asset": {
          "4": 100
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          }
        ]
      },
      "harness_id": "codex-text2cad-build123d",
      "harness_version": "0.1.3",
      "repetitions": 4,
      "optional_criteria": {
        "value": 73.4656862745098,
        "unit": "percent",
        "status": "known",
        "lower_bound_percent": 73.4656862745098,
        "upper_bound_percent": 73.4656862745098,
        "passed_checks": 14987,
        "failed_checks": 5413,
        "unknown_checks": 0,
        "total_checks": 20400,
        "checks_per_attempt": 51,
        "eligible_attempts": 400,
        "fully_scored_attempts": 400,
        "unknown_attempts": 0,
        "coverage_fraction": 1.0,
        "mean_over_fully_scored_attempts": 73.4656862745098,
        "source_field": "formal_report.optional_score",
        "scope": "all_optional_profile",
        "profile_inventory_sha256": "e47935d0d76c63b5e5dbe8b79a53408a8234544f6bbd5246686a65691a3231a9",
        "retries_excluded": true
      },
      "inference_spend": {
        "final": {
          "value": 0.7382938889999999,
          "sum_recorded": 295.3175556,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": null,
          "sum_recorded": null,
          "denominator": 400,
          "known_executions": 0,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.7382938889999999,
          "sum_recorded": 295.3175556,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "cost_kinds": [
          "estimated"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "harbor-evaluator-2.2.0-gpt-6.1-sol-high",
      "label": "ChatGPT 6.1 Sol \u00b7 high \u00b7 Codex",
      "model": "ChatGPT 6.1 Sol",
      "model_family": "Sol",
      "model_display_name": "ChatGPT 6.1 Sol",
      "reasoning_display": "high",
      "requested_model": "openai/gpt-6.1-sol",
      "reasoning": "high",
      "harness": "Codex",
      "authoring_label": "Codex",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "dc30f963-ebba-47f8-a052-535ce8a706f4",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 306,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/e586f232e0b1939f84c59956e88eed38c167bd2f/results/harbor/dc30f963-ebba-47f8-a052-535ce8a706f4",
      "scores": {
        "formal": {
          "value": 100.0,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known"
        },
        "visual": {
          "value": 56.86104470170295,
          "measured_attempts": 282,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        },
        "kinematic": {
          "value": 3.6766977151345324,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known"
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.7072036666666667,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 212.1611,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 0.7072036666666667,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 57.039052968280146,
              "x_value": 0.7072036666666667,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 4.03043724418612,
              "x_value": 0.7072036666666667,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_unverified_basis",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_unverified_basis"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 863.2201472466667,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 306,
          "total_attempts": 306,
          "unknown_execution_count": 0,
          "sum_recorded": 258966.044174,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 863.2201472466667,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 57.039052968280146,
              "x_value": 863.2201472466667,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 4.03043724418612,
              "x_value": 863.2201472466667,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 1990603.8366666667,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 306,
          "total_attempts": 306,
          "unknown_execution_count": 0,
          "sum_recorded": 597181151.0,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 1990603.8366666667,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 57.039052968280146,
              "x_value": 1990603.8366666667,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 4.03043724418612,
              "x_value": 1990603.8366666667,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 1990603.8366666667,
          "sum_recorded": 597181151.0,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "input": {
          "value": 1959382.44,
          "sum_recorded": 587814732.0,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "output": {
          "value": 31221.396666666667,
          "sum_recorded": 9366419.0,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cached_input": {
          "value": 1873388.4166666667,
          "sum_recorded": 562016525.0,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.7072036666666667,
        "sum_recorded": 212.1611,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 863.2201472466667,
        "sum_recorded": 258966.044174,
        "denominator": 300,
        "known_executions": 306,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.007072036666666667,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.012437401922119212,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.19234751438922407,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 100.0,
          "successes": 300,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_at_3": {
          "value": 100.0,
          "successes": 100,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_3": {
          "value": 100.0,
          "successes": 100,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "successes_per_asset": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 100
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          }
        ]
      },
      "optional_criteria": {
        "value": 72.84313725490196,
        "unit": "percent",
        "status": "known",
        "lower_bound_percent": 72.84313725490196,
        "upper_bound_percent": 72.84313725490196,
        "passed_checks": 11145,
        "failed_checks": 4155,
        "unknown_checks": 0,
        "total_checks": 15300,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "fully_scored_attempts": 300,
        "unknown_attempts": 0,
        "coverage_fraction": 1.0,
        "mean_over_fully_scored_attempts": 72.84313725490196,
        "source_field": "scores.formal.evidence.optional_score",
        "scope": "all_optional_profile",
        "profile_inventory_sha256": "e47935d0d76c63b5e5dbe8b79a53408a8234544f6bbd5246686a65691a3231a9",
        "retries_excluded": true
      },
      "repetitions": 3,
      "inference_spend": {
        "final": {
          "value": 0.7072036666666667,
          "sum_recorded": 212.1611,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 0.007278138333333333,
          "sum_recorded": 2.1834415,
          "denominator": 300,
          "known_executions": 6,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.714481805,
          "sum_recorded": 214.3445415,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_unverified_basis"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "harbor-evaluator-2.2.0-gpt-6.1-terra-low",
      "label": "ChatGPT 6.1 Terra \u00b7 low \u00b7 Codex",
      "model": "ChatGPT 6.1 Terra",
      "model_family": "Terra",
      "model_display_name": "ChatGPT 6.1 Terra",
      "reasoning_display": "low",
      "requested_model": "openai/gpt-6.1-terra",
      "reasoning": "low",
      "harness": "Codex",
      "authoring_label": "Codex",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "dc30f963-ebba-47f8-a052-535ce8a706f4",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 300,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/e586f232e0b1939f84c59956e88eed38c167bd2f/results/harbor/dc30f963-ebba-47f8-a052-535ce8a706f4",
      "scores": {
        "formal": {
          "value": 91.12789855072464,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known"
        },
        "visual": {
          "value": 47.80628554215278,
          "measured_attempts": 295,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        },
        "kinematic": {
          "value": 0.86649541310952,
          "measured_attempts": 299,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.09692058916666667,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 29.07617675,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 91.13021295474712,
              "x_value": 0.09692058916666667,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 47.27809010580624,
              "x_value": 0.09692058916666667,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.9486133682562952,
              "x_value": 0.09692058916666667,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_unverified_basis",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_unverified_basis"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 263.7662889233333,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 79129.886677,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 91.13021295474712,
              "x_value": 263.7662889233333,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 47.27809010580624,
              "x_value": 263.7662889233333,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.9486133682562952,
              "x_value": 263.7662889233333,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 359012.8933333333,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 107703868.0,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 91.13021295474712,
              "x_value": 359012.8933333333,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 47.27809010580624,
              "x_value": 359012.8933333333,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.9486133682562952,
              "x_value": 359012.8933333333,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 359012.8933333333,
          "sum_recorded": 107703868.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "input": {
          "value": 353438.25666666665,
          "sum_recorded": 106031477.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "output": {
          "value": 5574.636666666666,
          "sum_recorded": 1672391.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cached_input": {
          "value": 310615.98333333334,
          "sum_recorded": 93184795.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.09692058916666667,
        "sum_recorded": 29.07617675,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 263.7662889233333,
        "sum_recorded": 79129.886677,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.0010635665993407896,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.002027360797173999,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.11185355132908992,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 2.0,
          "successes": 6,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 2.0,
          "upper_bound": 2.0
        },
        "pass_at_3": {
          "value": 6.0,
          "successes": 6,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 6.0,
          "upper_bound": 6.0
        },
        "pass_all_3": {
          "value": 0.0,
          "successes": 0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 0.0,
          "upper_bound": 0.0
        },
        "successes_per_asset": {
          "0": 94,
          "1": 6,
          "2": 0,
          "3": 0
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          }
        ]
      },
      "optional_criteria": {
        "value": null,
        "unit": "percent",
        "status": "partial",
        "lower_bound_percent": 69.89542483660131,
        "upper_bound_percent": 70.0326797385621,
        "passed_checks": 10694,
        "failed_checks": 4585,
        "unknown_checks": 21,
        "total_checks": 15300,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "fully_scored_attempts": 293,
        "unknown_attempts": 7,
        "coverage_fraction": 0.9986274509803922,
        "mean_over_fully_scored_attempts": 69.95248611389948,
        "source_field": "scores.formal.evidence.optional_score",
        "scope": "all_optional_profile",
        "profile_inventory_sha256": "e47935d0d76c63b5e5dbe8b79a53408a8234544f6bbd5246686a65691a3231a9",
        "retries_excluded": true
      },
      "repetitions": 3,
      "inference_spend": {
        "final": {
          "value": 0.09692058916666667,
          "sum_recorded": 29.07617675,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 0.0,
          "sum_recorded": 0.0,
          "denominator": 300,
          "known_executions": 0,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.09692058916666667,
          "sum_recorded": 29.07617675,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_unverified_basis"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "e6465830-0d2a-4663-8811-76a32076b78e:codex-text2cad-build123d:gpt-6.1-terra:medium",
      "label": "ChatGPT 6.1 Terra \u00b7 medium \u00b7 Codex + CAD/build123d",
      "model": "ChatGPT 6.1 Terra",
      "model_family": "Terra",
      "model_display_name": "ChatGPT 6.1 Terra",
      "reasoning_display": "medium",
      "requested_model": "openai/gpt-6.1-terra",
      "reasoning": "medium",
      "harness": "Codex + CAD/build123d",
      "authoring_label": "Codex + CAD/build123d",
      "agent_revision": "cb68fc11a27a5c6d046946ec6de9ed876915d8a2",
      "evaluator_version": "2.2.0",
      "job_id": "e6465830-0d2a-4663-8811-76a32076b78e",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 400,
      "execution_count": 401,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/3f791f561dd935c8f1d4318b477f2908b7ba22bb/results/harbor/e6465830-0d2a-4663-8811-76a32076b78e",
      "scores": {
        "formal": {
          "value": 100.0,
          "measured_attempts": 400,
          "eligible_attempts": 400,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 100.0,
          "upper_bound_percent": 100.0
        },
        "visual": {
          "value": 54.10798611663832,
          "measured_attempts": 399,
          "eligible_attempts": 400,
          "complete": false,
          "status": "partial",
          "lower_bound_percent": 53.972716151346724,
          "upper_bound_percent": 54.18569352698347
        },
        "kinematic": {
          "value": 0.7129011519815553,
          "measured_attempts": 400,
          "eligible_attempts": 400,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 0.7129011519815553,
          "upper_bound_percent": 0.7129011519815553
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.3960623955,
          "unit": "USD/asset",
          "denominator": 400,
          "measured_attempts": 400,
          "total_attempts": 400,
          "unknown_execution_count": 0,
          "sum_recorded": 158.4249582,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 0.3960623955,
              "measured_attempts": 294,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 53.621230762418556,
              "x_value": 0.3960623955,
              "measured_attempts": 184,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.5068921265512137,
              "x_value": 0.3960623955,
              "measured_attempts": 271,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "kind": "estimated",
          "pricing_policy": "openai-standard-global-30m",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "estimated"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 513.6280323474999,
          "unit": "seconds/asset",
          "denominator": 400,
          "measured_attempts": 401,
          "total_attempts": 401,
          "unknown_execution_count": 0,
          "sum_recorded": 205451.212939,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 513.6280323474999,
              "measured_attempts": 294,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 53.621230762418556,
              "x_value": 513.6280323474999,
              "measured_attempts": 184,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.5068921265512137,
              "x_value": 513.6280323474999,
              "measured_attempts": 271,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 3917476.69,
          "unit": "tokens/asset",
          "denominator": 400,
          "measured_attempts": 400,
          "total_attempts": 401,
          "unknown_execution_count": 1,
          "sum_recorded": 1566990676,
          "complete": false,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 3917476.69,
              "measured_attempts": 294,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": true
            },
            "visual": {
              "value": 53.621230762418556,
              "x_value": 3917476.69,
              "measured_attempts": 184,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": true
            },
            "kinematic": {
              "value": 0.5068921265512137,
              "x_value": 3917476.69,
              "measured_attempts": 271,
              "eligible_attempts": 400,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": true
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 3917476.69,
          "sum_recorded": 1566990676.0,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "input": {
          "value": 3901989.5375,
          "sum_recorded": 1560795815.0,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "output": {
          "value": 15487.1525,
          "sum_recorded": 6194861.0,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "cached_input": {
          "value": 3798976.06,
          "sum_recorded": 1519590424.0,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.3960623955,
        "sum_recorded": 158.4249582,
        "denominator": 400,
        "known_executions": 400,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 513.6280323474999,
        "sum_recorded": 205451.212939,
        "denominator": 400,
        "known_executions": 401,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.003960623955,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.007319850985513023,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.5555642523498786,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "n": 4,
        "assets": 100,
        "final_attempts": 400,
        "pass_at_1": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_1": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_at_3": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_3": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_at_4": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_4": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "successes_per_asset": {
          "4": 100
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 4,
            "successes": 4,
            "unknown": 0
          }
        ]
      },
      "harness_id": "codex-text2cad-build123d",
      "harness_version": "0.1.3",
      "repetitions": 4,
      "optional_criteria": {
        "value": 70.8921568627451,
        "unit": "percent",
        "status": "known",
        "lower_bound_percent": 70.8921568627451,
        "upper_bound_percent": 70.8921568627451,
        "passed_checks": 14462,
        "failed_checks": 5938,
        "unknown_checks": 0,
        "total_checks": 20400,
        "checks_per_attempt": 51,
        "eligible_attempts": 400,
        "fully_scored_attempts": 400,
        "unknown_attempts": 0,
        "coverage_fraction": 1.0,
        "mean_over_fully_scored_attempts": 70.8921568627451,
        "source_field": "formal_report.optional_score",
        "scope": "all_optional_profile",
        "profile_inventory_sha256": "e47935d0d76c63b5e5dbe8b79a53408a8234544f6bbd5246686a65691a3231a9",
        "retries_excluded": true
      },
      "inference_spend": {
        "final": {
          "value": 0.3960623955,
          "sum_recorded": 158.4249582,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": null,
          "sum_recorded": null,
          "denominator": 400,
          "known_executions": 0,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.3960623955,
          "sum_recorded": 158.4249582,
          "denominator": 400,
          "known_executions": 400,
          "unknown_executions": 1,
          "partial": true,
          "includes_retries": true
        },
        "cost_kinds": [
          "estimated"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "harbor-evaluator-2.2.0-gpt-6.1-terra-high",
      "label": "ChatGPT 6.1 Terra \u00b7 high \u00b7 Codex",
      "model": "ChatGPT 6.1 Terra",
      "model_family": "Terra",
      "model_display_name": "ChatGPT 6.1 Terra",
      "reasoning_display": "high",
      "requested_model": "openai/gpt-6.1-terra",
      "reasoning": "high",
      "harness": "Codex",
      "authoring_label": "Codex",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "dc30f963-ebba-47f8-a052-535ce8a706f4",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 300,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/e586f232e0b1939f84c59956e88eed38c167bd2f/results/harbor/dc30f963-ebba-47f8-a052-535ce8a706f4",
      "scores": {
        "formal": {
          "value": 100.0,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known"
        },
        "visual": {
          "value": 54.0317300289362,
          "measured_attempts": 289,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        },
        "kinematic": {
          "value": 2.786428247049474,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known"
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.27350602183333333,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 82.05180655,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 0.27350602183333333,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 54.68142772798915,
              "x_value": 0.27350602183333333,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 3.065273465178823,
              "x_value": 0.27350602183333333,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_unverified_basis",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_unverified_basis"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 465.41184811000005,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 139623.554433,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 465.41184811000005,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 54.68142772798915,
              "x_value": 465.41184811000005,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 3.065273465178823,
              "x_value": 465.41184811000005,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 1653681.07,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 496104321.0,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 100.0,
              "x_value": 1653681.07,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 54.68142772798915,
              "x_value": 1653681.07,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 3.065273465178823,
              "x_value": 1653681.07,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 1653681.07,
          "sum_recorded": 496104321.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "input": {
          "value": 1633905.19,
          "sum_recorded": 490171557.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "output": {
          "value": 19775.88,
          "sum_recorded": 5932764.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cached_input": {
          "value": 1556437.42,
          "sum_recorded": 466931226.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.27350602183333333,
        "sum_recorded": 82.05180655,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 465.41184811000005,
        "sum_recorded": 139623.554433,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.0027350602183333333,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.005061951962057474,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.09815649196168845,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 100.0,
          "successes": 300,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_at_3": {
          "value": 100.0,
          "successes": 100,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_3": {
          "value": 100.0,
          "successes": 100,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "successes_per_asset": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 100
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          }
        ]
      },
      "optional_criteria": {
        "value": null,
        "unit": "percent",
        "status": "partial",
        "lower_bound_percent": 70.62091503267973,
        "upper_bound_percent": 70.640522875817,
        "passed_checks": 10805,
        "failed_checks": 4492,
        "unknown_checks": 3,
        "total_checks": 15300,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "fully_scored_attempts": 299,
        "unknown_attempts": 1,
        "coverage_fraction": 0.9998039215686274,
        "mean_over_fully_scored_attempts": 70.62758213653355,
        "source_field": "scores.formal.evidence.optional_score",
        "scope": "all_optional_profile",
        "profile_inventory_sha256": "e47935d0d76c63b5e5dbe8b79a53408a8234544f6bbd5246686a65691a3231a9",
        "retries_excluded": true
      },
      "repetitions": 3,
      "inference_spend": {
        "final": {
          "value": 0.27350602183333333,
          "sum_recorded": 82.05180655,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 0.0,
          "sum_recorded": 0.0,
          "denominator": 300,
          "known_executions": 0,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.27350602183333333,
          "sum_recorded": 82.05180655,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_unverified_basis"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "bb865915-d3cc-400f-bda1-0eaf4b3e4b4a:claude-opus-5-5:low:claude-code",
      "label": "Claude Opus 5.5 \u00b7 low \u00b7 Claude Code",
      "model": "anthropic/claude-opus-5-5",
      "model_family": "Claude",
      "model_display_name": "Claude Opus 5.5",
      "reasoning_display": "low",
      "requested_model": "anthropic/claude-opus-5-5",
      "reasoning": "low",
      "harness": "Claude Code",
      "authoring_label": "Claude Code",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "bb865915-d3cc-400f-bda1-0eaf4b3e4b4a",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 300,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/9b4845481387d2ba5f84ad2e7314b754e0c16416/results/harbor/bb865915-d3cc-400f-bda1-0eaf4b3e4b4a",
      "scores": {
        "formal": {
          "value": 97.12463768115941,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 97.12463768115941,
          "upper_bound_percent": 97.12463768115941
        },
        "visual": {
          "value": 51.19130645044229,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 51.19130645044229,
          "upper_bound_percent": 51.19130645044229
        },
        "kinematic": {
          "value": 2.4467528124592204,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 2.4467528124592204,
          "upper_bound_percent": 2.4467528124592204
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.312590902,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 93.7772706,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 97.06595681750962,
              "x_value": 0.312590902,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 51.273250475672924,
              "x_value": 0.312590902,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 2.611388860437899,
              "x_value": 0.312590902,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_agent_usage",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_agent_usage"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 245.72048925666667,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 73716.146777,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 97.06595681750962,
              "x_value": 245.72048925666667,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 51.273250475672924,
              "x_value": 245.72048925666667,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 2.611388860437899,
              "x_value": 245.72048925666667,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 381567.0733333333,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 114470122.0,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 97.06595681750962,
              "x_value": 381567.0733333333,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 51.273250475672924,
              "x_value": 381567.0733333333,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 2.611388860437899,
              "x_value": 381567.0733333333,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 381567.0733333333,
          "sum_recorded": 114470122.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "input": {
          "value": 374666.4866666667,
          "sum_recorded": 112399946.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "output": {
          "value": 6900.586666666667,
          "sum_recorded": 2070176.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cached_input": {
          "value": 353221.31,
          "sum_recorded": 105966393.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.312590902,
        "sum_recorded": 93.7772706,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 245.72048925666667,
        "sum_recorded": 73716.146777,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.0032184511516652743,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.006106327884063979,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.1277574507764911,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "samples_per_asset": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 67.0,
          "successes": 201.0,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 67.0,
          "upper_bound": 67.0
        },
        "pass_at_3": {
          "value": 94.0,
          "successes": 94.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 94.0,
          "upper_bound": 94.0
        },
        "pass_all_3": {
          "value": 37.0,
          "successes": 37.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 37.0,
          "upper_bound": 37.0
        },
        "successes_per_asset": {
          "0": 6,
          "1": 24,
          "2": 33,
          "3": 37
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          }
        ]
      },
      "optional_criteria": {
        "value": 73.77124183006536,
        "unit": "percent",
        "status": "known",
        "lower_bound_percent": 73.77124183006536,
        "upper_bound_percent": 73.77124183006536,
        "passed_checks": 11287.0,
        "unknown_checks": 0.0,
        "total_checks": 15300.0,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "fully_scored_attempts": 300,
        "unknown_attempts": 0,
        "retries_excluded": true
      },
      "job_level_costs": {
        "total_recorded_inference_usd": 93.7772706,
        "hosted_compute_cost_usd": "16.916789350851357550",
        "hosted_compute_cost_status": "partial",
        "reported_combined_cost_usd": "110.694059950851357550",
        "provider_reported_cost_usd": null,
        "cost_observed_at": "2026-11-04T02:15:31.109421+00:00"
      },
      "reasoning_effort": "low",
      "output_limit": null,
      "harness_id": "claude-code",
      "agent_version": "2.1.317",
      "harness_version": "2.1.317",
      "repetitions": 3,
      "max_authoring_turns": 64,
      "max_formal_feedback_calls": 5,
      "max_output_tokens": null,
      "reasoning_enabled": true,
      "inference_spend": {
        "final": {
          "value": 0.312590902,
          "sum_recorded": 93.7772706,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 0.0,
          "sum_recorded": 0.0,
          "denominator": 300,
          "known_executions": 0,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.312590902,
          "sum_recorded": 93.7772706,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_agent_usage"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "e03b16cc-bf75-4247-9b96-467a3685c942:claude-sonnet-5-5:low:claude-code",
      "label": "Claude Sonnet 5.5 \u00b7 low \u00b7 Claude Code",
      "model": "Claude Sonnet 5.5",
      "model_family": "Claude",
      "model_display_name": "Claude Sonnet 5.5",
      "reasoning_display": "low",
      "requested_model": "anthropic/claude-sonnet-5-5",
      "reasoning": "low",
      "harness": "Claude Code",
      "authoring_label": "Claude Code",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "e03b16cc-bf75-4247-9b96-467a3685c942",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 300,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/71bc6cbc6e6001aec9a71667f2c60f7aa72aa6af/results/harbor/e03b16cc-bf75-4247-9b96-467a3685c942",
      "scores": {
        "formal": {
          "value": 83.6,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 83.6,
          "upper_bound_percent": 83.6
        },
        "visual": {
          "value": 50.0645769239514,
          "measured_attempts": 298,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial",
          "lower_bound_percent": 49.73081307779172,
          "upper_bound_percent": 50.21987417105894
        },
        "kinematic": {
          "value": 0.6633233549895805,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 0.6633233549895805,
          "upper_bound_percent": 0.6633233549895805
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.05778986666666667,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 17.33696,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 83.61320615202602,
              "x_value": 0.05778986666666667,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 50.53786891345266,
              "x_value": 0.05778986666666667,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.718295189803457,
              "x_value": 0.05778986666666667,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_unverified_basis",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_unverified_basis"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 196.79211168666666,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 59037.633506,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 83.61320615202602,
              "x_value": 196.79211168666666,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 50.53786891345266,
              "x_value": 196.79211168666666,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.718295189803457,
              "x_value": 196.79211168666666,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 93819.46333333333,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 28145839.0,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 83.61320615202602,
              "x_value": 93819.46333333333,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 50.53786891345266,
              "x_value": 93819.46333333333,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.718295189803457,
              "x_value": 93819.46333333333,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 93819.46333333333,
          "sum_recorded": 28145839.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "input": {
          "value": 91460.49666666667,
          "sum_recorded": 27438149.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "output": {
          "value": 2358.9666666666667,
          "sum_recorded": 707690.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cached_input": {
          "value": 80386.76666666666,
          "sum_recorded": 24116030.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.05778986666666667,
        "sum_recorded": 17.33696,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 196.79211168666666,
        "sum_recorded": 59037.633506,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.0006912663476874004,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.0011543065020693185,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.0871217125583863,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "samples_per_asset": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 0.0,
          "successes": 0.0,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 0.0,
          "upper_bound": 0.0
        },
        "pass_at_3": {
          "value": 0.0,
          "successes": 0.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 0.0,
          "upper_bound": 0.0
        },
        "pass_all_3": {
          "value": 0.0,
          "successes": 0.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 0.0,
          "upper_bound": 0.0
        },
        "successes_per_asset": {
          "0": 100,
          "1": 0,
          "2": 0,
          "3": 0
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          }
        ]
      },
      "harness_id": "claude-code",
      "agent_version": "2.1.317",
      "harness_version": "2.1.317",
      "repetitions": 3,
      "optional_criteria": {
        "value": 70.01960784313725,
        "unit": "percent",
        "status": "known",
        "lower_bound_percent": 70.01960784313725,
        "upper_bound_percent": 70.01960784313725,
        "passed_checks": 10713,
        "unknown_checks": 0,
        "total_checks": 15300,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "fully_scored_attempts": 300,
        "unknown_attempts": 0,
        "retries_excluded": true
      },
      "inference_spend": {
        "final": {
          "value": 0.05778986666666667,
          "sum_recorded": 17.33696,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 0.0,
          "sum_recorded": 0.0,
          "denominator": 300,
          "known_executions": 0,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.05778986666666667,
          "sum_recorded": 17.33696,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_unverified_basis"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "0f2b2409-7945-44b8-abb4-c4faef268035:deepseek-v4.1-flash-1017:none:terminus-2",
      "label": "DeepSeek V4.1 Flash (1017) \u00b7 none \u00b7 Terminus 2",
      "model": "DeepSeek V4.1 Flash (1017)",
      "model_family": "DeepSeek",
      "model_display_name": "DeepSeek V4.1 Flash (1017)",
      "reasoning_display": "none",
      "requested_model": "openai/deepseek-v4.1-flash-1017",
      "reasoning": "none",
      "harness": "Terminus 2",
      "authoring_label": "Terminus 2",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "0f2b2409-7945-44b8-abb4-c4faef268035",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 301,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/c77d4ae599756ba985001ab126821e1c5ee31f51/results/harbor/0f2b2409-7945-44b8-abb4-c4faef268035",
      "scores": {
        "formal": {
          "value": 86.3243420095972,
          "measured_attempts": 299,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial",
          "lower_bound_percent": 86.31884057971014,
          "upper_bound_percent": 86.32572463768116
        },
        "visual": {
          "value": 35.36492790119117,
          "measured_attempts": 286,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial",
          "lower_bound_percent": 33.714564599135585,
          "upper_bound_percent": 37.290887906656856
        },
        "kinematic": {
          "value": 0.05241736631681492,
          "measured_attempts": 292,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial",
          "lower_bound_percent": 0.05101956988169986,
          "upper_bound_percent": 0.4795909984531284
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.027378095849999996,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 8.213428754999999,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 86.24704229517894,
              "x_value": 0.027378095849999996,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 35.67767239716405,
              "x_value": 0.027378095849999996,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.051288910507209066,
              "x_value": 0.027378095849999996,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_unverified_basis",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_unverified_basis"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 658.7579575766666,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 301,
          "total_attempts": 301,
          "unknown_execution_count": 0,
          "sum_recorded": 197627.387273,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 86.24704229517894,
              "x_value": 658.7579575766666,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 35.67767239716405,
              "x_value": 658.7579575766666,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.051288910507209066,
              "x_value": 658.7579575766666,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 2242278.0633333335,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 301,
          "total_attempts": 301,
          "unknown_execution_count": 0,
          "sum_recorded": 672683419.0,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 86.24704229517894,
              "x_value": 2242278.0633333335,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 35.67767239716405,
              "x_value": 2242278.0633333335,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.051288910507209066,
              "x_value": 2242278.0633333335,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 2242278.0633333335,
          "sum_recorded": 672683419.0,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "input": {
          "value": 2210364.4166666665,
          "sum_recorded": 663109325.0,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "output": {
          "value": 31913.646666666667,
          "sum_recorded": 9574094.0,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cached_input": {
          "value": 2012997.9333333333,
          "sum_recorded": 603899380.0,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.027378095849999996,
        "sum_recorded": 8.213428754999999,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 658.7579575766666,
        "sum_recorded": 197627.387273,
        "denominator": 300,
        "known_executions": 301,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.00031715383184682957,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.0007741595268197292,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.5223096422762736,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "samples_per_asset": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 2.3333333333333335,
          "successes": 7.0,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 2.3333333333333335,
          "upper_bound": 2.3333333333333335
        },
        "pass_at_3": {
          "value": 6.0,
          "successes": 6.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 6.0,
          "upper_bound": 6.0
        },
        "pass_all_3": {
          "value": 0.0,
          "successes": 0.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 0.0,
          "upper_bound": 0.0
        },
        "successes_per_asset": {
          "0": 94,
          "1": 5,
          "2": 1,
          "3": 0
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          }
        ]
      },
      "harness_id": "terminus-2",
      "agent_version": "2.0.0",
      "harness_version": "2.0.0",
      "repetitions": 3,
      "optional_criteria": {
        "value": null,
        "unit": "percent",
        "status": "partial",
        "lower_bound_percent": 71.640522875817,
        "upper_bound_percent": 73.22875816993464,
        "passed_checks": 10961,
        "unknown_checks": 243,
        "total_checks": 15300,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "fully_scored_attempts": 219,
        "unknown_attempts": 81,
        "retries_excluded": true
      },
      "inference_spend": {
        "final": {
          "value": 0.027378095849999996,
          "sum_recorded": 8.213428754999999,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 1.7981300000000003e-05,
          "sum_recorded": 0.005394390000000001,
          "denominator": 300,
          "known_executions": 1,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.02739607715,
          "sum_recorded": 8.218823145,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_unverified_basis"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "f5f94694-fb90-4593-9d21-8c5b1cb75e42:deepseek-v4.1-flash-1017:none:terminus-2-cad",
      "label": "DeepSeek V4.1 Flash (1017) \u00b7 none \u00b7 Terminus 2 + CAD/build123d",
      "model": "openai/deepseek-v4.1-flash-1017",
      "model_family": "DeepSeek",
      "model_display_name": "DeepSeek V4.1 Flash (1017)",
      "reasoning_display": "none",
      "requested_model": "openai/deepseek-v4.1-flash-1017",
      "reasoning": "none",
      "harness": "Terminus 2 + CAD/build123d",
      "authoring_label": "Terminus 2 + CAD/build123d",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "f5f94694-fb90-4593-9d21-8c5b1cb75e42",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 306,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/91b522cced9e869bc08182756788340616f2dc33/results/harbor/f5f94694-fb90-4593-9d21-8c5b1cb75e42",
      "scores": {
        "formal": {
          "value": 86.2626811594203,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 86.2626811594203,
          "upper_bound_percent": 86.2626811594203
        },
        "visual": {
          "value": 37.60889960134543,
          "measured_attempts": 292,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial",
          "lower_bound_percent": 36.60599561197622,
          "upper_bound_percent": 38.473179300254586
        },
        "kinematic": {
          "value": 0.093498715979961,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 0.093498715979961,
          "upper_bound_percent": 0.093498715979961
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.02504988907,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 7.514966721,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 86.09361135758651,
              "x_value": 0.02504988907,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 37.52755660600545,
              "x_value": 0.02504988907,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.10308233583584,
              "x_value": 0.02504988907,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_usage_times_recorded_tariff",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_usage_times_recorded_tariff"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 835.2293441633333,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 306,
          "total_attempts": 306,
          "unknown_execution_count": 0,
          "sum_recorded": 250568.803249,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 86.09361135758651,
              "x_value": 835.2293441633333,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 37.52755660600545,
              "x_value": 835.2293441633333,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.10308233583584,
              "x_value": 835.2293441633333,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 3468985.5633333335,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 306,
          "total_attempts": 306,
          "unknown_execution_count": 0,
          "sum_recorded": 1040695669.0,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 86.09361135758651,
              "x_value": 3468985.5633333335,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 37.52755660600545,
              "x_value": 3468985.5633333335,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.10308233583584,
              "x_value": 3468985.5633333335,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 3468985.5633333335,
          "sum_recorded": 1040695669.0,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "input": {
          "value": 3434797.42,
          "sum_recorded": 1030439226.0,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "output": {
          "value": 34188.14333333333,
          "sum_recorded": 10256443.0,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cached_input": {
          "value": 3300037.546666667,
          "sum_recorded": 990011264.0,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.02504988907,
        "sum_recorded": 7.514966721,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 835.2293441633333,
        "sum_recorded": 250568.803249,
        "denominator": 300,
        "known_executions": 306,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.0002903908009038789,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.0006660628025687798,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.26791693134447736,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "samples_per_asset": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 53.666666666666664,
          "successes": 161.0,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 53.666666666666664,
          "upper_bound": 53.666666666666664
        },
        "pass_at_3": {
          "value": 87.0,
          "successes": 87.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 87.0,
          "upper_bound": 87.0
        },
        "pass_all_3": {
          "value": 13.0,
          "successes": 13.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 13.0,
          "upper_bound": 13.0
        },
        "successes_per_asset": {
          "0": 13,
          "1": 26,
          "2": 48,
          "3": 13
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          }
        ]
      },
      "optional_criteria": {
        "value": null,
        "unit": "percent",
        "status": "partial",
        "lower_bound_percent": 64.42483660130719,
        "upper_bound_percent": 64.89542483660131,
        "passed_checks": 9857.0,
        "unknown_checks": 72.0,
        "total_checks": 15300.0,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "fully_scored_attempts": 276,
        "unknown_attempts": 24,
        "retries_excluded": true
      },
      "job_level_costs": {
        "total_recorded_inference_usd": 7.594046946,
        "hosted_compute_cost_usd": "61.029315732825024090",
        "hosted_compute_cost_status": "recorded",
        "reported_combined_cost_usd": "68.623362678825024090",
        "provider_reported_cost_usd": null,
        "cost_observed_at": "2026-11-03T23:24:27.889937+00:00"
      },
      "harness_id": "terminus-2-cad",
      "agent_version": "2.0.0",
      "harness_version": "2.0.0",
      "repetitions": 3,
      "max_authoring_turns": 64,
      "max_formal_feedback_calls": 5,
      "max_output_tokens": 4096,
      "reasoning_enabled": false,
      "inference_spend": {
        "final": {
          "value": 0.02504988907,
          "sum_recorded": 7.514966721,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 0.00026360075,
          "sum_recorded": 0.079080225,
          "denominator": 300,
          "known_executions": 6,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.02531348982,
          "sum_recorded": 7.594046946,
          "denominator": 300,
          "known_executions": 306,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_usage_times_recorded_tariff"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "ed242be1-ed4f-4caa-815d-9ce76615afe4:nemotron-3.5-super-vl:none:terminus-2",
      "label": "NVIDIA Nemotron 3.5 Super VL \u00b7 none \u00b7 Terminus 2",
      "model": "NVIDIA Nemotron 3.5 Super VL",
      "model_family": "Nemotron",
      "model_display_name": "NVIDIA Nemotron 3.5 Super VL",
      "reasoning_display": "none",
      "requested_model": "openai/nemotron-3.5-super-vl",
      "reasoning": "none",
      "harness": "Terminus 2",
      "authoring_label": "Terminus 2",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "ed242be1-ed4f-4caa-815d-9ce76615afe4",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 301,
      "source_url": "https://hub.harborframework.com/jobs/ed242be1-ed4f-4caa-815d-9ce76615afe4",
      "scores": {
        "formal": {
          "value": 76.68607221812822,
          "measured_attempts": 295,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        },
        "visual": {
          "value": 21.520957956961265,
          "measured_attempts": 227,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        },
        "kinematic": {
          "value": 0.0049902834876614606,
          "measured_attempts": 282,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial"
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.33568559526666664,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 100.70567858,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 76.65890269151139,
              "x_value": 0.33568559526666664,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 22.091779282956534,
              "x_value": 0.33568559526666664,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.004992065989401574,
              "x_value": 0.33568559526666664,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_unverified_basis",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_unverified_basis"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 670.7458901633333,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 301,
          "total_attempts": 301,
          "unknown_execution_count": 0,
          "sum_recorded": 201223.767049,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 76.65890269151139,
              "x_value": 670.7458901633333,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 22.091779282956534,
              "x_value": 670.7458901633333,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.004992065989401574,
              "x_value": 670.7458901633333,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 2511512.17,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 301,
          "total_attempts": 301,
          "unknown_execution_count": 0,
          "sum_recorded": 753453651.0,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 76.65890269151139,
              "x_value": 2511512.17,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 22.091779282956534,
              "x_value": 2511512.17,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": false,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 0.004992065989401574,
              "x_value": 2511512.17,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": false,
              "frontier_provisional": false
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 2511512.17,
          "sum_recorded": 753453651.0,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "input": {
          "value": 2464376.2866666666,
          "sum_recorded": 739312886.0,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "output": {
          "value": 47135.88333333333,
          "sum_recorded": 14140765.0,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cached_input": {
          "value": 2031610.2933333332,
          "sum_recorded": 609483088.0,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.33568559526666664,
        "sum_recorded": 100.70567858,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 670.7458901633333,
        "sum_recorded": 201223.767049,
        "denominator": 300,
        "known_executions": 301,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.004377399774913914,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.015598078670010332,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 67.26784081438531,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "samples_per_asset": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": null,
          "successes": 0.0,
          "denominator": 300,
          "unknown": 5,
          "lower_bound": 0.0,
          "upper_bound": 1.6666666666666667
        },
        "pass_at_3": {
          "value": null,
          "successes": 0.0,
          "denominator": 100,
          "unknown": 5,
          "lower_bound": 0.0,
          "upper_bound": 5.0
        },
        "pass_all_3": {
          "value": 0.0,
          "successes": 0.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 0.0,
          "upper_bound": 0.0
        },
        "successes_per_asset": {
          "0": 95,
          "1": 0,
          "2": 0,
          "3": 0
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 0,
            "unknown": 1
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 0,
            "unknown": 1
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 0,
            "unknown": 1
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 0,
            "unknown": 1
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 0,
            "unknown": 1
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 0,
            "unknown": 0
          }
        ]
      },
      "custom_agent": false,
      "repetitions": 3,
      "protocol": "64 turns; 4,096 output tokens per call; reasoning off",
      "optional_criteria": {
        "value": null,
        "status": "partial",
        "lower_bound_percent": 68.33333333333333,
        "upper_bound_percent": 69.62745098039215,
        "passed_checks": 10455,
        "failed_checks": 4647,
        "unknown_checks": 198,
        "total_checks": 15300,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "source_field": "formal_report.optional_score",
        "retries_excluded": true
      },
      "inference_spend": {
        "final": {
          "value": 0.33568559526666664,
          "sum_recorded": 100.70567858,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": 2.4840833333333334e-06,
          "sum_recorded": 0.000745225,
          "denominator": 300,
          "known_executions": 1,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.33568807935,
          "sum_recorded": 100.706423805,
          "denominator": 300,
          "known_executions": 301,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_unverified_basis"
        ],
        "missing_is_zero": false
      }
    },
    {
      "id": "03d80d61-938f-40cb-9ea7-31cd51e63b3b:gpt-6.2-astra:low:codex",
      "label": "ChatGPT 6.2 Astra \u00b7 low \u00b7 Codex",
      "model": "ChatGPT 6.2 Astra",
      "model_family": "Astra",
      "model_display_name": "ChatGPT 6.2 Astra",
      "reasoning_display": "low",
      "requested_model": "openai/gpt-6.2-astra",
      "reasoning": "low",
      "harness": "Codex",
      "authoring_label": "Codex",
      "agent_revision": null,
      "evaluator_version": "2.2.0",
      "job_id": "03d80d61-938f-40cb-9ea7-31cd51e63b3b",
      "frontier_group": "nv100-v1:2.2.0",
      "reference_count": 100,
      "final_slots": 300,
      "execution_count": 306,
      "source_url": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/tree/main/results/harbor/03d80d61-938f-40cb-9ea7-31cd51e63b3b",
      "scores": {
        "formal": {
          "value": 99.56521739130434,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 99.56521739130434,
          "upper_bound_percent": 99.56521739130434
        },
        "visual": {
          "value": 59.07945821800251,
          "measured_attempts": 298,
          "eligible_attempts": 300,
          "complete": false,
          "status": "partial",
          "lower_bound_percent": 58.68559516321582,
          "upper_bound_percent": 59.186165711116416
        },
        "kinematic": {
          "value": 5.354753897922746,
          "measured_attempts": 300,
          "eligible_attempts": 300,
          "complete": true,
          "status": "known",
          "lower_bound_percent": 5.354753897922746,
          "upper_bound_percent": 5.354753897922746
        }
      },
      "metrics": {
        "cost_usd_per_asset": {
          "value": 0.57959506,
          "unit": "USD/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 300,
          "unknown_execution_count": 0,
          "sum_recorded": 173.878518,
          "complete": true,
          "includes_retries": false,
          "paired_scores": {
            "formal": {
              "value": 99.55634427684117,
              "x_value": 0.57959506,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": false,
              "frontier_provisional": false
            },
            "visual": {
              "value": 57.997349843748296,
              "x_value": 0.57959506,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 5.8962826681936455,
              "x_value": 0.57959506,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "kind": "harbor_reported_agent_usage",
          "scope": "final_retained_outputs",
          "cost_kinds": [
            "harbor_reported_agent_usage"
          ]
        },
        "elapsed_seconds_per_asset": {
          "value": 286.77709938,
          "unit": "seconds/asset",
          "denominator": 300,
          "measured_attempts": 306,
          "total_attempts": 306,
          "unknown_execution_count": 0,
          "sum_recorded": 86033.129814,
          "complete": true,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 99.55634427684117,
              "x_value": 286.77709938,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": false
            },
            "visual": {
              "value": 57.997349843748296,
              "x_value": 286.77709938,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": false
            },
            "kinematic": {
              "value": 5.8962826681936455,
              "x_value": 286.77709938,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": false
            }
          },
          "timing_scope": "sum of individual trial durations, including setup and verification; not job wall time"
        },
        "tokens_per_asset": {
          "value": 276341.5133333333,
          "unit": "tokens/asset",
          "denominator": 300,
          "measured_attempts": 300,
          "total_attempts": 306,
          "unknown_execution_count": 6,
          "sum_recorded": 82902454.0,
          "complete": false,
          "includes_retries": true,
          "paired_scores": {
            "formal": {
              "value": 99.55634427684117,
              "x_value": 276341.5133333333,
              "measured_attempts": 294,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "4e53410325d3ac6e5f08fb30428374e33acc21444f8af7416e4c13641402d387",
              "frontier": true,
              "frontier_provisional": true
            },
            "visual": {
              "value": 57.997349843748296,
              "x_value": 276341.5133333333,
              "measured_attempts": 184,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "1b52f2d613865c7bcd2ef563674385839b4318887760069911ef1dc93c20f845",
              "frontier": true,
              "frontier_provisional": true
            },
            "kinematic": {
              "value": 5.8962826681936455,
              "x_value": 276341.5133333333,
              "measured_attempts": 271,
              "eligible_attempts": 300,
              "complete": false,
              "cohort_sha256": "b13e8506252996c6df05fca0fb6eec2b1edabd0a6dfb7038258d2d7f83ecda9e",
              "frontier": true,
              "frontier_provisional": true
            }
          }
        }
      },
      "tokens_per_asset": {
        "total": {
          "value": 276341.5133333333,
          "sum_recorded": 82902454.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 6,
          "partial": true,
          "includes_retries": true
        },
        "input": {
          "value": 271434.2966666667,
          "sum_recorded": 81430289.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 6,
          "partial": true,
          "includes_retries": true
        },
        "output": {
          "value": 4907.216666666666,
          "sum_recorded": 1472165.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 6,
          "partial": true,
          "includes_retries": true
        },
        "cached_input": {
          "value": 242808.56666666668,
          "sum_recorded": 72842570.0,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 6,
          "partial": true,
          "includes_retries": true
        }
      },
      "inference_cost_per_asset": {
        "value": 0.57959506,
        "sum_recorded": 173.878518,
        "denominator": 300,
        "known_executions": 300,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": false
      },
      "trial_seconds_per_asset": {
        "value": 286.77709938,
        "sum_recorded": 86033.129814,
        "denominator": 300,
        "known_executions": 306,
        "unknown_executions": 0,
        "partial": false,
        "includes_retries": true
      },
      "cost_per_point": {
        "formal": {
          "value": 0.005821260427947599,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "visual": {
          "value": 0.009810432889572228,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        },
        "kinematic": {
          "value": 0.1082393460182812,
          "status": "known",
          "unit": "USD per fidelity point on a 0\u2013100 scale"
        }
      },
      "pass_rates": {
        "criterion": "all_46_mandatory_requirements_pass",
        "k": 3,
        "samples_per_asset": 3,
        "assets": 100,
        "final_attempts": 300,
        "pass_at_1": {
          "value": 94.33333333333333,
          "successes": 283.0,
          "denominator": 300,
          "unknown": 0,
          "lower_bound": 94.33333333333333,
          "upper_bound": 94.33333333333333
        },
        "pass_at_3": {
          "value": 100.0,
          "successes": 100.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 100.0,
          "upper_bound": 100.0
        },
        "pass_all_3": {
          "value": 84.0,
          "successes": 84.0,
          "denominator": 100,
          "unknown": 0,
          "lower_bound": 84.0,
          "upper_bound": 84.0
        },
        "successes_per_asset": {
          "0": 0,
          "1": 1,
          "2": 15,
          "3": 84
        },
        "estimators": {
          "pass_at_k": "1 - C(n-c,k)/C(n,k)",
          "pass_all_k": "C(c,k)/C(n,k)"
        },
        "retries_excluded": true,
        "by_asset": [
          {
            "reference_id": "nv100-v1:nv100_04ed00317ccc5a90",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0d5a045d08832558",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0ee38dbeb8dccf6d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_0fdbdbbf0914a091",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_142ac642d87da17c",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_15d6296b43e32e18",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17a2f7ce1c73e24b",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_17b98c24982e1657",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_19607166b9604d89",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_198a9e9c4f7ce3f2",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_1d6ca6d90cd58961",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2124cde602dbdcb7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2442a6f885eb3cc3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_247345f2a336cc5f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_277538c0752a1186",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_29738d03bc5d9a68",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2999ac8a4e7af7e7",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2a6a57f1f660128a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2ab1c8d84ebe4eb9",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_2af97ae24a8d7fde",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3084bc1f12b79ce8",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_36db64b13e6813e1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_381f0169f4c4fbd1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_382b02d058a48b6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_3b37b9f58a70ca40",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_47447ae933c565c5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4afa194354477c47",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4b024c41c6d4c2bb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_4e10782bf98f9405",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_525b65ccbe56ffea",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_56025f338315d66a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58b1427ce07ba99a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_58fba5a2a836bcef",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5b8085baf6c090ed",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5c3d3461fa840d9d",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_5d12c190ef7457d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_600fc2ad768f1930",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6627b65375e93e5d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_69ecd52a60671756",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_6f6f63bdb94a83b4",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70a1243908f64ead",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_70c417d940a526ee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7777d0b26e69f86a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7876a753434cf7d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_78da45d42fdf538c",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_791e79d947ee35ae",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_7ac8dcd5266e9924",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_80416c2bc0a430eb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8574b6f4aa22c8e3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8817a4bd528f5549",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8cfd1e56a0b504ea",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8d81b2987b196041",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_8dd6e1f1cbb8b462",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_918619c7fe40bdc5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9499de2220b36cbe",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_968cd3dfe5ad21c1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_977905a7f68dc323",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9ba70a69917d2305",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e3f7dbd969acac5",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_9e6234280413cdcb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a194e402f44013d1",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a2939a17e3444d52",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a3fe4697f3e4c15e",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_a7a336d818a3d9d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_abb1b470da2eb131",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_adb626df31bf7509",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_af46aeafd295d310",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b4208a0e4f762579",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b6aeeb666424ae0d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_b83324e831779e19",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_bff10216da262e6a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c2f7f1f03e91af9b",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c35d376f74c032a5",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c6c60a8c7081041a",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c844b5efec42e5fa",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c87b9d6690f9e687",
            "samples": 3,
            "successes": 1,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c89ec640cbfbd302",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c8a8bac26c7b4874",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_c92d6c8478d7758f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ca63914168dce803",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_cb8c72f166e90540",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d152ce4f60b45ea9",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d37c0067281cdd26",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d5ddbfeda2fe7ad9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_d9edaf956545bd67",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_dea081707992a622",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_df1e96e685d276d3",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e006be66e4cc28f9",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e7a7d6fbf4f2326f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_e948d0fd08624353",
            "samples": 3,
            "successes": 2,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eadc1038f3bfd872",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb55c976d6cd528f",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_eb6800ce857dc2eb",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ec0dd39731982fa2",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_ef3a3efa5b480cd0",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f33ae88933be01fa",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_f7f391c5b6b8e44d",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fb8ddbbe3dbc2900",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fbb5620efff99eee",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          },
          {
            "reference_id": "nv100-v1:nv100_fd477c46fa524292",
            "samples": 3,
            "successes": 3,
            "unknown": 0
          }
        ]
      },
      "inference_spend": {
        "final": {
          "value": 0.57959506,
          "sum_recorded": 173.878518,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 0,
          "partial": false,
          "includes_retries": false
        },
        "retry_overhead": {
          "value": null,
          "sum_recorded": null,
          "denominator": 300,
          "known_executions": 0,
          "unknown_executions": 6,
          "partial": true,
          "includes_retries": true
        },
        "retry_inclusive": {
          "value": 0.57959506,
          "sum_recorded": 173.878518,
          "denominator": 300,
          "known_executions": 300,
          "unknown_executions": 6,
          "partial": true,
          "includes_retries": true
        },
        "cost_kinds": [
          "harbor_reported_agent_usage"
        ],
        "missing_is_zero": false
      },
      "custom_agent": false,
      "repetitions": 3,
      "protocol": "codex-single-image-usd-low-n3",
      "optional_criteria": {
        "value": null,
        "unit": "percent",
        "status": "partial",
        "lower_bound_percent": 72.63398692810458,
        "upper_bound_percent": 72.9281045751634,
        "passed_checks": 11113,
        "unknown_checks": 45,
        "total_checks": 15300,
        "checks_per_attempt": 51,
        "eligible_attempts": 300,
        "coverage_fraction": 0.9970588235294118,
        "retries_excluded": true,
        "failed_checks": 4142.0,
        "unknown_attempts": 15,
        "fully_scored_attempts": 285,
        "source_field": "formal_report.optional_score",
        "scope": "all_optional_profile"
      }
    }
  ],
  "leaderboard_definitions": {
    "pass": "all 46 mandatory requirements pass",
    "formal_success_source": "per-trial metrics.json: scores.formal.evidence.all_scored_mandatory_pass",
    "pass_at_1": "mean asset success rate over final attempts",
    "pass_at_3": "estimated probability of at least one formal pass in three attempts",
    "pass_all_3": "estimated probability of three formal passes in three attempts",
    "cost_per_point": "final retained inference cost per reconstruction divided by mean fidelity on the 0\u2013100 scale",
    "tokens": "total = input + output; cached input is already part of input",
    "resources": "cost uses final retained reconstructions; tokens and trial time retain retry-inclusive recorded sums per intended reconstruction"
  },
  "methodology": "Scores use the same object/repetition pairs across configurations.",
  "aggregation": {
    "scores": "means over available final scores",
    "pareto_scores": "common measured object/repetition slots across configurations, separately per fidelity metric",
    "resources": "cost excludes retries; tokens and trial time retain their published retry-inclusive scope",
    "missing": "null; never imputed as zero",
    "hosted_compute": "job-level only; not allocated to configurations",
    "inference_cost": "sum recorded inference cost of final retained outputs divided by final slots",
    "inference_spend": "separate final, retry overhead and retry-inclusive recorded spend; missing costs are never zero-filled",
    "pricing_basis": "measurement coverage is independent of reported versus estimated pricing; no invoice verification is implied"
  },
  "cost_reporting_version": "final-retained-outputs-v1",
  "cost_source_revision": "988adfa176c7e788164a8c4d07f32d5c59bf6018",
  "publication_source": "https://huggingface.co/datasets/hcltech-robotics/simreadybench/blob/5ecafa8612a5d16d83869587a1018bf5e3d75d00/results/harbor/03d80d61-938f-40cb-9ea7-31cd51e63b3b/presentation/leaderboard.json"
}
