{
  "schemaVersion": 1,
  "kind": "curated-historical-experiment-extract",
  "recomputedOn": "2026-11-02",
  "experimentDate": "2026-09-19",
  "nProjects": 150,
  "model": "Jev 1.13",
  "scope": "Top-10 model agreement, not accuracy. No independent human answer key. Not a new API run.",
  "sources": [
    {
      "path": "experiments/top10/compare.json",
      "generatedAt": "2026-09-19T16:46:55.021Z",
      "sha256": "773455b593a1782d43504f39deb8b8369dbf434af96265a8c49c2b265473eb4d"
    },
    {
      "path": "experiments/top10/v3.json",
      "generatedAt": "2026-09-19T17:24:17.462Z",
      "sha256": "159070acde02208df8ae6c69804fa3d6f8f8e9f5739747f22d9ff9a261e0fc4f"
    }
  ],
  "method": {
    "stage1": "Project summary, description and first 700 README characters; eight questions; weighted ranking in code; 0.4–0.6 no-judgment band.",
    "stage2": "Top 20; 190 pairs in both orders; final rank combines stage1 and pairwise rank.",
    "concurrency": 8,
    "positionBiasStatistic": -0.139
  },
  "lists": {
    "fable": [
      {
        "id": "p003",
        "repo": "browser-use/jev-ultrafast"
      },
      {
        "id": "p005",
        "repo": "tamaratran/fast-jev-compaction"
      },
      {
        "id": "p026",
        "repo": "coldteadotai/abide"
      },
      {
        "id": "p050",
        "repo": "DanRWilloughby/snifftest"
      },
      {
        "id": "p025",
        "repo": "jkudish/jev-browser"
      },
      {
        "id": "p016",
        "repo": "thruwire/foreman"
      },
      {
        "id": "p017",
        "repo": "devagrawal09/jev-review"
      },
      {
        "id": "p021",
        "repo": "superagents-lab/jev-search"
      },
      {
        "id": "p110",
        "repo": "CodeAlive-AI/mastra-jev-moderation"
      },
      {
        "id": "p020",
        "repo": "gargpratyush/jev-router"
      }
    ],
    "opus": [
      {
        "id": "p003",
        "repo": "browser-use/jev-ultrafast"
      },
      {
        "id": "p005",
        "repo": "tamaratran/fast-jev-compaction"
      },
      {
        "id": "p026",
        "repo": "coldteadotai/abide"
      },
      {
        "id": "p012",
        "repo": "TianyuCodings/NanoJev"
      },
      {
        "id": "p015",
        "repo": "jaredpalmer/kev"
      },
      {
        "id": "p050",
        "repo": "DanRWilloughby/snifftest"
      },
      {
        "id": "p051",
        "repo": "shiftynick/jev-axi"
      },
      {
        "id": "p017",
        "repo": "devagrawal09/jev-review"
      },
      {
        "id": "p016",
        "repo": "thruwire/foreman"
      },
      {
        "id": "p025",
        "repo": "jkudish/jev-browser"
      }
    ],
    "sonnet": [
      {
        "id": "p003",
        "repo": "browser-use/jev-ultrafast"
      },
      {
        "id": "p005",
        "repo": "tamaratran/fast-jev-compaction"
      },
      {
        "id": "p008",
        "repo": "vinnylarouge/jevlike"
      },
      {
        "id": "p009",
        "repo": "NandhaKishorM/laya"
      },
      {
        "id": "p012",
        "repo": "TianyuCodings/NanoJev"
      },
      {
        "id": "p017",
        "repo": "devagrawal09/jev-review"
      },
      {
        "id": "p015",
        "repo": "jaredpalmer/kev"
      },
      {
        "id": "p050",
        "repo": "DanRWilloughby/snifftest"
      },
      {
        "id": "p026",
        "repo": "coldteadotai/abide"
      },
      {
        "id": "p021",
        "repo": "superagents-lab/jev-search"
      }
    ],
    "haiku": [
      {
        "id": "p003",
        "repo": "browser-use/jev-ultrafast"
      },
      {
        "id": "p009",
        "repo": "NandhaKishorM/laya"
      },
      {
        "id": "p005",
        "repo": "tamaratran/fast-jev-compaction"
      },
      {
        "id": "p051",
        "repo": "shiftynick/jev-axi"
      },
      {
        "id": "p026",
        "repo": "coldteadotai/abide"
      },
      {
        "id": "p012",
        "repo": "TianyuCodings/NanoJev"
      },
      {
        "id": "p025",
        "repo": "jkudish/jev-browser"
      },
      {
        "id": "p050",
        "repo": "DanRWilloughby/snifftest"
      },
      {
        "id": "p010",
        "repo": "bastani-inc/atomic"
      },
      {
        "id": "p021",
        "repo": "superagents-lab/jev-search"
      }
    ],
    "jev_v1": [
      {
        "id": "p051",
        "repo": "shiftynick/jev-axi"
      },
      {
        "id": "p050",
        "repo": "DanRWilloughby/snifftest"
      },
      {
        "id": "p126",
        "repo": "edgardcham/huncho"
      },
      {
        "id": "p110",
        "repo": "CodeAlive-AI/mastra-jev-moderation"
      },
      {
        "id": "p001",
        "repo": "different-ai/openwork"
      },
      {
        "id": "p009",
        "repo": "NandhaKishorM/laya"
      },
      {
        "id": "p065",
        "repo": "Silbercue/public-browser"
      },
      {
        "id": "p144",
        "repo": "yodablocks/jev-orderby-bench"
      },
      {
        "id": "p125",
        "repo": "shitianfang/jev-use"
      },
      {
        "id": "p106",
        "repo": "WiktorB2004/llama-index-jev"
      }
    ],
    "jev_v2": [
      {
        "id": "p110",
        "repo": "CodeAlive-AI/mastra-jev-moderation"
      },
      {
        "id": "p025",
        "repo": "jkudish/jev-browser"
      },
      {
        "id": "p009",
        "repo": "NandhaKishorM/laya"
      },
      {
        "id": "p003",
        "repo": "browser-use/jev-ultrafast"
      },
      {
        "id": "p051",
        "repo": "shiftynick/jev-axi"
      },
      {
        "id": "p005",
        "repo": "tamaratran/fast-jev-compaction"
      },
      {
        "id": "p016",
        "repo": "thruwire/foreman"
      },
      {
        "id": "p064",
        "repo": "HyunjunJeon/pi-quiet-ask"
      },
      {
        "id": "p125",
        "repo": "shitianfang/jev-use"
      },
      {
        "id": "p067",
        "repo": "GodsBoy/jev-agent-skill-router"
      }
    ],
    "jev_v3": [
      {
        "id": "p051",
        "repo": "shiftynick/jev-axi"
      },
      {
        "id": "p110",
        "repo": "CodeAlive-AI/mastra-jev-moderation"
      },
      {
        "id": "p125",
        "repo": "shitianfang/jev-use"
      },
      {
        "id": "p098",
        "repo": "nexibeo/jev-cookbook"
      },
      {
        "id": "p025",
        "repo": "jkudish/jev-browser"
      },
      {
        "id": "p139",
        "repo": "shitianfang/wakegate"
      },
      {
        "id": "p067",
        "repo": "GodsBoy/jev-agent-skill-router"
      },
      {
        "id": "p050",
        "repo": "DanRWilloughby/snifftest"
      },
      {
        "id": "p026",
        "repo": "coldteadotai/abide"
      },
      {
        "id": "p017",
        "repo": "devagrawal09/jev-review"
      }
    ],
    "jev_v3_stage1": [
      {
        "id": "p025",
        "repo": "jkudish/jev-browser"
      },
      {
        "id": "p051",
        "repo": "shiftynick/jev-axi"
      },
      {
        "id": "p110",
        "repo": "CodeAlive-AI/mastra-jev-moderation"
      },
      {
        "id": "p125",
        "repo": "shitianfang/jev-use"
      },
      {
        "id": "p017",
        "repo": "devagrawal09/jev-review"
      },
      {
        "id": "p005",
        "repo": "tamaratran/fast-jev-compaction"
      },
      {
        "id": "p098",
        "repo": "nexibeo/jev-cookbook"
      },
      {
        "id": "p054",
        "repo": "devagrawal09/stanley-code"
      },
      {
        "id": "p139",
        "repo": "shitianfang/wakegate"
      },
      {
        "id": "p016",
        "repo": "thruwire/foreman"
      }
    ],
    "jev_v3_pairwise": [
      {
        "id": "p076",
        "repo": "abhixhek/jevcal"
      },
      {
        "id": "p067",
        "repo": "GodsBoy/jev-agent-skill-router"
      },
      {
        "id": "p110",
        "repo": "CodeAlive-AI/mastra-jev-moderation"
      },
      {
        "id": "p098",
        "repo": "nexibeo/jev-cookbook"
      },
      {
        "id": "p051",
        "repo": "shiftynick/jev-axi"
      },
      {
        "id": "p125",
        "repo": "shitianfang/jev-use"
      },
      {
        "id": "p026",
        "repo": "coldteadotai/abide"
      },
      {
        "id": "p139",
        "repo": "shitianfang/wakegate"
      },
      {
        "id": "p050",
        "repo": "DanRWilloughby/snifftest"
      },
      {
        "id": "p025",
        "repo": "jkudish/jev-browser"
      }
    ]
  },
  "overlapWithFable": {
    "fable": {
      "count": 10,
      "sharedIds": [
        "p003",
        "p005",
        "p016",
        "p017",
        "p020",
        "p021",
        "p025",
        "p026",
        "p050",
        "p110"
      ]
    },
    "opus": {
      "count": 7,
      "sharedIds": [
        "p003",
        "p005",
        "p016",
        "p017",
        "p025",
        "p026",
        "p050"
      ]
    },
    "sonnet": {
      "count": 6,
      "sharedIds": [
        "p003",
        "p005",
        "p017",
        "p021",
        "p026",
        "p050"
      ]
    },
    "haiku": {
      "count": 6,
      "sharedIds": [
        "p003",
        "p005",
        "p021",
        "p025",
        "p026",
        "p050"
      ]
    },
    "jev_v1": {
      "count": 2,
      "sharedIds": [
        "p050",
        "p110"
      ]
    },
    "jev_v2": {
      "count": 5,
      "sharedIds": [
        "p003",
        "p005",
        "p016",
        "p025",
        "p110"
      ]
    },
    "jev_v3": {
      "count": 5,
      "sharedIds": [
        "p017",
        "p025",
        "p026",
        "p050",
        "p110"
      ]
    },
    "jev_v3_stage1": {
      "count": 5,
      "sharedIds": [
        "p005",
        "p016",
        "p017",
        "p025",
        "p110"
      ]
    },
    "jev_v3_pairwise": {
      "count": 4,
      "sharedIds": [
        "p025",
        "p026",
        "p050",
        "p110"
      ]
    }
  },
  "recordedCostTime": {
    "fable": {
      "tokens": 94561,
      "ms": 104503,
      "cost": 0.946
    },
    "opus": {
      "tokens": 93218,
      "ms": 89806,
      "cost": 0.466
    },
    "jev_v1": {
      "tokens": 152458,
      "ms": 30068,
      "cost": 0.0064
    },
    "jev_v2": {
      "tokens": 181998,
      "ms": 10739,
      "cost": 0.0076
    },
    "jev_v3": {
      "tokens": 636431,
      "ms": 29633,
      "cost": 0.0267
    }
  },
  "costNote": "Historical estimates in USD, not billing receipts. Token counts are usage quantities, not authentication tokens.",
  "stage1": {
    "requests": 150,
    "wallMs": 9124,
    "tokens": 230743,
    "msP50": 400
  },
  "stage2": {
    "requests": 380,
    "wallMs": 20509,
    "tokens": 405688
  },
  "screeningCostUsdComputed": 0.00969121,
  "limitations": [
    "No human ground truth or repeated-run confidence intervals.",
    "Different judging procedures; not a matched-quality comparison.",
    "One collection and truncated material; do not generalize to all tasks or Jev 1.14."
  ]
}