{
  "schemaVersion": 1,
  "rubricVersion": "2026-09-27.1",
  "status": "frozen-prospective-review-rubric",
  "scope": "Independent review instructions for future authorized cases; not a reconstruction of a historical producing grader.",
  "correctness": {
    "correct": "The submitted solution satisfies every requirement in its frozen task contract, including required behavior, compatibility, documentation, and allowed changes. Judge the contract rather than matching a reference patch.",
    "incorrect": "Identify at least one concrete contract violation supported by the supplied submission and verification evidence. A grader decision is not evidence of correctness.",
    "unresolved": "The available evidence does not support either determination, including ambiguous contracts, absent execution evidence, or conflicting observations. Do not guess."
  },
  "testAdequacy": {
    "adequate": "The evidence demonstrates that tests exercise the requested behavior and relevant failure or boundary cases, use assertions that distinguish the intended behavior from plausible faulty behavior, and execute successfully. Explain the cases considered.",
    "inadequate": "The supplied evidence establishes missing relevant assertions, irrelevant tests, untested required behavior, or tests that fail to execute successfully. Explain the demonstrated gap.",
    "unresolved": "Evidence is insufficient to assess adequacy. Merely adding or changing a test file does not establish adequacy."
  },
  "separation": "Correctness and test adequacy receive separate labels. Test inadequacy affects correctness only where the frozen task contract explicitly requires those tests.",
  "reviewProcedure": [
    "Read the frozen task contract and redacted evidence without model identities, grader decisions, or another reviewer's labels.",
    "Record both labels and a private evidence-based rationale independently before discussing the case.",
    "A third reviewer may adjudicate disagreements using the same contract and both rationales, while remaining blind to model identity and grader decision.",
    "Keep initial labels, unresolved cases, and pre-adjudication disagreement counts. Do not replace original labels."
  ],
  "sampling": "Freeze the full authorized cohort before collecting labels. Exclude development controls from validation. Report authored controls separately from held-out cases.",
  "rates": {
    "falseAccept": "Accepted and finally incorrect / finally incorrect cases with an accept or reject grader decision.",
    "falseReject": "Rejected and finally correct / finally correct cases with an accept or reject grader decision.",
    "disagreement": "Different initial labels / double-reviewed cases, including unresolved labels in the denominator.",
    "zeroDenominator": null
  },
  "limitations": [
    "A local declaration or digest does not prove authorization, authentic provenance, independent selection, reviewer identity, or blinding. Those require an evidence steward's review.",
    "AI labels and authored controls are engineering diagnostics, not independent human validation.",
    "Repeated cases from the same task are clustered. Descriptive rates do not establish independent trials, confidence intervals, general accuracy, or a model ranking."
  ]
}