{
  "schemaVersion": 2,
  "studyId": "grader-audit",
  "artifactVersion": "2026-09-27.1",
  "correctionHistory": "/studies/correction-history.json",
  "status": "prospective",
  "empiricalResults": null,
  "question": "Does the evaluation accept correct solutions and reject broken ones?",
  "protocol": [
    "Sample both accepted and rejected attempts, recording the sampling rule before review.",
    "Freeze the grader and separate development controls from final validation. Include unseen known-good alternatives, naturally observed failures where permitted, and deliberately faulty controls.",
    "Abort control exports on execution errors or timeouts; never count them as grader false rejections.",
    "Use anonymous case identifiers and hide model identities and grader decisions from reviewers.",
    "Collect independent reviewer judgments against the same explicit behavioral contract.",
    "Report disagreement before adjudication; keep unresolved cases separate.",
    "Keep grader-development cases separate from final validation and freeze the revised grader before validation.",
    "Treat automated control checks as grader diagnostics, not as independent human review.",
    "Report documentation-rule disagreements separately because bounded semantic vocabularies can reject valid paraphrases."
  ],
  "outcomes": {
    "falseAcceptRate": "Incorrect solutions accepted / adjudicated incorrect solutions",
    "falseRejectRate": "Correct solutions rejected / adjudicated correct solutions",
    "reviewerDisagreement": "Disagreeing cases / independently double-reviewed cases",
    "missingDenominator": "Report null, not zero, when no qualifying cases exist"
  },
  "validation": {
    "developmentControls": "Useful for finding known rule failures; excluded from unseen accuracy estimates.",
    "unseenCases": "Required before any grader-accuracy claim. No unseen validation result is available.",
    "humanReview": "Independent, blinded, double-reviewed labels are required for reviewer-agreement reporting. No such labels are available.",
    "inconclusive": "Execution errors, timeouts, missing evidence, and unresolved labels stay separate from valid grader decisions."
  },
  "publication": {
    "modelIdentities": "withheld",
    "rawOutputs": "excluded",
    "configurationMapping": "private",
    "results": "Allowlisted aggregate fields only"
  },
  "validationPackage": {
    "status": "engineering prepared; human study pending",
    "rubric": "/studies/grader-audit/review-rubric.json",
    "rubricVersion": "2026-09-27.1",
    "heldOutCases": 0,
    "humanReviews": 0,
    "sampling": "Freeze the full authorized cohort before review. Exclude development controls; report authored controls separately. Existing API submissions would form a retrospective cohort audit, not a new prospective model experiment.",
    "blinding": "Two reviewers receive redacted contracts and evidence with no model identity, grader decision, or other reviewer labels. A separate third reviewer adjudicates disagreements; initial labels and unresolved cases remain recorded.",
    "reporting": "Separate correctness and test adequacy; confusion counts only for resolved correctness with valid accept/reject grader decisions. Report exact denominators, initial disagreement, invalid/error/missing exclusions, and task breakdowns. Rates with no denominator are null. Repeated tasks are clustered; no population confidence claim is supported.",
    "pending": [
      "Authorized independently selected cases not used in grader development",
      "Steward verification of evidence access, redaction, and frozen selection",
      "Independent human reviewers and adjudication"
    ]
  }
}