{
  "schema": "docsloth-benchmark-task-definitions/1",
  "version": "1.0.0",
  "note": "These are the exact task prompts and checker expressions from the public harness. Running them requires the repository scripts named per set; the live set additionally requires model credentials supplied by the operator.",
  "sets": [
    {
      "kind": "harness_verification_with_local_stub",
      "source": "docsloth-core/scripts/benchmark-harness.mjs",
      "seeds": [
        1,
        2,
        3,
        4,
        5
      ],
      "tasks": [
        {
          "id": "fix-import",
          "prompt": "Fix the missing import in the auth guide",
          "check": "out.includes('import') && out.includes('test')"
        },
        {
          "id": "add-guide",
          "prompt": "Write a guide for rotating keys",
          "check": "out.includes('document') && out.includes('evidence')"
        }
      ]
    },
    {
      "kind": "live_benchmark",
      "source": "scripts/benchmark-live.mjs",
      "seeds": [
        1,
        2,
        3
      ],
      "seeds_env": "BENCHMARK_SEEDS",
      "tasks": [
        {
          "id": "runnable-example",
          "prompt": "Write a runnable installation example for the SDK.",
          "check": "/install|npm|pip/.test(out) && out.length > 40"
        },
        {
          "id": "failure-mode",
          "prompt": "Document the failure mode when the token expires and how to recover.",
          "check": "/expire|refresh|re-?auth/i.test(out)"
        },
        {
          "id": "breaking-change",
          "prompt": "Update the guide after the parameter rename from user to username.",
          "check": "/username/.test(out)"
        }
      ]
    }
  ]
}