{
  "schemaVersion": 1,
  "updatedAt": "2026-11-12",
  "title": "PMDR Bench",
  "scope": "Curated full-gameplay development evaluations. Not a complete experiment archive.",
  "models": [
    {
      "id": "qwen35-4b-base", "name": "Qwen3.5-4B", "parameters": "4B", "variant": "Base",
      "configuration": "Untouched · BF16 / Q4_K_M",
      "description": "The untouched small-model baseline. Recorded configurations include native BF16 inference and a separately evaluated Q4_K_M deployment. Serving changes and protocols are kept explicit for every attempt.",
      "adapterSha256": null
    },
    {
      "id": "qwen35-4b-native-grpo", "name": "Qwen3.5-4B · Native GRPO", "parameters": "4B", "variant": "Trained",
      "configuration": "One native update · BF16",
      "description": "The small model after one acknowledged native GRPO update. This historical checkpoint is evaluated independently of later training checkpoints. A training gain remains unproven.",
      "adapterSha256": "3ba427e1d62671c9def1e1e46e500fda9e148bfa305c406a248f45b58c1c5f12"
    },
    {
      "id": "qwen38-27b-reference", "name": "Qwen3.8-27B", "parameters": "27B", "variant": "Reference",
      "configuration": "Deployed quantized reference",
      "description": "The deployed quantized 27B reference. Its native-capacity run reached floor 3 of Grotte Éclair. Backend and precision differ from the small-model BF16 pair; this is a configuration comparison, not an isolated model-size experiment.",
      "adapterSha256": null
    }
  ],
  "cohorts": [
    {
      "id": "mt-ardent-h12-quality-20261112",
      "name": "Mt Ardent · Recorded h12 · 12 Nov 2026",
      "scoreVersion": "QUALITY / EFFICIENCY V1",
      "scoringPolicy": "full-dungeon-quality-efficiency-v1",
      "protocolSha256": "12e9d8dea005ccbea772050ac6bf6dfdd435c9b78f60d78faa7cb5d292636cc3",
      "description": "Clear the first six floors and arrive on floor 7. One independently qualified, recorded 4B attempt under frozen harness h12. Quality and resource efficiency use the full-dungeon policy.",
      "caveat": "One familiar development slot; descriptive only. The simultaneous 27B attempt was excluded for a harness fault. No final common-build pair, normalized anchors, hosted-model result or superiority claim is available. Later repaired harnesses form separate groups."
    },
    {
      "id": "eclair-native-20261104", "name": "Grotte Éclair · Native capacity · 04 Nov 2026", "scoreVersion": "PROGRESS V2",
      "scoringPolicy": "eclair-depth-harness-progress-v2", "protocolSha256": null,
      "description": "Five-floor development objective. Three model configurations; one documented attempt each. Verified progress is shown before effort-adjusted score.",
      "caveat": "The two 4B arms form a matched BF16 pair. The 27B uses a different quantized serving backend. Automatic terminal completion is unqualified for this block. No stable ranking or training gain is established."
    },
    {
      "id": "mt-ardent-recovery-20261111", "name": "Mt Ardent · Recovery profile · 11 Nov 2026", "scoreVersion": "PROGRESS V1",
      "scoringPolicy": "eclair-dungeon-verified-progress-v1", "protocolSha256": null,
      "description": "Fresh-reset, full-gameplay attempt with the 29-tool recovery profile. Objective: advance through floor 6 and arrive on floor 7. One untouched Q4_K_M 4B attempt.",
      "caveat": "Different dungeon, serving configuration, tools and score revision from the Éclair block. Scores are not comparable across these groups. This development attempt ended at a qualified leader K.O.; evidence-retention gaps remain explicit."
    }
  ],
  "runs": [
    {
      "id": "eclair-native-reference-20261104", "modelId": "qwen38-27b-reference", "cohortId": "eclair-native-20261104", "date": "2026-11-04",
      "configuration": "Quantized vLLM reference · provider capacity",
      "progressScore": 27.5355, "adjustedScore": 22.9855, "furthestFloor": 3, "targetFloor": 5, "floorAdvances": 2,
      "wallSeconds": 2881.545, "completion": null, "outcome": "defeated", "stopReason": "Visually confirmed leader K.O.; TAL later stalled",
      "summary": "Reached floor 3, gaining 172 confirmed cohort XP, one item pile and 46 Poké. A retained native screenshot explicitly shows the leader K.O. The later TAL stop was stalled/no_reply. Manual visual outcome evidence is separate from automatic labels.",
      "xp": 172, "discoveredCells": 903, "itemPickups": 1, "money": 46, "generations": 399, "actions": 112, "promptTokens": 22926837, "completionTokens": 138815,
      "modelSha256": null, "templateSha256": null, "video": null,
      "bindings": { "experimentCommitPrefix": "dd037e4", "talCommitPrefix": "e26a247", "studioCommitPrefix": "884979f", "pmdrCommitPrefix": "7327b79" },
      "limitations": ["Leader K.O. is established by explicit manual visual evidence, not an automatic completion label.", "Automatic dungeon completion, complete K.O. history and complete game-turn accounting are unqualified.", "Resource gains are confirmed subsets. Final party and cash totals remain unknown.", "One familiar development fixture; backend and precision differ from the BF16 small-model pair."],
      "source": { "path": "docs/dungeon-baseline-native-20261104.md", "url": "./reports/native-baseline.md", "sha256": "ded7d97a025ea962b7b514af4a5e5f0b950208604ce75e27ced888f3d8c4c13a" }
    },
    {
      "id": "eclair-native-base-20261104", "modelId": "qwen35-4b-base", "cohortId": "eclair-native-20261104", "date": "2026-11-04",
      "configuration": "Native BF16 · 131,072 context · thinking enabled",
      "progressScore": 0.4922, "adjustedScore": -0.3378, "furthestFloor": 1, "targetFloor": 5, "floorAdvances": 0,
      "wallSeconds": 2748.555, "completion": null, "outcome": "model_stopped", "stopReason": "Model ended its turn on floor 1",
      "summary": "Discovered 63 new walkable cells without a verified floor advance. The model repeatedly polled request keys instead of receipt UUIDs. One response exhausted the provider context; compaction succeeded before the model ended its turn.",
      "xp": 0, "discoveredCells": 63, "itemPickups": 0, "money": 0, "generations": 71, "actions": 24, "promptTokens": 2668613, "completionTokens": 78833,
      "modelSha256": null, "templateSha256": null, "video": null, "bindings": { "experimentCommitPrefix": "34fe915" },
      "limitations": ["Completion is unknown because this block lacks a qualified automatic terminal decoder.", "Positive final party endpoints do not establish a clean prior K.O. history.", "This is one attempt on a familiar fixture, not held-out model-quality evidence.", "The 4B pair uses matching BF16 serving; the 27B reference uses a different backend and precision."],
      "source": { "path": "docs/dungeon-baseline-native-20261104.md", "url": "./reports/native-baseline.md", "sha256": "ded7d97a025ea962b7b514af4a5e5f0b950208604ce75e27ced888f3d8c4c13a" }
    },
    {
      "id": "eclair-native-trained-20261104", "modelId": "qwen35-4b-native-grpo", "cohortId": "eclair-native-20261104", "date": "2026-11-04",
      "configuration": "Native-GRPO BF16 merge · 131,072 context",
      "progressScore": 0.1656, "adjustedScore": -1.0594, "furthestFloor": 1, "targetFloor": 5, "floorAdvances": 0,
      "wallSeconds": 1087.585, "completion": null, "outcome": "model_stopped", "stopReason": "Model declared it could not complete the objective",
      "summary": "Earned 24 confirmed cohort XP and discovered two new walkable cells, then ended its turn after unsuccessful stairs attempts. There was no length truncation. This checkpoint scored below the untouched arm in this single attempt.",
      "xp": 24, "discoveredCells": 2, "itemPickups": 0, "money": 0, "generations": 105, "actions": 35, "promptTokens": 6095064, "completionTokens": 26672,
      "modelSha256": null, "templateSha256": null, "video": null, "bindings": { "experimentCommitPrefix": "34fe915" },
      "limitations": ["The model's explanation of a stairs problem is not proof that the tool was broken.", "Automatic completion and complete prior K.O. history remain unknown.", "The historical one-update checkpoint is not the latest trained model.", "One attempt cannot establish an improvement, degradation or stable model ranking."],
      "source": { "path": "docs/dungeon-baseline-native-20261104.md", "url": "./reports/native-baseline.md", "sha256": "ded7d97a025ea962b7b514af4a5e5f0b950208604ce75e27ced888f3d8c4c13a" }
    },
    {
      "id": "mt-ardent-base-recovery-20261111", "modelId": "qwen35-4b-base", "cohortId": "mt-ardent-recovery-20261111", "date": "2026-11-11",
      "configuration": "Q4_K_M · RTX 2080 Ti · 29-tool recovery profile",
      "progressScore": 0.5, "adjustedScore": null, "furthestFloor": 1, "targetFloor": 7, "floorAdvances": 0,
      "wallSeconds": 373.63165, "completion": false, "outcome": "defeated", "stopReason": "Automatic stop on qualified leader K.O. event",
      "summary": "The untouched model reached 71 newly verified walkable cells and no verified floor advance. The canonical outcome is defeated, finished:false. Automatic action admission closed on the qualified leader K.O., without operator coaching or censoring.",
      "xp": 0, "discoveredCells": 71, "itemPickups": 0, "money": 0, "generations": 27, "actions": 21, "promptTokens": 784833, "completionTokens": 7906,
      "modelSha256": "fcc8d8fed14b3070791b443bbbd6fa8bd91a7c2b26eff78fb2d2184b3076117f", "templateSha256": null, "video": null,
      "bindings": { "resetManifestSha256": "9605ee390602fe9362c947e03963c414a2a78b0b37b5756f960a42c8c8c3449c", "checkpointSha256": "0466e542301f8936a886e35c2d418f3c7f774a7cd6f6de5a239d127d263552b5", "experimentCommit": "3a624a19087ca4969562ca9135012acceaee23df", "pmdrCommit": "97b7f1803aa64cba636ace42ab0134d39d44a51c", "talCommit": "cc09fabade973b47787b71f3d91c0bd03bb48f39", "studioCommit": "163650c37cf7f8ca7990345cddbedfeffc3069ef" },
      "limitations": ["Two attack receipts remain uncertain; resolved inputs do not establish damage or kills.", "11 of 44 native frame records have explicit snapshot-retention gaps; 33 PNGs were retained.", "The K.O. outcome is supported by a qualified decoded event and a later visible message. The expired raw memory diagnostic is unqualified.", "Final party, bag, cash, total XP, complete turn accounting and prior K.O. history remain unknown.", "The recovery profile and scoring revision differ from the Éclair block; this is not a matched cross-protocol comparison."],
      "source": { "path": "docs/mt-ardent-recovery-profile-results-20261111.md", "url": "./reports/mt-ardent-recovery.md", "sha256": "34a3407a505f6ac314193419e53ee4219997967c0a9ad81ba92e2912cc8e1e3d" }
    },
    {
      "id": "mt-ardent-1112-h12-4b-s1-a1",
      "modelId": "qwen35-4b-base",
      "cohortId": "mt-ardent-h12-quality-20261112",
      "date": "2026-11-12",
      "configuration": "Q4_K_M · RTX 2080 Ti · 131,072 context · recorded h12",
      "progressScore": null,
      "adjustedScore": null,
      "quality": {
        "policy": "full-dungeon-quality-efficiency-v1",
        "score": 0.0,
        "floors_cleared": 0,
        "components": {
          "depth": {
            "points": 0.0,
            "max": 60.0,
            "status": "verified"
          },
          "completion": {
            "points": 0.0,
            "max": 20.0,
            "status": "verified"
          },
          "speed": {
            "points": 0.0,
            "max": 11.666666666666666,
            "status": "not_applicable"
          },
          "party": {
            "points": 0.0,
            "max": 4.166666666666667,
            "status": "lower_bound"
          },
          "items": {
            "points": 0.0,
            "max": 2.0833333333333335,
            "status": "unknown"
          },
          "money": {
            "points": 0.0,
            "max": 2.0833333333333335,
            "status": "unknown"
          }
        },
        "unverified_points": 8.333333333333334
      },
      "efficiency": {
        "policy": "full-dungeon-quality-efficiency-v1",
        "floors_cleared": 0,
        "spend": {
          "tokens": 2870864,
          "agent_turns": 57,
          "game_turns": null
        },
        "per_floor": {
          "tokens": null,
          "agent_turns": null,
          "game_turns": null
        },
        "status": "no_floor_cleared"
      },
      "furthestFloor": 1,
      "targetFloor": 7,
      "floorAdvances": 0,
      "wallSeconds": 643.6096360068768,
      "completion": false,
      "outcome": "defeated",
      "stopReason": "Qualified leader K.O. on floor 1",
      "summary": "The untouched 4B model cleared no floors. A qualified decoded leader K.O. event and independent visual review establish defeat. The complete recording preserves the French gameplay and terminal message. Quality is 0.0, with 8.33 points of unverified coverage; per-floor efficiency is undefined because no floor was cleared.",
      "xp": 0,
      "discoveredCells": 176,
      "itemPickups": 0,
      "money": 0,
      "generations": 57,
      "compactions": 1,
      "operationProofs": 35,
      "gameTurns": null,
      "actions": 36,
      "promptTokens": 2848088,
      "completionTokens": 22776,
      "modelSha256": "fcc8d8fed14b3070791b443bbbd6fa8bd91a7c2b26eff78fb2d2184b3076117f",
      "baseRevision": null,
      "templateSha256": "b5f6cf66bfc4c28e68867b5f5a4c2f32e6807a25b5fa626ae3478aaa6af545de",
      "actionDiagnostics": {
        "policy": "pmdr-action-refusals-v1",
        "scope": "unique_gateway_action_requests",
        "action_requests": 36,
        "gameplay_refusals": 4,
        "refusal_rate": null,
        "coverage_complete": false,
        "counts": {
          "completed": 16,
          "gameplay_refused": 4,
          "schema_error": 0,
          "context_guard": 2,
          "blocked_execution": 0,
          "normal_interruption": 12,
          "unverified_or_harness_error": 0,
          "unclassified": 2
        },
        "model_call_coverage": "not_assessed",
        "unknown_tool_calls_observed": 0,
        "schema_errors_observed": 0,
        "denied_gateway_requests": 1,
        "duplicate_admissions": 0,
        "issues": [
          "action_outcomes_unresolved_or_unclassified"
        ],
        "by_reason": [
          {
            "tool": "approach_dungeon_enemy",
            "category": "unclassified",
            "reason": "unclassified_reason",
            "count": 1
          },
          {
            "tool": "approach_dungeon_stairs",
            "category": "gameplay_refused",
            "reason": "stairs_unverified",
            "count": 2
          },
          {
            "tool": "attack",
            "category": "completed",
            "reason": "completed",
            "count": 6
          },
          {
            "tool": "attack",
            "category": "gameplay_refused",
            "reason": "target_not_adjacent",
            "count": 1
          },
          {
            "tool": "close_dungeon_menu",
            "category": "completed",
            "reason": "completed",
            "count": 1
          },
          {
            "tool": "explore_dungeon",
            "category": "normal_interruption",
            "reason": "floor_item_encountered",
            "count": 1
          },
          {
            "tool": "move_to_dungeon_coordinate",
            "category": "completed",
            "reason": "completed",
            "count": 7
          },
          {
            "tool": "move_to_dungeon_coordinate",
            "category": "gameplay_refused",
            "reason": "destination_unknown_or_blocked",
            "count": 1
          },
          {
            "tool": "move_to_dungeon_coordinate",
            "category": "normal_interruption",
            "reason": "enemy_encountered",
            "count": 11
          },
          {
            "tool": "open_dungeon_menu",
            "category": "completed",
            "reason": "completed",
            "count": 1
          },
          {
            "tool": "reposition_in_dungeon",
            "category": "completed",
            "reason": "completed",
            "count": 1
          },
          {
            "tool": "reposition_in_dungeon",
            "category": "unclassified",
            "reason": "unknown_response",
            "count": 1
          },
          {
            "tool": "take_stairs",
            "category": "context_guard",
            "reason": "menu_unverified",
            "count": 2
          }
        ]
      },
      "video": {
        "url": "./media/mt-ardent-1112-h12-4b-s1-a1/74b14367deaee7443622ca2ceb0facab308c610933a7f115eaf362a5bd4032d8.mp4",
        "sha256": "74b14367deaee7443622ca2ceb0facab308c610933a7f115eaf362a5bd4032d8",
        "timelineUrl": "./media/mt-ardent-1112-h12-4b-s1-a1/3951757cb84807080649741e43a76e215a4330b05b2958db76f0e7a67d5362b4.timeline.json",
        "caption": "Complete audited native-resolution recording. Action-focused view retains 4m 09s; full source is 10m 47s.",
        "provenance": "38,618 decoded frames, 240 × 160. One unchanged compressed source; 37 reviewed between-operation cuts. Full playback retains all passive game activity. Recording duration differs from model wall time."
      },
      "bindings": {
        "frozenHarnessSha256": "41beb7f89319cbbb6af849680a229a063ab0b2ed7037a526a405ddf5e03201e2",
        "specSha256": "af102197d4dc25de666309e8d5ebb254c3e17158ea1cbb4bd82047c5cd53aeaf",
        "specIdentitySha256": "5da11a24938ebaadfd0fd4938bb78cca9d950ffadf62be059a1c7115a861c9f9",
        "qualificationSha256": "094a475bd76cc8c6dabfc7babde53bf4a9ec4a534924b14ed02a930ebdccfe3f",
        "operatorQualificationSha256": "027872518739dbdefc2bba7ecca4e5cb35aa863e180d999c713bb6f5d021ee15",
        "resultSha256": "19cebc907ac996a1256f13d796aa12d463a7911afe3bbad1ec0326e8e35ce8d9",
        "sourceManifestSha256": "2867beaee1f82fc10e0f3be64730d06057f26729e54de7bddabb783180ac7214",
        "resetIdentitySha256": "479a1340ee8df0ed88089e7ae6bb541a0cd3471e10fd443d3656f3e5d6d84b94",
        "resetManifestSha256": "13504caa736df53170d98adb7ce7741f4758bfb8b07509c6d43733deba5bdbe6",
        "projectorSha256": "5466575346d6822e5bde533578b3190ef041e6880d1f7082b84800e2c87067a9",
        "actionDiagnosticsSha256": "92530a6bbd48cc642b81101a3b1b64843b5688c9996f20d4e6d58656cf945f00",
        "videoAuditSha256": "146c62cea376eb3316bfc556192e1d3c6f5f53287da6ada6744ccb1a0782b51a",
        "timelineSha256": "3951757cb84807080649741e43a76e215a4330b05b2958db76f0e7a67d5362b4"
      },
      "limitations": [
        "One familiar development slot; this qualified h12 attempt is not a final matched comparison or evidence of model superiority.",
        "Quality components for party progression are lower bounds; endpoint bag and cash coverage are unknown. Unverified quality coverage is 8.33 points.",
        "No floors cleared: tokens and agent turns per floor are undefined. Complete game-turn accounting and prior K.O. history remain unknown.",
        "The 27B h12 attempt was excluded for a use_move harness fault. This 4B arm did not exercise use_move; a later repaired harness is not the same build.",
        "Four gameplay refusals are observed among 36 action requests. Two outcomes are unclassified under diagnostic policy v1, so no refusal rate is reported.",
        "Original executed-source and frozen-harness identities are retained; public history rewriting does not redefine this historical run."
      ],
      "source": {
        "path": "operator-qualification-h12.json (private reviewed source)",
        "url": "./reports/mt-ardent-h12-4b-20261112.md",
        "sha256": "027872518739dbdefc2bba7ecca4e5cb35aa863e180d999c713bb6f5d021ee15"
      }
    }
  ]
}