{
 "schema": "telltale-accuracy-report/1",
 "date": "2026-11-03",
 "classifier": "heuristic-v4.33",
 "manifestSha256": "da8d370870f4a59b4aea9e07ca920ff78e9e8c92f08ed4585e01a79ad2f2e7a6",
 "frozenAt": "2026-11-03",
 "inputs": {
  "manifest": "da8d370870f4a59b4aea9e07ca920ff78e9e8c92f08ed4585e01a79ad2f2e7a6",
  "adversarial": "c8687c01d0a56936857887a7b761a19d75e33f47fc6358f2f0fb01c55578fc8d",
  "adversarialBefore": "856cffc4c355c51cc6c68890ecc93ac8b44ff1002d012e2cec8da8656c7c9a7b",
  "fpAgentPages": "2deb5244175a424dcee99aa12814450eda0763c07d853d65d120e0e09e3f92b6",
  "fpAgentEvents": "3b85780dffd71da5f1e37f8f5a61bffee42730a1aa7d078eeb0310bf0c674ccd"
 },
 "reading": {
  "heldOutOnly": "Every number is computed on held-out rows only (evaluation/holdout-manifest.json). Tuning rows are never shown.",
  "unknown": "Unknown is never counted as Human or as correct. Unknown share is stated beside every rate.",
  "intervals": "Wilson 95% intervals assume independent sessions; sessions from one person, agent family or run are correlated, so true uncertainty is wider.",
  "tuned": "Corpora marked \"tuned on this data\" were looked at while the current rules were chosen, held-out split included. Their numbers show the rules fit that data; they are not an independent accuracy estimate.",
  "synthetic": "Synthetic fixtures are rule checks written from how each input path behaves. They are listed and never counted in a rate.",
  "categories": "Verdicts are categories (agent, human, unknown), not calibrated probabilities."
 },
 "clean": [
  {
   "id": "adversarial-bench",
   "title": "Adversarial humaniser agents (headless bench)",
   "provenance": "scripted",
   "exposure": "clean",
   "exposureNote": "Written on 2026-11-03 with rules frozen at the time; no rule has been chosen from them.",
   "labelSource": "controlled automation (the bench)",
   "heldOutItems": 20,
   "metrics": {
    "agents": {
     "n": 20,
     "final": {
      "agent": 15,
      "unknown": 5,
      "human": 0
     },
     "agent": {
      "count": 15,
      "of": 20,
      "ci95": [
       0.5313,
       0.8881
      ]
     },
     "agentHigh": {
      "count": 15,
      "of": 20,
      "ci95": [
       0.5313,
       0.8881
      ]
     },
     "unknownShare": {
      "count": 5,
      "of": 20,
      "ci95": [
       0.1119,
       0.4687
      ]
     },
     "human": {
      "count": 0,
      "of": 20,
      "ci95": [
       0,
       0.1611
      ]
     },
     "everHuman": {
      "count": 0,
      "of": 20,
      "ci95": [
       0,
       0.1611
      ]
     },
     "actionAgent": {
      "count": 15,
      "of": 20,
      "ci95": [
       0.5313,
       0.8881
      ]
     }
    },
    "humans": null
   },
   "byFamily": {
    "humaniser-fill": {
     "agents": {
      "n": 5,
      "final": {
       "agent": 0,
       "unknown": 5,
       "human": 0
      },
      "agent": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "agentHigh": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "unknownShare": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      },
      "human": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "actionAgent": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      }
     },
     "humans": null
    },
    "humaniser-jitter": {
     "agents": {
      "n": 5,
      "final": {
       "agent": 5,
       "unknown": 0,
       "human": 0
      },
      "agent": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      },
      "agentHigh": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      },
      "unknownShare": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "human": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "actionAgent": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      }
     },
     "humans": null
    },
    "humaniser-minjerk": {
     "agents": {
      "n": 5,
      "final": {
       "agent": 5,
       "unknown": 0,
       "human": 0
      },
      "agent": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      },
      "agentHigh": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      },
      "unknownShare": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "human": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "actionAgent": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      }
     },
     "humans": null
    },
    "humaniser-trace-replay": {
     "agents": {
      "n": 5,
      "final": {
       "agent": 5,
       "unknown": 0,
       "human": 0
      },
      "agent": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      },
      "agentHigh": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      },
      "unknownShare": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "human": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "actionAgent": {
       "count": 5,
       "of": 5,
       "ci95": [
        0.5655,
        1
       ]
      }
     },
     "humans": null
    }
   },
   "adversarialNote": "Adversarial humanisers: 20 held-out runs (4 kinds × seeds 101, 102, 103, 104, 105), headless and muted, normal user agent, navigator.webdriver hidden. These are scripted agents, not real ones: they show which humanising tricks the live rules miss. 0 of 20 read Human and 0 of 20 would be treated as a person by a protected action. Under heuristic-v4.25 and v4.26, 15 of these 20 read Human; since heuristic-v4.27 Human also needs a physical-device input trait, which these runs do not show. 15 of 20 read AI agent (their input arrived without hardware delay: injected by software) and 5 read Unknown. Agents that only humanise behaviour, without injected input or an automation trace, are why a single behaviour pattern is medium confidence: held for review, never charged."
  },
  {
   "id": "real-agent-runs",
   "title": "Real named-agent runs (Claude in Chrome, ChatGPT agent, Atlas, Comet)",
   "provenance": "real",
   "exposure": "clean",
   "exposureNote": "Collected after the freeze with tools/real-agent-runs.js.",
   "labelSource": "controlled run by the operator, consented",
   "heldOutItems": 0,
   "metrics": {
    "agents": null,
    "humans": null
   },
   "byFamily": {}
  },
  {
   "id": "os-level-runs",
   "title": "OS-level computer-use runs (real mouse and keyboard)",
   "provenance": "real",
   "exposure": "clean",
   "exposureNote": "Collected after the freeze with tools/os-agent-harness.js.",
   "labelSource": "controlled run by the operator",
   "heldOutItems": 0,
   "metrics": {
    "agents": null,
    "humans": null
   },
   "byFamily": {}
  },
  {
   "id": "assistive-recordings",
   "title": "Assistive-technology recordings (consenting people)",
   "provenance": "real",
   "exposure": "clean",
   "exposureNote": "Collected after the freeze under docs/public/accuracy-assistive-protocol.md.",
   "labelSource": "reviewer (participant consented, session observed)",
   "heldOutItems": 0,
   "metrics": {
    "agents": null,
    "humans": null
   },
   "byFamily": {}
  }
 ],
 "seenDuringTuning": [
  {
   "id": "fp-agent",
   "title": "FP-Agent research recordings",
   "provenance": "real",
   "exposure": "seen",
   "exposureNote": "Rules heuristic-v4.18 to v4.25 were chosen after looking at all 7,728 sessions, test split included. Its held-out numbers are tuned on this data.",
   "labelSource": "research study (controlled agents, consenting participants)",
   "heldOutItems": 1569,
   "tunedOnThisData": true,
   "metrics": {
    "agents": {
     "n": 1410,
     "final": {
      "agent": 1386,
      "unknown": 24,
      "human": 0
     },
     "agent": {
      "count": 1386,
      "of": 1410,
      "ci95": [
       0.9748,
       0.9885
      ]
     },
     "agentHigh": {
      "count": 1178,
      "of": 1410,
      "ci95": [
       0.8152,
       0.8539
      ]
     },
     "unknownShare": {
      "count": 24,
      "of": 1410,
      "ci95": [
       0.0115,
       0.0252
      ]
     },
     "human": {
      "count": 0,
      "of": 1410,
      "ci95": [
       0,
       0.0027
      ]
     },
     "everHuman": {
      "count": 0,
      "of": 1410,
      "ci95": [
       0,
       0.0027
      ]
     },
     "actionAgent": {
      "count": 1386,
      "of": 1410,
      "ci95": [
       0.9748,
       0.9885
      ]
     }
    },
    "humans": {
     "n": 159,
     "final": {
      "agent": 0,
      "unknown": 159,
      "human": 0
     },
     "falsePositive": {
      "count": 0,
      "of": 159,
      "ci95": [
       0,
       0.0236
      ]
     },
     "falsePositiveHigh": {
      "count": 0,
      "of": 159,
      "ci95": [
       0,
       0.0236
      ]
     },
     "everAgent": {
      "count": 0,
      "of": 159,
      "ci95": [
       0,
       0.0236
      ]
     },
     "actionFalsePositive": {
      "count": 0,
      "of": 159,
      "ci95": [
       0,
       0.0236
      ]
     },
     "unknownShare": {
      "count": 159,
      "of": 159,
      "ci95": [
       0.9764,
       1
      ]
     },
     "human": {
      "count": 0,
      "of": 159,
      "ci95": [
       0,
       0.0236
      ]
     }
    }
   },
   "byFamily": {
    "Atlas Agent": {
     "agents": {
      "n": 196,
      "final": {
       "agent": 196,
       "unknown": 0,
       "human": 0
      },
      "agent": {
       "count": 196,
       "of": 196,
       "ci95": [
        0.9808,
        1
       ]
      },
      "agentHigh": {
       "count": 196,
       "of": 196,
       "ci95": [
        0.9808,
        1
       ]
      },
      "unknownShare": {
       "count": 0,
       "of": 196,
       "ci95": [
        0,
        0.0192
       ]
      },
      "human": {
       "count": 0,
       "of": 196,
       "ci95": [
        0,
        0.0192
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 196,
       "ci95": [
        0,
        0.0192
       ]
      },
      "actionAgent": {
       "count": 196,
       "of": 196,
       "ci95": [
        0.9808,
        1
       ]
      }
     },
     "humans": null
    },
    "Browser Use": {
     "agents": {
      "n": 223,
      "final": {
       "agent": 221,
       "unknown": 2,
       "human": 0
      },
      "agent": {
       "count": 221,
       "of": 223,
       "ci95": [
        0.9679,
        0.9975
       ]
      },
      "agentHigh": {
       "count": 206,
       "of": 223,
       "ci95": [
        0.8813,
        0.9519
       ]
      },
      "unknownShare": {
       "count": 2,
       "of": 223,
       "ci95": [
        0.0025,
        0.0321
       ]
      },
      "human": {
       "count": 0,
       "of": 223,
       "ci95": [
        0,
        0.0169
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 223,
       "ci95": [
        0,
        0.0169
       ]
      },
      "actionAgent": {
       "count": 221,
       "of": 223,
       "ci95": [
        0.9679,
        0.9975
       ]
      }
     },
     "humans": null
    },
    "ChatGPT Agent": {
     "agents": {
      "n": 204,
      "final": {
       "agent": 204,
       "unknown": 0,
       "human": 0
      },
      "agent": {
       "count": 204,
       "of": 204,
       "ci95": [
        0.9815,
        1
       ]
      },
      "agentHigh": {
       "count": 204,
       "of": 204,
       "ci95": [
        0.9815,
        1
       ]
      },
      "unknownShare": {
       "count": 0,
       "of": 204,
       "ci95": [
        0,
        0.0185
       ]
      },
      "human": {
       "count": 0,
       "of": 204,
       "ci95": [
        0,
        0.0185
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 204,
       "ci95": [
        0,
        0.0185
       ]
      },
      "actionAgent": {
       "count": 204,
       "of": 204,
       "ci95": [
        0.9815,
        1
       ]
      }
     },
     "humans": null
    },
    "Claude": {
     "agents": {
      "n": 186,
      "final": {
       "agent": 167,
       "unknown": 19,
       "human": 0
      },
      "agent": {
       "count": 167,
       "of": 186,
       "ci95": [
        0.846,
        0.9336
       ]
      },
      "agentHigh": {
       "count": 71,
       "of": 186,
       "ci95": [
        0.315,
        0.4533
       ]
      },
      "unknownShare": {
       "count": 19,
       "of": 186,
       "ci95": [
        0.0664,
        0.154
       ]
      },
      "human": {
       "count": 0,
       "of": 186,
       "ci95": [
        0,
        0.0202
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 186,
       "ci95": [
        0,
        0.0202
       ]
      },
      "actionAgent": {
       "count": 167,
       "of": 186,
       "ci95": [
        0.846,
        0.9336
       ]
      }
     },
     "humans": null
    },
    "Comet": {
     "agents": {
      "n": 203,
      "final": {
       "agent": 203,
       "unknown": 0,
       "human": 0
      },
      "agent": {
       "count": 203,
       "of": 203,
       "ci95": [
        0.9814,
        1
       ]
      },
      "agentHigh": {
       "count": 203,
       "of": 203,
       "ci95": [
        0.9814,
        1
       ]
      },
      "unknownShare": {
       "count": 0,
       "of": 203,
       "ci95": [
        0,
        0.0186
       ]
      },
      "human": {
       "count": 0,
       "of": 203,
       "ci95": [
        0,
        0.0186
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 203,
       "ci95": [
        0,
        0.0186
       ]
      },
      "actionAgent": {
       "count": 203,
       "of": 203,
       "ci95": [
        0.9814,
        1
       ]
      }
     },
     "humans": null
    },
    "Human": {
     "agents": null,
     "humans": {
      "n": 159,
      "final": {
       "agent": 0,
       "unknown": 159,
       "human": 0
      },
      "falsePositive": {
       "count": 0,
       "of": 159,
       "ci95": [
        0,
        0.0236
       ]
      },
      "falsePositiveHigh": {
       "count": 0,
       "of": 159,
       "ci95": [
        0,
        0.0236
       ]
      },
      "everAgent": {
       "count": 0,
       "of": 159,
       "ci95": [
        0,
        0.0236
       ]
      },
      "actionFalsePositive": {
       "count": 0,
       "of": 159,
       "ci95": [
        0,
        0.0236
       ]
      },
      "unknownShare": {
       "count": 159,
       "of": 159,
       "ci95": [
        0.9764,
        1
       ]
      },
      "human": {
       "count": 0,
       "of": 159,
       "ci95": [
        0,
        0.0236
       ]
      }
     }
    },
    "Manus": {
     "agents": {
      "n": 210,
      "final": {
       "agent": 207,
       "unknown": 3,
       "human": 0
      },
      "agent": {
       "count": 207,
       "of": 210,
       "ci95": [
        0.9588,
        0.9951
       ]
      },
      "agentHigh": {
       "count": 119,
       "of": 210,
       "ci95": [
        0.499,
        0.6319
       ]
      },
      "unknownShare": {
       "count": 3,
       "of": 210,
       "ci95": [
        0.0049,
        0.0412
       ]
      },
      "human": {
       "count": 0,
       "of": 210,
       "ci95": [
        0,
        0.018
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 210,
       "ci95": [
        0,
        0.018
       ]
      },
      "actionAgent": {
       "count": 207,
       "of": 210,
       "ci95": [
        0.9588,
        0.9951
       ]
      }
     },
     "humans": null
    },
    "Skyvern": {
     "agents": {
      "n": 188,
      "final": {
       "agent": 188,
       "unknown": 0,
       "human": 0
      },
      "agent": {
       "count": 188,
       "of": 188,
       "ci95": [
        0.98,
        1
       ]
      },
      "agentHigh": {
       "count": 179,
       "of": 188,
       "ci95": [
        0.9115,
        0.9746
       ]
      },
      "unknownShare": {
       "count": 0,
       "of": 188,
       "ci95": [
        0,
        0.02
       ]
      },
      "human": {
       "count": 0,
       "of": 188,
       "ci95": [
        0,
        0.02
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 188,
       "ci95": [
        0,
        0.02
       ]
      },
      "actionAgent": {
       "count": 188,
       "of": 188,
       "ci95": [
        0.98,
        1
       ]
      }
     },
     "humans": null
    }
   },
   "note": "Frozen test split (1569 of 7728 sessions). Verdict combines the movement-shape visit replay (Agent) with the event replay (Human); both are approximate replays of a research recorder, not live traffic. The replay has no device timing, so a movement-replay AI agent verdict is high confidence (deny under the AI agent rule, chargeable) when two independent behaviour families agree on one page (heuristic-v4.28.1) and medium (held for review, never charged) otherwise.",
   "byTask": {
    "Flight-booking": {
     "agents": {
      "n": 455,
      "final": {
       "agent": 452,
       "unknown": 3,
       "human": 0
      },
      "agent": {
       "count": 452,
       "of": 455,
       "ci95": [
        0.9808,
        0.9978
       ]
      },
      "agentHigh": {
       "count": 375,
       "of": 455,
       "ci95": [
        0.7865,
        0.8564
       ]
      },
      "unknownShare": {
       "count": 3,
       "of": 455,
       "ci95": [
        0.0022,
        0.0192
       ]
      },
      "human": {
       "count": 0,
       "of": 455,
       "ci95": [
        0,
        0.0084
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 455,
       "ci95": [
        0,
        0.0084
       ]
      },
      "actionAgent": {
       "count": 452,
       "of": 455,
       "ci95": [
        0.9808,
        0.9978
       ]
      }
     },
     "humans": {
      "n": 53,
      "final": {
       "agent": 0,
       "unknown": 53,
       "human": 0
      },
      "falsePositive": {
       "count": 0,
       "of": 53,
       "ci95": [
        0,
        0.0676
       ]
      },
      "falsePositiveHigh": {
       "count": 0,
       "of": 53,
       "ci95": [
        0,
        0.0676
       ]
      },
      "everAgent": {
       "count": 0,
       "of": 53,
       "ci95": [
        0,
        0.0676
       ]
      },
      "actionFalsePositive": {
       "count": 0,
       "of": 53,
       "ci95": [
        0,
        0.0676
       ]
      },
      "unknownShare": {
       "count": 53,
       "of": 53,
       "ci95": [
        0.9324,
        1
       ]
      },
      "human": {
       "count": 0,
       "of": 53,
       "ci95": [
        0,
        0.0676
       ]
      }
     }
    },
    "Forums": {
     "agents": {
      "n": 466,
      "final": {
       "agent": 463,
       "unknown": 3,
       "human": 0
      },
      "agent": {
       "count": 463,
       "of": 466,
       "ci95": [
        0.9812,
        0.9978
       ]
      },
      "agentHigh": {
       "count": 402,
       "of": 466,
       "ci95": [
        0.8284,
        0.891
       ]
      },
      "unknownShare": {
       "count": 3,
       "of": 466,
       "ci95": [
        0.0022,
        0.0188
       ]
      },
      "human": {
       "count": 0,
       "of": 466,
       "ci95": [
        0,
        0.0082
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 466,
       "ci95": [
        0,
        0.0082
       ]
      },
      "actionAgent": {
       "count": 463,
       "of": 466,
       "ci95": [
        0.9812,
        0.9978
       ]
      }
     },
     "humans": {
      "n": 57,
      "final": {
       "agent": 0,
       "unknown": 57,
       "human": 0
      },
      "falsePositive": {
       "count": 0,
       "of": 57,
       "ci95": [
        0,
        0.0631
       ]
      },
      "falsePositiveHigh": {
       "count": 0,
       "of": 57,
       "ci95": [
        0,
        0.0631
       ]
      },
      "everAgent": {
       "count": 0,
       "of": 57,
       "ci95": [
        0,
        0.0631
       ]
      },
      "actionFalsePositive": {
       "count": 0,
       "of": 57,
       "ci95": [
        0,
        0.0631
       ]
      },
      "unknownShare": {
       "count": 57,
       "of": 57,
       "ci95": [
        0.9369,
        1
       ]
      },
      "human": {
       "count": 0,
       "of": 57,
       "ci95": [
        0,
        0.0631
       ]
      }
     }
    },
    "Shopping": {
     "agents": {
      "n": 489,
      "final": {
       "agent": 471,
       "unknown": 18,
       "human": 0
      },
      "agent": {
       "count": 471,
       "of": 489,
       "ci95": [
        0.9426,
        0.9766
       ]
      },
      "agentHigh": {
       "count": 401,
       "of": 489,
       "ci95": [
        0.7835,
        0.8516
       ]
      },
      "unknownShare": {
       "count": 18,
       "of": 489,
       "ci95": [
        0.0234,
        0.0574
       ]
      },
      "human": {
       "count": 0,
       "of": 489,
       "ci95": [
        0,
        0.0078
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 489,
       "ci95": [
        0,
        0.0078
       ]
      },
      "actionAgent": {
       "count": 471,
       "of": 489,
       "ci95": [
        0.9426,
        0.9766
       ]
      }
     },
     "humans": {
      "n": 49,
      "final": {
       "agent": 0,
       "unknown": 49,
       "human": 0
      },
      "falsePositive": {
       "count": 0,
       "of": 49,
       "ci95": [
        0,
        0.0727
       ]
      },
      "falsePositiveHigh": {
       "count": 0,
       "of": 49,
       "ci95": [
        0,
        0.0727
       ]
      },
      "everAgent": {
       "count": 0,
       "of": 49,
       "ci95": [
        0,
        0.0727
       ]
      },
      "actionFalsePositive": {
       "count": 0,
       "of": 49,
       "ci95": [
        0,
        0.0727
       ]
      },
      "unknownShare": {
       "count": 49,
       "of": 49,
       "ci95": [
        0.9273,
        1
       ]
      },
      "human": {
       "count": 0,
       "of": 49,
       "ci95": [
        0,
        0.0727
       ]
      }
     }
    }
   }
  },
  {
   "id": "recorded-browser",
   "title": "Recorded browser agents (Codex in-app browser and others)",
   "provenance": "real",
   "exposure": "seen",
   "exposureNote": "Development recordings; rules v4.9 to v4.17 were tuned on them.",
   "labelSource": "controlled automation",
   "heldOutItems": 12,
   "tunedOnThisData": true,
   "metrics": {
    "agents": {
     "n": 12,
     "final": {
      "agent": 9,
      "unknown": 3,
      "human": 0
     },
     "agent": {
      "count": 9,
      "of": 12,
      "ci95": [
       0.4677,
       0.9111
      ]
     },
     "agentHigh": {
      "count": 0,
      "of": 12,
      "ci95": [
       0,
       0.2425
      ]
     },
     "unknownShare": {
      "count": 3,
      "of": 12,
      "ci95": [
       0.0889,
       0.5323
      ]
     },
     "human": {
      "count": 0,
      "of": 12,
      "ci95": [
       0,
       0.2425
      ]
     },
     "everHuman": {
      "count": 0,
      "of": 12,
      "ci95": [
       0,
       0.2425
      ]
     },
     "actionAgent": {
      "count": 9,
      "of": 12,
      "ci95": [
       0.4677,
       0.9111
      ]
     }
    },
    "humans": null
   },
   "byFamily": {
    "Codex in-app browser": {
     "agents": {
      "n": 5,
      "final": {
       "agent": 3,
       "unknown": 2,
       "human": 0
      },
      "agent": {
       "count": 3,
       "of": 5,
       "ci95": [
        0.2307,
        0.8824
       ]
      },
      "agentHigh": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "unknownShare": {
       "count": 2,
       "of": 5,
       "ci95": [
        0.1176,
        0.7693
       ]
      },
      "human": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 5,
       "ci95": [
        0,
        0.4345
       ]
      },
      "actionAgent": {
       "count": 3,
       "of": 5,
       "ci95": [
        0.2307,
        0.8824
       ]
      }
     },
     "humans": null
    },
    "Codex native browser controls": {
     "agents": {
      "n": 7,
      "final": {
       "agent": 6,
       "unknown": 1,
       "human": 0
      },
      "agent": {
       "count": 6,
       "of": 7,
       "ci95": [
        0.4869,
        0.9743
       ]
      },
      "agentHigh": {
       "count": 0,
       "of": 7,
       "ci95": [
        0,
        0.3543
       ]
      },
      "unknownShare": {
       "count": 1,
       "of": 7,
       "ci95": [
        0.0257,
        0.5131
       ]
      },
      "human": {
       "count": 0,
       "of": 7,
       "ci95": [
        0,
        0.3543
       ]
      },
      "everHuman": {
       "count": 0,
       "of": 7,
       "ci95": [
        0,
        0.3543
       ]
      },
      "actionAgent": {
       "count": 6,
       "of": 7,
       "ci95": [
        0.4869,
        0.9743
       ]
      }
     },
     "humans": null
    }
   }
  },
  {
   "id": "recorded-human",
   "title": "Confirmed human recordings (one person)",
   "provenance": "real",
   "exposure": "seen",
   "exposureNote": "One person; used as regression checks while tuning.",
   "labelSource": "one person who confirmed each recording",
   "heldOutItems": 1,
   "tunedOnThisData": true,
   "metrics": {
    "agents": null,
    "humans": {
     "n": 1,
     "final": {
      "agent": 0,
      "unknown": 1,
      "human": 0
     },
     "falsePositive": {
      "count": 0,
      "of": 1,
      "ci95": [
       0,
       0.7935
      ]
     },
     "falsePositiveHigh": {
      "count": 0,
      "of": 1,
      "ci95": [
       0,
       0.7935
      ]
     },
     "everAgent": {
      "count": 0,
      "of": 1,
      "ci95": [
       0,
       0.7935
      ]
     },
     "actionFalsePositive": {
      "count": 0,
      "of": 1,
      "ci95": [
       0,
       0.7935
      ]
     },
     "unknownShare": {
      "count": 1,
      "of": 1,
      "ci95": [
       0.2065,
       1
      ]
     },
     "human": {
      "count": 0,
      "of": 1,
      "ci95": [
       0,
       0.7935
      ]
     }
    }
   },
   "byFamily": {
    "Human in in-app browser": {
     "agents": null,
     "humans": {
      "n": 1,
      "final": {
       "agent": 0,
       "unknown": 1,
       "human": 0
      },
      "falsePositive": {
       "count": 0,
       "of": 1,
       "ci95": [
        0,
        0.7935
       ]
      },
      "falsePositiveHigh": {
       "count": 0,
       "of": 1,
       "ci95": [
        0,
        0.7935
       ]
      },
      "everAgent": {
       "count": 0,
       "of": 1,
       "ci95": [
        0,
        0.7935
       ]
      },
      "actionFalsePositive": {
       "count": 0,
       "of": 1,
       "ci95": [
        0,
        0.7935
       ]
      },
      "unknownShare": {
       "count": 1,
       "of": 1,
       "ci95": [
        0.2065,
        1
       ]
      },
      "human": {
       "count": 0,
       "of": 1,
       "ci95": [
        0,
        0.7935
       ]
      }
     }
    }
   }
  },
  {
   "id": "automation-startup",
   "title": "Automation start-up smoke corpus",
   "provenance": "real",
   "exposure": "seen",
   "exposureNote": "Smoke corpus from 2026-10-24, checked on every rule change.",
   "labelSource": "controlled automation",
   "heldOutItems": 1,
   "tunedOnThisData": true,
   "metrics": {
    "agents": {
     "n": 1,
     "final": {
      "agent": 0,
      "unknown": 1,
      "human": 0
     },
     "agent": {
      "count": 0,
      "of": 1,
      "ci95": [
       0,
       0.7935
      ]
     },
     "agentHigh": {
      "count": 0,
      "of": 1,
      "ci95": [
       0,
       0.7935
      ]
     },
     "unknownShare": {
      "count": 1,
      "of": 1,
      "ci95": [
       0.2065,
       1
      ]
     },
     "human": {
      "count": 0,
      "of": 1,
      "ci95": [
       0,
       0.7935
      ]
     },
     "everHuman": null,
     "actionAgent": {
      "count": 0,
      "of": 1,
      "ci95": [
       0,
       0.7935
      ]
     }
    },
    "humans": null
   },
   "byFamily": {
    "agent-browser executing timed DOM dispatch; synthetic events": {
     "agents": {
      "n": 1,
      "final": {
       "agent": 0,
       "unknown": 1,
       "human": 0
      },
      "agent": {
       "count": 0,
       "of": 1,
       "ci95": [
        0,
        0.7935
       ]
      },
      "agentHigh": {
       "count": 0,
       "of": 1,
       "ci95": [
        0,
        0.7935
       ]
      },
      "unknownShare": {
       "count": 1,
       "of": 1,
       "ci95": [
        0.2065,
        1
       ]
      },
      "human": {
       "count": 0,
       "of": 1,
       "ci95": [
        0,
        0.7935
       ]
      },
      "everHuman": null,
      "actionAgent": {
       "count": 0,
       "of": 1,
       "ci95": [
        0,
        0.7935
       ]
      }
     },
     "humans": null
    }
   }
  },
  {
   "id": "sapimouse",
   "title": "SapiMouse human pointer movement",
   "provenance": "real-with-synthetic-timing",
   "exposure": "seen",
   "exposureNote": "Reported on every evaluate run since 2026-10-24; pointer movement only, timing reconstructed.",
   "labelSource": "research study",
   "heldOutItems": 30,
   "tunedOnThisData": true,
   "metrics": {
    "agents": null,
    "humans": {
     "n": 30,
     "final": {
      "agent": 0,
      "unknown": 30,
      "human": 0
     },
     "falsePositive": {
      "count": 0,
      "of": 30,
      "ci95": [
       0,
       0.1135
      ]
     },
     "falsePositiveHigh": {
      "count": 0,
      "of": 30,
      "ci95": [
       0,
       0.1135
      ]
     },
     "everAgent": null,
     "actionFalsePositive": {
      "count": 0,
      "of": 30,
      "ci95": [
       0,
       0.1135
      ]
     },
     "unknownShare": {
      "count": 30,
      "of": 30,
      "ci95": [
       0.8865,
       1
      ]
     },
     "human": {
      "count": 0,
      "of": 30,
      "ci95": [
       0,
       0.1135
      ]
     }
    }
   },
   "byFamily": {
    "pointer movement only": {
     "agents": null,
     "humans": {
      "n": 30,
      "final": {
       "agent": 0,
       "unknown": 30,
       "human": 0
      },
      "falsePositive": {
       "count": 0,
       "of": 30,
       "ci95": [
        0,
        0.1135
       ]
      },
      "falsePositiveHigh": {
       "count": 0,
       "of": 30,
       "ci95": [
        0,
        0.1135
       ]
      },
      "everAgent": null,
      "actionFalsePositive": {
       "count": 0,
       "of": 30,
       "ci95": [
        0,
        0.1135
       ]
      },
      "unknownShare": {
       "count": 30,
       "of": 30,
       "ci95": [
        0.8865,
        1
       ]
      },
      "human": {
       "count": 0,
       "of": 30,
       "ci95": [
        0,
        0.1135
       ]
      }
     }
    }
   }
  }
 ],
 "pooled": {
  "clean": {
   "agents": {
    "n": 20,
    "final": {
     "agent": 15,
     "unknown": 5,
     "human": 0
    },
    "agent": {
     "count": 15,
     "of": 20,
     "ci95": [
      0.5313,
      0.8881
     ]
    },
    "agentHigh": {
     "count": 15,
     "of": 20,
     "ci95": [
      0.5313,
      0.8881
     ]
    },
    "unknownShare": {
     "count": 5,
     "of": 20,
     "ci95": [
      0.1119,
      0.4687
     ]
    },
    "human": {
     "count": 0,
     "of": 20,
     "ci95": [
      0,
      0.1611
     ]
    },
    "everHuman": {
     "count": 0,
     "of": 20,
     "ci95": [
      0,
      0.1611
     ]
    },
    "actionAgent": {
     "count": 15,
     "of": 20,
     "ci95": [
      0.5313,
      0.8881
     ]
    }
   },
   "humans": null
  },
  "seenDuringTuning": {
   "agents": {
    "n": 1423,
    "final": {
     "agent": 1395,
     "unknown": 28,
     "human": 0
    },
    "agent": {
     "count": 1395,
     "of": 1423,
     "ci95": [
      0.9717,
      0.9864
     ]
    },
    "agentHigh": {
     "count": 1178,
     "of": 1423,
     "ci95": [
      0.8073,
      0.8466
     ]
    },
    "unknownShare": {
     "count": 28,
     "of": 1423,
     "ci95": [
      0.0136,
      0.0283
     ]
    },
    "human": {
     "count": 0,
     "of": 1423,
     "ci95": [
      0,
      0.0027
     ]
    },
    "everHuman": {
     "count": 0,
     "of": 1422,
     "ci95": [
      0,
      0.0027
     ]
    },
    "actionAgent": {
     "count": 1395,
     "of": 1423,
     "ci95": [
      0.9717,
      0.9864
     ]
    }
   },
   "humans": {
    "n": 190,
    "final": {
     "agent": 0,
     "unknown": 190,
     "human": 0
    },
    "falsePositive": {
     "count": 0,
     "of": 190,
     "ci95": [
      0,
      0.0198
     ]
    },
    "falsePositiveHigh": {
     "count": 0,
     "of": 190,
     "ci95": [
      0,
      0.0198
     ]
    },
    "everAgent": {
     "count": 0,
     "of": 160,
     "ci95": [
      0,
      0.0234
     ]
    },
    "actionFalsePositive": {
     "count": 0,
     "of": 190,
     "ci95": [
      0,
      0.0198
     ]
    },
    "unknownShare": {
     "count": 190,
     "of": 190,
     "ci95": [
      0.9802,
      1
     ]
    },
    "human": {
     "count": 0,
     "of": 190,
     "ci95": [
      0,
      0.0198
     ]
    }
   }
  }
 },
 "synthetic": {
  "label": "SYNTHETIC approximations of assistive-technology input, not recordings of people. Rule checks only: never counted in the human false-positive rate.",
  "classifier": "heuristic-v4.33",
  "seeds": [
   7,
   11,
   23
  ],
  "scenarios": [
   {
    "id": "switch-scanning-activate",
    "label": "Switch Control auto-scanning: the highlight steps every 1.2 s; each selection activates the control at its centre with no pointer movement; text through the scanning keyboard (short OS key presses seconds apart)",
    "assumption": "A Switch Control selection reaches the page as an accessibility click at the element centre (no pointer events); scanning-keyboard letters arrive as key presses with 10-25 ms holds.",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "switch-point-mode",
    "label": "Switch Control point mode: a vertical line sweeps across at constant speed and stops on a switch press, then a horizontal line does; then a real mouse click",
    "assumption": "Point-mode sweeps move the real cursor at a constant speed in straight lines; the click is a trusted mouse click where the cursor stopped.",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "head-pointer-dwell",
    "label": "Head pointer with dwell clicking: slow jittery travel to each control, the pointer held still for 1 s, then the OS clicks",
    "assumption": "Dwell clicks are trusted mouse clicks with no movement in the dwell time (lead about 1000 ms).",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "eye-gaze-dwell",
    "label": "Eye-gaze control: saccades jump the pointer in one or two samples, fixations jitter a few pixels, an 800 ms dwell on the target clicks",
    "assumption": "The gaze system moves the real cursor; fixation jitter continues during the dwell, so the press follows recent small movements.",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "voice-control-click-name",
    "label": "macOS Voice Control \"click Submit\" and dictation: each command activates the named control with no pointer movement; text dictated as phrases",
    "assumption": "A Voice Control \"click <name>\" reaches the page as an accessibility click at the element centre; the grid variant is the corpus scenario voice-control.",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "voice-control-mousegrid-fast",
    "label": "Voice Control mouse grid used quickly: \"show grid\", a number, \"click\"; the cursor jumps onto the target and clicks within a fraction of a second",
    "assumption": "The grid jump is one pointer move and the click follows it within 150-300 ms (faster than the corpus voice-control scenario).",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "dictation",
    "label": "Desktop dictation: clicks, then whole phrases inserted by the operating system with no key presses",
    "assumption": "corpus scenario (tools/detection-bench-corpus.js)",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "keyboard-only",
    "label": "Keyboard-only person: Tab between fields, typing, Space and Enter to activate controls (clicks with detail 0)",
    "assumption": "corpus scenario (tools/detection-bench-corpus.js)",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "assistive-tech",
    "label": "Screen-reader user: activations land at element centres with no pointer movement, typing in between",
    "assumption": "corpus scenario (tools/detection-bench-corpus.js)",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "screen-reader-clicks-only",
    "label": "Screen-reader user activating controls only (no typing): accessibility clicks at element centres with no pointer type and no pointer movement, no keys reaching the page",
    "assumption": "corpus scenario (tools/detection-bench-corpus.js)",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "voice-control",
    "label": "Voice-control user: the system moves the pointer onto each named control, then a mouse click lands at its centre; dictated text with no keys",
    "assumption": "corpus scenario (tools/detection-bench-corpus.js)",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   },
   {
    "id": "screen-reader-form",
    "label": "Screen-reader user filling a form: the browse cursor moves focus to each field (no Tab key reaches the page), typing after a moment, activations at control centres",
    "assumption": "corpus scenario (tools/detection-bench-corpus.js)",
    "truth": "human",
    "runs": 3,
    "final": {
     "agent": 0,
     "unknown": 3,
     "human": 0
    },
    "everAgent": 0,
    "everHuman": 0,
    "liveRules": []
   }
  ]
 },
 "notCovered": [
  "Real named-agent runs (Claude in Chrome, ChatGPT agent, Atlas, Comet) in a normal visible browser: pending. The run kit is tools/real-agent-runs.js (docs/maintainers/accuracy-real-agent-runs.md); until those rows exist the clean held-out real-agent table is empty.",
  "Assistive-technology recordings of consenting people (screen readers, voice control, switch access, keyboard-only): pending. The protocol is docs/public/accuracy-assistive-protocol.md; until those rows exist there is no measured human false-positive rate for assistive-technology users.",
  "OS-level computer-use agents that drive the real mouse and keyboard: harness written (tools/os-agent-harness.js) and dry-run scored on emulated input, not yet run on a real screen.",
  "Phones and tablets, non-Chrome browsers, and people other than the one person we recorded and the FP-Agent study participants.",
  "Production traffic: every figure here is from labelled test sessions, not from a customer site."
 ],
 "change": {
  "from": "heuristic-v4.32",
  "to": "heuristic-v4.33",
  "rows": [
   {
    "corpus": "fp-agent",
    "scope": "all",
    "title": "FP-Agent, all 7,728 sessions (no-regression check)",
    "exposure": "tuned on this data, not an accuracy estimate",
    "before": {
     "agents": {
      "n": 7182,
      "agent": 7025,
      "high": 5993,
      "medium": 1032,
      "unknown": 157,
      "human": 0
     },
     "people": {
      "n": 546,
      "agent": 0,
      "high": 0,
      "medium": 0,
      "unknown": 546,
      "human": 0
     }
    },
    "after": {
     "agents": {
      "n": 7182,
      "agent": 7025,
      "high": 5993,
      "medium": 1032,
      "unknown": 157,
      "human": 0
     },
     "people": {
      "n": 546,
      "agent": 0,
      "high": 0,
      "medium": 0,
      "unknown": 546,
      "human": 0
     }
    }
   },
   {
    "corpus": "fp-agent",
    "scope": "heldOut",
    "title": "FP-Agent, frozen held-out test split (the grade)",
    "exposure": "tuned on this data, not an accuracy estimate",
    "before": {
     "agents": {
      "n": 1410,
      "agent": 1386,
      "high": 1178,
      "medium": 208,
      "unknown": 24,
      "human": 0
     },
     "people": {
      "n": 159,
      "agent": 0,
      "high": 0,
      "medium": 0,
      "unknown": 159,
      "human": 0
     }
    },
    "after": {
     "agents": {
      "n": 1410,
      "agent": 1386,
      "high": 1178,
      "medium": 208,
      "unknown": 24,
      "human": 0
     },
     "people": {
      "n": 159,
      "agent": 0,
      "high": 0,
      "medium": 0,
      "unknown": 159,
      "human": 0
     }
    }
   },
   {
    "corpus": "fp-agent",
    "scope": "tuning",
    "title": "FP-Agent, tuning portion (train and validation splits)",
    "exposure": "tuned on this data, not an accuracy estimate",
    "before": {
     "agents": {
      "n": 5772,
      "agent": 5639,
      "high": 4815,
      "medium": 824,
      "unknown": 133,
      "human": 0
     },
     "people": {
      "n": 387,
      "agent": 0,
      "high": 0,
      "medium": 0,
      "unknown": 387,
      "human": 0
     }
    },
    "after": {
     "agents": {
      "n": 5772,
      "agent": 5639,
      "high": 4815,
      "medium": 824,
      "unknown": 133,
      "human": 0
     },
     "people": {
      "n": 387,
      "agent": 0,
      "high": 0,
      "medium": 0,
      "unknown": 387,
      "human": 0
     }
    }
   },
   {
    "corpus": "adversarial-bench",
    "scope": "heldOut",
    "title": "Adversarial humanisers, held-out seeds (independent scripted agents)",
    "exposure": "independent of FP-Agent; see the note on thresholds",
    "before": {
     "agents": {
      "n": 20,
      "agent": 15,
      "high": 15,
      "medium": 0,
      "unknown": 5,
      "human": 0
     },
     "people": {
      "n": 0,
      "agent": 0,
      "high": 0,
      "medium": 0,
      "unknown": 0,
      "human": 0
     }
    },
    "after": {
     "agents": {
      "n": 20,
      "agent": 15,
      "high": 15,
      "medium": 0,
      "unknown": 5,
      "human": 0
     },
     "people": {
      "n": 0,
      "agent": 0,
      "high": 0,
      "medium": 0,
      "unknown": 0,
      "human": 0
     }
    }
   }
  ],
  "notes": [
   "What changed: two readings that heuristic-v4.32 held at medium confidence now read Unknown. (1) Injected input that is the only sign of software, on pages whose input is otherwise shaped like a person's, from a published data-centre address (remote browser isolation, a cloud browser's live view). (2) A screen-reader page: every mouse press with no pointer travel (VoiceOver's activate command, or the pointer resting after following the VoiceOver cursor) and nothing typed; its jump clicks, alone or with a paste, are no longer AI agent evidence. Proof is unchanged: an automation trace, a verified signature, an AI agent's cursor, sustained machine input, and injected input from any other address stay high.",
   "Effect on these test sets: no held-out figure changes. No held-out corpus records the connection address, the adversarial runs come from a local address, and FP-Agent replays carry no press-shape summary, so neither rule can apply. Fresh adversarial run under heuristic-v4.33: 15 high, 5 Unknown (the heuristic-v4.32 run read 15 high, 5 Unknown too; one run that read an automation trace then read injected input now, both high). Worst case, the same runs re-scored with a data-centre address assumed: 13 of the 15 high runs read Unknown (heuristic-v4.32: medium), 2 stay high (their input fails the person-shape checks), and the 5 Unknown stay Unknown. People read AI agent: 0 of 159 held-out, 0 of 546 overall; the VoiceOver and isolation people fixtures now read Unknown instead of medium."
  ]
 },
 "checkpoints": {
  "checkpointsS": [
   2,
   5,
   10,
   20,
   30,
   60
  ],
  "corpora": {
   "fp-agent-all": {
    "agents": {
     "2": {
      "n": 7175,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 7174,
      "human": 0
     },
     "5": {
      "n": 7175,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 7174,
      "human": 0
     },
     "10": {
      "n": 7175,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 7174,
      "human": 0
     },
     "20": {
      "n": 7129,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 7128,
      "human": 0
     },
     "30": {
      "n": 6771,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 6770,
      "human": 0
     },
     "60": {
      "n": 5121,
      "high": 1,
      "medium": 0,
      "agentAny": 1,
      "unknown": 5120,
      "human": 0
     }
    },
    "people": {
     "2": {
      "n": 517,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 517,
      "human": 0
     },
     "5": {
      "n": 504,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 504,
      "human": 0
     },
     "10": {
      "n": 483,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 483,
      "human": 0
     },
     "20": {
      "n": 394,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 394,
      "human": 0
     },
     "30": {
      "n": 343,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 343,
      "human": 0
     },
     "60": {
      "n": 218,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 218,
      "human": 0
     }
    }
   },
   "fp-agent-heldout": {
    "agents": {
     "2": {
      "n": 1409,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 1409,
      "human": 0
     },
     "5": {
      "n": 1409,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 1409,
      "human": 0
     },
     "10": {
      "n": 1409,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 1409,
      "human": 0
     },
     "20": {
      "n": 1402,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 1402,
      "human": 0
     },
     "30": {
      "n": 1332,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 1332,
      "human": 0
     },
     "60": {
      "n": 995,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 995,
      "human": 0
     }
    },
    "people": {
     "2": {
      "n": 154,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 154,
      "human": 0
     },
     "5": {
      "n": 149,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 149,
      "human": 0
     },
     "10": {
      "n": 139,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 139,
      "human": 0
     },
     "20": {
      "n": 115,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 115,
      "human": 0
     },
     "30": {
      "n": 101,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 101,
      "human": 0
     },
     "60": {
      "n": 60,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 60,
      "human": 0
     }
    }
   },
   "fp-agent-tuning": {
    "agents": {
     "2": {
      "n": 5766,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 5765,
      "human": 0
     },
     "5": {
      "n": 5766,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 5765,
      "human": 0
     },
     "10": {
      "n": 5766,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 5765,
      "human": 0
     },
     "20": {
      "n": 5727,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 5726,
      "human": 0
     },
     "30": {
      "n": 5439,
      "high": 0,
      "medium": 1,
      "agentAny": 1,
      "unknown": 5438,
      "human": 0
     },
     "60": {
      "n": 4126,
      "high": 1,
      "medium": 0,
      "agentAny": 1,
      "unknown": 4125,
      "human": 0
     }
    },
    "people": {
     "2": {
      "n": 363,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 363,
      "human": 0
     },
     "5": {
      "n": 355,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 355,
      "human": 0
     },
     "10": {
      "n": 344,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 344,
      "human": 0
     },
     "20": {
      "n": 279,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 279,
      "human": 0
     },
     "30": {
      "n": 242,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 242,
      "human": 0
     },
     "60": {
      "n": 158,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 158,
      "human": 0
     }
    }
   },
   "adversarial-heldout": {
    "agents": {
     "2": {
      "n": 20,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 20,
      "human": 0
     },
     "5": {
      "n": 20,
      "high": 5,
      "medium": 0,
      "agentAny": 5,
      "unknown": 15,
      "human": 0
     },
     "10": {
      "n": 20,
      "high": 15,
      "medium": 0,
      "agentAny": 15,
      "unknown": 5,
      "human": 0
     },
     "20": {
      "n": 20,
      "high": 15,
      "medium": 0,
      "agentAny": 15,
      "unknown": 5,
      "human": 0
     },
     "30": {
      "n": 20,
      "high": 15,
      "medium": 0,
      "agentAny": 15,
      "unknown": 5,
      "human": 0
     },
     "60": {
      "n": 0,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 0,
      "human": 0
     }
    }
   },
   "user-recordings": {
    "people": {
     "2": {
      "n": 4,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 4,
      "human": 0
     },
     "5": {
      "n": 4,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 2,
      "human": 2
     },
     "10": {
      "n": 4,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 1,
      "human": 3
     },
     "20": {
      "n": 4,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 2,
      "human": 2
     },
     "30": {
      "n": 4,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 2,
      "human": 2
     },
     "60": {
      "n": 4,
      "high": 0,
      "medium": 0,
      "agentAny": 0,
      "unknown": 2,
      "human": 2
     }
    }
   }
  }
 },
 "adversarialHistory": [
  {
   "classifier": "heuristic-v4.25",
   "file": "evaluation/bench/adversarial-2026-11-03.json",
   "sha256": "d2942bb30e6d89d4c4ee8f3e65c085f157ec4ac6df4591080d8826dddc19b014",
   "n": 20,
   "high": 0,
   "medium": 0,
   "unknown": 5,
   "human": 15,
   "change": "Smoothed, human-like paths passed the checks for a person."
  },
  {
   "classifier": "heuristic-v4.27",
   "file": "evaluation/bench/adversarial-2026-11-03-v427.json",
   "sha256": "6b36276d755950928590be9ecf2a1cf563aba80c098e032acb8a57c4aae724c4",
   "n": 20,
   "high": 0,
   "medium": 0,
   "unknown": 20,
   "human": 0,
   "change": "A person now needs a real device's input delay, so none of them passes as one."
  },
  {
   "classifier": "heuristic-v4.28",
   "file": "evaluation/bench/adversarial-2026-11-03-v428.json",
   "sha256": "949170b87e11df8593bcb2bfe8d654422da84ddce29f33408550cdaab0dbc2f7",
   "n": 20,
   "high": 15,
   "medium": 0,
   "unknown": 5,
   "human": 0,
   "change": "Input injected by software, with no hardware delay, is caught at high confidence."
  },
  {
   "classifier": "heuristic-v4.28.1",
   "file": "evaluation/bench/adversarial-2026-11-03-v4281.json",
   "sha256": "663c469ea84bef90eb376c941317a8c302717e118ace2edf93be02339f85218f",
   "n": 20,
   "high": 15,
   "medium": 0,
   "unknown": 5,
   "human": 0,
   "change": "Several behaviour signals agreeing can reach high confidence. No change for these runs."
  },
  {
   "classifier": "heuristic-v4.30",
   "file": "evaluation/bench/adversarial-2026-11-03-v430.json",
   "sha256": "c3c8f0d154315194dc0331337af094bf7857b8da79128b2923ff9d1bea966d6b",
   "n": 20,
   "high": 15,
   "medium": 0,
   "unknown": 5,
   "human": 0,
   "change": "Mouse-hardware clues and Claude's cursor. No change for these runs."
  },
  {
   "classifier": "heuristic-v4.31",
   "file": "evaluation/bench/adversarial-2026-11-03-v431.json",
   "sha256": "45366fd54e692dc2d62bf79e4e910a43454b45a9eccb36883f2efafa067b45ad",
   "n": 20,
   "high": 15,
   "medium": 0,
   "unknown": 5,
   "human": 0,
   "change": "A person on real hardware reaches Human after 3 clicks with pointer travel. None of these runs does."
  },
  {
   "classifier": "heuristic-v4.32",
   "file": "evaluation/bench/adversarial-2026-11-03-v432.json",
   "sha256": "856cffc4c355c51cc6c68890ecc93ac8b44ff1002d012e2cec8da8656c7c9a7b",
   "n": 20,
   "high": 15,
   "medium": 0,
   "unknown": 5,
   "human": 0,
   "change": "Remote browser isolation: person-like injected input from a cloud address is held at medium. These runs come from a local address, so no change."
  },
  {
   "classifier": "heuristic-v4.33",
   "file": "evaluation/bench/adversarial-2026-11-03-v433.json",
   "sha256": "c8687c01d0a56936857887a7b761a19d75e33f47fc6358f2f0fb01c55578fc8d",
   "n": 20,
   "high": 15,
   "medium": 0,
   "unknown": 5,
   "human": 0,
   "change": "Edge people read Unknown: person-like injected input from a cloud address, and screen-reader activate-and-paste. These runs come from a local address, so no change."
  }
 ],
 "humanEstimate": {
  "assumption": "Assuming real hardware, which live sessions show: the FP-Agent replay has no device timing, so every page is given the input-pipeline delay and merged pointer samples a physical mouse and keyboard leave on a live Chromium page. An estimate, not a measurement.",
  "heldOut": {
   "people": {
    "n": 159,
    "human": 130,
    "everHuman": 135,
    "agent": 0,
    "humanBy": {
     "2": 0,
     "5": 15,
     "10": 65,
     "20": 104,
     "30": 115,
     "60": 125
    },
    "medianFirstHumanS": 11,
    "unknownWhy": {
     "fewInteractions": 13,
     "fewIntervals": 5,
     "noTravel": 2,
     "fewKinds": 2,
     "noInteraction": 2
    }
   },
   "agents": {
    "n": 1410,
    "human": 0,
    "everHuman": 0
   }
  },
  "all": {
   "people": {
    "n": 546,
    "human": 457,
    "everHuman": 470,
    "agent": 0,
    "humanBy": {
     "2": 1,
     "5": 53,
     "10": 242,
     "20": 344,
     "30": 384,
     "60": 435
    },
    "medianFirstHumanS": 10,
    "unknownWhy": {
     "fewInteractions": 39,
     "noInteraction": 15,
     "fewIntervals": 8,
     "fewKinds": 6,
     "notVaried": 5,
     "noTravel": 3
    }
   },
   "agents": {
    "n": 7182,
    "human": 0,
    "everHuman": 0
   }
  },
  "classifier": "heuristic-v4.33",
  "before": {
   "classifier": "heuristic-v4.32",
   "assumption": "Assuming real hardware, which live sessions show: the FP-Agent replay has no device timing, so every page is given the input-pipeline delay and merged pointer samples a physical mouse and keyboard leave on a live Chromium page. An estimate, not a measurement.",
   "heldOut": {
    "people": {
     "n": 159,
     "human": 130,
     "everHuman": 135,
     "agent": 0,
     "humanBy": {
      "2": 0,
      "5": 15,
      "10": 65,
      "20": 104,
      "30": 115,
      "60": 125
     },
     "medianFirstHumanS": 11,
     "unknownWhy": {
      "fewInteractions": 13,
      "fewIntervals": 5,
      "noTravel": 2,
      "fewKinds": 2,
      "noInteraction": 2
     }
    },
    "agents": {
     "n": 1410,
     "human": 0,
     "everHuman": 0
    }
   },
   "all": {
    "people": {
     "n": 546,
     "human": 457,
     "everHuman": 470,
     "agent": 0,
     "humanBy": {
      "2": 1,
      "5": 53,
      "10": 242,
      "20": 344,
      "30": 384,
      "60": 435
     },
     "medianFirstHumanS": 10,
     "unknownWhy": {
      "fewInteractions": 39,
      "noInteraction": 15,
      "fewIntervals": 8,
      "fewKinds": 6,
      "notVaried": 5,
      "noTravel": 3
     }
    },
    "agents": {
     "n": 7182,
     "human": 0,
     "everHuman": 0
    }
   }
  }
 },
 "recognisedAsPerson": {
  "count": 130,
  "n": 159,
  "basis": "People in the research recordings' test set, given the input timing a real mouse and keyboard show on live sites."
 },
 "source": {
  "name": "FP-Agent",
  "description": "A research study of seven commercial AI agents and consenting people doing the same shopping, flight-booking and forum tasks.",
  "paper": "https://arxiv.org/abs/2605.01247",
  "code": "https://github.com/ethanbwang/fp-agent"
 },
 "reproduce": [
  "# Adversarial humanisers: the committed runs are evaluation/bench/adversarial-2026-11-03.json (heuristic-v4.25), evaluation/bench/adversarial-2026-11-03-v432.json (heuristic-v4.32) and evaluation/bench/adversarial-2026-11-03-v433.json (heuristic-v4.33). To re-run (headless, muted; about 20 minutes):",
  "node tools/detection-bench.js --kinds adversarial --seeds held-out --ports 4761-4769 --out .scratch/accuracy/adversarial.json",
  "# FP-Agent page summaries and event replay input (needs the research dataset; see tools/fp-agent-benchmark/README.md)",
  ".scratch/fp-agent-data/venv/bin/python -I tools/fp-agent-benchmark/pages.py RAW.json .scratch/accuracy/fp-pages.jsonl",
  ".scratch/fp-agent-data/venv/bin/python -I tools/fp-agent-benchmark/convert.py RAW.json .scratch/accuracy/fp-events.jsonl",
  "# The verdict over time",
  "node tools/evidence-checkpoints.js --fp-agent-events .scratch/accuracy/fp-events.jsonl --adversarial evaluation/bench/adversarial-2026-11-03-v433.json --out .scratch/accuracy/checkpoints.json",
  "# The report: the data file (JSON), this page (Markdown) and the /accuracy page, which renders from the JSON",
  "node tools/accuracy-bench.js --date 2026-11-03 --adversarial evaluation/bench/adversarial-2026-11-03-v433.json --adversarial-before evaluation/bench/adversarial-2026-11-03-v432.json --compare heuristic-v4.32 --fp-agent-pages .scratch/accuracy/fp-pages.jsonl --fp-agent-events .scratch/accuracy/fp-events.jsonl --checkpoints .scratch/accuracy/checkpoints.json --json docs/public/accuracy-report-2026-11.json --markdown docs/public/accuracy-report-2026-11.md"
 ]
}