{
  "docId": "019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "docSlug": "5df6bd1b0447b5f6",
  "documentTitle": "2025 Air Street Capital The State of AI Report 2025",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 257,
  "pageCount": 313,
  "prevPage": 256,
  "nextPage": 258,
  "slideType": "diagnosis",
  "function": "diagnose",
  "density": "dense",
  "nDataPoints": 5,
  "notes": "The slide discusses a specific research method for detecting hallucinations in LLMs using linear probes.",
  "elementsJson": [
    "headline_text",
    "action_title",
    "paragraph",
    "bullet_list",
    "screenshot"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0d0f98caffe1/257",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1#slide-257",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "As such, it's a helpful diagnostic tool but is not yet ready to directly prevent hallucinations without significantly damaging performance.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d06-cea3964e6c7f",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.38,
        "x": 0.6,
        "y": 0.4
      },
      "kind": "image",
      "text": "Visual example of token-level hallucination detection on a text response about Riley v. California.",
      "attrs": null,
      "subkind": "screenshot",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "fc966d40-5db0-48e4-a02d-9312ea2c3309",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.35,
        "w": 0.55,
        "x": 0.02,
        "y": 0.42
      },
      "kind": "list",
      "text": "The probes detect fabricated names/dates/citations in long-form text with ~70% recall at 10% false positive rate, and generalizes to mathematical reasoning (0.87 AUC) despite only being trained on factual entities.\nProbes trained on one model detect hallucinations in others' outputs (only 2-4% AUC drop), but selective answering experiments show you must sacrifice ~50% of correct answers to meaningfully reduce hallucinations. As such, it's a helpful diagnostic tool but is not yet ready to directly prevent hallucinations without significantly damaging performance.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "1baac613-1046-478a-b0b4-f0087ebc7022",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "AUC: 0.87",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d06-d1eeeddcc229",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.18,
        "w": 0.55,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "Token-level hallucination detection is far more helpful than broad hallucination classification of overall responses (consider a response that says “The Eiffel Tower is in Paris and is made of rubber”). Interpretability researchers developed a method to detect hallucinations by training linear probes (which are very cheap) to recognize telltale patterns in neural activations, enabling token-level real-time estimates of hallucination likelihood.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "20376b24-2fec-4f96-8ec8-d53fa52dda10",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.6,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Until hallucinations disappear, can we detect them in real time?",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "f678eeea-8499-4044-a033-fb8c529299b4",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Diagnostic test",
      "slug": "diagnostic-test",
      "agent": null,
      "layer": "slide",
      "matchId": "6518e343-b033-4711-b418-cccb12ef121c",
      "evidence": "Token-level hallucination detection is far more helpful than broad hallucination classification of overall responses",
      "confidence": 0.8
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 282,
      "from": 190,
      "beatId": "019dd95a-0682-776c-8e35-a256fe16ea9c",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Implications (So What)",
      "beatSlug": "triple-take-the-implications-so-what",
      "evidence": "Sections 3-4 frame political and safety consequences of the technical/industry trends.",
      "position": 2,
      "confidence": 70,
      "parentBeatName": "Reflection",
      "parentBeatSlug": "reflection"
    },
    {
      "to": 282,
      "from": 190,
      "beatId": "019dd95a-0682-776c-8e35-b713af60bf88",
      "arcName": "The Onion",
      "arcSlug": "onion",
      "beatName": "Core Insight",
      "beatSlug": "onion-core-insight",
      "evidence": "Politics and Safety reveal the structural risks at the core.",
      "position": 4,
      "confidence": 45,
      "parentBeatName": "Turn",
      "parentBeatSlug": "turn"
    }
  ],
  "loops": [
    {
      "to": 282,
      "from": 247,
      "name": "Pre Mortem",
      "slug": "19-pre-mortem",
      "bestFor": "Project kick-offs, risk management, building trust with skeptical stakeholders",
      "matchId": "019dd95a-07fe-70ce-8d3e-bb4d43115ab4",
      "evidence": "Vibe hacking, AI psychosis, alignment faking, scheming, prompt injection — what could go wrong + mitigations.",
      "position": 12,
      "objective": "Surface AI safety failure modes before they become catastrophic",
      "structure": "The Future Disaster (Hypothetical) -> What Went Wrong -> The Prevention Plan",
      "confidence": 75,
      "description": "Build trust by anticipating failure modes before they happen and solving them in advance"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}