{
  "docId": "019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "docSlug": "5df6bd1b0447b5f6",
  "documentTitle": "2025 Air Street Capital The State of AI Report 2025",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 24,
  "pageCount": 313,
  "prevPage": 23,
  "nextPage": 25,
  "slideType": "key_takeaways",
  "function": "summarize",
  "density": "dense",
  "nDataPoints": 3,
  "notes": "The slide highlights that CoT utility stems from structural encoding of reasoning rather than literal faithfulness.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "scatter_plot"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0d0f98caffe1/24",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1#slide-24",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "In red-teaming evaluations of frontier models, CoT-based monitors flagged ~99% of reward-hacking attempts (only 3 failures out of 21,000 completions).",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-41ac-7180-a81b-fd7593d96217",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.47,
        "x": 0.51,
        "y": 0.33
      },
      "kind": "chart",
      "text": "Clue-Usage Detector Sensitivity of Various LLMs",
      "attrs": null,
      "subkind": "scatter",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "402554f6-f214-4c6a-9dc5-68d28da95bde",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.45,
        "x": 0.02,
        "y": 0.32
      },
      "kind": "list",
      "text": "In red-teaming evaluations of frontier models, CoT-based monitors flagged ~99% of reward-hacking attempts (only 3 failures out of 21,000 completions). This held true even when models were prompted to conceal intent, suggesting that CoT traces still reveal useful signals, even if they aren't perfectly faithful. CoTs work not because models “mean what they say”, but because reasoning traces encode useful structure for detecting hacks, especially in maths and logic. As such, CoT traces offer a surprisingly reliable window into model behavior, even under adversarial conditions.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "c1f6cb82-bd52-49c7-b765-9d07e83cafef",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Detector Sensitivity: 99%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-41ac-7180-a81c-03f89e612634",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.08,
        "w": 0.45,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "New findings from METR show that even when models generate unfaithful or misleading reasoning traces, CoT is still highly diagnostic for oversight.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "b23f2c2d-dab8-4f48-a947-8b51886e3998",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.45,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "CoTs still work, even when models aren't honest...",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "f027ecd8-c524-4b9f-99b7-1fc10cab5590",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Concrete Language",
      "slug": "concrete-language",
      "agent": "Storyteller",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a95b-ff6ddc34cfd2",
      "evidence": "99% catch rate from 21,000 completions makes claim concrete.",
      "confidence": 85
    },
    {
      "name": "Waterfall chart",
      "slug": "waterfall-chart",
      "agent": null,
      "layer": "slide",
      "matchId": "a74c8169-5f94-4550-8944-0322da1ed0b5",
      "evidence": "chart/scatter: Clue-Usage Detector Sensitivity of Various LLMs",
      "confidence": 0.7
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 189,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-9f380673831f",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Sections 1-2 lay out research findings and industry data with charts and case studies.",
      "position": 1,
      "confidence": 70,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 89,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-ac68557d958b",
      "arcName": "The Onion",
      "arcSlug": "onion",
      "beatName": "First Layer",
      "beatSlug": "onion-first-layer",
      "evidence": "Research section unpacks technical layer beneath the headlines.",
      "position": 2,
      "confidence": 45,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 30,
      "from": 13,
      "name": "Zoom In",
      "slug": "06-zoom-in",
      "bestFor": "Technical deep-dives, case studies, detailed analysis",
      "matchId": "019dd95a-07fe-70ce-8d3e-9252f6f92bd7",
      "evidence": "Big picture 'reasoning race' (p.13) zooms into o1, R1, DeepSeek, parallel reasoning, CoT mechanics.",
      "position": 2,
      "objective": "Drill from reasoning paradigm into specific reasoning research findings",
      "structure": "The Big Picture -> Key Area of Focus -> Specific Detail -> Implication",
      "confidence": 75,
      "description": "Start broad, then progressively focus on specific details that prove your point"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}