{
  "docId": "019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "docSlug": "5df6bd1b0447b5f6",
  "documentTitle": "2025 Air Street Capital The State of AI Report 2025",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 21,
  "pageCount": 313,
  "prevPage": 20,
  "nextPage": 22,
  "slideType": "industry_trends",
  "function": "diagnose",
  "density": "overcrowded",
  "nDataPoints": 6,
  "notes": "The slide presents a debate regarding the performance of reasoning models (LRMs) versus standard LLMs across different complexity levels.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "line_chart"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0d0f98caffe1/21",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1#slide-21",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "One widely discussed paper suggests that large reasoning models (LRMs) paradoxically give up on complex problems and only outperform standard models in a narrow complexity window.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-41ac-7180-a81a-c34597db6772",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.45,
        "x": 0.52,
        "y": 0.38
      },
      "kind": "chart",
      "text": "Pass@k performance of thinking models (Claude 3.7 Sonnet with extended thinking, DeepSeek-R1) versus their non-thinking counterparts across equivalent inference compute budgets in puzzle environments of low, medium, and high complexity.",
      "attrs": null,
      "subkind": "line",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "431b691e-27fb-4b8b-b56c-4a8e72bc93b3",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.45,
        "x": 0.04,
        "y": 0.35
      },
      "kind": "list",
      "text": "The paper shows that LRMs exhibit a surprising defeatist behavior: they reason more as problems get harder but then give up entirely on very complex tasks, and are outperformed by LLMs on simple tasks.\nDespite generating reasoning traces, the authors claim LRMs fail to use explicitly given algorithms and reason inconsistently across different difficulty levels.\nHowever, critics found these results stem from flawed experimental design: the supposed \"accuracy collapse\" occurred when models hit token limits or were asked to solve mathematically impossible puzzles, not from actual reasoning failures.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "802e8dc5-eb44-4de3-a1eb-9e12925f82cf",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "pass@k",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-41ac-7180-a81a-c5fc57824368",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.45,
        "x": 0.04,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "One widely discussed paper suggests that large reasoning models (LRMs) paradoxically give up on complex problems and only outperform standard models in a narrow complexity window. However, critics argue these results stem from flawed experimental design rather than genuine reasoning failures.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "32cf6fe3-331a-43ad-ba63-50854d2f4c08",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.08,
        "w": 0.45,
        "x": 0.52,
        "y": 0.78
      },
      "kind": "source-note",
      "text": "Figure 4: Pass@k performance of thinking models (Claude 3.7 Sonnet with extended thinking, DeepSeek-R1) versus their non-thinking counterparts (Claude 3.7 Sonnet, DeepSeek-V3) across equivalent inference compute budgets in puzzle environments of low, medium, and high complexity.",
      "attrs": null,
      "subkind": null,
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "7117c2dd-ad7b-452a-99e1-347d137542c9",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.25,
        "x": 0.04,
        "y": 0.14
      },
      "kind": "title",
      "text": "How far have we come?",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "6f3a2bcb-06e9-4479-a1dc-0bff76d48ac4",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 189,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-9f380673831f",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Sections 1-2 lay out research findings and industry data with charts and case studies.",
      "position": 1,
      "confidence": 70,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 89,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-ac68557d958b",
      "arcName": "The Onion",
      "arcSlug": "onion",
      "beatName": "First Layer",
      "beatSlug": "onion-first-layer",
      "evidence": "Research section unpacks technical layer beneath the headlines.",
      "position": 2,
      "confidence": 45,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 30,
      "from": 13,
      "name": "Zoom In",
      "slug": "06-zoom-in",
      "bestFor": "Technical deep-dives, case studies, detailed analysis",
      "matchId": "019dd95a-07fe-70ce-8d3e-9252f6f92bd7",
      "evidence": "Big picture 'reasoning race' (p.13) zooms into o1, R1, DeepSeek, parallel reasoning, CoT mechanics.",
      "position": 2,
      "objective": "Drill from reasoning paradigm into specific reasoning research findings",
      "structure": "The Big Picture -> Key Area of Focus -> Specific Detail -> Implication",
      "confidence": 75,
      "description": "Start broad, then progressively focus on specific details that prove your point"
    },
    {
      "to": 23,
      "from": 20,
      "name": "Myth Buster",
      "slug": "12-myth-buster",
      "bestFor": "Rebranding, changing market perception, correcting false assumptions",
      "matchId": "019dd95a-07fe-70ce-8d3e-94ced6245fc4",
      "evidence": "'The illusion of reasoning gains' (p.20), 'How reasoning breaks' (pp.22-23) directly challenge consensus.",
      "position": 3,
      "objective": "Bust the 'reasoning gains are real' belief",
      "structure": "The Common Belief -> The Friction/Failure of that Belief -> The New Truth",
      "confidence": 80,
      "description": "Address a common misconception head-on to clear the room for a new truth"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}