{
  "docId": "019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "docSlug": "5df6bd1b0447b5f6",
  "documentTitle": "2025 Air Street Capital The State of AI Report 2025",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 25,
  "pageCount": 313,
  "prevPage": 24,
  "nextPage": 26,
  "slideType": "diagnosis",
  "function": "diagnose",
  "density": "dense",
  "nDataPoints": 0,
  "notes": "References a specific research finding regarding model activations and test awareness.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "bar_chart_stacked"
  ],
  "metadataConfidence": 0.95,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0d0f98caffe1/25",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1#slide-25",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "This raises concerns that evaluations may be systematically manipulated to overestimate real-world safety: developers could reduce awareness and mask behavior changes, potentially letting models act differently under evaluation vs deployment.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-41ac-7180-a81b-e3a503c4826c",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.38,
        "w": 0.22,
        "x": 0.74,
        "y": 0.48
      },
      "kind": "chart",
      "text": "Fine-grained overview of how the inputs change after steering.",
      "attrs": null,
      "subkind": "bar-stacked",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "d75196d5-df8c-4720-bcca-16d938629c8c",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.65,
        "x": 0.04,
        "y": 0.48
      },
      "kind": "list",
      "text": "Using white-box probing, researchers identify a linear direction in a model’s internal activations that encodes “test awareness”.\nA model can be made more or less aware that it is in an evaluation. The more it is aware, the more its behavior appears aligned and cautious.\nThe effect shows up across multiple reasoning models and tasks and even on “fake” evaluations that look like tests but aren’t.\nThis raises concerns that evaluations may be systematically manipulated to overestimate real-world safety: developers could reduce awareness and mask behavior changes, potentially letting models act differently under evaluation vs deployment.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "c73b6380-acf1-47b6-805f-560c40b02761",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Proportion of behavior change",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-41ac-7180-a81b-e60692c814b8",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.25,
        "w": 0.95,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "Researchers find a single, steerable direction in model activations that encodes “test awareness.” By nudging the model along this direction, they can make it act more or less like it’s under evaluation. Turning awareness up increases refusals and safety-compliant behavior and turning it down does the opposite. This means reported safety can be inflated by evaluation setup rather than true robustness. This “test awareness” is akin to the Hawthorne effect, where humans change behavior when being observed and change their behavior accordingly.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "240dcbf5-f885-4573-b1d4-4c2d10303246",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.6,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "...but there are safety concerns, like the “AI Hawthorne effect”",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "362bedc1-ff2a-4850-afaa-d6a853a5cffa",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Audience Definition",
      "slug": "audience-definition",
      "agent": "Storyteller",
      "layer": "slide",
      "matchId": "8e3d44e2-e42f-4277-8d8b-857730b2eac8",
      "evidence": "list/bullet: Using white-box probing, researchers identify a linear direction in a model’s internal activations that encodes “test awareness”.",
      "confidence": 0.6
    }
  ],
  "frameworks": [
    {
      "name": "Hawthorne effect",
      "slug": null,
      "matchId": "aa55e0e5-9348-414b-9347-85cfdf72fd07",
      "evidence": "Explicitly mentioned in title and body text as the core analogy.",
      "confidence": 1
    }
  ],
  "arcBeats": [
    {
      "to": 189,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-9f380673831f",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Sections 1-2 lay out research findings and industry data with charts and case studies.",
      "position": 1,
      "confidence": 70,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 89,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-ac68557d958b",
      "arcName": "The Onion",
      "arcSlug": "onion",
      "beatName": "First Layer",
      "beatSlug": "onion-first-layer",
      "evidence": "Research section unpacks technical layer beneath the headlines.",
      "position": 2,
      "confidence": 45,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 30,
      "from": 13,
      "name": "Zoom In",
      "slug": "06-zoom-in",
      "bestFor": "Technical deep-dives, case studies, detailed analysis",
      "matchId": "019dd95a-07fe-70ce-8d3e-9252f6f92bd7",
      "evidence": "Big picture 'reasoning race' (p.13) zooms into o1, R1, DeepSeek, parallel reasoning, CoT mechanics.",
      "position": 2,
      "objective": "Drill from reasoning paradigm into specific reasoning research findings",
      "structure": "The Big Picture -> Key Area of Focus -> Specific Detail -> Implication",
      "confidence": 75,
      "description": "Start broad, then progressively focus on specific details that prove your point"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}