{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 187,
  "pageCount": 213,
  "prevPage": 186,
  "nextPage": 188,
  "slideType": "problem_statement",
  "function": "diagnose",
  "density": "balanced",
  "nDataPoints": 0,
  "notes": "Includes a technical diagram (Figure 1) illustrating the transformation of model outputs into accuracy metrics.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "process_diagram",
    "paragraph"
  ],
  "metadataConfidence": 0.95,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/187",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-187",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "This would appear to strengthen the argument that 'emergent capabilities' are the artificial product of poor metric construction, rather than real capability jumps.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d21-6b6bef2e67d4",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.75,
        "x": 0.22,
        "y": 0.78
      },
      "kind": "image",
      "text": "Figure 1: Multiple-choice benchmark accuracy computation process",
      "attrs": null,
      "subkind": "infographic",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "141f1ea5-89db-485d-9bf3-00505a532f03",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.95,
        "x": 0.03,
        "y": 0.22
      },
      "kind": "list",
      "text": "While there's a significant body of work on how pre-training performance scales, there's much less clarity on how downstreaming training does. A team of researchers have scrutinized the role of multiple-choice questions.\nThey argue that standard performance metrics like accuracy mask the clear scaling trends visible in raw model outputs, making capability prediction difficult. These metrics compress and distort the original probability data, obscuring subtle improvements that occur as models get larger.\nThis would appear to strengthen the argument that 'emergent capabilities' are the artificial product of poor metric construction, rather than real capability jumps.\nAs the metrics rely on comparing the correct choice against specific incorrect choices, the researchers argue that we need to understand how probabilities change for both correct and incorrect answers as scale increases.\nThis will also involve developing new evaluation techniques that preserve more of the raw probability information.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "ba8d0fba-198c-4006-87ba-08fb8bd19155",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.65,
        "x": 0.03,
        "y": 0.14
      },
      "kind": "title",
      "text": "Why is it so hard to predict the downstream capabilities of frontier models?",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "8a497f21-fd17-41e0-8497-673da420948c",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a95a-a72cc868a233",
      "evidence": "Title 'Why is it so hard to predict the downstream capabilities of frontier models?'",
      "confidence": 80
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 204,
      "from": 154,
      "beatId": "019dd95a-0682-776c-8e35-605159a069e6",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Implications (So What)",
      "beatSlug": "triple-take-the-implications-so-what",
      "evidence": "Politics + Safety sections explore consequences and risks",
      "position": 2,
      "confidence": 78,
      "parentBeatName": "Reflection",
      "parentBeatSlug": "reflection"
    },
    {
      "to": 204,
      "from": 175,
      "beatId": "019dd95a-0682-776c-8e35-74f05e198faf",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Return",
      "beatSlug": "voyage-return-the-return",
      "evidence": "Safety section returns to risks/governance debate",
      "position": 4,
      "confidence": 55,
      "parentBeatName": "Resolution",
      "parentBeatSlug": "resolution"
    }
  ],
  "loops": [
    {
      "to": 201,
      "from": 187,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-79a3afd2ac42",
      "evidence": "Emergent capabilities, sycophancy, DPO, hybrid RLHF, critiques, transparency, reward tampering, SAEs",
      "position": 23,
      "objective": "Catalogue alignment, RLHF, sycophancy and interpretability research",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 76,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}