{
  "docId": "019dd923-5e88-73ef-bd5d-06b04d219fea",
  "docSlug": "dd91c78f6570bf29",
  "documentTitle": "2023 Air Street Capital The State of AI Report 2023",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 22,
  "pageCount": 163,
  "prevPage": 21,
  "nextPage": 23,
  "slideType": "diagnosis",
  "function": "diagnose",
  "density": "dense",
  "nDataPoints": 6,
  "notes": "The slide presents a critique of the 'emergent capabilities' phenomenon in LLMs, citing Stanford research that suggests these are artifacts of metric choice.",
  "elementsJson": [
    "headline_text",
    "action_title",
    "paragraph",
    "bullet_list",
    "line_chart"
  ],
  "metadataConfidence": 0.95,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-06b04d219fea/22",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea#slide-22",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Emergent capabilities might be merely artifacts of researchers' choice of evaluation metrics.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a223-0019f70242f1",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.4,
        "x": 0.55,
        "y": 0.45
      },
      "kind": "chart",
      "text": "Emergent Abilities charts (C, D) and No Emergent Abilities charts (A, B, E, F)",
      "attrs": null,
      "subkind": "line",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "b9c88e2e-5aea-4212-8bc4-21b0181e0476",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.4,
        "w": 0.45,
        "x": 0.03,
        "y": 0.45
      },
      "kind": "list",
      "text": "Stanford researchers found that emergent abilities appeared only under metrics that nonlinearly or discontinuously scale the model's per-token error rate.\nFor example, >92% of reported emergent abilities on BIG-Bench (a comprehensive LLM benchmark) appeared under one of two discontinuous metrics.\nThey test their hypotheses on new models and confirm that replacing nonlinear or discontinuous metrics with linear or continuous proxies results in continuous improvements, rather than emerging capabilities.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "83476b07-a791-4e93-955b-511d900f03a5",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Model accuracy: 92%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a223-06e4f55fac86",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.2,
        "w": 0.95,
        "x": 0.03,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "Scaling laws that researchers developed for all types of ML models generally predict a smooth decrease in a model's loss as a function of its parameter count and number of training tokens. In contrast, it has often been observed that some of the models' capabilities actually emerge unpredictably when a given (unpredictable) scale is surpassed. Some call this observation into question: Emergent capabilities might be merely artifacts of researchers' choice of evaluation metrics. Others are not convinced and offer counterarguments to the points below.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "a1665aad-7c48-4dfd-9942-9ee784b460b4",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.5,
        "x": 0.03,
        "y": 0.14
      },
      "kind": "title",
      "text": "Are emergent capabilities of language models a mirage?",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "f52f693e-f786-4fb3-839e-373f846b5d28",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Information Gap Theory",
      "slug": "information-gap-theory",
      "agent": "Storyteller",
      "layer": "loop",
      "matchId": "019dd95a-1055-74e0-a957-2dedf65afdf1",
      "evidence": "Opens loop with a question that demands resolution.",
      "confidence": 70
    },
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a957-282ba59fd542",
      "evidence": "Provocative question title ('Are emergent capabilities... a mirage?').",
      "confidence": 80
    }
  ],
  "frameworks": [
    {
      "name": "hypothesis-driven-structure",
      "slug": null,
      "matchId": "34108b60-2cc4-4685-adc7-69a137a50d9f",
      "evidence": "The slide presents a hypothesis (emergent capabilities are artifacts) and describes the testing/confirmation process.",
      "confidence": 0.8
    }
  ],
  "arcBeats": [
    {
      "to": 120,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-41afd44ef59f",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Research + Industry sections inventory model, compute, funding facts.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 120,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-523bfb7f96e6",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Escalating capabilities, compute concentration and capital flows.",
      "position": 2,
      "confidence": 45,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 31,
      "from": 22,
      "name": "Zoom In",
      "slug": "06-zoom-in",
      "bestFor": "Technical deep-dives, case studies, detailed analysis",
      "matchId": "019dd95a-07fe-70ce-8d3d-ea04ad01cb62",
      "evidence": "Slides drill from emergent-capability mirage into context-length, memory and data-exhaustion details.",
      "position": 3,
      "objective": "Zoom into the data and context-length bottleneck",
      "structure": "The Big Picture -> Key Area of Focus -> Specific Detail -> Implication",
      "confidence": 72,
      "description": "Start broad, then progressively focus on specific details that prove your point"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}