{
  "docId": "019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "docSlug": "cc8183ec02431b7a",
  "documentTitle": "2022 Air Street Capital The State of AI Report 2022",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 27,
  "pageCount": 114,
  "prevPage": 26,
  "nextPage": 28,
  "slideType": "industry_trends",
  "function": "analyze_data",
  "density": "dense",
  "nDataPoints": 4,
  "notes": "The slide discusses the non-linear performance gains in LLMs as they scale, using four specific task examples (Arithmetics, Fig. of speech, Multi-task NLU, Transliteration).",
  "elementsJson": [
    "headline_text",
    "paragraph",
    "bullet_list",
    "line_chart"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc/27",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc#slide-27",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "While model loss can be reasonably predicted as a function of size and compute using well-calibrated scaling laws, many LLM capabilities emerge unpredictably when models reach a critical size.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a216-0d802a237a5a",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.5,
        "w": 0.2,
        "x": 0.75,
        "y": 0.35
      },
      "kind": "chart",
      "text": "Arithmetics, Fig. of speech, Multi-task NLU, Transliteration performance vs Training FLOPs",
      "attrs": null,
      "subkind": "line",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "2ddd0837-6bc1-400e-bb55-f8e5378de385",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.5,
        "w": 0.7,
        "x": 0.05,
        "y": 0.38
      },
      "kind": "list",
      "text": "Emergence is not fully understood: it could be that for multi-step reasoning tasks, models need to be deeper to encode the reasoning steps. For memorization tasks, having more parameters is a natural solution. The metrics themselves may be part of the explanation, as an answer on a reasoning task is only considered correct if its conclusion is. Thus despite continuous improvements with model size, we only consider a model successful when increments accumulate past a certain point.\nA possible consequence of emergence is that there are a range of tasks that are out of reach of current LLMs that could soon be successfully tackled.\nAlternatively, deploying LLMs on real-world tasks at larger scales is more uncertain as unsafe and undesirable abilities can emerge. Alongside the brittle nature of ML models, this is another feature practitioners will need to account for.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "7b786410-ccf5-4c5c-972c-5717e82527cd",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Accuracy",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a216-1022a9c01e03",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.7,
        "x": 0.05,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "While model loss can be reasonably predicted as a function of size and compute using well-calibrated scaling laws, many LLM capabilities emerge unpredictably when models reach a critical size. These acquired capabilities are exciting, but the emergence phenomenon makes evaluating model safety more difficult.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "e0ea34de-c47c-498b-9b1a-72dd43c061eb",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.5,
        "x": 0.05,
        "y": 0.14
      },
      "kind": "title",
      "text": "Ducking language model scaling laws: emergence",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "1ae6c4b7-0d5a-45ab-ba7d-5a9c80946fe8",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Aha! Moment",
      "slug": "aha-moment",
      "agent": "Storyteller",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a955-dbd11622b6d5",
      "evidence": "'Capabilities emerge unpredictably when models reach critical size' — insight reveal.",
      "confidence": 75
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 80,
      "from": 5,
      "beatId": "019dd95a-0682-776c-8e35-2557e3799e96",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Definitions, exec summary, Research and Industry sections inventory the state of AI.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 93,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-35abf646aa8b",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Research, Industry, Politics sections accumulate signals.",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 29,
      "from": 22,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-a20c665fe3de",
      "evidence": "Codex, Transformer alternatives, Minerva, BIG-Bench, Chinchilla, emergence, tool-use, compute eras stack as evidence.",
      "position": 3,
      "objective": "Survey LLM research frontier — transformers, scaling, emergence, compute eras",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 78,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}