{
  "docId": "019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "docSlug": "cc8183ec02431b7a",
  "documentTitle": "2022 Air Street Capital The State of AI Report 2022",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 26,
  "pageCount": 114,
  "prevPage": 25,
  "nextPage": 27,
  "slideType": "industry_trends",
  "function": "analyze_data",
  "density": "dense",
  "nDataPoints": 4,
  "notes": "The chart illustrates the relationship between model parameters and compute (FLOPs), comparing DeepMind's findings with previous OpenAI scaling laws.",
  "elementsJson": [
    "headline_text",
    "action_title",
    "bullet_list",
    "line_chart"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc/26",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc#slide-26",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "DeepMind revisited LM scaling laws and found that current LMs are significantly undertrained: they're not trained on enough data given their large size.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a216-c62e2f6a02aa",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.35,
        "w": 0.35,
        "x": 0.6,
        "y": 0.38
      },
      "kind": "chart",
      "text": "Chart showing Parameters vs FLOPs for various models and scaling approaches.",
      "attrs": null,
      "subkind": "scatter",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "6f865b5b-8839-43ce-ab45-2b69633ece3f",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.5,
        "w": 0.55,
        "x": 0.03,
        "y": 0.38
      },
      "kind": "list",
      "text": "Empirical LM scaling laws determine, for a fixed compute budget, the model and training data sizes that should be used. Past work from OpenAI had established that model size should increase faster than training data size as the compute budget increases.\nDeepMind claims that the model size and the number of training tokens should instead increase at roughly the same rate.\nCompared to OpenAI's work, DeepMind uses larger models to derive their scaling laws. They emphasize that data scaling leads to better predictions from multibillion parameter models.\nFollowing these new scaling laws, Chinchilla (70B params) is trained on 1.4T tokens. Gopher (230B) on 300B.\nThough trained with the same compute budget, the lighter Chinchilla should be faster to run.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "56b7b68b-eaaf-4094-bb59-c9a1184025a2",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Parameters vs FLOPs",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a216-c9e70286c894",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.55,
        "x": 0.03,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "DeepMind revisited LM scaling laws and found that current LMs are significantly undertrained: they're not trained on enough data given their large size. They train Chinchilla, a 4x smaller version of their Gopher, on 4.6x more data, and find that Chinchilla outperforms Gopher and other large models on BIG-bench.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "49c8827b-6aeb-48ca-9bea-769551d5975c",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.5,
        "x": 0.03,
        "y": 0.14
      },
      "kind": "title",
      "text": "Ducking language model scaling laws: more data please",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "5ca3b7f5-024b-41f9-8f12-26eb09f7f763",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Unexpected Pattern",
      "slug": "unexpected-pattern",
      "agent": "Storyteller",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a955-d562f0aa84b6",
      "evidence": "Reveals current LMs are 'undertrained', breaking received scaling-law assumption.",
      "confidence": 75
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 80,
      "from": 5,
      "beatId": "019dd95a-0682-776c-8e35-2557e3799e96",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Definitions, exec summary, Research and Industry sections inventory the state of AI.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 93,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-35abf646aa8b",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Research, Industry, Politics sections accumulate signals.",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 29,
      "from": 22,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-a20c665fe3de",
      "evidence": "Codex, Transformer alternatives, Minerva, BIG-Bench, Chinchilla, emergence, tool-use, compute eras stack as evidence.",
      "position": 3,
      "objective": "Survey LLM research frontier — transformers, scaling, emergence, compute eras",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 78,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}