{
  "docId": "019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "docSlug": "cc8183ec02431b7a",
  "documentTitle": "2022 Air Street Capital The State of AI Report 2022",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 25,
  "pageCount": 114,
  "prevPage": 24,
  "nextPage": 26,
  "slideType": "industry_trends",
  "function": "summarize",
  "density": "dense",
  "nDataPoints": 5,
  "notes": "The slide includes three small scatter plots showing different benchmark saturation dynamics.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "line_chart"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc/25",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc#slide-25",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Only 66% of machine learning benchmarks have received more than 3 results at different time points, and many are solved or saturated soon after their release.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a216-6a4fc6a7047e",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.25,
        "w": 0.46,
        "x": 0.52,
        "y": 0.38
      },
      "kind": "chart",
      "text": "Three scatter plots showing benchmark saturation dynamics: continuous growth, saturation/stagnation, and stagnation followed by burst.",
      "attrs": null,
      "subkind": "scatter",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "3dc05e48-2fc0-45d7-b61b-6b0feea8635d",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.96,
        "x": 0.02,
        "y": 0.21
      },
      "kind": "list",
      "text": "Only 66% of machine learning benchmarks have received more than 3 results at different time points, and many are solved or saturated soon after their release. BIG (Beyond the Imitation Game), a new benchmark designed by 444 authors across 132 institutions, aims to challenge current and future language models.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "14575353-1553-42c4-97cd-28fe8771a0d6",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.5,
        "x": 0.02,
        "y": 0.38
      },
      "kind": "list",
      "text": "A study from the University of Vienna, Oxford, and FHI examined 1,688 benchmarks for 406 AI tasks and identified different submission dynamics (see right). They note that language benchmarks in particular tend to be quickly saturated. Rapid LLM progress and emerging capabilities seem to outrun current benchmarks. As a result, much of this progress is only captured through circumstantial evidence like demos or one-off breakthroughs, and/or evaluated on disparate dedicated benchmarks, making it difficult to identify actual progress. The new BIG benchmark contains 204 tasks, all with strong human expert baselines, which evaluate a large set of LLM capabilities from memorization to multi-step reasoning. They show that, for now, even the best models perform poorly on the BIG benchmark.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "17ba4bbb-e444-4d25-8283-902de1727da2",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Benchmark saturation: 66%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a216-6d09b5de4884",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.96,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Fast progress in LLM research renders benchmarks obsolete, but a BIG one comes to help",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "20d307fd-fed2-411c-b14c-4c3f268736d3",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 80,
      "from": 5,
      "beatId": "019dd95a-0682-776c-8e35-2557e3799e96",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Definitions, exec summary, Research and Industry sections inventory the state of AI.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 93,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-35abf646aa8b",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Research, Industry, Politics sections accumulate signals.",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 29,
      "from": 22,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-a20c665fe3de",
      "evidence": "Codex, Transformer alternatives, Minerva, BIG-Bench, Chinchilla, emergence, tool-use, compute eras stack as evidence.",
      "position": 3,
      "objective": "Survey LLM research frontier — transformers, scaling, emergence, compute eras",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 78,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}