{
  "docId": "019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "docSlug": "cc8183ec02431b7a",
  "documentTitle": "2022 Air Street Capital The State of AI Report 2022",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 24,
  "pageCount": 114,
  "prevPage": 23,
  "nextPage": 25,
  "slideType": "key_takeaways",
  "function": "summarize",
  "density": "overcrowded",
  "nDataPoints": 7,
  "notes": "Includes a specific example of a math problem solved by Minerva.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "screenshot"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc/24",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc#slide-24",
  "components": [
    {
      "bbox": {
        "h": 0.12,
        "w": 0.96,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "callout",
      "text": "Built on Google's 540B parameter LM PaLM, Google's Minerva achieves a 50.3% score on the MATH benchmark (43.4 pct points better than previous SOTA), beating forecasters expectations for best score in 2022 (13%). Meanwhile, OpenAI trained a network to solve two mathematical olympiad problems (IMO).",
      "attrs": null,
      "subkind": "primary",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "8be02ae8-6969-4a4b-a584-56f3459faa0e",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "callout",
      "text": "Built on Google's 540B parameter LM PaLM, Google's Minerva achieves a 50.3% score on the MATH benchmark (43.4 pct points better than previous SOTA), beating forecasters expectations for best score in 2022 (13%).",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a216-2ea95c9e3071",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.52,
        "w": 0.24,
        "x": 0.74,
        "y": 0.36
      },
      "kind": "image",
      "text": "Question: Assume that the variance of the first n natural numbers is 10, and the variance of the first m even natural numbers is 16. Compute m + n. Model output: ...",
      "attrs": null,
      "subkind": "infographic",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "d4a8e1fe-ba81-461b-96f5-033b969740be",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.2,
        "w": 0.7,
        "x": 0.02,
        "y": 0.36
      },
      "kind": "list",
      "text": "Google trained its (pre-trained) LLM PaLM on an additional 118GB dataset of scientific papers from arXiv and web pages using LaTeX and MathJax. Using chain of thought prompting (including intermediate reasoning steps in prompts rather than the final answer only) and other techniques like majority voting, Minerva improves the SOTA on most datasets by at least double digit pct points.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "2cc5dff9-f58e-4248-ab0d-405e0f5e071a",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.7,
        "x": 0.02,
        "y": 0.58
      },
      "kind": "list",
      "text": "Minerva only uses a language model and doesn't explicitly encode formal mathematics. It is more flexible but can only be automatically evaluated on its final answer rather than its whole reasoning, which might justify some score inflation.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "31eac231-7033-463c-ab55-085b0d6cd95c",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.7,
        "x": 0.02,
        "y": 0.72
      },
      "kind": "list",
      "text": "In contrast, OpenAI built a (transformer-based) theorem prover built in the Lean formal environment. Different versions of their model were able to solve a number of problems from AMC12 (26), AIME (6) and IMO (2) (increasing order of difficulty).",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "3d90ec0e-495f-4f23-b925-6bae44e21cae",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "MATH benchmark score: 50.3%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a216-3140fa343f46",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.6,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Mathematical abilities of Language Models largely surpass expectations",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "6bb33a8a-f523-4edf-b301-4a3b9bd4ff2e",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Von Restorff Effect",
      "slug": "von-restorff-effect",
      "agent": "Designer",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a955-d095adb494ae",
      "evidence": "50.3% MATH score highlighted as breakout against expectations.",
      "confidence": 75
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 80,
      "from": 5,
      "beatId": "019dd95a-0682-776c-8e35-2557e3799e96",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Definitions, exec summary, Research and Industry sections inventory the state of AI.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 93,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-35abf646aa8b",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Research, Industry, Politics sections accumulate signals.",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 29,
      "from": 22,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-a20c665fe3de",
      "evidence": "Codex, Transformer alternatives, Minerva, BIG-Bench, Chinchilla, emergence, tool-use, compute eras stack as evidence.",
      "position": 3,
      "objective": "Survey LLM research frontier — transformers, scaling, emergence, compute eras",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 78,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}