{
  "docId": "019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "docSlug": "bf350dd574c19997",
  "documentTitle": "2021 Air Street Capital The State of AI Report 2021",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 42,
  "pageCount": 188,
  "prevPage": 41,
  "nextPage": 43,
  "slideType": "problem_statement",
  "function": "diagnose",
  "density": "dense",
  "nDataPoints": 9,
  "notes": "Includes a visual example of a coding problem (H-Index) and a bar chart comparing model performance across difficulty levels.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "bar_chart_grouped",
    "other"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5c-fd5012384ae3/42",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3#slide-42",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Codex vastly outperforms the other language models on APPS problems, but all models achieve low scores, especially on intermediate and hard level problems (well below 5% accuracy).",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d27-e85973e5dc66",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.3,
        "w": 0.4,
        "x": 0.55,
        "y": 0.65
      },
      "kind": "chart",
      "text": "Model Test Case Averages by Problem Difficulty",
      "attrs": null,
      "subkind": "bar-grouped",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "ec548caa-b7a6-4aab-bd46-3d7d4149391f",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.35,
        "w": 0.4,
        "x": 0.55,
        "y": 0.25
      },
      "kind": "image",
      "text": "Example of H-Index problem, generated code, and test cases.",
      "attrs": null,
      "subkind": "infographic",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "8e0b4a45-17f9-4051-89e0-9f6fe0a78271",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.6,
        "w": 0.5,
        "x": 0.03,
        "y": 0.22
      },
      "kind": "list",
      "text": "Code generation models can generate snippets of code, but they struggle to generate entire programs.\nIn coding challenges, participants are required to write programs that solve problems which are described in natural language.\nAPPS, a benchmark of 10,000 coding questions, tests how well code generation models can solve these challenges. Generated code is then tested using human-written test cases.\nGPT-2 and GPT-Neo, two general-purpose language models, are fine-tuned on Github and APPS training data, while OpenAI's Codex, which is trained on code, is used without further fine-tuning.\nCodex vastly outperforms the other language models on APPS problems, but all models achieve low scores, especially on intermediate and hard level problems (well below 5% accuracy).",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "6aec4128-44c5-435a-b7b1-fd2ac73df560",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Test Case Average (%)",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d27-eee5c33500aa",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.6,
        "x": 0.03,
        "y": 0.13
      },
      "kind": "title",
      "text": "Yet, code generation models still cannot crack the coding interview",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "c5cffff2-02a9-4b0e-b429-0e9e7e1cc5a9",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-0fd5-7148-8ed1-0a4485d7cff1",
      "evidence": "Title 'still cannot crack the coding interview' states the limit.",
      "confidence": 80
    },
    {
      "name": "Problem Statement Canvas",
      "slug": "problem-statement-canvas",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "ff72f156-56ef-48d3-95e8-fe5a6baf03b5",
      "evidence": "The slide clearly states a problem and provides context, which matches the Problem Statement Canvas tool.",
      "confidence": 0.7
    },
    {
      "name": "Unexpected Pattern",
      "slug": "unexpected-pattern",
      "agent": "Storyteller",
      "layer": "slide",
      "matchId": "019dd95a-0fd5-7148-8ed1-0fb44f4f622a",
      "evidence": "Counter to the Codex hype slide before.",
      "confidence": 65
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 152,
      "from": 5,
      "beatId": "019dd95a-0682-776c-8e35-0affbadec38e",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Definitions p.5-6, Exec Summary p.7, then Sections 1-3 catalog evidence.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 152,
      "from": 10,
      "beatId": "019dd95a-0682-776c-8e35-1b3ee3a1c012",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Three sections accumulating evidence of accelerating progress.",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 51,
      "from": 41,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-576c8c35c986",
      "evidence": "Codex, math limits, truthfulness, prompt engineering, SuperGLUE, multilingual LLMs.",
      "position": 6,
      "objective": "Trace LLM frontier and the prompting paradigm",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 78,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}