{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 13,
  "pageCount": 213,
  "prevPage": 12,
  "nextPage": 14,
  "slideType": "industry_trends",
  "function": "analyze_data",
  "density": "overcrowded",
  "nDataPoints": 35,
  "notes": "The slide highlights the convergence of performance between GPT-4, Claude 3.5 Sonnet, Gemini 1.5, and Grok 2.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "data_table"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/13",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-13",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "For much of the year, both benchmarks and community leaderboards pointed to a chasm between GPT-4 and 'the best of the rest'. However, Claude 3.5 Sonnet, Gemini 1.5, and Grok 2 have all but eliminated this gap as model performance now begin to converge.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d0e-4361c7b01c62",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.28,
        "x": 0.7,
        "y": 0.45
      },
      "kind": "chart",
      "text": "Claude 3.5 Sonnet benchmarks table",
      "attrs": null,
      "subkind": "data",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "8663ef73-39d4-488b-a306-1c4dac964577",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.65,
        "x": 0.03,
        "y": 0.35
      },
      "kind": "list",
      "text": "On both formal benchmarks and vibes-based analysis, the best-funded frontier labs are able to rack up scores within low single digits of each other on individual capabilities.\nModels are now consistently highly capable coders, are strong at factual recall and math, but less good at open-ended question-answering and multi-modal problem solving.\nMany of the variations are sufficiently small that they are now likely to be the product of differences in implementation. For example, GPT-4o outperforms Claude 3.5 Sonnet on MMLU, but apparently underperforms it on MMLU-Pro - a benchmark designed to be more challenging.\nConsidering the relatively subtle technical differences between architectures and likely heavy overlaps in pre-training data, model builders are now increasingly having to compete on new capabilities and product features.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "34cd26a2-1939-41c4-b223-63379097c0d7",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Benchmark score",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d0e-44d6c5c7615d",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.65,
        "x": 0.03,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "For much of the year, both benchmarks and community leaderboards pointed to a chasm between GPT-4 and 'the best of the rest'. However, Claude 3.5 Sonnet, Gemini 1.5, and Grok 2 have all but eliminated this gap as model performance now begin to converge.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "69ba7f30-43f9-4467-9f52-821bcd01ffad",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.03,
        "w": 0.15,
        "x": 0.82,
        "y": 0.95
      },
      "kind": "source-note",
      "text": "state.ai 2024",
      "attrs": null,
      "subkind": null,
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "d347c409-e278-44ab-9db4-909582eaad12",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.5,
        "x": 0.03,
        "y": 0.13
      },
      "kind": "title",
      "text": "OpenAI's reign of terror came to an end, until...",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "bbcfeda8-cf8f-4aad-8d5a-9292a49c9cbf",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Inductive Reasoning",
      "slug": "inductive-reasoning",
      "agent": "Architect",
      "layer": "loop",
      "matchId": "019dd95a-1055-74e0-a958-509fb98f9198",
      "evidence": "Opens evidence-stacking loop on frontier convergence",
      "confidence": 70
    },
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a958-4ffb4799a4b6",
      "evidence": "Title 'OpenAI's reign of terror came to an end, until...' states the insight",
      "confidence": 88
    },
    {
      "name": "Unexpected Pattern",
      "slug": "unexpected-pattern",
      "agent": "Storyteller",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a958-54bd559bc8d6",
      "evidence": "'until...' signals pattern reversal",
      "confidence": 65
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 153,
      "from": 9,
      "beatId": "019dd95a-0682-776c-8e35-5f2398b8d1d0",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Exec summary + Research + Industry sections inventory the year",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 85,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-6d9d8ee06d0a",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Unknown",
      "beatSlug": "voyage-return-the-unknown",
      "evidence": "Research section explores frontier model uncharted territory",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Complication",
      "parentBeatSlug": "complication"
    }
  ],
  "loops": [
    {
      "to": 20,
      "from": 13,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-27658cc21e8b",
      "evidence": "Series of analyze_data slides on GPT-4, o1, Claude 3.5, Llama 3, openness, contamination, leaderboards",
      "position": 2,
      "objective": "Show frontier model gap is closing through multiple data points",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 78,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}