{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 16,
  "pageCount": 213,
  "prevPage": 15,
  "nextPage": 17,
  "slideType": "industry_trends",
  "function": "summarize",
  "density": "dense",
  "nDataPoints": 12,
  "notes": "Includes a stacked bar chart showing human evaluation win/tie/loss rates against GPT-4o and Claude 3.5 Sonnet.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "bar_chart_stacked",
    "logo_grid"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/16",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-16",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "This marks the first time an open model has closed the gap with the proprietary frontier.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d0e-9cec975ed4ac",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.3,
        "w": 0.4,
        "x": 0.58,
        "y": 0.6
      },
      "kind": "chart",
      "text": "Llama 3.1 405B Human Evaluation",
      "attrs": null,
      "subkind": "bar-stacked-100pct",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "46fb124a-90f8-4c02-a424-11ce64c5448b",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.7,
        "w": 0.55,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "list",
      "text": "In April, Meta dropped the Llama 3 family, 3.1 in July, and 3.2 in September. Llama 3.1 405B, their largest to-date, is able to hold its own against GPT-4o and Claude 3.5 Sonnet across reasoning, math, multilingual, and long-context tasks. This marks the first time an open model has closed the gap with the proprietary frontier.\nMeta stuck to the same decoder-only transformer architecture that it's used since Llama 1, with minor adaptations, namely more transformer layers and attention heads.\nMeta used an incredible 15T tokens to train the family. While this blew through the \"Chinchilla-optimal\" amount of training compute, they found that both the 8B and 70B models improved log-linearly up to 15T.\nLlama 3.1 405B was trained over 16,000 H100 GPUs, the first Llama model trained at this scale.\nMeta followed up with Llama 3.2 in September, which incorporated 11B and 90B VLMs (Llama's multimodal debut). The former was competitive with Claude 3 Haiku, the latter with GPT-4o-mini. The company also released 1B and 3B text-only models, designed to operate on-device.\nLlama-based models have now racked up over 440M downloads on Hugging Face.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "e5b42cd2-ed7c-4a6f-9236-4826bf30df52",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Human Evaluation Win Rate: 52.2%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d0e-a3014abeecf3",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.6,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Llama 3 closes the gap between open and closed models",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "2b052a5f-8a4e-4dfa-a481-ec1f85e00321",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a958-674c390980e8",
      "evidence": "Title 'Llama 3 closes the gap between open and closed models'",
      "confidence": 88
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 153,
      "from": 9,
      "beatId": "019dd95a-0682-776c-8e35-5f2398b8d1d0",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Exec summary + Research + Industry sections inventory the year",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 85,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-6d9d8ee06d0a",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Unknown",
      "beatSlug": "voyage-return-the-unknown",
      "evidence": "Research section explores frontier model uncharted territory",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Complication",
      "parentBeatSlug": "complication"
    }
  ],
  "loops": [
    {
      "to": 20,
      "from": 13,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-27658cc21e8b",
      "evidence": "Series of analyze_data slides on GPT-4, o1, Claude 3.5, Llama 3, openness, contamination, leaderboards",
      "position": 2,
      "objective": "Show frontier model gap is closing through multiple data points",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 78,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}