{
  "docId": "019dd923-5e88-73ef-bd5d-06b04d219fea",
  "docSlug": "dd91c78f6570bf29",
  "documentTitle": "2023 Air Street Capital The State of AI Report 2023",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 36,
  "pageCount": 163,
  "prevPage": 35,
  "nextPage": 37,
  "slideType": "industry_trends",
  "function": "analyze_data",
  "density": "dense",
  "nDataPoints": 12,
  "notes": "The slide uses bar charts to compare accuracy, verbosity, and mismatch metrics for two model versions over time.",
  "elementsJson": [
    "headline_text",
    "paragraph",
    "bar_chart_vertical"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-06b04d219fea/36",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea#slide-36",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Downstream tasks are highly dependent on underlying LLM performance. However, changes to the same version of GPT models are not announced by OpenAI, despite them being continuously updated.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a227-01085ace983d",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.28,
        "w": 0.36,
        "x": 0.51,
        "y": 0.61
      },
      "kind": "chart",
      "text": "GPT-3.5 performance metrics (Accuracy, Verbosity, Mismatch) for March vs June 2023",
      "attrs": null,
      "subkind": "bar-grouped",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "9f0e76d6-090a-474d-b50e-0ef71390624e",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.28,
        "w": 0.36,
        "x": 0.13,
        "y": 0.61
      },
      "kind": "chart",
      "text": "GPT-4 performance metrics (Accuracy, Verbosity, Mismatch) for March vs June 2023",
      "attrs": null,
      "subkind": "bar-grouped",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "fcf12272-a547-47f9-817d-ae5843a31a6e",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Accuracy: 84.0%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a227-05067c41c404",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.1,
        "w": 0.96,
        "x": 0.02,
        "y": 0.45
      },
      "kind": "paragraph",
      "text": "How is ChatGPT's Behaviour Changing over Time? report shows that March 2023 and June 2023 versions of GPT3.5 and GPT4 varied in performance on tasks like math questions (figure below), sensitive questions, opinion surveys, knowledge questions, generating code, US Medical License tests and visual reasoning.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "7a947696-2c2a-4ffa-a626-e42fbd0fa4fe",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.96,
        "x": 0.02,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "Downstream tasks are highly dependent on underlying LLM performance. However, changes to the same version of GPT models are not announced by OpenAI, despite them being continuously updated. The same LLM version has been reported by users to have drastically different performance over time. Everyone had to continuously monitor performance as well as update carefully curated prompts.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "c19c9205-05ac-4be5-b8e9-28d3f73f1776",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.35,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Prompt engineering trial and error",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "37d66dd7-7807-47cf-ba22-f21583199fe3",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Data Story Arc",
      "slug": "data-story-arc",
      "agent": "Storyteller",
      "layer": "slide",
      "matchId": "a8077c66-1364-402f-9f05-a294be6d950c",
      "evidence": "Downstream tasks are highly dependent on underlying LLM performance.",
      "confidence": 0.6
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 120,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-41afd44ef59f",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Research + Industry sections inventory model, compute, funding facts.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 120,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-523bfb7f96e6",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Escalating capabilities, compute concentration and capital flows.",
      "position": 2,
      "confidence": 45,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 40,
      "from": 32,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-ec11d4feb687",
      "evidence": "Vibe-check, code, prompting, Agent Smith, Voyager, RAP, GPT-4-RL slides accumulate into 'LLMs as agents'.",
      "position": 4,
      "objective": "Show LLMs becoming agentic across prompting, tools and reasoning",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 70,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}