{
  "docId": "019dd923-5e88-73ef-bd5d-06b04d219fea",
  "docSlug": "dd91c78f6570bf29",
  "documentTitle": "2023 Air Street Capital The State of AI Report 2023",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 42,
  "pageCount": 163,
  "prevPage": 41,
  "nextPage": 43,
  "slideType": "industry_trends",
  "function": "present_solution",
  "density": "overcrowded",
  "nDataPoints": 0,
  "notes": "The slide explains a technical approach to visual reasoning using LLMs as code generators for visual APIs.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "flowchart",
    "icon_grid"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-06b04d219fea/42",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea#slide-42",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Two methods VisProg and ViperGPT show how given an input natural language query about an image, an LLM can decompose this into a sequence of interpretable steps that call predefined API functions for visual tasks.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a223-b3ee6ee7aa4a",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.3,
        "x": 0.68,
        "y": 0.43
      },
      "kind": "diagram",
      "text": "ViperGPT workflow diagram showing Query -> Code LLM -> API Specification -> Code Execution -> Result",
      "attrs": null,
      "subkind": "process",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "7f189032-49e9-4a86-ad4c-400878293970",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.1,
        "w": 0.65,
        "x": 0.03,
        "y": 0.21
      },
      "kind": "list",
      "text": "Two methods VisProg and ViperGPT show how given an input natural language query about an image, an LLM can decompose this into a sequence of interpretable steps that call predefined API functions for visual tasks.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "09eca39c-6a5b-4a22-b9bf-6a6778ba179a",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.07,
        "w": 0.65,
        "x": 0.03,
        "y": 0.43
      },
      "kind": "list",
      "text": "An API for visual primitives calls into existing SOTA models (e.g. semantic segmentation, object detection, depth estimation).",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "118d32f1-f296-4636-9fac-31bde2f03ea5",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.1,
        "w": 0.65,
        "x": 0.03,
        "y": 0.32
      },
      "kind": "list",
      "text": "The visual programming approach aims to build general-purpose vision systems via compositional multi step reasoning instead of end-to-end multitask training. Both methods use entirely off-the-shelf components.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "b8cac771-a96a-4f5c-b810-fd886b1cdc22",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.65,
        "x": 0.03,
        "y": 0.51
      },
      "kind": "list",
      "text": "ViperGPT uses Codex to directly generate python programs based on the API which can be executed using a python interpreter. VisProg prompts GPT-3 with examples of pseudocode instructions and interprets them as a 'visual program,' relying on LLM in-context learning from examples.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "bba250ff-dff7-4290-adaf-73563a06abf8",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.65,
        "x": 0.03,
        "y": 0.67
      },
      "kind": "list",
      "text": "World knowledge in LLMs from training on internet scale data is shown to aid in visual reasoning tasks (e.g. querying for non alcoholic drink in an image based on detected brand). Both methods show state-of-the-art results across various complex visual tasks.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "d352cba0-422a-4947-949e-5154e9948418",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.75,
        "x": 0.03,
        "y": 0.13
      },
      "kind": "title",
      "text": "Leveraging LLMs and world knowledge for compositional visual reasoning",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "3a41a08d-a360-468a-aba0-87c2b54191ba",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Analytical method",
      "slug": "analytical-method",
      "agent": null,
      "layer": "slide",
      "matchId": "2e1d1730-3fe4-4347-9ef7-9caa7cd2787f",
      "evidence": "Two methods VisProg and ViperGPT show how given an input natural language query about an image, an LLM can decompose this into a sequence of interpretable steps",
      "confidence": 0.7
    },
    {
      "name": "Canvas framework",
      "slug": "canvas-framework",
      "agent": null,
      "layer": "slide",
      "matchId": "9f2db2bf-6862-48ea-9054-6e9cc03e075e",
      "evidence": "ViperGPT uses Codex to directly generate python programs based on the API",
      "confidence": 0.5
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 120,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-41afd44ef59f",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Research + Industry sections inventory model, compute, funding facts.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 120,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-523bfb7f96e6",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Escalating capabilities, compute concentration and capital flows.",
      "position": 2,
      "confidence": 45,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 49,
      "from": 41,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-f0e6f3668fb1",
      "evidence": "VLM, autonomous driving, PaLM-E, RT-2, RoboCat, drone-racing, CICERO stack as parallel evidence.",
      "position": 5,
      "objective": "Multimodal models extend from vision-language to embodied robots",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 75,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}