{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 71,
  "pageCount": 213,
  "prevPage": 70,
  "nextPage": 72,
  "slideType": "case_study",
  "function": "illustrate_case",
  "density": "dense",
  "nDataPoints": 2,
  "notes": "The slide describes a specific technical implementation (TestGen-LLM) and provides quantitative results (10% improvement, 73% acceptance rate).",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "process_diagram"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/71",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-71",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "In deployment, TestGen-LLM improved about 10% of test classes it was applied to, with 73% of its recommendations accepted by developers.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d15-bfe559853581",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.25,
        "w": 0.6,
        "x": 0.2,
        "y": 0.65
      },
      "kind": "diagram",
      "text": "TestGen-LLM top level architecture",
      "attrs": null,
      "subkind": "process",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "b96d0d2b-56db-42fc-a951-638aca55aca7",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.4,
        "w": 0.9,
        "x": 0.05,
        "y": 0.22
      },
      "kind": "list",
      "text": "Meta's TestGen-LLM combines multiple LLMs, prompts and configurations to leverage different models' strengths to improve unit testing coverage for Android code on Instagram and Facebook.\nIt uses an \"assured\" approach, filtering generated tests to ensure they build successfully, pass reliably, and increase coverage before recommending them. This is the first large-scale industrial deployment of an approach that combines LLMs with verifiable guarantees of code improvement, addressing concerns about LLM hallucinations and reliability in a software engineering context.\nIn deployment, TestGen-LLM improved about 10% of test classes it was applied to, with 73% of its recommendations accepted by developers.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "5fb5b6d6-c83a-4c92-a234-3cd281507c02",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "test class improvement rate: 10%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d15-c29c4a422d11",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.03,
        "w": 0.5,
        "x": 0.25,
        "y": 0.92
      },
      "kind": "source-note",
      "text": "Figure 1: TestGen-LLM top level architecture (an instance of Assured Offline LLMSE [6]).",
      "attrs": null,
      "subkind": null,
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "0be86ec6-6825-45ec-be4f-33a753018c01",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.85,
        "x": 0.05,
        "y": 0.14
      },
      "kind": "title",
      "text": "An ensemble approach appears to drive strong performance improvements in code",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "2801d9ae-4efc-4919-a972-679e06f10745",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [
    {
      "name": "process",
      "slug": null,
      "matchId": "2ad77177-41cc-432a-9cdd-ac7708839df5",
      "evidence": "The slide includes a flowchart showing the TestGen-LLM architecture steps.",
      "confidence": 0.8
    }
  ],
  "arcBeats": [
    {
      "to": 153,
      "from": 9,
      "beatId": "019dd95a-0682-776c-8e35-5f2398b8d1d0",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Exec summary + Research + Industry sections inventory the year",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 85,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-6d9d8ee06d0a",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Unknown",
      "beatSlug": "voyage-return-the-unknown",
      "evidence": "Research section explores frontier model uncharted territory",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Complication",
      "parentBeatSlug": "complication"
    }
  ],
  "loops": [
    {
      "to": 71,
      "from": 59,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-3f24b9b4c1e3",
      "evidence": "ARC challenge, planning failures, COT, open-endedness, MCTS, FunSearch, RL-LLM combos",
      "position": 8,
      "objective": "Catalogue progress and limits in reasoning research",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 76,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}