{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 193,
  "pageCount": 213,
  "prevPage": 192,
  "nextPage": 194,
  "slideType": "diagnosis",
  "function": "diagnose",
  "density": "balanced",
  "nDataPoints": 0,
  "notes": "The slide references specific research methodologies (semantic entropy and Google DeepMind's SAFE) to diagnose and mitigate LLM reliability issues.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "flowchart"
  ],
  "metadataConfidence": 0.95,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/193",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-193",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "LLMs suffer from two main reliability errors: response that are inconsistent with their internal knowledge (hallucinations) and ones that share information that does not accord with established external knowledge.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d22-a22adce3bbbd",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.33,
        "x": 0.65,
        "y": 0.45
      },
      "kind": "image",
      "text": "Diagram showing the SAFE process: Factoid decomposition, possible questions, clustering by meaning, and semantic entropy probability.",
      "attrs": null,
      "subkind": "infographic",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "2aab9f82-faed-4ebb-890a-8c3d4572f60e",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.65,
        "w": 0.62,
        "x": 0.03,
        "y": 0.22
      },
      "kind": "list",
      "text": "LLMs suffer from two main reliability errors: response that are inconsistent with their internal knowledge (hallucinations) and ones that share information that does not accord with established external knowledge.\nA recent paper from the University of Oxford focuses on a subset of hallucinations called confabulations, where LLMs produce incorrect generalizations.\nThey measure the LLM’s uncertainty by generating multiple answers to a question, and using another model to group them together by similar meaning. Higher entropy scores across groups suggest confabulation.\nMeanwhile, Google DeepMind have introduced SAFE, which evaluates the factuality of LLM responses by breaking them down into individual facts, using search engines to verify facts, and clustering semantically similar statements.\nThey’ve also curated LongFact, a new benchmark dataset for evaluating long-form factuality across 38 topics.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "1eba84ff-c91e-492b-8d8e-defd987da47d",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.45,
        "x": 0.03,
        "y": 0.14
      },
      "kind": "title",
      "text": "Can LLMs improve the reliability of...LLMs?",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "ca83c878-98ab-4774-9fb8-0927467a0451",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 204,
      "from": 154,
      "beatId": "019dd95a-0682-776c-8e35-605159a069e6",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Implications (So What)",
      "beatSlug": "triple-take-the-implications-so-what",
      "evidence": "Politics + Safety sections explore consequences and risks",
      "position": 2,
      "confidence": 78,
      "parentBeatName": "Reflection",
      "parentBeatSlug": "reflection"
    },
    {
      "to": 204,
      "from": 175,
      "beatId": "019dd95a-0682-776c-8e35-74f05e198faf",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Return",
      "beatSlug": "voyage-return-the-return",
      "evidence": "Safety section returns to risks/governance debate",
      "position": 4,
      "confidence": 55,
      "parentBeatName": "Resolution",
      "parentBeatSlug": "resolution"
    }
  ],
  "loops": [
    {
      "to": 201,
      "from": 187,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-79a3afd2ac42",
      "evidence": "Emergent capabilities, sycophancy, DPO, hybrid RLHF, critiques, transparency, reward tampering, SAEs",
      "position": 23,
      "objective": "Catalogue alignment, RLHF, sycophancy and interpretability research",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 76,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}