{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 35,
  "pageCount": 213,
  "prevPage": 34,
  "nextPage": 36,
  "slideType": "diagnosis",
  "function": "diagnose",
  "density": "dense",
  "nDataPoints": 0,
  "notes": "The slide includes a schematic diagram of the Ragnarök framework and a list of example research questions.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "process_diagram",
    "infographic"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/35",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-35",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Many commonly used RAG benchmarks are repurposed retrieval or question answering datasets. They don't effectively evaluate the accuracy of citations, the importance of each piece of text to the overall answer, or the impact of conflicting points of information.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d10-69030db9fee9",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.25,
        "w": 0.45,
        "x": 0.02,
        "y": 0.65
      },
      "kind": "image",
      "text": "Schematic diagram of the Ragnarök framework",
      "attrs": null,
      "subkind": "infographic",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "19186b64-c09e-4cd0-b680-c0eb20ab553a",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.25,
        "w": 0.45,
        "x": 0.52,
        "y": 0.65
      },
      "kind": "image",
      "text": "Examples of Researchy Questions and other datasets",
      "attrs": null,
      "subkind": "infographic",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "963d8912-d1c1-4a4f-92e6-27bab224dbd0",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.08,
        "w": 0.95,
        "x": 0.02,
        "y": 0.5
      },
      "kind": "list",
      "text": "Meanwhile, Researchy Questions provides a large-scale collection of complex, multi-faceted questions that require in-depth research and analysis to answer, drawn from real user queries.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "12f94f5a-d7be-4216-9aa2-73c03b242f17",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.95,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "list",
      "text": "Many commonly used RAG benchmarks are repurposed retrieval or question answering datasets. They don't effectively evaluate the accuracy of citations, the importance of each piece of text to the overall answer, or the impact of conflicting points of information.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "1ddedac7-00f9-47cd-86c8-27dc38b33272",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.95,
        "x": 0.02,
        "y": 0.36
      },
      "kind": "list",
      "text": "Researchers are now pioneering novel approaches, like Ragnarök, which introduces a novel web-based arena for human evaluation through pairwise system comparisons. This addresses the challenge of assessing RAG quality beyond traditional automated metrics.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "44983d91-397d-4b11-a993-05ed37741864",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.4,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Evaluation for RAG remains unsolved",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "400737df-0a86-49c4-9848-4862b9094795",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a958-bcbe53982cc0",
      "evidence": "Title 'Evaluation for RAG remains unsolved'",
      "confidence": 85
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 153,
      "from": 9,
      "beatId": "019dd95a-0682-776c-8e35-5f2398b8d1d0",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Exec summary + Research + Industry sections inventory the year",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 85,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-6d9d8ee06d0a",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Unknown",
      "beatSlug": "voyage-return-the-unknown",
      "evidence": "Research section explores frontier model uncharted territory",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Complication",
      "parentBeatSlug": "complication"
    }
  ],
  "loops": [
    {
      "to": 37,
      "from": 30,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-2e9bfb3e3bea",
      "evidence": "Synthetic data, FineWeb, embeddings, RAG, power grid, JEST data curation",
      "position": 4,
      "objective": "Survey data, retrieval and training-infra advances",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 72,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}