{
  "docId": "019dd923-5e88-73ef-bd5d-06b04d219fea",
  "docSlug": "dd91c78f6570bf29",
  "documentTitle": "2023 Air Street Capital The State of AI Report 2023",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 155,
  "pageCount": 163,
  "prevPage": 154,
  "nextPage": 156,
  "slideType": "industry_trends",
  "function": "analyze_data",
  "density": "overcrowded",
  "nDataPoints": 36,
  "notes": "The chart shows win rates across various models, comparing human evaluation against GPT-4 and GPT-3.5 as judges.",
  "elementsJson": [
    "headline_text",
    "paragraph",
    "bullet_list",
    "line_chart",
    "logo_grid"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-06b04d219fea/155",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea#slide-155",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Concerns that LLMs judges may favour they've generated have been raised.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a230-6b42673ab06f",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.3,
        "w": 0.45,
        "x": 0.05,
        "y": 0.6
      },
      "kind": "chart",
      "text": "Win rate comparison of models evaluated by different judges",
      "attrs": null,
      "subkind": "line",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "ef40bfc8-5fb9-48ec-b5c9-55bd240b2444",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.45,
        "x": 0.05,
        "y": 0.38
      },
      "kind": "list",
      "text": "Judging-LLM-as-a-judge paper shows that GPT-4 reaches 80% agreement with humans (about the same level of agreement between humans!) on MT-Bench and Chatbot Arena. MT-Bench is a smaller case study with controlled human evaluation. Chatbot Arena is a large-scale crowdsourced human evaluation benchmark.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "2afd92eb-3ff5-48ec-9122-8717a9cd7521",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.25,
        "w": 0.45,
        "x": 0.52,
        "y": 0.58
      },
      "kind": "list",
      "text": "Concerns that LLMs judges may favour they've generated have been raised. Judging-LLM-as-a-judge shows that GPT-4 favours itself with a 10% higher win rate and Claude-v1 does so with 25%. Conducting a controlled study on this is challenging, because it would require rephrasing a response to fit the style of another model.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "707609e2-65f1-45c3-a4a6-e427ca330918",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Win rate: 80%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a230-6fbc65534bd3",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.9,
        "x": 0.05,
        "y": 0.2
      },
      "kind": "paragraph",
      "text": "Evaluation metrics are strongly tied to their implementation, making it hard to assess the same metric evaluated using another library. Good assessments of performance are based on human pairwise comparison, but SOTA LLMs are making it increasingly difficult for human to discern the differences (in addition to it being slow and costly). A recent approach is to use LMs to evaluate other LMs.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "a7e6862a-446b-41aa-89d9-76e24d2c52bb",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.5,
        "x": 0.05,
        "y": 0.14
      },
      "kind": "title",
      "text": "Standard LLM benchmarks struggle with consistency",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "47955c75-57fc-4bc1-8bfd-e7e2a9c38871",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 155,
      "from": 121,
      "beatId": "019dd95a-0682-776c-8e35-478c8c58938c",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Implications (So What)",
      "beatSlug": "triple-take-the-implications-so-what",
      "evidence": "Politics + Safety sections explore regulatory, geopolitical and risk implications.",
      "position": 2,
      "confidence": 78,
      "parentBeatName": "Reflection",
      "parentBeatSlug": "reflection"
    },
    {
      "to": 155,
      "from": 121,
      "beatId": "019dd95a-0682-776c-8e35-554ec1797b3a",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Climax",
      "beatSlug": "mountain-climax",
      "evidence": "Regulatory divergence and the x-risk debate erupt into the mainstream.",
      "position": 3,
      "confidence": 45,
      "parentBeatName": "Turn",
      "parentBeatSlug": "turn"
    }
  ],
  "loops": [
    {
      "to": 155,
      "from": 149,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-1a62463741ad",
      "evidence": "RLHF survey, conditional pretraining, Constitutional AI, scalable supervision, process eval, interpretability, benchmark drift.",
      "position": 15,
      "objective": "Show alignment research progress on RLHF problems",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 70,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}