{
  "docId": "019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "docSlug": "bf350dd574c19997",
  "documentTitle": "2021 Air Street Capital The State of AI Report 2021",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 51,
  "pageCount": 188,
  "prevPage": 50,
  "nextPage": 52,
  "slideType": "diagnosis",
  "function": "diagnose",
  "density": "dense",
  "nDataPoints": 3,
  "notes": "The slide highlights the unreliability of human benchmarks in NLP, specifically referencing GPT-2 and GPT-3 evaluation.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "infographic",
    "paragraph"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5c-fd5012384ae3/51",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3#slide-51",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Researchers show that human evaluators are often in disagreement on Natural Language Generation (NLG) tasks. This calls into question the idea of beating current human baselines as the gold standard for NLP tasks.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d28-bc30bffadc56",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.25,
        "x": 0.72,
        "y": 0.33
      },
      "kind": "image",
      "text": "Excerpts from human evaluators' explanations for why they believe a GPT3-generated story was written by a human or a machine.",
      "attrs": null,
      "subkind": "infographic",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "08c15ee5-d8a8-47c0-9e4f-c4d8a695143e",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.65,
        "x": 0.03,
        "y": 0.33
      },
      "kind": "list",
      "text": "780 evaluators were asked to determine whether text passages were written by humans or state-of-the-art generative models: GPT-2 and GPT-3. They correctly distinguished 57.9% of the time for GPT-2, but only 49.9% of the time for GPT-3, pointing to the improvement in NLG models.\nA way to improve their performance was to train human experts to better identify GPT-3-authored text, but this improved the accuracy to only 55%.\nWhat is striking, however, is the justifications of their classification: human evaluators often gave contradicting explanations on the same examples, “sometimes using the same aspect of the text to come to opposite conclusions.”\nMost evaluators systematically underestimated current NLG models, and focused on form rather than content in their evaluation. The researchers call the community to think better about how to collect human evaluation of NLG models.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "dff30f77-a9cb-441b-8f43-60012fd9506c",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Human classification accuracy: 49.9%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d28-c0daf53c7110",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.1,
        "w": 0.65,
        "x": 0.03,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "Researchers show that human evaluators are often in disagreement on Natural Language Generation (NLG) tasks. This calls into question the idea of beating current human baselines as the gold standard for NLP tasks.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "4650bf4d-064f-456b-a1d8-6e2c71b3dba5",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.6,
        "x": 0.03,
        "y": 0.14
      },
      "kind": "title",
      "text": "New study suggests human evaluation should be re-evaluated",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "b177743b-b0f8-466e-8c17-1028da5362fd",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 152,
      "from": 5,
      "beatId": "019dd95a-0682-776c-8e35-0affbadec38e",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Definitions p.5-6, Exec Summary p.7, then Sections 1-3 catalog evidence.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 152,
      "from": 10,
      "beatId": "019dd95a-0682-776c-8e35-1b3ee3a1c012",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Three sections accumulating evidence of accelerating progress.",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 51,
      "from": 41,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-576c8c35c986",
      "evidence": "Codex, math limits, truthfulness, prompt engineering, SuperGLUE, multilingual LLMs.",
      "position": 6,
      "objective": "Trace LLM frontier and the prompting paradigm",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 78,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}