{
  "docId": "019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "docSlug": "cc8183ec02431b7a",
  "documentTitle": "2022 Air Street Capital The State of AI Report 2022",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 101,
  "pageCount": 114,
  "prevPage": 100,
  "nextPage": 102,
  "slideType": "diagnosis",
  "function": "present_framework",
  "density": "overcrowded",
  "nDataPoints": 4,
  "notes": "Includes a process diagram for RLHF and a bar chart comparing model performance.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "process_diagram",
    "bar_chart_vertical"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc/101",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-01cf0d8a8fbc#slide-101",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Notably, their method was highly data-efficient, with only 100 samples of human feedback they were able to finetune GPT-3 and improve it's abilities on a summarization task to human-level.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a21e-a4382889fd26",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.2,
        "w": 0.2,
        "x": 0.44,
        "y": 0.75
      },
      "kind": "chart",
      "text": "Win Rate vs. Human Summaries (%)",
      "attrs": null,
      "subkind": "bar-vertical",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "ed8489fa-2138-4f8f-8306-ed2e21de6f00",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.65,
        "w": 0.28,
        "x": 0.71,
        "y": 0.21
      },
      "kind": "diagram",
      "text": "Process flow for RLHF: Generate output, condition model, choose refinement, finetune.",
      "attrs": null,
      "subkind": "process",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "5ecab686-bb27-4c06-b2d9-e74756c5ec58",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.65,
        "x": 0.02,
        "y": 0.42
      },
      "kind": "list",
      "text": "Notably, their method was highly data-efficient, with only 100 samples of human feedback they were able to finetune GPT-3 and improve it's abilities on a summarization task to human-level.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "22b774ad-fbbe-4542-a52d-8d8efefe5553",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.65,
        "x": 0.02,
        "y": 0.58
      },
      "kind": "list",
      "text": "On a synthetic task for removing offensive words from sentences, they observe that only the largest GPT-3 models are capable of incorporating feedback effectively, demonstrating another example of emergent behaviour at scale.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "f39da41e-29f3-4899-9b49-145921ee8e96",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Win Rate vs. Human Summaries (%): 100",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a21e-aba47f7dbeb7",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.65,
        "x": 0.02,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "An RLHF preference model provides limited learning signal, compared to the full expressiveness of language that humans use. NYU researchers demonstrated that language models could be improved directly using human feedback written in language.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "f6b4933e-ecfd-4a7f-87a0-705e2c743bf3",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.65,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Language Model Alignment: Reinforcement Learning from Human Feedback",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "2a7fcd6a-c73d-423d-898a-a8cdf3fbcf1c",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Diagnostic test",
      "slug": "diagnostic-test",
      "agent": null,
      "layer": "slide",
      "matchId": "52ec0efc-7496-4b3c-bbdf-ee95e60d0b63",
      "evidence": "The slide presents a diagnostic analysis of RLHF's limitations and potential improvements for language model alignment.",
      "confidence": 0.7
    }
  ],
  "frameworks": [
    {
      "name": "Reinforcement Learning from Human Feedback (RLHF)",
      "slug": null,
      "matchId": "51615f91-1cf7-4c3c-9671-cece925c0344",
      "evidence": "Explicitly mentioned in title and body text.",
      "confidence": 1
    }
  ],
  "arcBeats": [
    {
      "to": 106,
      "from": 81,
      "beatId": "019dd95a-0682-776c-8e35-28fc5bded874",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Implications (So What)",
      "beatSlug": "triple-take-the-implications-so-what",
      "evidence": "Politics and Safety sections interpret what the facts mean for governance and risk.",
      "position": 2,
      "confidence": 78,
      "parentBeatName": "Reflection",
      "parentBeatSlug": "reflection"
    },
    {
      "to": 106,
      "from": 94,
      "beatId": "019dd95a-0682-776c-8e35-381428a808af",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Climax",
      "beatSlug": "mountain-climax",
      "evidence": "Safety section confronts existential risk and alignment urgency.",
      "position": 3,
      "confidence": 55,
      "parentBeatName": "Turn",
      "parentBeatSlug": "turn"
    }
  ],
  "loops": [
    {
      "to": 106,
      "from": 100,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-db1f31ffae02",
      "evidence": "RLHF, red-teaming, mechanistic interpretability, goal misgeneralization, moral behavior, Conjecture lab.",
      "position": 17,
      "objective": "Catalogue alignment-research techniques and example labs",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 78,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}