{
  "docId": "019dd923-5e88-73ef-bd5d-06b04d219fea",
  "docSlug": "dd91c78f6570bf29",
  "documentTitle": "2023 Air Street Capital The State of AI Report 2023",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 13,
  "pageCount": 163,
  "prevPage": 12,
  "nextPage": 14,
  "slideType": "industry_trends",
  "function": "present_framework",
  "density": "dense",
  "nDataPoints": 0,
  "notes": "Includes a process diagram illustrating the RLHF training pipeline.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "process_diagram",
    "footnote"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-06b04d219fea/13",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea#slide-13",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "RLHF is now central to the success of state of the art LLMs, especially those designed for chat applications.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a221-81ad5b2ef8af",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.7,
        "w": 0.3,
        "x": 0.68,
        "y": 0.2
      },
      "kind": "diagram",
      "text": "Typical steps of RLHF, which follow an initial step of supervised fine-tuning of a pre-trained language model, e.g. GPT-3.",
      "attrs": null,
      "subkind": "process",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "648acbc8-1a5e-4411-934d-bd81f0e10928",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.5,
        "w": 0.65,
        "x": 0.02,
        "y": 0.35
      },
      "kind": "list",
      "text": "RLHF involves humans ranking language model outputs sampled for a given input, using these rankings to learn a reward model of human preferences, and then using this as a reward signal to finetune the language model with using RL. In its modern form, it dates back to 2017, when OpenAI and DeepMind researchers applied it to incorporate human feedback in training agents on Atari games and to other RL applications.\nRLHF is now central to the success of state of the art LLMs, especially those designed for chat applications. These include Anthropic’s Claude, Google’s Bard, Meta’s LLaMA-2-chat, and, of course, OpenAI’s ChatGPT.\nRLHF requires hiring humans to evaluate and rank model outputs, and then models their preferences. This makes this technique hard, expensive, and biased. This motivated researchers to look for alternatives.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "c52638ea-ea5c-44ca-8fee-ee6f17682378",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.1,
        "w": 0.65,
        "x": 0.02,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "In last year’s Safety section (Slide 100), we highlighted how Reinforcement Learning from Human Feedback (RLHF) – used in InstructGPT – helped make OpenAI’s models safer and more helpful for users. Despite a few hiccups, ChatGPT’s success proved the technique’s viability at a massive scale.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "46ea6fe4-897e-45c1-adc5-127bbb61eac9",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.03,
        "w": 0.3,
        "x": 0.02,
        "y": 0.95
      },
      "kind": "source-note",
      "text": "1 We will cover other issues of RLHF in the Safety section.",
      "attrs": {
        "numbered": true
      },
      "subkind": null,
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "b5866cc3-7d07-4eb0-8746-5d8d24713950",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.45,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Fueled by ChatGPT’s success, RLHF becomes MVP",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "43c516f9-e208-436d-8aac-0c605a79447f",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a957-091004741f7e",
      "evidence": "'RLHF becomes MVP' is a verdict not a topic.",
      "confidence": 88
    }
  ],
  "frameworks": [
    {
      "name": "process",
      "slug": null,
      "matchId": "2551ddd1-3b95-4d3f-8823-5329d1af7b60",
      "evidence": "The slide illustrates the RLHF training pipeline as a step-by-step process.",
      "confidence": 0.9
    }
  ],
  "arcBeats": [
    {
      "to": 120,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-41afd44ef59f",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Research + Industry sections inventory model, compute, funding facts.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 120,
      "from": 11,
      "beatId": "019dd95a-0682-776c-8e35-523bfb7f96e6",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Escalating capabilities, compute concentration and capital flows.",
      "position": 2,
      "confidence": 45,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 21,
      "from": 12,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3d-e61d00ffc6e4",
      "evidence": "Sequential GPT-4, RLHF, LLaMa, popularity-mention slides converge on a single conclusion.",
      "position": 2,
      "objective": "Pile up evidence that GPT-4 and RLHF dominate the LLM landscape",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 82,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}