{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 191,
  "pageCount": 213,
  "prevPage": 190,
  "nextPage": 192,
  "slideType": "industry_trends",
  "function": "summarize",
  "density": "dense",
  "nDataPoints": 4,
  "notes": "The slide discusses the persistence of RLHF in AI training, citing Google DeepMind research and Cohere's RLOO optimization.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "scatter_plot"
  ],
  "metadataConfidence": 0.95,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/191",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-191",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Due to a combination of innate advantages and innovation designed to improve its efficiency, offline direct alignment methods don’t look set to displace RLHF at scale anytime soon.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d20-f6d270d2e7d4",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.48,
        "w": 0.3,
        "x": 0.68,
        "y": 0.42
      },
      "kind": "chart",
      "text": "Comparison of online vs offline performance across four metrics (summarization, helpfulness, chat, harmlessness)",
      "attrs": null,
      "subkind": "scatter",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "dcca7fc3-bdce-4e74-b271-07e62f7a4637",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.65,
        "x": 0.02,
        "y": 0.32
      },
      "kind": "list",
      "text": "Testing online vs. offline approaches across datasets covering summarization, helpfulness, conversational ability, and harmlessness, a Google DeepMind team found that RLHF emerged as the winner across all of them.\nThey argue that that this stems from on-policy sampling, which more effectively improves generative tasks and cannot be easily replicated by offline algorithms, even with similar data or model scaling.\nCohere for AI has explored scrapping the Proximal Policy Optimization algorithm in RLHF (which treats each token as an individual action), in favor of their RLOO (REINFORCE Leave One-Out) Trainer, which entire generation as one action, distributing rewards across the full sequence.\nThey find this leads to a 50-75% GPU use reduction and a 2-3x increase in training speed versus PPO, depending on model size.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "9b93c0fd-0aa3-470b-a2de-94c88fbabcb8",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Win rate against golden policy: 50-75%",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d20-f893078b6cba",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.08,
        "w": 0.65,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "Due to a combination of innate advantages and innovation designed to improve its efficiency, offline direct alignment methods don't look set to displace RLHF at scale anytime soon.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "4e6561ef-fe45-4482-b5c5-15e87cb9f929",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.35,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "RLHF isn't going anywhere fast",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "4f9c9475-0ad5-428f-b06f-9e05f7b09dee",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a95a-b5cdcd28ff08",
      "evidence": "Title 'RLHF isn't going anywhere fast'",
      "confidence": 85
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 204,
      "from": 154,
      "beatId": "019dd95a-0682-776c-8e35-605159a069e6",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Implications (So What)",
      "beatSlug": "triple-take-the-implications-so-what",
      "evidence": "Politics + Safety sections explore consequences and risks",
      "position": 2,
      "confidence": 78,
      "parentBeatName": "Reflection",
      "parentBeatSlug": "reflection"
    },
    {
      "to": 204,
      "from": 175,
      "beatId": "019dd95a-0682-776c-8e35-74f05e198faf",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Return",
      "beatSlug": "voyage-return-the-return",
      "evidence": "Safety section returns to risks/governance debate",
      "position": 4,
      "confidence": 55,
      "parentBeatName": "Resolution",
      "parentBeatSlug": "resolution"
    }
  ],
  "loops": [
    {
      "to": 201,
      "from": 187,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-79a3afd2ac42",
      "evidence": "Emergent capabilities, sycophancy, DPO, hybrid RLHF, critiques, transparency, reward tampering, SAEs",
      "position": 23,
      "objective": "Catalogue alignment, RLHF, sycophancy and interpretability research",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 76,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}