{
  "docId": "019dd923-5e88-73ef-bd5d-06b04d219fea",
  "docSlug": "dd91c78f6570bf29",
  "documentTitle": "2023 Air Street Capital The State of AI Report 2023",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 149,
  "pageCount": 163,
  "prevPage": 148,
  "nextPage": 150,
  "slideType": "diagnosis",
  "function": "diagnose",
  "density": "dense",
  "nDataPoints": 0,
  "notes": "The slide categorizes challenges into three pillars based on RLHF components, referencing specific sections (§3.1, §3.2, §3.3) of a research paper.",
  "elementsJson": [
    "headline_text",
    "paragraph",
    "icon_grid",
    "bullet_list"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-06b04d219fea/149",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-06b04d219fea#slide-149",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "A survey from leading institutions in AI Safety identifies open problems and limitations of RLHF.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a22f-7c6a7265a751",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.5,
        "w": 0.3,
        "x": 0.05,
        "y": 0.4
      },
      "kind": "list",
      "text": "Human Feedback, §3.1: Misaligned Evaluators, Difficulty of Oversight, Data Quality, Feedback Type Limitations",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "7869c474-b614-4ecd-a9fe-97003041026a",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.5,
        "w": 0.3,
        "x": 0.38,
        "y": 0.4
      },
      "kind": "list",
      "text": "Reward Model, §3.2: Problem Misspecification, Misgeneralization/Hacking, Evaluation Difficulty",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "88476eb3-53e6-438e-ad81-799cc032213b",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.5,
        "w": 0.3,
        "x": 0.7,
        "y": 0.4
      },
      "kind": "list",
      "text": "Policy, §3.3: RL Difficulties, Policy Misgeneralization, Distributional Challenges",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "8ccad5a0-bcf5-4f87-994f-d42440cb2215",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.1,
        "w": 0.9,
        "x": 0.05,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "A survey from leading institutions in AI Safety identifies open problems and limitations of RLHF. For each component of RLHF, the authors list problems that they classify as tractable (they can be solved within the RLHF framework) and fundamental (where we need a different approach). Below we list the fundamental ones.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "5803f871-872d-40a1-bc2d-77b33ffb935f",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.4,
        "x": 0.05,
        "y": 0.14
      },
      "kind": "title",
      "text": "Fundamental challenges with RLHF",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "ea208d40-a223-4948-b39b-94f20fee1012",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Authority Bias",
      "slug": "authority-bias",
      "agent": "Storyteller",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a958-0f7d8f8d090c",
      "evidence": "Cites 'survey from leading institutions in AI Safety'.",
      "confidence": 70
    },
    {
      "name": "Problem Statement Canvas",
      "slug": "problem-statement-canvas",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "8cbce0ee-16d9-4ef8-a9de-8cb24540e39d",
      "evidence": "The slide presents a list of challenges with RLHF, which can be seen as a problem statement canvas.",
      "confidence": 0.6
    }
  ],
  "frameworks": [
    {
      "name": "RLHF (Reinforcement Learning from Human Feedback)",
      "slug": null,
      "matchId": "019dd95a-1ca5-70bb-bba0-a402f2a9e159",
      "evidence": "Title 'Fundamental challenges with RLHF' uses RLHF as the analytical frame.",
      "confidence": 82
    }
  ],
  "arcBeats": [
    {
      "to": 155,
      "from": 121,
      "beatId": "019dd95a-0682-776c-8e35-478c8c58938c",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Implications (So What)",
      "beatSlug": "triple-take-the-implications-so-what",
      "evidence": "Politics + Safety sections explore regulatory, geopolitical and risk implications.",
      "position": 2,
      "confidence": 78,
      "parentBeatName": "Reflection",
      "parentBeatSlug": "reflection"
    },
    {
      "to": 155,
      "from": 121,
      "beatId": "019dd95a-0682-776c-8e35-554ec1797b3a",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Climax",
      "beatSlug": "mountain-climax",
      "evidence": "Regulatory divergence and the x-risk debate erupt into the mainstream.",
      "position": 3,
      "confidence": 45,
      "parentBeatName": "Turn",
      "parentBeatSlug": "turn"
    }
  ],
  "loops": [
    {
      "to": 155,
      "from": 149,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-1a62463741ad",
      "evidence": "RLHF survey, conditional pretraining, Constitutional AI, scalable supervision, process eval, interpretability, benchmark drift.",
      "position": 15,
      "objective": "Show alignment research progress on RLHF problems",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 70,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}