{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 192,
  "pageCount": 213,
  "prevPage": 191,
  "nextPage": 193,
  "slideType": "case_study",
  "function": "illustrate_case",
  "density": "balanced",
  "nDataPoints": 0,
  "notes": "The slide describes 'Direct Alignment from AI Feedback' (OAIF), a method using an LLM as an annotator to perform online DPO.",
  "elementsJson": [
    "paragraph",
    "process_diagram"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/192",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-192",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "They found it outperformed traditional RLHF and offline DPO across summarization, harmfulness, and helpfulness tasks.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d21-8555d0057ded",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.35,
        "w": 0.56,
        "x": 0.22,
        "y": 0.5
      },
      "kind": "image",
      "text": "Diagram showing the OAIF process: prompt x -> LLM -> y1/y2 -> LLM Annotator -> (y+, y-) -> Direct Alignment from Preferences -> update parameters",
      "attrs": null,
      "subkind": "infographic",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "065ea888-0e20-4c1c-9459-ef7d04e771b7",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.22,
        "w": 0.96,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "A Google DeepMind team has combined the simplicity of direct alignment from preferences (DAP) with the on-line policy learning of RLHF to create direct alignment from AI feedback. Here, an LLM serves as an annotator, choosing between two responses during each training iteration. This keeps the advantages of online learning without requiring a separate reward model. This is essentially a form of online DPO. They found it outperformed traditional RLHF and offline DPO across summarization, harmfulness, and helpfulness tasks.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "b16045ca-97ca-442c-bab7-54985006af5a",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.08,
        "w": 0.56,
        "x": 0.22,
        "y": 0.88
      },
      "kind": "source-note",
      "text": "Figure 1: Summary of the proposed online AI feedback (OAIF) approach for making direct alignment from preferences (DAP) methods online and on-policy.",
      "attrs": null,
      "subkind": null,
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "fbc1bd52-598d-48a9-becf-46a7826c1d62",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.3,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Is a happy middle possible?",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "7d60a65a-838b-4919-a4e2-d3e1a20321cc",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 204,
      "from": 154,
      "beatId": "019dd95a-0682-776c-8e35-605159a069e6",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Implications (So What)",
      "beatSlug": "triple-take-the-implications-so-what",
      "evidence": "Politics + Safety sections explore consequences and risks",
      "position": 2,
      "confidence": 78,
      "parentBeatName": "Reflection",
      "parentBeatSlug": "reflection"
    },
    {
      "to": 204,
      "from": 175,
      "beatId": "019dd95a-0682-776c-8e35-74f05e198faf",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Return",
      "beatSlug": "voyage-return-the-return",
      "evidence": "Safety section returns to risks/governance debate",
      "position": 4,
      "confidence": 55,
      "parentBeatName": "Resolution",
      "parentBeatSlug": "resolution"
    }
  ],
  "loops": [
    {
      "to": 201,
      "from": 187,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-79a3afd2ac42",
      "evidence": "Emergent capabilities, sycophancy, DPO, hybrid RLHF, critiques, transparency, reward tampering, SAEs",
      "position": 23,
      "objective": "Catalogue alignment, RLHF, sycophancy and interpretability research",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 76,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}