{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 190,
  "pageCount": 213,
  "prevPage": 189,
  "nextPage": 191,
  "slideType": "diagnosis",
  "function": "diagnose",
  "density": "dense",
  "nDataPoints": 6,
  "notes": "The slide uses a series of scatter plots with fitted curves to demonstrate the relationship between KL divergence and win rates, suggesting that DPO is susceptible to over-optimization.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "scatter_plot"
  ],
  "metadataConfidence": 0.95,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/190",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-190",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "There are signs that the “over-optimization” that’s traditionally associated with RLHF can also happen with DPO and other kinds of direct alignment algorithms (DAAs), despite the absence of a reward model.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d21-b6ffd87eb902",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.4,
        "w": 0.4,
        "x": 0.55,
        "y": 0.5
      },
      "kind": "chart",
      "text": "Over-optimization results for √Forward KL vs. winrates.",
      "attrs": null,
      "subkind": "scatter",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "76f325d4-dd99-479b-9c7f-2f9ec55767d9",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.6,
        "w": 0.5,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "list",
      "text": "First proposed as an alternative to RLHF in 2023, DPO has no explicit reward function and comes with efficiency advantages because it doesn't sample from a policy during training or require extensive hyperparameter tuning. Despite its novelty, the method has already been used to align Llama 3.1 and Qwen2.\nHowever, there are signs that the “over-optimization” that’s traditionally associated with RLHF can also happen with DPO and other kinds of direct alignment algorithms (DAAs), despite the absence of a reward model. This worsens the more models are allowed to deviate from their starting point as they learn to align with human preferences.\nThis could be the result of underconstrained objectives, where the algorithm unintentionally assigns high probabilities to out-of-distribution data.\nThis is inherent to DAAs, but can be partially mitigated through careful parameter tuning and increased model size.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "0992b1c1-c543-49aa-8a7c-77c6bede9cd1",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "winrates",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d21-bbdae487e4c5",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.1,
        "w": 0.4,
        "x": 0.55,
        "y": 0.85
      },
      "kind": "source-note",
      "text": "Figure 5: Over-optimization results for √Forward KL vs. winrates. The top row shows the final performance after 1 epoch of training, while the second row also includes 4 intermediate checkpoints. The fitted dotted curves are scaling laws from [21] applied to DAAs, with GPT4 winrates taking the place of the gold reward model score.",
      "attrs": null,
      "subkind": null,
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "07d15c97-ed40-4dcf-bad7-403387f9fb64",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.8,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Direct Preference Optimization offers an escape from “reward hacking”…or does it?",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "16fa374a-2c08-4ba8-8cef-a33981ac315f",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a95a-b1c1f31fbea2",
      "evidence": "Title 'Direct Preference Optimization offers an escape from reward hacking…or does it?'",
      "confidence": 85
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 204,
      "from": 154,
      "beatId": "019dd95a-0682-776c-8e35-605159a069e6",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Implications (So What)",
      "beatSlug": "triple-take-the-implications-so-what",
      "evidence": "Politics + Safety sections explore consequences and risks",
      "position": 2,
      "confidence": 78,
      "parentBeatName": "Reflection",
      "parentBeatSlug": "reflection"
    },
    {
      "to": 204,
      "from": 175,
      "beatId": "019dd95a-0682-776c-8e35-74f05e198faf",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Return",
      "beatSlug": "voyage-return-the-return",
      "evidence": "Safety section returns to risks/governance debate",
      "position": 4,
      "confidence": 55,
      "parentBeatName": "Resolution",
      "parentBeatSlug": "resolution"
    }
  ],
  "loops": [
    {
      "to": 201,
      "from": 187,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-79a3afd2ac42",
      "evidence": "Emergent capabilities, sycophancy, DPO, hybrid RLHF, critiques, transparency, reward tampering, SAEs",
      "position": 23,
      "objective": "Catalogue alignment, RLHF, sycophancy and interpretability research",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 76,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}