{
  "docId": "019de06f-84e4-70e4-bce4-3c7c4f09a8ea",
  "docSlug": "602cc25b036280012a1a042017f6a35d",
  "documentTitle": "OpenAI | Product Presentation Deck | 20 slides",
  "authorId": "openai",
  "authorName": "OpenAI",
  "documentKindSlug": "conference-presentation",
  "documentKindLabel": "Conference presentation",
  "sourceTypeSlug": "investor_relations",
  "sourceTypeLabel": "Investor relations",
  "presentationDate": "2017-08-01 00:00:00",
  "orientation": "landscape",
  "aspectRatio": 1.7977529,
  "pageNumber": 11,
  "pageCount": 20,
  "prevPage": 10,
  "nextPage": 12,
  "slideType": "propose_solution",
  "function": "present_framework",
  "density": "balanced",
  "nDataPoints": 1,
  "notes": "The diagram shows a standard RL loop (RL Algorithm <-> Environment) augmented with a Reward Predictor that receives human feedback.",
  "elementsJson": null,
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019de06f-84e4-70e4-bce4-3c7c4f09a8ea/11",
  "deckHref": "/decks/019de06f-84e4-70e4-bce4-3c7c4f09a8ea",
  "deckJsonHref": "/decks/019de06f-84e4-70e4-bce4-3c7c4f09a8ea.json",
  "deckAnchorHref": "/decks/019de06f-84e4-70e4-bce4-3c7c4f09a8ea#slide-11",
  "components": [
    {
      "bbox": {
        "h": 0.38,
        "w": 0.54,
        "x": 0.23,
        "y": 0.18
      },
      "kind": "diagram",
      "text": "RL loop with reward predictor",
      "attrs": null,
      "subkind": "process",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "7437ecdd-5d76-404f-a2c8-e52685ba95f2",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.28,
        "w": 0.88,
        "x": 0.06,
        "y": 0.64
      },
      "kind": "list",
      "text": "Bottom 2 boxes are standard reinforcement learning\nInjecting reward predictor modifies RL to learn the human's preference from sparse supervision (\"surrogate human\")\nRequest feedback on only 0.1% of experience; use active learning",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "191a97bf-b575-4b04-8030-9f11fbcf66cf",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.06,
        "w": 0.25,
        "x": 0.04,
        "y": 0.06
      },
      "kind": "title",
      "text": "Behind the hood",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "3d30c46e-594e-413a-a427-3e823c982edc",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [
    {
      "name": "Reinforcement Learning from Human Feedback (RLHF)",
      "slug": null,
      "matchId": "11d8debb-9c89-4b9c-984f-2bdce2f78139",
      "evidence": "Diagram shows RL algorithm, environment, and reward predictor with human feedback.",
      "confidence": 1
    }
  ],
  "arcBeats": [
    {
      "to": 11,
      "from": 8,
      "beatId": "72a1fd78-bc62-4d21-a603-262dd269d030",
      "arcName": "Problem-Agitate-Solution",
      "arcSlug": "problem-agitate-solution",
      "beatName": "Solution",
      "beatSlug": "problem-agitate-solution-solution-provide-relief",
      "evidence": "The deck proposes a solution using human feedback training processes and provides evidence to support it.",
      "position": 2,
      "confidence": 0.9,
      "parentBeatName": "Resolution",
      "parentBeatSlug": "resolution"
    }
  ],
  "loops": [
    {
      "to": 14,
      "from": 8,
      "name": "Jobs To Be Done",
      "slug": "53-jobs-to-be-done",
      "bestFor": "Product innovation, market entry, competitive positioning",
      "matchId": "fdcd1287-c4f6-4816-863c-2281b6ffccea",
      "evidence": "The deck proposes a solution using human feedback training processes and provides evidence to support it.",
      "position": 1,
      "objective": "How can we train an ML system to align with human preferences and values?",
      "structure": "The Customer's Job -> Current Solutions (Hired/Fired) -> Unmet Needs -> Our Solution Fit",
      "confidence": 0.7,
      "description": "Reframe the problem around what the customer is trying to accomplish, not what they're buying"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}