{
  "docId": "019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "docSlug": "bf350dd574c19997",
  "documentTitle": "2021 Air Street Capital The State of AI Report 2021",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 107,
  "pageCount": 188,
  "prevPage": 106,
  "nextPage": 108,
  "slideType": "industry_trends",
  "function": "present_solution",
  "density": "dense",
  "nDataPoints": 0,
  "notes": "The slide explains the concept of dynamic benchmarking using the Dynabench platform as a case study.",
  "elementsJson": [
    "headline_text",
    "paragraph",
    "bullet_list",
    "screenshot",
    "logo_grid"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5c-fd5012384ae3/107",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3#slide-107",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Dynamic Benchmarking, where datasets are continuously updated by human users, are a solution to make benchmarks more useful.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a20b-50037712f8b7",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.65,
        "x": 0.02,
        "y": 0.093
      },
      "kind": "image",
      "text": "Institutional logos",
      "attrs": null,
      "subkind": "logo-grid",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "c156f1fb-5966-4b12-ac7c-1737e6784798",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.48,
        "w": 0.37,
        "x": 0.61,
        "y": 0.38
      },
      "kind": "image",
      "text": "Dynabench interface screenshot",
      "attrs": null,
      "subkind": "screenshot",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "ac57ea8d-bb92-4f84-a6df-5a068ac9320e",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.55,
        "x": 0.02,
        "y": 0.36
      },
      "kind": "list",
      "text": "Dynabench is a web-based open-source tool that allows users to propose difficult examples that fool the model or make it very uncertain. These examples are then validated by expert linguists and crowdworkers.\nThe collected data can be used to both evaluate current state-of-the-art models and train other models.\nThe aim of dynamic benchmarking is to create a virtuous cycle where models are improved to be able to deal with harder examples. Then, it becomes increasingly harder to fool the models, which hopefully evolve to be robust to the worst case scenarios that are encountered in the real world.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "d5f5a4b8-b9a6-4c0e-8f81-f36d0e70b27b",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.55,
        "x": 0.02,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "Due to the rapid progress in model development, beating benchmarks has become a matter of months. The high-performing models nonetheless often fail in real-world scenarios. Dynamic Benchmarking, where datasets are continuously updated by human users, are a solution to make benchmarks more useful.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "d19858b6-6462-4b0a-aecd-d6b59317d672",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.5,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Machine Learning in production: active benchmarking",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "891e7e1e-4d40-49c4-bf9e-d8a86247cecc",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [
    {
      "name": "flywheel",
      "slug": null,
      "matchId": "1b32bbe2-02ff-468f-9087-ebec498f7797",
      "evidence": "The slide describes a 'virtuous cycle' where model improvement leads to harder examples, which leads to further improvement.",
      "confidence": 0.8
    }
  ],
  "arcBeats": [
    {
      "to": 152,
      "from": 5,
      "beatId": "019dd95a-0682-776c-8e35-0affbadec38e",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Definitions p.5-6, Exec Summary p.7, then Sections 1-3 catalog evidence.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 152,
      "from": 10,
      "beatId": "019dd95a-0682-776c-8e35-1b3ee3a1c012",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Three sections accumulating evidence of accelerating progress.",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 111,
      "from": 106,
      "name": "Iceberg",
      "slug": "10-iceberg",
      "bestFor": "Consulting, complex problem solving, organizational change",
      "matchId": "019dd95a-07fe-70ce-8d3d-77cba6708c34",
      "evidence": "Data-centric pivot, benchmarking, distribution shifts, underspecification, bad data, data-driven AI.",
      "position": 14,
      "objective": "Diagnose ML-in-production challenges",
      "structure": "The Symptom (Visible) -> The System (Hidden) -> The Root Cause",
      "confidence": 78,
      "description": "Reveal that the visible problem is merely a symptom of a deeper root cause"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}