{
  "docId": "019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "docSlug": "bf350dd574c19997",
  "documentTitle": "2021 Air Street Capital The State of AI Report 2021",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 109,
  "pageCount": 188,
  "prevPage": 108,
  "nextPage": 110,
  "slideType": "industry_trends",
  "function": "diagnose",
  "density": "overcrowded",
  "nDataPoints": 2,
  "notes": "The slide references a specific research paper/finding regarding underspecification in ML models.",
  "elementsJson": [
    "headline_text",
    "paragraph",
    "bullet_list",
    "line_chart",
    "screenshot",
    "footnote"
  ],
  "metadataConfidence": 0.95,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5c-fd5012384ae3/109",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3#slide-109",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "A more pernicious problem in ML systems is underspecification: Models trained and tested successfully on the same data, but using different random seeds, can behave differently on real-world data.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a20b-8c61ee2185d5",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.48,
        "x": 0.48,
        "y": 0.28
      },
      "kind": "chart",
      "text": "Parallel axis plots showing accuracy variation across different corruption types for Resnet-50 and BIT models.",
      "attrs": null,
      "subkind": "line",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "2d63d009-d75f-45f5-bc5c-e9705b671f67",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.26,
        "x": 0.03,
        "y": 0.93
      },
      "kind": "image",
      "text": "Google AI, MIT, UCSD, Stanford University",
      "attrs": null,
      "subkind": "logo-grid",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "c4f64334-00ba-4379-b05c-32e297e68a74",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.35,
        "w": 0.45,
        "x": 0.03,
        "y": 0.31
      },
      "kind": "list",
      "text": "Researchers from Google, MIT, UCSD and Stanford illustrate this problem with examples from computer vision, medical imaging, NLP, clinical risk prediction based on health records, and medical genomics.\nWhile they identify the problem and illustrate it, they do not have a definitive solution, and hope to spur interest in improving the machine learning pipeline to tackle the underspecification challenge. But it is unclear whether it can be tackled at all.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "5f791f44-63b7-4be2-bce6-97da5538ac49",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Relative accuracy",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c4-7719-a20b-92490ac5f15d",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.08,
        "w": 0.45,
        "x": 0.03,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "A more pernicious problem in ML systems is underspecification: Models trained and tested successfully on the same data, but using different random seeds, can behave differently on real-world data.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "21df4b10-d7e8-4400-8ed1-4ad9295f9350",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.48,
        "x": 0.48,
        "y": 0.73
      },
      "kind": "source-note",
      "text": "Figure 4: Image classification model performance on stress tests is sensitive to random initialization in ways that are not apparent in iid evaluation.",
      "attrs": null,
      "subkind": null,
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "6b66e47a-40b2-4351-9b85-2211008c3cf3",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.45,
        "x": 0.03,
        "y": 0.14
      },
      "kind": "title",
      "text": "Machine Learning in production: underspecification",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "b4b7afea-6dd7-4701-98d3-bf6a11e59c89",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 152,
      "from": 5,
      "beatId": "019dd95a-0682-776c-8e35-0affbadec38e",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Definitions p.5-6, Exec Summary p.7, then Sections 1-3 catalog evidence.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 152,
      "from": 10,
      "beatId": "019dd95a-0682-776c-8e35-1b3ee3a1c012",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Three sections accumulating evidence of accelerating progress.",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 111,
      "from": 106,
      "name": "Iceberg",
      "slug": "10-iceberg",
      "bestFor": "Consulting, complex problem solving, organizational change",
      "matchId": "019dd95a-07fe-70ce-8d3d-77cba6708c34",
      "evidence": "Data-centric pivot, benchmarking, distribution shifts, underspecification, bad data, data-driven AI.",
      "position": 14,
      "objective": "Diagnose ML-in-production challenges",
      "structure": "The Symptom (Visible) -> The System (Hidden) -> The Root Cause",
      "confidence": 78,
      "description": "Reveal that the visible problem is merely a symptom of a deeper root cause"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}