{
  "docId": "019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "docSlug": "bf350dd574c19997",
  "documentTitle": "2021 Air Street Capital The State of AI Report 2021",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 58,
  "pageCount": 188,
  "prevPage": 57,
  "nextPage": 59,
  "slideType": "industry_trends",
  "function": "diagnose",
  "density": "dense",
  "nDataPoints": 1,
  "notes": "The slide uses a diagram to propose a documentation framework for CommonCrawl-based datasets.",
  "elementsJson": [
    "headline_text",
    "action_title",
    "bullet_list",
    "paragraph",
    "process_diagram"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5c-fd5012384ae3/58",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5c-fd5012384ae3#slide-58",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "The researchers recommend a documentation methodology where the excluded data is explicitly described.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d29-9cdc3970d051",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.2,
        "w": 0.25,
        "x": 0.735,
        "y": 0.485
      },
      "kind": "diagram",
      "text": "Documentation Levels for CommonCrawl-based Datasets: Metadata (Provenance, Utterance Date), Included data (Machine or human authored, Social biases, Data contamination), Excluded data (Medical or health data, Demographic identities)",
      "attrs": null,
      "subkind": "process",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "14d9d65a-8ce4-4df4-a5d2-d91d042a9c88",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.68,
        "x": 0.03,
        "y": 0.38
      },
      "kind": "list",
      "text": "Among the most frequent identity mentions, those of “sexual orientations (lesbian, gay, heterosexual, homosexual, bisexual) had the highest likelihood of being filtered out”. Moreover, African American English and Hispanic-aligned English were disproportionately removed from the text due to the blocklist filter.\nInterestingly, the dataset contains machine-generated translations. With the proliferation of machine-generated text online, many practitioners fear that new LLMs will inherit the flaws of older ones, further perpetuating their biases.\nThe researchers recommend a documentation methodology where the excluded data is explicitly described. They put this to practice and host a documented version of the C4 corpus, which had not been made easily available before.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "b1e5f208-1fc5-4d5e-a215-74f2b18f2cdf",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.94,
        "x": 0.03,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "As Large Language Models (LLMs) become ever-more successful and ubiquitous, better documentation of the large training text corpora becomes critical. Researchers dissected C4, a 305 GB dataset that Google obtained by filtering a snapshot of Common Crawl. They found that the filtering disproportionately removed text about minority individuals.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "08b98895-782b-4b81-aee9-62aacb8e2988",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.6,
        "x": 0.03,
        "y": 0.14
      },
      "kind": "title",
      "text": "Large language training datasets need better documentation",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "6e03b605-6480-4dea-b879-41f52e34dcf4",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [
    {
      "name": "documentation-methodology",
      "slug": null,
      "matchId": "df4b18c1-4572-45cd-9e8c-32232f103fb5",
      "evidence": "Proposed documentation framework for dataset transparency",
      "confidence": 0.8
    }
  ],
  "arcBeats": [
    {
      "to": 152,
      "from": 5,
      "beatId": "019dd95a-0682-776c-8e35-0affbadec38e",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Definitions p.5-6, Exec Summary p.7, then Sections 1-3 catalog evidence.",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 152,
      "from": 10,
      "beatId": "019dd95a-0682-776c-8e35-1b3ee3a1c012",
      "arcName": "The Mountain",
      "arcSlug": "mountain",
      "beatName": "Rising Action",
      "beatSlug": "mountain-rising-action",
      "evidence": "Three sections accumulating evidence of accelerating progress.",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [
    {
      "to": 63,
      "from": 56,
      "name": "Iceberg",
      "slug": "10-iceberg",
      "bestFor": "Consulting, complex problem solving, organizational change",
      "matchId": "019dd95a-07fe-70ce-8d3d-5f004461126b",
      "evidence": "Code availability, data cascades, dataset documentation, collusion rings, code+conferences.",
      "position": 8,
      "objective": "Diagnose openness and trust crisis in ML research",
      "structure": "The Symptom (Visible) -> The System (Hidden) -> The Root Cause",
      "confidence": 75,
      "description": "Reveal that the visible problem is merely a symptom of a deeper root cause"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}