{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 32,
  "pageCount": 213,
  "prevPage": 31,
  "nextPage": 33,
  "slideType": "case_study",
  "function": "illustrate_case",
  "density": "dense",
  "nDataPoints": 3,
  "notes": "Includes a process flow diagram for the FineWeb pipeline.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "process_diagram"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/32",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-32",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "FineWeb-edu, which filtered out samples scored below 3, outperformed FineWeb and all other open datasets, despite being significantly smaller.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d10-1a0924aa676f",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.2,
        "w": 0.35,
        "x": 0.63,
        "y": 0.63
      },
      "kind": "diagram",
      "text": "The FineWeb pipeline",
      "attrs": null,
      "subkind": "process",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "a8e657f2-a607-4290-8af5-e53abd3e1dfd",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.65,
        "w": 0.6,
        "x": 0.02,
        "y": 0.22
      },
      "kind": "list",
      "text": "Team Hugging Face built a 15T token dataset for LLM pre-training, using 96 CommonCrawl snapshots, which produces LLMs that outperform other open pre-training datasets. They also released an instruction manual.\nFineWeb, the dataset, was created through a multi-step process including base filtering, independent MinHash deduplication per dump, selected filters derived from the C4 dataset, and the team’s custom filters.\nThe text extraction using the trafilatura library produced higher quality data than default CommonCrawl WET files, even though the resulting dataset was meaningfully smaller.\nThey found deduplication drove performance improvements, up to a point, before hitting a point of diminishing returns, and then worsening it.\nThe team also used llama-3-70b-instruct to annotate 500k samples from FineWeb, scoring scoring each for their educational quality on a scale from 0 to 5. FineWeb-edu, which filtered out samples scored below 3, outperformed FineWeb and all other open datasets, despite being significantly smaller.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "30a1bde0-8053-4e1b-b667-247096422cdb",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.6,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "Web data is decanted openly at scale - proving quality is key",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "78802c06-22dd-4b70-b6ed-c0d6e87b23e9",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a958-b04a2b95e851",
      "evidence": "Title 'Web data is decanted openly at scale - proving quality is key'",
      "confidence": 88
    },
    {
      "name": "Process flow",
      "slug": "process-flow",
      "agent": null,
      "layer": "slide",
      "matchId": "c2cd8f75-51b7-491d-8bde-cbc5946d636c",
      "evidence": "The FineWeb pipeline",
      "confidence": 0.7
    }
  ],
  "frameworks": [
    {
      "name": "process",
      "slug": null,
      "matchId": "9b8596a7-bbda-4eb5-9914-00e17faf83e5",
      "evidence": "The FineWeb pipeline diagram illustrates a sequential data processing workflow.",
      "confidence": 0.8
    }
  ],
  "arcBeats": [
    {
      "to": 153,
      "from": 9,
      "beatId": "019dd95a-0682-776c-8e35-5f2398b8d1d0",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Exec summary + Research + Industry sections inventory the year",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 85,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-6d9d8ee06d0a",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Unknown",
      "beatSlug": "voyage-return-the-unknown",
      "evidence": "Research section explores frontier model uncharted territory",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Complication",
      "parentBeatSlug": "complication"
    }
  ],
  "loops": [
    {
      "to": 37,
      "from": 30,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-2e9bfb3e3bea",
      "evidence": "Synthetic data, FineWeb, embeddings, RAG, power grid, JEST data curation",
      "position": 4,
      "objective": "Survey data, retrieval and training-infra advances",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 72,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}