{
  "docId": "019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "docSlug": "5df6bd1b0447b5f6",
  "documentTitle": "2025 Air Street Capital The State of AI Report 2025",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 39,
  "pageCount": 313,
  "prevPage": 38,
  "nextPage": 40,
  "slideType": "industry_trends",
  "function": "present_solution",
  "density": "dense",
  "nDataPoints": 4,
  "notes": "The slide compares memory usage of standard cross-entropy vs. CCE across various LLM models.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "bubble_chart"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0d0f98caffe1/39",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1#slide-39",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "CCE achieves a remarkable 24 reduction in memory consumption, taking Gemma 2's loss computation from 24GB down to just 1 MB, while actually running ~5% faster than the best existing methods.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-41ac-7180-a81d-388d103c9bf9",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.45,
        "x": 0.52,
        "y": 0.38
      },
      "kind": "chart",
      "text": "Comparison of memory usage for Regular cross-entropy vs. Cut cross-entropy across various LLM models.",
      "attrs": null,
      "subkind": "bubble",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "8bc4d343-7dba-4902-86fa-e41642670a78",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.5,
        "w": 0.45,
        "x": 0.05,
        "y": 0.38
      },
      "kind": "list",
      "text": "Cut Cross Entropy (CCE) computes the cross-entropy loss by calculating only the logits for correct tokens directly, while evaluating the normalization term over the vocabulary in fast on-chip memory. This makes global memory consumption for the cross-entropy computation negligible.\nCCE achieves a remarkable 24 reduction in memory consumption, taking Gemma 2's loss computation from 24GB down to just 1 MB, while actually running ~5% faster than the best existing methods.\nThe practical impact of this is that it allows researchers to train models much more efficiently - either using fewer GPUs for the same batch size, or achieving better GPU utilization with larger batches on the same hardware.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "9db2ba7a-d135-4e8f-9f27-dddda4e1aa0b",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Memory consumption: 24x",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-41ac-7180-a81d-3f9ae1690683",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.15,
        "w": 0.9,
        "x": 0.05,
        "y": 0.2
      },
      "kind": "paragraph",
      "text": "As vocabularies grow, the loss layer consumes up to 90% of training memory in modern LLMs. Apple researchers show that this bottleneck can be eliminated by computing the loss without materializing the massive logit matrix, enabling dramatically larger batch sizes and more efficient training.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "95c7309e-3e8c-4cb8-9937-53a161860b99",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.6,
        "x": 0.05,
        "y": 0.13
      },
      "kind": "title",
      "text": "Cutting your losses: significant memory reduction for LLM training",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "3bf7a8d6-e937-476f-8d7b-58f7d49c07a3",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Data-Ink Ratio",
      "slug": "data-ink-ratio",
      "agent": "Designer",
      "layer": "slide",
      "matchId": "85bf72cd-7f6e-41ca-bec0-5a5f15590c90",
      "evidence": "The components mention computing the loss without materializing the entire vocabulary, which relates to maximizing data-ink ratio for efficient communication.",
      "confidence": 0.6
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 189,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-9f380673831f",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Sections 1-2 lay out research findings and industry data with charts and case studies.",
      "position": 1,
      "confidence": 70,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 89,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-ac68557d958b",
      "arcName": "The Onion",
      "arcSlug": "onion",
      "beatName": "First Layer",
      "beatSlug": "onion-first-layer",
      "evidence": "Research section unpacks technical layer beneath the headlines.",
      "position": 2,
      "confidence": 45,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}