{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 22,
  "pageCount": 213,
  "prevPage": 21,
  "nextPage": 23,
  "slideType": "key_takeaways",
  "function": "summarize",
  "density": "balanced",
  "nDataPoints": 3,
  "notes": "Discusses model pruning techniques (layer removal, knowledge distillation) and their impact on performance benchmarks.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "paragraph",
    "process_diagram",
    "line_chart"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/22",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-22",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "A Meta/MIT team looking at open-weight pre-trained LLMs concluded that it’s possible to do away with up to half a model’s layers and suffer only negligible performance drops on question-answering benchmark.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d0f-3a773d930feb",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.45,
        "w": 0.43,
        "x": 0.54,
        "y": 0.48
      },
      "kind": "chart",
      "text": "Diagrams showing layer removal process and performance metrics (MMLU, BoolQ, C4 validation loss) vs fraction of layers dropped.",
      "attrs": null,
      "subkind": "process",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "39f568e8-4a55-4d75-b656-5327d1a02f97",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.65,
        "w": 0.5,
        "x": 0.03,
        "y": 0.22
      },
      "kind": "list",
      "text": "Research suggests that models are robust in the face of deeper layers being pruned intelligently. A Meta/MIT team found up to half of layers can be removed with negligible performance drops. NVIDIA researchers used pruning and knowledge distillation to create MINITRON models that outperform smaller models with 40x fewer training tokens.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "d0fbe9e2-511b-499c-9eba-fc4d178f31ef",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Model performance: 40x",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d0f-3fb71587f60e",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.6,
        "x": 0.03,
        "y": 0.14
      },
      "kind": "title",
      "text": "It's possible to shrink models with minimal impact on performance...",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "63587e44-eeef-495f-ac18-54824749a1f0",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Action Titles",
      "slug": "action-titles",
      "agent": "Architect",
      "layer": "slide",
      "matchId": "019dd95a-1055-74e0-a958-8e3d68f7a4d0",
      "evidence": "Title 'It's possible to shrink models with minimal impact on performance'",
      "confidence": 88
    },
    {
      "name": "Chartjunk Elimination",
      "slug": "chartjunk-elimination",
      "agent": "Designer",
      "layer": "slide",
      "matchId": "53d38a01-eaea-4d8b-9200-8864283f1381",
      "evidence": "Diagrams showing layer removal process and performance metrics (MMLU, BoolQ, C4 validation loss) vs fraction of layers dropped.",
      "confidence": 0.7
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 153,
      "from": 9,
      "beatId": "019dd95a-0682-776c-8e35-5f2398b8d1d0",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Exec summary + Research + Industry sections inventory the year",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 85,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-6d9d8ee06d0a",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Unknown",
      "beatSlug": "voyage-return-the-unknown",
      "evidence": "Research section explores frontier model uncharted territory",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Complication",
      "parentBeatSlug": "complication"
    }
  ],
  "loops": [
    {
      "to": 29,
      "from": 21,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-2b2229583eee",
      "evidence": "Eight slides on shrinking models, on-device, hybrids, transformers reigning",
      "position": 3,
      "objective": "Demonstrate efficiency frontier across distillation, quantization, hybrids",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 75,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}