{
  "docId": "019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "docSlug": "8757f1b44ef7f176",
  "documentTitle": "2024 Air Street Capital The State of AI Report 2024",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 28,
  "pageCount": 213,
  "prevPage": 27,
  "nextPage": 29,
  "slideType": "industry_trends",
  "function": "analyze_data",
  "density": "overcrowded",
  "nDataPoints": 12,
  "notes": "The slide discusses model distillation techniques in AI research, specifically comparing token budget vs. accuracy.",
  "elementsJson": [
    "headline_text",
    "bullet_list",
    "scatter_plot"
  ],
  "metadataConfidence": 0.95,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0856e1444fb9/28",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0856e1444fb9#slide-28",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Mohawk can train Phi-Mamba and Hybrid-Phi-Mamba to achieve performance close to the teacher model.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d0f-df2923e75779",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.35,
        "w": 0.35,
        "x": 0.6,
        "y": 0.52
      },
      "kind": "chart",
      "text": "Token Budget vs Common Sense and Language Understanding",
      "attrs": null,
      "subkind": "scatter",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "fcf3e4c7-0d80-4c8c-a019-d69e9ff15ccb",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.55,
        "x": 0.03,
        "y": 0.32
      },
      "kind": "list",
      "text": "MOHAWK is a new method for distilling knowledge from a large, pre-trained transformer model (teacher) to a smaller, subquadratic model (student) like a state-space model (SSM).\nIt aligns i) the sequence transformation matrices of the student and teacher models ii) and the hidden states of each layer, then iii) transfers the remaining weights of the teacher model to the student model to finetune it.\nThe authors create Phi-Mamba, a new student model combining Mamba-2 and an MLP block and a variant called Hybrid-Phi-Mamba that retains some attention layers from the teacher model.\nMohawk can train Phi-Mamba and Hybrid-Phi-Mamba to achieve performance close to the teacher model. Phi-Mamba is distilled with only 3B tokens, less than 1% of the data used to train either the previously best-performing Mamba models and 2% for the Phi-1.5 model itself.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "4769e542-341e-48e3-9d19-39050d33e42a",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": null,
      "kind": "metric",
      "text": "Average Accuracy (%): 3B",
      "attrs": null,
      "subkind": "primary",
      "toolName": "Quantification",
      "toolSlug": "quantification",
      "confidence": null,
      "componentId": "019dd952-47c3-7407-8d0f-e386c0b8e6c5",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.08,
        "w": 0.85,
        "x": 0.03,
        "y": 0.22
      },
      "kind": "paragraph",
      "text": "By transferring knowledge from a larger, more powerful model, one could improve the performance of subquadratic models, allowing us to harness their efficiency on downstream tasks.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "fb581fa1-95a4-4274-84f0-7a98c5fe4f6f",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.08,
        "w": 0.38,
        "x": 0.6,
        "y": 0.88
      },
      "kind": "source-note",
      "text": "Figure 1: Plot of trained token budget to averaged accuracy on Winogrande, Arc-E, Arc-C, PIQA, and Hellaswag on various open-source models (mainly non-Transformer-based models). Our model (Phi-Mamba) uses more than 35x less token budget to achieve 5% higher average accuracy than the next best model.",
      "attrs": null,
      "subkind": null,
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "c660b9ae-eba8-4640-ba68-b738dbbda063",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.05,
        "w": 0.75,
        "x": 0.03,
        "y": 0.14
      },
      "kind": "title",
      "text": "And could we distill transformers into hybrid models? It's...complicated.",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "f0cbe82d-f491-4954-8148-3fb33f590742",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [
    {
      "name": "Waterfall chart",
      "slug": "waterfall-chart",
      "agent": null,
      "layer": "slide",
      "matchId": "80e7a7ed-08e1-4e66-8bb9-2c2070a90869",
      "evidence": "Token Budget vs Common Sense and Language Understanding",
      "confidence": 0.5
    }
  ],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 153,
      "from": 9,
      "beatId": "019dd95a-0682-776c-8e35-5f2398b8d1d0",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Exec summary + Research + Industry sections inventory the year",
      "position": 1,
      "confidence": 78,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 85,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-6d9d8ee06d0a",
      "arcName": "Voyage and Return",
      "arcSlug": "voyage-return",
      "beatName": "The Unknown",
      "beatSlug": "voyage-return-the-unknown",
      "evidence": "Research section explores frontier model uncharted territory",
      "position": 2,
      "confidence": 55,
      "parentBeatName": "Complication",
      "parentBeatSlug": "complication"
    }
  ],
  "loops": [
    {
      "to": 29,
      "from": 21,
      "name": "Pattern Hunter",
      "slug": "02-pattern-hunter",
      "bestFor": "Time-pressed audiences, building consensus, when data is strong",
      "matchId": "019dd95a-07fe-70ce-8d3e-2b2229583eee",
      "evidence": "Eight slides on shrinking models, on-device, hybrids, transformers reigning",
      "position": 3,
      "objective": "Demonstrate efficiency frontier across distillation, quantization, hybrids",
      "structure": "Evidence A -> Evidence B -> Evidence C -> Pattern/Conclusion",
      "confidence": 75,
      "description": "Group multiple pieces of evidence that together point to a pattern or conclusion"
    }
  ],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}