{
  "docId": "019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "docSlug": "5df6bd1b0447b5f6",
  "documentTitle": "2025 Air Street Capital The State of AI Report 2025",
  "authorId": "AirStreetCapital",
  "authorName": "Air Street Capital",
  "documentKindSlug": "consulting-deck",
  "documentKindLabel": "Consulting deck",
  "sourceTypeSlug": "vc_research",
  "sourceTypeLabel": "VC research",
  "presentationDate": null,
  "orientation": "landscape",
  "aspectRatio": 1.777,
  "pageNumber": 54,
  "pageCount": 313,
  "prevPage": 53,
  "nextPage": 55,
  "slideType": "industry_trends",
  "function": "summarize",
  "density": "dense",
  "nDataPoints": 4,
  "notes": "The slide highlights the current state of AI research agent evaluation, noting that while benchmarks exist, performance remains limited.",
  "elementsJson": [
    "headline_text",
    "paragraph",
    "bullet_list"
  ],
  "metadataConfidence": 1,
  "imagePath": null,
  "slideHref": "/slides/019dd923-5e88-73ef-bd5d-0d0f98caffe1/54",
  "deckHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1",
  "deckJsonHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1.json",
  "deckAnchorHref": "/decks/019dd923-5e88-73ef-bd5d-0d0f98caffe1#slide-54",
  "components": [
    {
      "bbox": null,
      "kind": "callout",
      "text": "Open endedness describes a learning system that continually proposes and solves new tasks without a fixed endpoint, selecting tasks that are both novel and learnable, and accumulating the resulting skills so they can be reused to reach further, harder tasks.",
      "attrs": null,
      "subkind": null,
      "toolName": "Visual emphasis",
      "toolSlug": "visual-emphasis",
      "confidence": null,
      "componentId": "019dd952-41ac-7180-a81e-b6d8b16d7beb",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.55,
        "w": 0.94,
        "x": 0.04,
        "y": 0.35
      },
      "kind": "list",
      "text": "Meta's MLGym is a gym for AI-research agents with 13 open-ended tasks across vision, language, RL, and game theory. It supports RL training and logs reproducible traces. Early results indicate that most gains come from hyperparameter tuning rather than genuinely new method design.\nOpenAI's PaperBench evaluates replication of 20 ICML 2024 spotlight and oral papers. It decomposes each paper into thousands of graded subtasks. Current agents achieve low replication scores, which highlights a significant gap to human research practice.\nMichigan's EXP-Bench contains 461 tasks derived from 51 top papers. It requires agents to design, implement, run, and analyze complete experiments starting from provided code. End-to-end success is rare while partial component scores are higher.\nMLR-Bench offers 201 real research tasks with an LLM reviewer calibrated to expert judgment. It evaluates literature synthesis, experiment execution, and report quality. The authors report reasonable judge alignment and frequent failure modes such as fabrication and invalid runs.",
      "attrs": null,
      "subkind": "bullet",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "510d7ec0-8293-4da5-b371-492c6bcf8fc5",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.12,
        "w": 0.96,
        "x": 0.02,
        "y": 0.21
      },
      "kind": "paragraph",
      "text": "Open endedness describes a learning system that continually proposes and solves new tasks without a fixed endpoint, selecting tasks that are both novel and learnable, and accumulating the resulting skills so they can be reused to reach further, harder tasks. Interactive and persistent world models make this increasingly feasible.",
      "attrs": null,
      "subkind": "paragraph",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "a4d31972-d411-4f88-b605-dbd94d27b8f6",
      "frameworkName": null,
      "frameworkSlug": null
    },
    {
      "bbox": {
        "h": 0.04,
        "w": 0.5,
        "x": 0.02,
        "y": 0.14
      },
      "kind": "title",
      "text": "How should we measure progress on open-endedness?",
      "attrs": null,
      "subkind": "headline",
      "toolName": null,
      "toolSlug": null,
      "confidence": null,
      "componentId": "ea98981c-7412-4ce6-b14f-422814b85eef",
      "frameworkName": null,
      "frameworkSlug": null
    }
  ],
  "metrics": [],
  "tools": [],
  "frameworks": [],
  "arcBeats": [
    {
      "to": 189,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-9f380673831f",
      "arcName": "The Triple Take",
      "arcSlug": "triple-take",
      "beatName": "The Facts (What)",
      "beatSlug": "triple-take-the-facts-what",
      "evidence": "Sections 1-2 lay out research findings and industry data with charts and case studies.",
      "position": 1,
      "confidence": 70,
      "parentBeatName": "Setup",
      "parentBeatSlug": "setup"
    },
    {
      "to": 89,
      "from": 12,
      "beatId": "019dd95a-0682-776c-8e35-ac68557d958b",
      "arcName": "The Onion",
      "arcSlug": "onion",
      "beatName": "First Layer",
      "beatSlug": "onion-first-layer",
      "evidence": "Research section unpacks technical layer beneath the headlines.",
      "position": 2,
      "confidence": 45,
      "parentBeatName": "Development",
      "parentBeatSlug": "development"
    }
  ],
  "loops": [],
  "imagePathAlt": null,
  "thumbSrc": null,
  "thumbSrcAlt": null,
  "locked": true
}