{
  "schema_version": 1,
  "title": "Narwhal serving evaluation, September 2026",
  "evaluation_dates": {
    "start": "2026-09-14",
    "end": "2026-09-18"
  },
  "scope": {
    "selected_runs_per_cell": 1,
    "cells": 12,
    "request_records": 172014,
    "narwhal_version": "v0.1.0",
    "sampling": "One canonical run per system, workload and prefix-cache condition. These figures describe the selected runs; repeated-run uncertainty is outside this evaluation.",
    "public_export": "Aggregate results only."
  },
  "definitions": {
    "offers": "All requests in the workload schedule, including completed, failed and refused requests.",
    "completion_percent": "100 * completed / offers. Completed includes natural completion and output-cap completion.",
    "slo_qualified": "Completed naturally, nonempty answer, at least two output tokens, scheduled TTFT <= 2.1 seconds and TPOT <= 45 milliseconds.",
    "slo_qualified_percent": "100 * slo_qualified_count / offers.",
    "joint_latency_slo": "Requests with both recorded TTFT <= 2.1 seconds and TPOT <= 45 milliseconds, before completion, natural-ending and answer-content conditions are applied.",
    "latency_population": "Completed requests with a recorded value for the metric. Each percentile object includes its sample count.",
    "ttft": "Schedule-adjusted time-to-first-token estimate, in seconds; includes load-generator scheduling delay.",
    "tpot": "Average time per output token after the first token, in milliseconds.",
    "percentiles": "Linear interpolation of sorted values at rank (n - 1) * p / 100.",
    "document_f1": "Token-overlap F1 on document requests, shown on a 0-100 scale. Failed and refused document offers have zero F1.",
    "completed_only_f1_percent": "100 * sum(F1 for completed documents) / completed_documents.",
    "all_offer_f1_percent": "100 * sum(F1 for all documents) / offered_documents.",
    "slo_delivered_f1_percent": "100 * sum(F1 for SLO-qualified documents) / offered_documents; all other document offers contribute zero.",
    "outcome_labels": "Recorded evaluator outcome categories. Framework HTTP and streaming error handling differ; counts preserve those categories."
  },
  "workloads": [
    {
      "id": "chat-document-cycles",
      "label": "Chat/document cycles",
      "description": "Five chat/document/chat cycles.",
      "evaluation_dates": {
        "start": "2026-09-14",
        "end": "2026-09-17"
      },
      "offers_per_cell": 14250,
      "document_offers_per_cell": 7440
    },
    {
      "id": "mixed-payload",
      "label": "Mixed payload",
      "description": "Five request classes (short chat, standard chat, conversation history, long generation and document) across nine consecutive load phases over one hour.",
      "evaluation_dates": {
        "start": "2026-09-17",
        "end": "2026-09-18"
      },
      "offers_per_cell": 14419,
      "document_offers_per_cell": 6310
    }
  ],
  "cells": [
    {
      "workload": "chat-document-cycles",
      "system": "Dynamo Planner",
      "prefix_cache": "off",
      "offers": 14250,
      "outcomes": {
        "completed": 6973,
        "failed": 9,
        "refused": 7268
      },
      "completion_percent": 48.93333333333333,
      "slo_qualified_count": 3812,
      "slo_qualified_percent": 26.750877192982458,
      "joint_latency_slo_count": 4033,
      "joint_latency_slo_percent": 28.301754385964912,
      "natural_completions": 6674,
      "output_cap_completions": 299,
      "completed_request_ttft_seconds": {
        "count": 6973,
        "p50": 1.796222048,
        "p95": 9.115661017599997,
        "p99": 15.263195929439986
      },
      "completed_request_tpot_milliseconds": {
        "count": 6973,
        "p50": 32.47263506382979,
        "p95": 36.01859937173194,
        "p99": 36.81101903081144
      },
      "document_quality": {
        "offered_documents": 7440,
        "completed_documents": 3269,
        "slo_qualified_documents": 1505,
        "completed_only_f1_percent": 48.06868611590967,
        "all_offer_f1_percent": 21.120502004423216,
        "slo_delivered_f1_percent": 9.841018063854946
      }
    },
    {
      "workload": "chat-document-cycles",
      "system": "Dynamo Planner",
      "prefix_cache": "on",
      "offers": 14250,
      "outcomes": {
        "completed": 9380,
        "failed": 47,
        "refused": 4823
      },
      "completion_percent": 65.82456140350877,
      "slo_qualified_count": 8088,
      "slo_qualified_percent": 56.757894736842104,
      "joint_latency_slo_count": 8303,
      "joint_latency_slo_percent": 58.266666666666666,
      "natural_completions": 9142,
      "output_cap_completions": 238,
      "completed_request_ttft_seconds": {
        "count": 9380,
        "p50": 0.6266817775,
        "p95": 7.943264546899999,
        "p99": 14.288360654549912
      },
      "completed_request_tpot_milliseconds": {
        "count": 9380,
        "p50": 31.80084586529049,
        "p95": 36.12889556901672,
        "p99": 36.79514928266756
      },
      "document_quality": {
        "offered_documents": 7440,
        "completed_documents": 6125,
        "slo_qualified_documents": 5970,
        "completed_only_f1_percent": 48.66386773991155,
        "all_offer_f1_percent": 40.06265993373095,
        "slo_delivered_f1_percent": 39.08701197927006
      }
    },
    {
      "workload": "chat-document-cycles",
      "system": "Narwhal",
      "prefix_cache": "off",
      "offers": 14250,
      "outcomes": {
        "completed": 13044,
        "failed": 807,
        "refused": 399
      },
      "completion_percent": 91.53684210526316,
      "slo_qualified_count": 5743,
      "slo_qualified_percent": 40.30175438596491,
      "joint_latency_slo_count": 6072,
      "joint_latency_slo_percent": 42.61052631578947,
      "natural_completions": 12608,
      "output_cap_completions": 436,
      "completed_request_ttft_seconds": {
        "count": 13044,
        "p50": 2.2896235925,
        "p95": 11.684103350700004,
        "p99": 17.653129520199997
      },
      "completed_request_tpot_milliseconds": {
        "count": 13044,
        "p50": 32.65410841875639,
        "p95": 35.985183188924985,
        "p99": 36.326626581730764
      },
      "document_quality": {
        "offered_documents": 7440,
        "completed_documents": 7377,
        "slo_qualified_documents": 1758,
        "completed_only_f1_percent": 48.719282832734315,
        "all_offer_f1_percent": 48.306740518424874,
        "slo_delivered_f1_percent": 12.027478295828534
      }
    },
    {
      "workload": "chat-document-cycles",
      "system": "Narwhal",
      "prefix_cache": "on",
      "offers": 14250,
      "outcomes": {
        "completed": 13074,
        "failed": 765,
        "refused": 411
      },
      "completion_percent": 91.74736842105263,
      "slo_qualified_count": 10855,
      "slo_qualified_percent": 76.17543859649123,
      "joint_latency_slo_count": 11187,
      "joint_latency_slo_percent": 78.50526315789473,
      "natural_completions": 12615,
      "output_cap_completions": 459,
      "completed_request_ttft_seconds": {
        "count": 13074,
        "p50": 0.631438881,
        "p95": 3.600505416050003,
        "p99": 14.096415601090005
      },
      "completed_request_tpot_milliseconds": {
        "count": 13074,
        "p50": 34.52325382052052,
        "p95": 36.17401074720665,
        "p99": 36.73638337870625
      },
      "document_quality": {
        "offered_documents": 7440,
        "completed_documents": 7375,
        "slo_qualified_documents": 6894,
        "completed_only_f1_percent": 48.836363880520985,
        "all_offer_f1_percent": 48.409702099306756,
        "slo_delivered_f1_percent": 45.420555808578946
      }
    },
    {
      "workload": "chat-document-cycles",
      "system": "Ray Serve LLM",
      "prefix_cache": "off",
      "offers": 14250,
      "outcomes": {
        "completed": 7731,
        "failed": 6519,
        "refused": 0
      },
      "completion_percent": 54.252631578947366,
      "slo_qualified_count": 2262,
      "slo_qualified_percent": 15.873684210526315,
      "joint_latency_slo_count": 2409,
      "joint_latency_slo_percent": 16.905263157894737,
      "natural_completions": 7465,
      "output_cap_completions": 266,
      "completed_request_ttft_seconds": {
        "count": 7731,
        "p50": 5.202786981,
        "p95": 30.526806184,
        "p99": 33.490222593199995
      },
      "completed_request_tpot_milliseconds": {
        "count": 7731,
        "p50": 26.764702951612904,
        "p95": 29.665138598275988,
        "p99": 29.773009854228988
      },
      "document_quality": {
        "offered_documents": 7440,
        "completed_documents": 3837,
        "slo_qualified_documents": 99,
        "completed_only_f1_percent": 48.22199622725699,
        "all_offer_f1_percent": 24.869327893008744,
        "slo_delivered_f1_percent": 0.653512400043768
      }
    },
    {
      "workload": "chat-document-cycles",
      "system": "Ray Serve LLM",
      "prefix_cache": "on",
      "offers": 14250,
      "outcomes": {
        "completed": 10059,
        "failed": 4191,
        "refused": 0
      },
      "completion_percent": 70.58947368421053,
      "slo_qualified_count": 7361,
      "slo_qualified_percent": 51.656140350877195,
      "joint_latency_slo_count": 7502,
      "joint_latency_slo_percent": 52.64561403508772,
      "natural_completions": 9860,
      "output_cap_completions": 199,
      "completed_request_ttft_seconds": {
        "count": 10059,
        "p50": 0.67576395,
        "p95": 4.1119866227,
        "p99": 6.832740565940005
      },
      "completed_request_tpot_milliseconds": {
        "count": 10059,
        "p50": 27.72870557471264,
        "p95": 51.60205579247478,
        "p99": 59.879872541303136
      },
      "document_quality": {
        "offered_documents": 7440,
        "completed_documents": 6629,
        "slo_qualified_documents": 5612,
        "completed_only_f1_percent": 48.35352874279291,
        "all_offer_f1_percent": 43.08273414462019,
        "slo_delivered_f1_percent": 36.454235819532116
      }
    },
    {
      "workload": "mixed-payload",
      "system": "Dynamo Planner",
      "prefix_cache": "off",
      "offers": 14419,
      "outcomes": {
        "completed": 8298,
        "failed": 19,
        "refused": 6102
      },
      "completion_percent": 57.54906720299605,
      "slo_qualified_count": 2144,
      "slo_qualified_percent": 14.86926971357237,
      "joint_latency_slo_count": 4554,
      "joint_latency_slo_percent": 31.583327553921908,
      "natural_completions": 4404,
      "output_cap_completions": 3894,
      "completed_request_ttft_seconds": {
        "count": 8298,
        "p50": 1.953946694,
        "p95": 3.960070723249999,
        "p99": 5.018424992750001
      },
      "completed_request_tpot_milliseconds": {
        "count": 8298,
        "p50": 35.91641684731543,
        "p95": 36.6479364517934,
        "p99": 36.98746672224584
      },
      "document_quality": {
        "offered_documents": 6310,
        "completed_documents": 3539,
        "slo_qualified_documents": 1141,
        "completed_only_f1_percent": 48.761954364860244,
        "all_offer_f1_percent": 27.348424167549986,
        "slo_delivered_f1_percent": 10.355722776668483
      }
    },
    {
      "workload": "mixed-payload",
      "system": "Dynamo Planner",
      "prefix_cache": "on",
      "offers": 14419,
      "outcomes": {
        "completed": 8131,
        "failed": 11,
        "refused": 6277
      },
      "completion_percent": 56.39087315347805,
      "slo_qualified_count": 3135,
      "slo_qualified_percent": 21.742145779873777,
      "joint_latency_slo_count": 6027,
      "joint_latency_slo_percent": 41.799015188293225,
      "natural_completions": 4267,
      "output_cap_completions": 3864,
      "completed_request_ttft_seconds": {
        "count": 8131,
        "p50": 1.405489002,
        "p95": 3.51987843,
        "p99": 4.667198612699999
      },
      "completed_request_tpot_milliseconds": {
        "count": 8131,
        "p50": 36.03961963274336,
        "p95": 36.68674785456885,
        "p99": 37.09066037939588
      },
      "document_quality": {
        "offered_documents": 6310,
        "completed_documents": 3259,
        "slo_qualified_documents": 1966,
        "completed_only_f1_percent": 48.276844043413455,
        "all_offer_f1_percent": 24.93411010102765,
        "slo_delivered_f1_percent": 17.60843157871126
      }
    },
    {
      "workload": "mixed-payload",
      "system": "Narwhal",
      "prefix_cache": "off",
      "offers": 14419,
      "outcomes": {
        "completed": 13999,
        "failed": 48,
        "refused": 372
      },
      "completion_percent": 97.08717664193078,
      "slo_qualified_count": 6327,
      "slo_qualified_percent": 43.87960330119981,
      "joint_latency_slo_count": 12167,
      "joint_latency_slo_percent": 84.38171856578126,
      "natural_completions": 7444,
      "output_cap_completions": 6555,
      "completed_request_ttft_seconds": {
        "count": 13999,
        "p50": 0.5761630080000001,
        "p95": 11.203989442500003,
        "p99": 13.45993339134
      },
      "completed_request_tpot_milliseconds": {
        "count": 13999,
        "p50": 32.70146895906433,
        "p95": 36.18781994250544,
        "p99": 36.659642907710406
      },
      "document_quality": {
        "offered_documents": 6310,
        "completed_documents": 6065,
        "slo_qualified_documents": 4006,
        "completed_only_f1_percent": 48.24230282538871,
        "all_offer_f1_percent": 46.36918647162956,
        "slo_delivered_f1_percent": 36.15004540444795
      }
    },
    {
      "workload": "mixed-payload",
      "system": "Narwhal",
      "prefix_cache": "on",
      "offers": 14419,
      "outcomes": {
        "completed": 13973,
        "failed": 51,
        "refused": 395
      },
      "completion_percent": 96.90685900547888,
      "slo_qualified_count": 6579,
      "slo_qualified_percent": 45.627297316041336,
      "joint_latency_slo_count": 12427,
      "joint_latency_slo_percent": 86.1848949303003,
      "natural_completions": 7470,
      "output_cap_completions": 6503,
      "completed_request_ttft_seconds": {
        "count": 13973,
        "p50": 0.384450788,
        "p95": 11.268143746199996,
        "p99": 14.202872779680028
      },
      "completed_request_tpot_milliseconds": {
        "count": 13973,
        "p50": 32.759776660333465,
        "p95": 36.255968078612206,
        "p99": 36.772160723760535
      },
      "document_quality": {
        "offered_documents": 6310,
        "completed_documents": 6058,
        "slo_qualified_documents": 4207,
        "completed_only_f1_percent": 47.9766194711585,
        "all_offer_f1_percent": 46.060595999410175,
        "slo_delivered_f1_percent": 37.87701752285577
      }
    },
    {
      "workload": "mixed-payload",
      "system": "Ray Serve LLM",
      "prefix_cache": "off",
      "offers": 14419,
      "outcomes": {
        "completed": 12364,
        "failed": 2055,
        "refused": 0
      },
      "completion_percent": 85.74797142658991,
      "slo_qualified_count": 3670,
      "slo_qualified_percent": 25.45252791455718,
      "joint_latency_slo_count": 7943,
      "joint_latency_slo_percent": 55.08703793605659,
      "natural_completions": 6443,
      "output_cap_completions": 5921,
      "completed_request_ttft_seconds": {
        "count": 12364,
        "p50": 0.8926562650000001,
        "p95": 16.89826491595,
        "p99": 18.622867519120003
      },
      "completed_request_tpot_milliseconds": {
        "count": 12364,
        "p50": 23.508376176765214,
        "p95": 26.1251323212741,
        "p99": 28.424963987288944
      },
      "document_quality": {
        "offered_documents": 6310,
        "completed_documents": 5015,
        "slo_qualified_documents": 1774,
        "completed_only_f1_percent": 47.803292653387196,
        "all_offer_f1_percent": 37.99263275067144,
        "slo_delivered_f1_percent": 16.038259481175434
      }
    },
    {
      "workload": "mixed-payload",
      "system": "Ray Serve LLM",
      "prefix_cache": "on",
      "offers": 14419,
      "outcomes": {
        "completed": 13569,
        "failed": 850,
        "refused": 0
      },
      "completion_percent": 94.10500034676468,
      "slo_qualified_count": 5353,
      "slo_qualified_percent": 37.1246272279631,
      "joint_latency_slo_count": 10767,
      "joint_latency_slo_percent": 74.67230737221722,
      "natural_completions": 7185,
      "output_cap_completions": 6384,
      "completed_request_ttft_seconds": {
        "count": 13569,
        "p50": 0.489238018,
        "p95": 1.1827449978000002,
        "p99": 1.916541664079994
      },
      "completed_request_tpot_milliseconds": {
        "count": 13569,
        "p50": 26.227336033153428,
        "p95": 49.746646974769185,
        "p99": 50.31315122777252
      },
      "document_quality": {
        "offered_documents": 6310,
        "completed_documents": 5828,
        "slo_qualified_documents": 3159,
        "completed_only_f1_percent": 48.08463160178709,
        "all_offer_f1_percent": 44.41160585978053,
        "slo_delivered_f1_percent": 28.34747022226756
      }
    }
  ]
}