{ "community": [], "cross_family_note": "Each board is one result-sealed bundle, shown with its interval and its provenance. Benchmark families share no denominators.", "gateway": { "bundle_count": 1, "bundles": [ { "arms": [ { "arm_id": "direct-openai", "baseline": true, "conditions": { "cold": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.395e-05, "p95": 1.4399999999999998e-05 }, "cold_end_to_end_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.2618867295095697, "p95": 0.9161389394081192 }, "cold_end_to_end_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.26175627097836696, "p95": 0.9146265079878497 }, "cold_end_to_end_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.5556630629871506, "p95": 1.384148770500905 }, "cold_end_to_end_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.7653896670381073, "p95": 1.7911753040534668 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 37.0, "p95": 39.55 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.395e-05, "p95": 1.4399999999999998e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 14.0, "p95": 14.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.7349729375273455, "p95": 1.7633883562404655 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.2320579580264166, "p95": 0.8883519915951177 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.2319540415192023, "p95": 0.8868395601748482 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.5268897500063758, "p95": 1.3563618226879037 }, "setup_dns_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0016002080228645355, "p95": 0.002891178955906066 }, "setup_tcp_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.011436458502430469, "p95": 0.012133112322771922 }, "setup_tls_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.01643262500874698, "p95": 0.022621695898124017 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 92.29271620360342, "p95": 123.19624845830998 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 51.0, "p95": 54.0 } } }, "warm": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 2.835e-05, "p95": 3.0300000000000005e-05 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 38.0, "p95": 40.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.41e-05, "p95": 1.5457499999999996e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 14.0, "p95": 16.19999999999999 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.6562053334782831, "p95": 1.2209905081253964 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.18428431247593835, "p95": 0.43743395827186743 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.1841714994807262, "p95": 0.4361857331299687 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.5037482295010705, "p95": 0.9772484541055746 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 95.10098558742803, "p95": 127.21719451628391 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 52.0, "p95": 54.55 } } } }, "role": "direct" }, { "arm_id": "cloudflare-openai", "baseline": false, "conditions": { "cold": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.395e-05, "p95": 1.4332499999999999e-05 }, "cold_end_to_end_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.3778885630017612, "p95": 0.8001176937425039 }, "cold_end_to_end_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.37778033351060003, "p95": 0.7999479419959239 }, "cold_end_to_end_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.684436937503051, "p95": 1.0906790498644112 }, "cold_end_to_end_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.8488696874992456, "p95": 1.247032909898553 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 37.0, "p95": 39.55 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.395e-05, "p95": 1.4332499999999999e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 14.0, "p95": 14.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.8121094164962415, "p95": 1.2154791437787935 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.34092250050161965, "p95": 0.7672035979398056 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.34081427101045847, "p95": 0.7670338461932256 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.6555284794885665, "p95": 1.054727060740697 }, "setup_dns_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0013577920035459101, "p95": 0.0017405648017302154 }, "setup_tcp_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.01132097901427187, "p95": 0.0123495559731964 }, "setup_tls_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.016649396013235673, "p95": 0.0229986027610721 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 84.58266220944367, "p95": 113.73809479723356 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 51.0, "p95": 53.55 } } }, "warm": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 2.835e-05, "p95": 3.0300000000000005e-05 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 38.0, "p95": 40.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.41e-05, "p95": 1.5457499999999996e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 14.0, "p95": 16.19999999999999 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.7536033329670317, "p95": 0.8933024374855449 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.26018185398424976, "p95": 0.4104486996948252 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.2600222500041127, "p95": 0.41031452936585977 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.5606690415006597, "p95": 0.739080439400277 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 96.57889187473157, "p95": 126.43455910330448 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 52.0, "p95": 54.55 } } } }, "role": "gateway" }, { "arm_id": "concentrate-openai", "baseline": false, "conditions": { "cold": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.395e-05, "p95": 1.4399999999999998e-05 }, "cold_end_to_end_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.4355404380185064, "p95": 0.5690667329850838 }, "cold_end_to_end_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.4354242920235265, "p95": 0.5689299000136088 }, "cold_end_to_end_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.793651020998368, "p95": 0.9779021225287579 }, "cold_end_to_end_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.9667132294853218, "p95": 1.8331581669684933 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 37.0, "p95": 39.55 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.395e-05, "p95": 1.4399999999999998e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 14.0, "p95": 14.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.9297308755049016, "p95": 1.8025811024388503 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.40371929202228785, "p95": 0.5377796893997584 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.4036018130136654, "p95": 0.5376428564282832 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.7626268334861379, "p95": 0.9463682267582043 }, "setup_dns_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0012215210008434951, "p95": 0.0016650413163006301 }, "setup_tcp_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.01148339599603787, "p95": 0.014478962498833422 }, "setup_tls_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.017997916496824473, "p95": 0.02259496236511039 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 84.86000426935587, "p95": 122.97578848982381 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 51.0, "p95": 54.0 } } }, "warm": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 2.835e-05, "p95": 3.0300000000000005e-05 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 38.0, "p95": 40.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.41e-05, "p95": 1.5457499999999996e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 14.0, "p95": 16.19999999999999 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.8962873334821779, "p95": 1.0960268963506674 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.36147922949749045, "p95": 0.43618071058008334 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.3613480414787773, "p95": 0.43604239198320993 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.7036326875095256, "p95": 0.9330836377979717 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 85.26395326652326, "p95": 131.6394817915579 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 52.0, "p95": 54.55 } } } }, "role": "gateway" }, { "arm_id": "openrouter-openai", "baseline": false, "conditions": { "cold": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 0, "ratio": 0.0, "total": 30 }, "p50": null, "p95": null }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.575e-05, "p95": 1.61325e-05 }, "cold_end_to_end_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.499478020996321, "p95": 0.6956120435992488 }, "cold_end_to_end_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.48873749951599166, "p95": 0.6954786559566855 }, "cold_end_to_end_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.5005642494943459, "p95": 0.7029175142524763 }, "cold_end_to_end_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.6901099374808837, "p95": 0.8744508600619156 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 37.0, "p95": 39.55 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.575e-05, "p95": 1.61325e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 17.0, "p95": 17.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.6617980415176135, "p95": 0.8448033626715187 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.46975895849755034, "p95": 0.667759970485349 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.459018437017221, "p95": 0.6676265828427856 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.4708451869955752, "p95": 0.6750654411385766 }, "setup_dns_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0011508960160426795, "p95": 0.0015771440521348263 }, "setup_tcp_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0111028749961406, "p95": 0.011712333545438013 }, "setup_tls_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.015560562023892999, "p95": 0.020372152770869435 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 100.57065537140309, "p95": 156.85007680269499 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 54.0, "p95": 56.55 } } }, "warm": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 0, "ratio": 0.0, "total": 30 }, "p50": null, "p95": null }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 3.135e-05, "p95": 3.24e-05 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 38.0, "p95": 40.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.56e-05, "p95": 1.62e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 17.0, "p95": 17.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.5871571250027046, "p95": 0.7637681001157034 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.43211764548323117, "p95": 0.6315843268821475 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.4233607085188851, "p95": 0.4849301542155444 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.4328554795065429, "p95": 0.6547882412996835 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 105.94427864870052, "p95": 415.5487273251547 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 54.5, "p95": 57.0 } } } }, "role": "gateway" }, { "arm_id": "vercel-openai", "baseline": false, "conditions": { "cold": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.395e-05, "p95": 1.4399999999999998e-05 }, "cold_end_to_end_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.8351534999965224, "p95": 2.5231243521120583 }, "cold_end_to_end_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.8292382500076201, "p95": 2.5188931835087667 }, "cold_end_to_end_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.8415064374858048, "p95": 2.5277202040539075 }, "cold_end_to_end_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.004722916521132, "p95": 2.752883272795584 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 37.0, "p95": 39.55 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.395e-05, "p95": 1.4399999999999998e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 14.0, "p95": 14.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.9506968329951633, "p95": 2.719361543457485 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.7828884375048801, "p95": 2.489602622773959 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.7733296249934938, "p95": 2.4853714541706675 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.7888890624744818, "p95": 2.4941984747158084 }, "setup_dns_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.001817666518036276, "p95": 0.14762862310744762 }, "setup_tcp_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.005584062513662502, "p95": 0.006914308253908529 }, "setup_tls_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.024053103988990188, "p95": 0.028209633677033705 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 87.90931630091568, "p95": 345.2667300252649 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 51.0, "p95": 54.0 } } }, "warm": { "availability": { "attempted": 30, "rate": 1.0, "successes": 30, "wilson95": { "confidence": 0.95, "high": 1.0, "low": 0.886482908609522 } }, "denominators": { "attempted": 30, "request_failed": 0, "route_failed": 0, "route_unverifiable": 0, "route_verified": 30, "scheduled": 30, "success": 30 }, "metrics": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.0, "p95": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 2.835e-05, "p95": 3.0382500000000003e-05 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 38.0, "p95": 40.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 1.4175e-05, "p95": 1.62825e-05 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 14.0, "p95": 18.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.7264135834993795, "p95": 2.4583423890871896 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.5721195829974022, "p95": 1.6005666395387363 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.5588160619954579, "p95": 1.5710355376475476 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 0.5721195829974022, "p95": 1.6050971270393333 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 94.37047761218236, "p95": 449.19378363279077 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "p50": 52.5, "p95": 55.0 } } } }, "role": "gateway" } ], "baseline_arm_id": "direct-openai", "complete_blocks": { "cold": 30, "warm": 30 }, "contrasts": [ { "arm_id": "cloudflare-openai", "conditions": { "cold": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "cold_end_to_end_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.12170043648802675, "low": 0.06726702000014484, "replicates": 2000 }, "median_gateway_minus_direct": 0.09042116702767089 }, "cold_end_to_end_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.1216524159826804, "low": 0.06718133254471469, "replicates": 2000 }, "median_gateway_minus_direct": 0.09037456254009157 }, "cold_end_to_end_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.1595789585262537, "low": 0.007124416501028463, "replicates": 2000 }, "median_gateway_minus_direct": 0.09380425000563264 }, "cold_end_to_end_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.18731377049698494, "low": 0.011488624528283253, "replicates": 2000 }, "median_gateway_minus_direct": 0.11504197953036055 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.1890914993367912, "low": 0.015864896005950868, "replicates": 2000 }, "median_gateway_minus_direct": 0.11766906300908886 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.11944389549898915, "low": 0.07481614596326835, "replicates": 2000 }, "median_gateway_minus_direct": 0.0899678754794877 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.12039591697975993, "low": 0.07240795850520954, "replicates": 2000 }, "median_gateway_minus_direct": 0.08992870850488544 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.1626327504927758, "low": 0.011061979486839846, "replicates": 2000 }, "median_gateway_minus_direct": 0.09514824996585958 }, "setup_dns_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": -0.0001253119553439319, "low": -0.0003124999930150807, "replicates": 2000 }, "median_gateway_minus_direct": -0.00027477103867568076 }, "setup_tcp_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.00021356251090765, "low": -0.0003019794530700892, "replicates": 2000 }, "median_gateway_minus_direct": -1.8708029529079795e-05 }, "setup_tls_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0019422090263105929, "low": -0.0012832079664804041, "replicates": 2000 }, "median_gateway_minus_direct": 0.00019310397328808904 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 20.860273527171852, "low": -20.957597029407538, "replicates": 2000 }, "median_gateway_minus_direct": 1.9003459749691913 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 } }, "warm": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.12259283295134082, "low": -0.012794332986231893, "replicates": 2000 }, "median_gateway_minus_direct": 0.05114497951581143 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.11200035398360342, "low": 0.051706875558011234, "replicates": 2000 }, "median_gateway_minus_direct": 0.07910862501012161 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.11193162450217642, "low": 0.04981233406579122, "replicates": 2000 }, "median_gateway_minus_direct": 0.08056397899053991 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.11582944041074303, "low": 0.0025938330218195915, "replicates": 2000 }, "median_gateway_minus_direct": 0.037403958005597815 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 24.219404585049492, "low": -20.987276097366294, "replicates": 2000 }, "median_gateway_minus_direct": -0.2077389019195195 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 } } }, "direct_arm": "direct-openai" }, { "arm_id": "concentrate-openai", "conditions": { "cold": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "cold_end_to_end_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.19392779149347916, "low": 0.14470543750212528, "replicates": 2000 }, "median_gateway_minus_direct": 0.16657393801142462 }, "cold_end_to_end_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.18906864503514953, "low": 0.1446818329859525, "replicates": 2000 }, "median_gateway_minus_direct": 0.1665636670368258 }, "cold_end_to_end_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.23637485451763496, "low": 0.08468770905165002, "replicates": 2000 }, "median_gateway_minus_direct": 0.19116072947508655 }, "cold_end_to_end_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.2920525830122642, "low": 0.05204991699429229, "replicates": 2000 }, "median_gateway_minus_direct": 0.19803156197303906 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.27374437503749505, "low": 0.06171727098990232, "replicates": 2000 }, "median_gateway_minus_direct": 0.19892531199730001 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.1927400419372134, "low": 0.14380220850580372, "replicates": 2000 }, "median_gateway_minus_direct": 0.16858725002384745 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.1900320767555964, "low": 0.1426413330482319, "replicates": 2000 }, "median_gateway_minus_direct": 0.16857697904924862 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.23524766700575128, "low": 0.0965253540489357, "replicates": 2000 }, "median_gateway_minus_direct": 0.19317883349140175 }, "setup_dns_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": -0.00024479100829921663, "low": -0.000430583517299965, "replicates": 2000 }, "median_gateway_minus_direct": -0.00032902153907343745 }, "setup_tcp_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0005222299951128662, "low": -0.00036754150642082095, "replicates": 2000 }, "median_gateway_minus_direct": 0.000377853459212929 }, "setup_tls_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0027130624803248793, "low": -0.00022147901472635567, "replicates": 2000 }, "median_gateway_minus_direct": 0.0011074999929405749 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 27.941385781837674, "low": -32.401504564727, "replicates": 2000 }, "median_gateway_minus_direct": 3.817695001056485 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 } }, "warm": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.2706499999621883, "low": 0.156668666517362, "replicates": 2000 }, "median_gateway_minus_direct": 0.20070966650382616 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.20377760447445326, "low": 0.14063983474479755, "replicates": 2000 }, "median_gateway_minus_direct": 0.16801072948146611 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.20367116702254862, "low": 0.14306247950298712, "replicates": 2000 }, "median_gateway_minus_direct": 0.16795933348475955 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.23001316701993346, "low": 0.1422742500435561, "replicates": 2000 }, "median_gateway_minus_direct": 0.20233466601348482 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 7.422840232342814, "low": -15.673043016897921, "replicates": 2000 }, "median_gateway_minus_direct": -5.29159021579887 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 } } }, "direct_arm": "direct-openai" }, { "arm_id": "openrouter-openai", "conditions": { "cold": { "cache_write_input_tokens": { "coverage": { "covered": 0, "ratio": 0.0, "total": 30 }, "interval": null, "median_gateway_minus_direct": null }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 1.7999999999999997e-06, "low": 1.7999999999999997e-06, "replicates": 2000 }, "median_gateway_minus_direct": 1.7999999999999997e-06 }, "cold_end_to_end_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.2527823124837596, "low": 0.2226975834928453, "replicates": 2000 }, "median_gateway_minus_direct": 0.2416072295163758 }, "cold_end_to_end_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.24588058301014826, "low": 0.20653462497284636, "replicates": 2000 }, "median_gateway_minus_direct": 0.23038235399872065 }, "cold_end_to_end_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": -0.009755562525242567, "low": -0.17344358400441706, "replicates": 2000 }, "median_gateway_minus_direct": -0.08814597950549796 }, "cold_end_to_end_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.002021333493757993, "low": -0.2061335839680396, "replicates": 2000 }, "median_gateway_minus_direct": -0.09709120853221975 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 1.7999999999999997e-06, "low": 1.7999999999999997e-06, "replicates": 2000 }, "median_gateway_minus_direct": 1.7999999999999997e-06 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 3.0, "low": 3.0, "replicates": 2000 }, "median_gateway_minus_direct": 3.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.014045981059461138, "low": -0.17590204198495485, "replicates": 2000 }, "median_gateway_minus_direct": -0.0945279790321365 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.2520663960312959, "low": 0.2309046603484603, "replicates": 2000 }, "median_gateway_minus_direct": 0.24342847950174473 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.24409531254786998, "low": 0.2121493960148655, "replicates": 2000 }, "median_gateway_minus_direct": 0.23311016647494398 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": -0.00858166697435081, "low": -0.17558802096755244, "replicates": 2000 }, "median_gateway_minus_direct": -0.08753445904585533 }, "setup_dns_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": -0.0002869585296139121, "low": -0.0005898955278098583, "replicates": 2000 }, "median_gateway_minus_direct": -0.00044333349796943367 }, "setup_tcp_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": -8.949998300522566e-05, "low": -0.0006410834612324834, "replicates": 2000 }, "median_gateway_minus_direct": -0.0003782699932344258 }, "setup_tls_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.00126479149912484, "low": -0.0020909784943796694, "replicates": 2000 }, "median_gateway_minus_direct": -0.0005678745219483972 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 47.94634096848077, "low": -2.0912354822923547, "replicates": 2000 }, "median_gateway_minus_direct": 27.41498365396285 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 3.0, "low": 3.0, "replicates": 2000 }, "median_gateway_minus_direct": 3.0 } }, "warm": { "cache_write_input_tokens": { "coverage": { "covered": 0, "ratio": 0.0, "total": 30 }, "interval": null, "median_gateway_minus_direct": null }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 3.5999999999999994e-06, "low": 2.6999999999999996e-06, "replicates": 2000 }, "median_gateway_minus_direct": 3.0000000000000018e-06 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 1.7999999999999997e-06, "low": 1.2000000000000004e-06, "replicates": 2000 }, "median_gateway_minus_direct": 1.7999999999999997e-06 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 3.0, "low": 2.0, "replicates": 2000 }, "median_gateway_minus_direct": 3.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.005193853547098115, "low": -0.12351249950006604, "replicates": 2000 }, "median_gateway_minus_direct": -0.055522812996059656 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.2694512909511104, "low": 0.2287311260588467, "replicates": 2000 }, "median_gateway_minus_direct": 0.2501706040056888 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.25157781250891276, "low": 0.2024219169979915, "replicates": 2000 }, "median_gateway_minus_direct": 0.2296486044942867 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": -0.018306938029127195, "low": -0.1089712499815505, "replicates": 2000 }, "median_gateway_minus_direct": -0.054442000488052145 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 27.1157458830417, "low": 4.206847860683574, "replicates": 2000 }, "median_gateway_minus_direct": 16.020852197066944 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 3.0, "low": 2.0, "replicates": 2000 }, "median_gateway_minus_direct": 3.0 } } }, "direct_arm": "direct-openai" }, { "arm_id": "vercel-openai", "conditions": { "cold": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "cold_end_to_end_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.6373836660059169, "low": 0.4620176244643517, "replicates": 2000 }, "median_gateway_minus_direct": 0.5194777495053131 }, "cold_end_to_end_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.6328783954959363, "low": 0.4505084067335702, "replicates": 2000 }, "median_gateway_minus_direct": 0.5110465420002583 }, "cold_end_to_end_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.29971533350180835, "low": 0.17978658198262565, "replicates": 2000 }, "median_gateway_minus_direct": 0.2288713965099305 }, "cold_end_to_end_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.3379475625115447, "low": 0.16034895749180578, "replicates": 2000 }, "median_gateway_minus_direct": 0.238228269998217 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.29213631301536225, "low": 0.14674216747516766, "replicates": 2000 }, "median_gateway_minus_direct": 0.23466897898470052 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.6006696879921947, "low": 0.45213562549906783, "replicates": 2000 }, "median_gateway_minus_direct": 0.5163948324916419 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.5961508960172068, "low": 0.43974770803470165, "replicates": 2000 }, "median_gateway_minus_direct": 0.5079636249865871 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.2967062824573074, "low": 0.17082722901250236, "replicates": 2000 }, "median_gateway_minus_direct": 0.22735554198152386 }, "setup_dns_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.00042562547605484724, "low": -5.208348738960922e-05, "replicates": 2000 }, "median_gateway_minus_direct": 0.00022552150767296553 }, "setup_tcp_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": -0.005180478998227045, "low": -0.006166687497170642, "replicates": 2000 }, "median_gateway_minus_direct": -0.005376624991185963 }, "setup_tls_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.009415479988092557, "low": 0.005901666474528611, "replicates": 2000 }, "median_gateway_minus_direct": 0.008299104461912066 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 32.281512534957905, "low": -8.347501492040571, "replicates": 2000 }, "median_gateway_minus_direct": 9.449165425328447 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 } }, "warm": { "cache_write_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "cached_input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "charged_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "input_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "measured_cost_usd": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "output_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 }, "request_stream_total_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.22041020850883797, "low": -0.04030391649575904, "replicates": 2000 }, "median_gateway_minus_direct": 0.07145018698065542 }, "request_to_first_body_byte_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.4653479169937782, "low": 0.33173170901136473, "replicates": 2000 }, "median_gateway_minus_direct": 0.36314587452216074 }, "request_to_response_headers_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.46518966602161527, "low": 0.32805150002241135, "replicates": 2000 }, "median_gateway_minus_direct": 0.3480733534961473 }, "request_to_semantic_ttft_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.21406552052940242, "low": 0.013774916500551626, "replicates": 2000 }, "median_gateway_minus_direct": 0.05843802049639635 }, "throughput_tokens_per_s": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 31.988273684160433, "low": -12.96412002218355, "replicates": 2000 }, "median_gateway_minus_direct": 10.75924396155878 }, "total_tokens": { "coverage": { "covered": 30, "ratio": 1.0, "total": 30 }, "interval": { "confidence": 0.95, "high": 0.0, "low": 0.0, "replicates": 2000 }, "median_gateway_minus_direct": 0.0 } } }, "direct_arm": "direct-openai" } ], "date": "2026-07-27", "experiment_digest": "e3649dbc6dc423c5209cdc09c73f0d1275bfa6c231a5fa8300e444dc89e6f943", "experiment_id": "gpt-4o-mini-five-way-managed-probe-r30-2026-07-27", "family": "gateway_probe", "id": "2026-07-27-gpt4o-mini-managed-30", "kind": "release", "link": null, "model_match": "rolling_alias", "path": null, "price_digest": "56ef4049554cc2dc3cb79debc2f5c49d503cfa97c5744d35ffe912815179aea5", "result_count": 300, "results_path": "gateway-probe/2026-07-27-gpt4o-mini-managed-30/results.jsonl", "schedule_digest": "e6b8e2ac79abf1e3b6c6b4ce6b691747a1f9f6bf76c3c78bdb00051e99c097be", "scheduled_blocks_per_condition": 30, "submitter": "OpenBench", "title": "GPT-4o mini managed gateway request benchmark", "track": "request_probe", "verified_with_commit": "2cd722da4d657d1a7d7aeeee7e92de1f7e0261ff" } ], "note": "Gateway Bench measures request-level transport and serving telemetry. Cold and warm denominators are separate and are never merged.", "skipped": [] }, "generated_by": "obench site", "harness": { "bundle_count": 5, "bundles": [ { "also_seen_as": [], "arms": [ { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 5518.583333333333, "harness": "cursor", "median_wall_s": 42.908500000000004, "model": "gpt-5.6-sol", "n": 42, "solve_rate": 0.8571428571428571, "solved": 36, "token_telemetry_bases": [ "harness_reported" ], "token_telemetry_coverage": { "covered_rows": 42, "native_covered_rows": 42, "proxy_covered_rows": 0, "ratio": 1.0, "total_rows": 42 }, "token_telemetry_source": "native", "tokens_cache_read_per_solve": 407067.22222222225, "tokens_cache_write_per_solve": 45776.333333333336, "tokens_input_uncached_per_solve": 72.13888888888889, "tokens_output_per_solve": 5446.444444444444, "wilson95": [ 0.7215861764782122, 0.9328412431581007 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": null, "harness": "devin", "median_wall_s": 68.4285, "model": "gpt-5.6-sol", "n": 42, "solve_rate": 0.8095238095238095, "solved": 34, "token_telemetry_bases": [], "token_telemetry_coverage": { "covered_rows": 0, "native_covered_rows": 0, "proxy_covered_rows": 0, "ratio": 0.0, "total_rows": 42 }, "token_telemetry_source": null, "tokens_cache_read_per_solve": null, "tokens_cache_write_per_solve": null, "tokens_input_uncached_per_solve": null, "tokens_output_per_solve": null, "wilson95": [ 0.6669893264500356, 0.9001811039014356 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 46215.58823529412, "harness": "grokbuild", "median_wall_s": 86.2995, "model": "gpt-5.6-sol", "n": 42, "solve_rate": 0.8095238095238095, "solved": 34, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 42, "native_covered_rows": 0, "proxy_covered_rows": 42, "ratio": 1.0, "total_rows": 42 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 368079.0588235294, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 38726.82352941176, "tokens_output_per_solve": 7488.764705882353, "wilson95": [ 0.6669893264500356, 0.9001811039014356 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 42634.5, "harness": "opencode", "median_wall_s": 60.524, "model": "gpt-5.6-sol", "n": 42, "solve_rate": 0.8095238095238095, "solved": 34, "token_telemetry_bases": [ "vendor_split" ], "token_telemetry_coverage": { "covered_rows": 42, "native_covered_rows": 42, "proxy_covered_rows": 0, "ratio": 1.0, "total_rows": 42 }, "token_telemetry_source": "native", "tokens_cache_read_per_solve": 313916.23529411765, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 36707.55882352941, "tokens_output_per_solve": 5926.941176470588, "wilson95": [ 0.6669893264500356, 0.9001811039014356 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 40255.0, "harness": "claude", "median_wall_s": 58.793, "model": "gpt-5.6-sol", "n": 42, "solve_rate": 0.7619047619047619, "solved": 32, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 42, "native_covered_rows": 42, "proxy_covered_rows": 42, "ratio": 1.0, "total_rows": 42 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 153488.0, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 34455.125, "tokens_output_per_solve": 5799.875, "wilson95": [ 0.6147216491768112, 0.8651917918898182 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 37602.0625, "harness": "pi", "median_wall_s": 39.670500000000004, "model": "gpt-5.6-sol", "n": 42, "solve_rate": 0.7619047619047619, "solved": 32, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 42, "native_covered_rows": 42, "proxy_covered_rows": 42, "ratio": 1.0, "total_rows": 42 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 111408.0, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 32915.71875, "tokens_output_per_solve": 4686.34375, "wilson95": [ 0.6147216491768112, 0.8651917918898182 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 117107.35483870968, "harness": "codex", "median_wall_s": 94.603, "model": "gpt-5.6-sol", "n": 42, "solve_rate": 0.7380952380952381, "solved": 31, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 42, "native_covered_rows": 0, "proxy_covered_rows": 42, "ratio": 1.0, "total_rows": 42 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 1164510.9677419355, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 102174.32258064517, "tokens_output_per_solve": 14933.032258064517, "wilson95": [ 0.5893098001502446, 0.8469751462739642 ] } ], "caveats": [], "claim": null, "countable_rows": 312, "date": "2026-07-21", "excluded_counts": { "infra": 2, "rate_limited": 1 }, "family": "harness", "has_caveats": false, "id": "2026-07-21-gpt56", "kind": "release", "link": null, "matched": { "duplicate_cells_excluded": 0, "duplicate_rows_excluded": 0, "groups_compared": 7, "matched_cells_per_group": 42, "matched_rows": 294, "unmatched_countable_rows": 18 }, "models": [ "gpt-5.6-sol" ], "path": "releases/2026-07-21-gpt56/index.html", "raw_rows": 315, "results_path": "releases/2026-07-21-gpt56/results.jsonl", "results_sha256": "546f167576aac07d8d4c889d559ac86d9f9ad427c5fa53e5a421059f9db498b6", "submitter": null, "table": "matched", "task_set_digest": "d949d46e8aae81489a91d1925ec14f91e4667f5c69ccb314e8fdc611d966c059", "title": "OpenBench: gpt-5.6 across 7 harnesses" }, { "also_seen_as": [], "arms": [ { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 83322.96666666666, "harness": "claude", "median_wall_s": 47.6005, "model": "deepseek-v4-flash", "n": 38, "solve_rate": 0.7894736842105263, "solved": 30, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 38, "native_covered_rows": 37, "proxy_covered_rows": 38, "ratio": 1.0, "total_rows": 38 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 2053623.4666666666, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 35233.7, "tokens_output_per_solve": 48089.26666666667, "wilson95": [ 0.6365391162355784, 0.8892533964790406 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 69624.10714285714, "harness": "pi", "median_wall_s": 24.4575, "model": "deepseek-v4-flash", "n": 38, "solve_rate": 0.7368421052631579, "solved": 28, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 38, "native_covered_rows": 35, "proxy_covered_rows": 38, "ratio": 1.0, "total_rows": 38 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 1729696.0, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 37986.607142857145, "tokens_output_per_solve": 31637.5, "wilson95": [ 0.5799084979362467, 0.8502853761029869 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 71318.76923076923, "harness": "opencode", "median_wall_s": 40.271, "model": "deepseek-v4-flash", "n": 38, "solve_rate": 0.6842105263157895, "solved": 26, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 38, "native_covered_rows": 37, "proxy_covered_rows": 38, "ratio": 1.0, "total_rows": 38 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 1584019.6923076923, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 37710.61538461538, "tokens_output_per_solve": 33608.153846153844, "wilson95": [ 0.5254394356005149, 0.8091557997633336 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 131326.68, "harness": "grokbuild", "median_wall_s": 27.67, "model": "deepseek-v4-flash", "n": 38, "solve_rate": 0.6578947368421053, "solved": 25, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 38, "native_covered_rows": 0, "proxy_covered_rows": 38, "ratio": 1.0, "total_rows": 38 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 1745745.92, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 81795.08, "tokens_output_per_solve": 49531.6, "wilson95": [ 0.4989181162907454, 0.7878777997354104 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 92723.33333333333, "harness": "codex", "median_wall_s": 27.182, "model": "deepseek-v4-flash", "n": 38, "solve_rate": 0.631578947368421, "solved": 24, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 38, "native_covered_rows": 33, "proxy_covered_rows": 38, "ratio": 1.0, "total_rows": 38 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 1212576.0, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 40351.541666666664, "tokens_output_per_solve": 52371.791666666664, "wilson95": [ 0.4728367282426011, 0.766159868445862 ] } ], "caveats": [ "Archived first-party matrix published without rerunning models. The source contains 225 rows across 5 harnesses, 15 tasks, and 3 trials.", "The bundled results SHA is sealed. Current-tree task checks report expected post-run definition drift; recorded task digests are retained as run-time fingerprints.", "The unified board ranks only task/trial cells shared by every harness. The historical release page retains its older per-arm countable view, so its denominators differ.", "Harness and host/container version stamps vary in the archive. Token columns use the complete counting-proxy lane for every displayed harness." ], "claim": null, "countable_rows": 217, "date": "2026-07-20", "excluded_counts": { "infra": 8 }, "family": "harness", "has_caveats": true, "id": "2026-07-20-deepseek", "kind": "release", "link": null, "matched": { "duplicate_cells_excluded": 0, "duplicate_rows_excluded": 0, "groups_compared": 5, "matched_cells_per_group": 38, "matched_rows": 190, "unmatched_countable_rows": 27 }, "models": [ "deepseek-v4-flash" ], "path": "releases/2026-07-20-deepseek/index.html", "raw_rows": 225, "results_path": "releases/2026-07-20-deepseek/results.jsonl", "results_sha256": "ec6b9d58fd3aa6d001db67dc8333e1590c7b1acaa65595f05347413ad23ef414", "submitter": null, "table": "matched", "task_set_digest": "1c4805e7fcd0fd8ff18912c3f745ad243edd2dfd2448f933efe5d0a79966f5bc", "title": "OpenBench: deepseek across 5 harnesses" }, { "also_seen_as": [], "arms": [ { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 36706.57142857143, "harness": "cursor", "median_wall_s": 25.938, "model": "grok-4.5", "n": 39, "solve_rate": 0.8974358974358975, "solved": 35, "token_telemetry_bases": [ "harness_reported" ], "token_telemetry_coverage": { "covered_rows": 39, "native_covered_rows": 39, "proxy_covered_rows": 0, "ratio": 1.0, "total_rows": 39 }, "token_telemetry_source": "native", "tokens_cache_read_per_solve": 247504.45714285714, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 28600.485714285714, "tokens_output_per_solve": 8106.085714285714, "wilson95": [ 0.7642084129775517, 0.9593873444171301 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": null, "harness": "grokbuild", "median_wall_s": 42.2565, "model": "grok-4.5", "n": 39, "solve_rate": 0.8717948717948718, "solved": 34, "token_telemetry_bases": [], "token_telemetry_coverage": { "covered_rows": 0, "native_covered_rows": 0, "proxy_covered_rows": 0, "ratio": 0.0, "total_rows": 39 }, "token_telemetry_source": null, "tokens_cache_read_per_solve": null, "tokens_cache_write_per_solve": null, "tokens_input_uncached_per_solve": null, "tokens_output_per_solve": null, "wilson95": [ 0.7329395096236953, 0.9439726505197168 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": null, "harness": "pi", "median_wall_s": 25.284, "model": "grok-4.5", "n": 39, "solve_rate": 0.7948717948717948, "solved": 31, "token_telemetry_bases": [], "token_telemetry_coverage": { "covered_rows": 0, "native_covered_rows": 38, "proxy_covered_rows": 0, "ratio": 0.0, "total_rows": 39 }, "token_telemetry_source": null, "tokens_cache_read_per_solve": null, "tokens_cache_write_per_solve": null, "tokens_input_uncached_per_solve": null, "tokens_output_per_solve": null, "wilson95": [ 0.6446572703687137, 0.8922040980208888 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 55008.13333333333, "harness": "opencode", "median_wall_s": 21.2275, "model": "grok-4.5", "n": 39, "solve_rate": 0.7692307692307693, "solved": 30, "token_telemetry_bases": [ "vendor_split" ], "token_telemetry_coverage": { "covered_rows": 39, "native_covered_rows": 39, "proxy_covered_rows": 0, "ratio": 1.0, "total_rows": 39 }, "token_telemetry_source": "native", "tokens_cache_read_per_solve": 250734.93333333332, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 45493.666666666664, "tokens_output_per_solve": 9514.466666666667, "wilson95": [ 0.6166343954261705, 0.8735433757121625 ] } ], "caveats": [ "Archived first-party matrix published without rerunning models. The source contains 180 rows across 4 harnesses, 15 tasks, and 3 trials.", "The bundled results SHA is sealed. Current-tree task checks report expected post-run definition drift; recorded task digests are retained as run-time fingerprints.", "The unified board ranks only task/trial cells shared by every harness. The historical release page retains its older per-arm countable view, so its denominators differ.", "No counting-proxy telemetry was retained for this run. Complete native split telemetry is available only where the board reports 100% native coverage." ], "claim": null, "countable_rows": 174, "date": "2026-07-20", "excluded_counts": { "infra": 6 }, "family": "harness", "has_caveats": true, "id": "2026-07-20-grok45", "kind": "release", "link": null, "matched": { "duplicate_cells_excluded": 0, "duplicate_rows_excluded": 0, "groups_compared": 4, "matched_cells_per_group": 39, "matched_rows": 156, "unmatched_countable_rows": 18 }, "models": [ "grok-4.5" ], "path": "releases/2026-07-20-grok45/index.html", "raw_rows": 180, "results_path": "releases/2026-07-20-grok45/results.jsonl", "results_sha256": "3905ac339eea53b2e66fd2eb6b3a670174d364a40111777d4428619aba3f0081", "submitter": null, "table": "matched", "task_set_digest": "1c4805e7fcd0fd8ff18912c3f745ad243edd2dfd2448f933efe5d0a79966f5bc", "title": "OpenBench: grok-4.5 across 4 harnesses" }, { "also_seen_as": [], "arms": [ { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 5856.0, "harness": "aider", "median_wall_s": 39.0355, "model": "deepseek-v4-flash", "n": 8, "solve_rate": 1.0, "solved": 8, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 8, "native_covered_rows": 0, "proxy_covered_rows": 8, "ratio": 1.0, "total_rows": 8 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 336.0, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 2307.625, "tokens_output_per_solve": 3548.375, "wilson95": [ 0.6755843804891231, 1.0 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 13669.375, "harness": "opencode", "median_wall_s": 31.720999999999997, "model": "deepseek-v4-flash", "n": 8, "solve_rate": 1.0, "solved": 8, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 8, "native_covered_rows": 8, "proxy_covered_rows": 8, "ratio": 1.0, "total_rows": 8 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 91600.0, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 10327.875, "tokens_output_per_solve": 3341.5, "wilson95": [ 0.6755843804891231, 1.0 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 7742.25, "harness": "pi", "median_wall_s": 26.444, "model": "deepseek-v4-flash", "n": 8, "solve_rate": 1.0, "solved": 8, "token_telemetry_bases": [ "proxy_measured" ], "token_telemetry_coverage": { "covered_rows": 8, "native_covered_rows": 8, "proxy_covered_rows": 8, "ratio": 1.0, "total_rows": 8 }, "token_telemetry_source": "proxy", "tokens_cache_read_per_solve": 35456.0, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 4927.75, "tokens_output_per_solve": 2814.5, "wilson95": [ 0.6755843804891231, 1.0 ] } ], "caveats": [ "Harness mode is not apples-to-apples: aider ran as a one-shot --message invocation, while pi and opencode ran full agentic loops. Token comparison reflects that difference in harness mode, not a pure like-for-like agent loop contest.", "One aider cell excluded: 1 of 9 aider cells was infrastructure-classified and is excluded from solve-rate denominators (aider reported as 8/8). pi and opencode are 9/9.", "Uniform total-token basis: totals include uncached input, output, and cache reads from split fields; the vendor aggregate is never used. Values are 6,192 vs 48,262 vs 111,380 tokens/solve (aider vs pi vs opencode).", "Verifiable bundle: this release includes results.jsonl and provenance.json. Re-check digests with obench verify docs/releases/2026-07-20-aider-showcase." ], "claim": "Seed community submission: the verified aider showcase bundle (same artifact as docs/releases/2026-07-20-aider-showcase).", "countable_rows": 26, "date": "2026-07-20", "excluded_counts": { "infra": 1 }, "family": "harness", "has_caveats": true, "id": "openbench-aider-showcase", "kind": "community", "link": "https://github.com/minghinmatthewlam/openbench/tree/main/docs/releases/2026-07-20-aider-showcase", "matched": { "duplicate_cells_excluded": 0, "duplicate_rows_excluded": 0, "groups_compared": 3, "matched_cells_per_group": 8, "matched_rows": 24, "unmatched_countable_rows": 2 }, "models": [ "deepseek-v4-flash" ], "path": "../data/community/openbench-aider-showcase/index.html", "raw_rows": 27, "results_path": "../data/community/openbench-aider-showcase/results.jsonl", "results_sha256": "677642cc0649ecfd42231dbdf49fc5aa0b2c2a8161ccb879358bfeec40e45192", "submitter": "openbench", "table": "matched", "task_set_digest": "8b757f8abf3816cac2ddc80e5ceaefa3f4a323c47c2431258a655d4d3b7f2e43", "title": "OpenBench showcase: BYO aider vs pi vs opencode (deepseek-v4-flash)" }, { "also_seen_as": [], "arms": [ { "cost_per_solve_usd": null, "fresh_tokens_per_solve": null, "harness": "pi", "median_wall_s": 693.4925000000001, "model": "glm-5.2", "n": 8, "solve_rate": 1.0, "solved": 8, "token_telemetry_bases": [], "token_telemetry_coverage": { "covered_rows": 0, "native_covered_rows": 6, "proxy_covered_rows": 0, "ratio": 0.0, "total_rows": 8 }, "token_telemetry_source": null, "tokens_cache_read_per_solve": null, "tokens_cache_write_per_solve": null, "tokens_input_uncached_per_solve": null, "tokens_output_per_solve": null, "wilson95": [ 0.6755843804891231, 1.0 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": 94197.42857142857, "harness": "opencode", "median_wall_s": 661.219, "model": "glm-5.2", "n": 8, "solve_rate": 0.875, "solved": 7, "token_telemetry_bases": [ "vendor_split" ], "token_telemetry_coverage": { "covered_rows": 8, "native_covered_rows": 8, "proxy_covered_rows": 0, "ratio": 1.0, "total_rows": 8 }, "token_telemetry_source": "native", "tokens_cache_read_per_solve": 973165.7142857143, "tokens_cache_write_per_solve": 0.0, "tokens_input_uncached_per_solve": 54762.0, "tokens_output_per_solve": 39435.42857142857, "wilson95": [ 0.5291051942301385, 0.9775830911367039 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": null, "harness": "claude", "median_wall_s": 568.97, "model": "glm-5.2", "n": 8, "solve_rate": 0.625, "solved": 5, "token_telemetry_bases": [], "token_telemetry_coverage": { "covered_rows": 0, "native_covered_rows": 4, "proxy_covered_rows": 0, "ratio": 0.0, "total_rows": 8 }, "token_telemetry_source": null, "tokens_cache_read_per_solve": null, "tokens_cache_write_per_solve": null, "tokens_input_uncached_per_solve": null, "tokens_output_per_solve": null, "wilson95": [ 0.3057378545838018, 0.863158240538479 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": null, "harness": "grokbuild", "median_wall_s": 267.572, "model": "glm-5.2", "n": 8, "solve_rate": 0.625, "solved": 5, "token_telemetry_bases": [], "token_telemetry_coverage": { "covered_rows": 0, "native_covered_rows": 0, "proxy_covered_rows": 0, "ratio": 0.0, "total_rows": 8 }, "token_telemetry_source": null, "tokens_cache_read_per_solve": null, "tokens_cache_write_per_solve": null, "tokens_input_uncached_per_solve": null, "tokens_output_per_solve": null, "wilson95": [ 0.3057378545838018, 0.863158240538479 ] }, { "cost_per_solve_usd": null, "fresh_tokens_per_solve": null, "harness": "codex", "median_wall_s": 801.077, "model": "glm-5.2", "n": 8, "solve_rate": 0.5, "solved": 4, "token_telemetry_bases": [], "token_telemetry_coverage": { "covered_rows": 0, "native_covered_rows": 4, "proxy_covered_rows": 0, "ratio": 0.0, "total_rows": 8 }, "token_telemetry_source": null, "tokens_cache_read_per_solve": null, "tokens_cache_write_per_solve": null, "tokens_input_uncached_per_solve": null, "tokens_output_per_solve": null, "wilson95": [ 0.2152125268244419, 0.7847874731755581 ] } ], "caveats": [ "Quarantine-safe derived bundle: all 15 terminal-bench/cancel-async-tasks rows were excluded because its load-sensitive checker is binding-quarantined. No model was rerun.", "The bundled results SHA is sealed. Current-tree task checks report expected post-run definition drift; recorded task digests are retained as run-time fingerprints.", "The retained archive lacks image digests, version-source labels, and an explicit per-row timeout field; contemporaneous version-stamp drift is documented in the bundle README.", "The unified board ranks only task/trial cells shared by every harness. The release page also shows the per-arm countable view, so its denominators differ." ], "claim": null, "countable_rows": 44, "date": "2026-07-09", "excluded_counts": { "infra": 1 }, "family": "harness", "has_caveats": true, "id": "2026-07-09-glm52", "kind": "release", "link": null, "matched": { "duplicate_cells_excluded": 0, "duplicate_rows_excluded": 0, "groups_compared": 5, "matched_cells_per_group": 8, "matched_rows": 40, "unmatched_countable_rows": 4 }, "models": [ "glm-5.2" ], "path": "releases/2026-07-09-glm52/index.html", "raw_rows": 45, "results_path": "releases/2026-07-09-glm52/results.jsonl", "results_sha256": "5434f3f8aa66a708936cc322e26b23dd5670caa5532163f1c27fe3cb1a57349c", "submitter": null, "table": "matched", "task_set_digest": "46075fa27b81914497dc37b6ff595ef49dd09b04154ac9f9e93c33892aa0168f", "title": "OpenBench: GLM 5.2 quarantine-safe archive" } ], "note": "Scores are not comparable across bundles: task sets, trial counts, and timeout caps differ.", "skipped": [ { "id": "2026-07-02-m3", "kind": "release", "path": "releases/2026-07-02-m3/index.html", "reason": "no results.jsonl (HTML-only release page)" }, { "id": "2026-07-20-kimi-k3", "kind": "release", "path": "releases/2026-07-20-kimi-k3/index.html", "reason": "no results.jsonl (HTML-only release page)" } ] }, "packs": [ { "content_sha256": "3b1e576398a8c8f6d42a514ce83ec610f81742edd79f733453bbff681dbeca58", "description": "Tiny polarity-checked smoke tasks (make-it-run, fix-failing-test)", "id": "openbench/core-smoke", "kind": "tasks", "latest": "1.0.0", "license": "Apache-2.0", "name": "core-smoke", "org": "openbench", "source": "data/packs/openbench-core-smoke", "tasks": [ "make-it-run", "fix-failing-test" ] } ], "releases": [ { "date": "2026-07-02", "id": "2026-07-02-m3", "models": [ "gpt-5.5-medium" ], "path": "releases/2026-07-02-m3/index.html", "title": "OpenBench M3 — Harness Comparison" }, { "date": "2026-07-09", "id": "2026-07-09-glm52", "models": [ "glm-5.2" ], "path": "releases/2026-07-09-glm52/index.html", "title": "OpenBench: GLM 5.2 quarantine-safe archive" }, { "date": "2026-07-20", "id": "2026-07-20-grok45", "models": [ "grok-4.5" ], "path": "releases/2026-07-20-grok45/index.html", "title": "OpenBench: grok-4.5 across 4 harnesses" }, { "date": "2026-07-20", "id": "2026-07-20-kimi-k3", "models": [ "kimi-k3" ], "path": "releases/2026-07-20-kimi-k3/index.html", "title": "OpenBench: kimi-k3 across 3 harnesses" }, { "date": "2026-07-20", "id": "2026-07-20-deepseek", "models": [ "deepseek-v4-flash" ], "path": "releases/2026-07-20-deepseek/index.html", "title": "OpenBench: deepseek across 5 harnesses" }, { "date": "2026-07-21", "id": "2026-07-21-gpt56", "models": [ "gpt-5.6-sol" ], "path": "releases/2026-07-21-gpt56/index.html", "title": "OpenBench: gpt-5.6 across 7 harnesses" } ], "schema_version": 5, "site_metadata": { "canonical_url": "https://openbench.run/openbench/", "social_image_url": "https://openbench.run/openbench/og.png" } }