{"schema_version":1,"generated_at":"2026-10-06T19:05:19+00:00","revision":"e874bd125ad989f24cbc134f87ce5b24dde2e6f0","alpha":{"ngram":{"target":"qwen2.5-1.5b-instruct","split":"heldout","n_blocks":700,"n_runs":14,"n_tokens":56990,"max_tokens":256,"ctx_budget":1024,"bootstrap":{"n_resamples":1000,"seed":0,"unit":"runs (with replacement)"},"overall":{"n_tokens":56990,"n_runs":14,"alpha":0.3228337810341852,"alpha_ci95":[0.30735509975170394,0.3389067423903147],"greedy":0.3586079104310954,"greedy_ci95":[0.34289353105576914,0.3749171097568578],"greedy_corpus":0.32299182217406963,"greedy_corpus_ci95":[0.30720817416725277,0.33932525867756674],"coverage":0.7390326137456411,"greedy_valid":0.4503314054225021,"greedy_valid_ci95":[0.374396258235602,0.4970487163930428],"alpha_valid":0.4055527577041534,"alpha_valid_ci95":[0.33605862183338525,0.449215151202422],"tokens_per_call":{"1":1.3228337810341855,"2":1.4271236350084255,"3":1.460836037276166,"4":1.4717409530820469,"5":1.4752706735279704,"6":1.4764139310433682,"7":1.4767844700601294,"8":1.4769046439991649},"speedup":{"1":1.3172721343425697,"2":1.4151736428938648,"3":1.442564113682392,"4":1.4472984333213765,"5":1.4447708816304783,"6":1.4399366443888613,"7":1.4343915198856798,"8":1.4286495047053347},"tokens_per_call_ci95":{"1":[1.307355099751704,1.3389067423903147],"2":[1.4018222583129536,1.4537645226736688],"3":[1.4308572220041846,1.4926905989942718],"4":[1.4397812665106982,1.5058829087988228],"5":[1.4425241172414829,1.510353871577566],"6":[1.4433671464557136,1.5118691110244766],"7":[1.443626255803955,1.5123826358959673],"8":[1.4437058943907026,1.5125566729399094]},"speedup_ci95":{"1":[1.301858530742394,1.3332775194592408],"2":[1.3900841267860682,1.4415914536022731],"3":[1.412960269049212,1.4740202431993648],"4":[1.4158695298806654,1.4808733630098776],"5":[1.412701328669563,1.4791287685498444],"6":[1.407706336135565,1.4745158445811282],"7":[1.4021851537517833,1.4689677957293248],"8":[1.3965354630996856,1.4631366692591783]},"tpc_valid_ci95":{"1":[1.3360586218333852,1.449215151202422],"2":[1.4489940193419544,1.651009403889798],"3":[1.4869469335193055,1.7416584401774657],"4":[1.4997013376200559,1.7823793610937704],"5":[1.5039875651148131,1.8006718159643287],"6":[1.5054279888323143,1.8088890639706925],"7":[1.5059120556468146,1.8125803763441766],"8":[1.5060747304753859,1.814238569825725]}},"by_kind":{"text":{"n_tokens":2149,"n_runs":14,"alpha":0.1982511106400509,"alpha_ci95":[0.1355513236669709,0.2680912574881537],"greedy":0.25085455433619164,"greedy_ci95":[0.18184626379693256,0.3320039823811917],"greedy_corpus":0.1830587454640551,"greedy_corpus_ci95":[0.11999904030710172,0.2543201043122307],"coverage":0.6830241361320813,"greedy_valid":0.3286770681523644,"greedy_valid_ci95":[0.259024966145056,0.39884770508746065],"alpha_valid":0.24820729646240405,"alpha_valid_ci95":[0.19122912354272192,0.30638644391704156],"tokens_per_call":{"1":1.1982511106400509,"2":1.2387404028930724,"3":1.2472464343056666,"4":1.2490818621180517,"5":1.2494880687482501,"6":1.2495801456711049,"7":1.2496014940310616,"8":1.2496065502485771},"speedup":{"1":1.1932132522025336,"2":1.228367834123673,"3":1.2316460582410582,"4":1.2283372412433116,"5":1.223656113460963,"6":1.2187071687822904,"7":1.2137301160823082,"8":1.2087779575633735},"tokens_per_call_ci95":{"1":[1.135551323666971,1.2680912574881538],"2":[1.1539254892361799,1.339964181987987],"3":[1.1564161328431097,1.3592326858543264],"4":[1.1567537431128245,1.36439840375162],"5":[1.1567995066736398,1.3657832877251428],"6":[1.1568057099919684,1.366154563066942],"7":[1.1568065508611296,1.3662540987581802],"8":[1.156806664842239,1.366280783412436]},"speedup_ci95":{"1":[1.1307770766277891,1.262759767131624],"2":[1.1442631165034336,1.3287440178650267],"3":[1.1419518489106601,1.3422316021266834],"4":[1.1375424979782411,1.3417386178189883],"5":[1.1328837976083377,1.3375470414551125],"6":[1.1282248814047395,1.3324014193424456],"7":[1.1235989681255554,1.3270340615025509],"8":[1.1190101375073134,1.32164007503229]},"tpc_valid_ci95":{"1":[1.1912291235427217,1.3063864439170414],"2":[1.2277977088796077,1.4002590969530038],"3":[1.2347906903229728,1.4290204053099034],"4":[1.236127952871813,1.437832480305361],"5":[1.2363836766299556,1.4405323806290313],"6":[1.236432578510934,1.441359593489005],"7":[1.2364419299864233,1.4416130402958562],"8":[1.2364437182634804,1.4416906929618445]}},"thinking":{"n_tokens":18273,"n_runs":13,"alpha":0.16315060336126905,"alpha_ci95":[0.14604828480535592,0.18161853483039972],"greedy":0.2069900771540762,"greedy_ci95":[0.18798770717984223,0.22603239380280907],"greedy_corpus":0.1650310009779379,"greedy_corpus_ci95":[0.14597131810943453,0.18551958895822182],"coverage":0.6599578703779622,"greedy_valid":0.30250872792300104,"greedy_valid_ci95":[0.2783257797880369,0.3256213865629409],"alpha_valid":0.23752796898155148,"alpha_valid_ci95":[0.21364219261922462,0.2613253344385884],"tokens_per_call":{"1":1.163150603361269,"2":1.1898546856591932,"3":1.194239530188198,"4":1.1949618133112931,"5":1.1950811649468651,"6":1.195100948735991,"7":1.1951042383492485,"8":1.1951047870429539},"speedup":{"1":1.1582603194890373,"2":1.1798914604961424,"3":1.1793021567311808,"4":1.17511601254456,"5":1.1703740196849548,"6":1.1655739719366673,"7":1.160797272467118,"8":1.1560569390970485},"tokens_per_call_ci95":{"1":[1.146048284805356,1.1816185348303996],"2":[1.167378386309559,1.214603827086011],"3":[1.170493611051781,1.2205945675588312],"4":[1.1709485842827811,1.221682597072158],"5":[1.1710150323429207,1.2218802033996992],"6":[1.1710247369681546,1.2219160923717163],"7":[1.1710261543120286,1.2219226104743037],"8":[1.1710263613126712,1.221923794282561]},"speedup_ci95":{"1":[1.1412299049431207,1.176650605443343],"2":[1.1576033660037497,1.20443336546661],"3":[1.155853248079892,1.2053275491472404],"4":[1.1515015927112517,1.2013930203245178],"5":[1.1468054310567493,1.1966190139813295],"6":[1.1420926034304284,1.191726602397384],"7":[1.1374103800273045,1.1868457895887472],"8":[1.1327652315833494,1.1819996846664607]},"tpc_valid_ci95":{"1":[1.2136421926192247,1.2613253344385884],"2":[1.2592851794434827,1.3296162650426062],"3":[1.2690364473788531,1.3474624154183088],"4":[1.2711197296902552,1.3521260666715076],"5":[1.2715648067050258,1.353344796908137],"6":[1.2716598939380617,1.3536632819991212],"7":[1.2716802085839698,1.3537465102233872],"8":[1.2716845486497015,1.3537682598673402]}},"tool_result":{"n_tokens":12634,"n_runs":13,"alpha":0.2733356887349496,"alpha_ci95":[0.23862487117513964,0.30830895950741805],"greedy":0.3049667908620116,"greedy_ci95":[0.26946849809148615,0.33951874015158273],"greedy_corpus":0.27052812467554355,"greedy_corpus_ci95":[0.2352074390068669,0.306457726076876],"coverage":0.7167333409230844,"greedy_valid":0.4242852509996297,"greedy_valid_ci95":[0.3883434886934337,0.45662245583998007],"alpha_valid":0.3804624544396679,"alpha_valid_ci95":[0.3450518529555296,0.41429144264642886],"tokens_per_call":{"1":1.2733356887349496,"2":1.348391770527272,"3":1.3690947879176174,"4":1.3748307214449769,"5":1.3764268268436346,"6":1.3768728633119993,"7":1.3769980318455133,"8":1.37703330149159},"speedup":{"1":1.2679821489916325,"2":1.337101038154919,"3":1.351970350459113,"4":1.3519976766716264,"5":1.3479705357140463,"6":1.3428549059723864,"7":1.3374695765171842,"8":1.3320412743856367},"tokens_per_call_ci95":{"1":[1.2386248711751398,1.308308959507418],"2":[1.2955667061735183,1.4033633740239009],"3":[1.309154447006066,1.4326695016624922],"4":[1.3123968209100318,1.4417048433830602],"5":[1.3131705322824634,1.4444905201881841],"6":[1.3133551591535875,1.4453493693056736],"7":[1.3133992157439736,1.4456141601835266],"8":[1.3134097287497015,1.4456957975835878]},"speedup_ci95":{"1":[1.2334172676079413,1.3028083801447057],"2":[1.2847183034543175,1.3916123379944612],"3":[1.29277973456897,1.4147498809784915],"4":[1.2906006718243126,1.417761160189595],"5":[1.286022003758708,1.4146270781404395],"6":[1.2809065134097988,1.4096395848397307],"7":[1.2756964441879262,1.4041159928432079],"8":[1.2704964847104174,1.3984603498704584]},"tpc_valid_ci95":{"1":[1.3450518529555295,1.414291442646429],"2":[1.4641126371402422,1.5859288423850586],"3":[1.505194783376945,1.6570367485739914],"4":[1.5193702551143669,1.6864961457611967],"5":[1.5242615283890495,1.6987009220034872],"6":[1.52594927150495,1.7037572564026093],"7":[1.5265316304809677,1.7058520524964693],"8":[1.5267325745595584,1.706719908602466]}},"tool_use:Bash":{"n_tokens":22279,"n_runs":13,"alpha":0.47687491729722464,"alpha_ci95":[0.45825466595951214,0.495545585735289],"greedy":0.5075922436026107,"greedy_ci95":[0.49017361840407264,0.5250583674960697],"greedy_corpus":0.47883923505549847,"greedy_corpus_ci95":[0.45848376386079115,0.49921527798436477],"coverage":0.8148643618677642,"greedy_valid":0.6214038876278121,"greedy_valid_ci95":[0.6078535529872223,0.6351599332897861],"alpha_valid":0.5834598003172102,"alpha_valid_ci95":[0.567736909220404,0.5999453885245389],"tokens_per_call":{"1":1.4768749172972249,"2":1.7043754685011017,"3":1.812951523519057,"4":1.8647908598431842,"5":1.8895513202874226,"6":1.901382608908088,"7":1.9070382204358909,"8":1.909742810355887},"speedup":{"1":1.4706656288624416,"2":1.690103913529119,"3":1.7902753908993259,"4":1.8338206083557989,"5":1.8504866773832305,"6":1.854405757087122,"7":1.852294296797245,"8":1.8473454811149306},"tokens_per_call_ci95":{"1":[1.4582546659595121,1.4955455857352893],"2":[1.6682520048979579,1.7411110173262192],"3":[1.7644842653649517,1.862799886974979],"4":[1.8085831477805903,1.9231022721479005],"5":[1.828791666436089,1.9529848549029318],"6":[1.838052314416382,1.9677930381003483],"7":[1.8422960495710126,1.9751311686440103],"8":[1.8442407610109166,1.9787675472631852]},"speedup_ci95":{"1":[1.4521236634444976,1.4892577994099643],"2":[1.6542829290487493,1.7265318579477111],"3":[1.7424143541247554,1.8395002583125029],"4":[1.7785463880941332,1.891163590826076],"5":[1.7909831705098598,1.9126087851264972],"6":[1.7926401439205613,1.9191754050515981],"7":[1.789410631136954,1.9184325515350975],"8":[1.7839835906001293,1.9141149618635356]},"tpc_valid_ci95":{"1":[1.5677369092204039,1.5999453885245387],"2":[1.8900621079520097,1.959879859283907],"3":[2.0730580207708806,2.175820887044222],"4":[2.1769515553337815,2.3053737125139433],"5":[2.2359357500031285,2.3830983340623098],"6":[2.2694232547100803,2.429728863338288],"7":[2.2884353473564203,2.457704635064266],"8":[2.2992292142232773,2.474488570807415]}},"tool_use:Edit":{"n_tokens":1348,"n_runs":4,"alpha":0.5691141042758421,"alpha_ci95":[0.5394123442495616,0.6140749936848018],"greedy":0.5940391981717801,"greedy_ci95":[0.5627062706270627,0.6416184971098265],"greedy_corpus":0.5729354368826657,"greedy_corpus_ci95":[0.5394495412844037,0.615606936416185],"coverage":0.8491998285110385,"greedy_valid":0.7027290792909568,"greedy_valid_ci95":[0.6669569209822206,0.7393831904621144],"alpha_valid":0.6739810941290313,"alpha_valid_ci95":[0.639944017321287,0.7087332298631375],"tokens_per_call":{"1":1.5691141042758423,"2":1.8933379205143275,"3":2.078241510115112,"4":2.183804035768551,"5":2.244136040154862,"6":2.2786559121430274,"7":2.2984294415203594,"8":2.309769182758745},"speedup":{"1":1.562517010678799,"2":1.8774840921105875,"3":2.0522471691259363,"4":2.1475356468337155,"5":2.197740701697612,"6":2.2223578895176996,"7":2.232450142056374,"8":2.2343017285100997},"tokens_per_call_ci95":{"1":[1.5394123442495615,1.6140749936848018],"2":[1.8303780213783691,1.991163091553791],"3":[1.9873284993745806,2.2227234628713046],"4":[2.071989524641606,2.364918896425757],"5":[2.1176567267474637,2.4522375563877152],"6":[2.1422901792907028,2.50585776195242],"7":[2.1555777676740115,2.5387845893459438],"8":[2.1627452568732757,2.559004130669683]},"speedup_ci95":{"1":[1.532940127033628,1.6072888690957938],"2":[1.8150513864705198,1.9744901259752117],"3":[1.962471285033121,2.1949219627389556],"4":[2.037578139408425,2.3256425708351576],"5":[2.0738762255587124,2.401539920696104],"6":[2.0893613012003303,2.443946335911104],"7":[2.0936992046509086,2.4659055939462826],"8":[2.0920815386355875,2.475393383502839]},"tpc_valid_ci95":{"1":[1.6399440173212867,1.7087332298631375],"2":[2.0494723626265947,2.211036020975372],"3":[2.311547577128213,2.5670347004896152],"4":[2.479261042736716,2.819342794448757],"5":[2.586588271677097,2.9981619248010314],"6":[2.655271689733166,3.1248969846169157],"7":[2.6992252322073242,3.214718332897126],"8":[2.727353038753739,3.2783777071744207]}},"tool_use:Read":{"n_tokens":307,"n_runs":5,"alpha":0.5601281141481335,"alpha_ci95":[0.44978788069316317,0.6095507748921712],"greedy":0.590541593760969,"greedy_ci95":[0.48214285714285715,0.6409287089132365],"greedy_corpus":0.5289716340647455,"greedy_corpus_ci95":[0.41964285714285715,0.5821596244131455],"coverage":0.8279115312792408,"greedy_valid":0.7153762944148541,"greedy_valid_ci95":[0.6637209929843497,0.7640238704177322],"alpha_valid":0.670068913774156,"alpha_valid_ci95":[0.6109990866648719,0.719048333066402],"tokens_per_call":{"1":1.5601281141481336,"2":1.8757361472038978,"3":2.054449544753679,"4":2.156073514053592,"5":2.2140726688113554,"6":2.247280727151446,"7":2.266348943527873,"8":2.2773263146233513},"speedup":{"1":1.5535688007340438,"2":1.860029706908029,"3":2.028752790189083,"4":2.120265670721956,"5":2.168298861430516,"6":2.1917578811840412,"7":2.201290554989306,"8":2.202918871342459},"tokens_per_call_ci95":{"1":[1.4497878806931634,1.6095507748921714],"2":[1.6520970183116106,1.9811029220638177],"3":[1.7430932165658735,2.2075828212851447],"4":[1.7840222037297933,2.3456338193530057],"5":[1.8024315661251704,2.4297829121999075],"6":[1.8107118742218995,2.4810760569512103],"7":[1.8144362564522136,2.512341833081023],"8":[1.816111438442478,2.5313999111485557]},"speedup_ci95":{"1":[1.4436924754458786,1.6027836716698995],"2":[1.638263215929663,1.9645141951195986],"3":[1.7212908614368863,2.179970698084336],"4":[1.7543933496323225,2.3066777783046346],"5":[1.7651680397345701,2.37954950452228],"6":[1.7659751952351095,2.419776904567105],"7":[1.7623505883172002,2.4402219101604814],"8":[1.7567733418421065,2.4486910810171154]},"tpc_valid_ci95":{"1":[1.610999086664872,1.719048333066402],"2":[1.9843189705701796,2.2360788383519736],"3":[2.2124170786701587,2.6078487613220434],"4":[2.351784814389231,2.8751693047178963],"5":[2.4369383736241357,3.0673856958410894],"6":[2.488967120542925,3.205598571466261],"7":[2.5207566373906234,3.304980309292855],"8":[2.5401800031500845,3.376440582214309]}}}},"lora":{"target":"qwen2.5-1.5b-instruct","split":"heldout","n_blocks":700,"n_runs":14,"n_tokens":56990,"max_tokens":256,"ctx_budget":1024,"bootstrap":{"n_resamples":1000,"seed":0,"unit":"runs (with replacement)"},"overall":{"n_tokens":56990,"n_runs":14,"alpha":0.7119553246441913,"alpha_ci95":[0.7060653965165569,0.7187037139746982],"greedy":0.6964866788394445,"greedy_ci95":[0.6876688232440973,0.7066849905385792],"greedy_corpus":0.5829131924873775,"greedy_corpus_ci95":[0.5729801796835053,0.5947366966817499],"coverage":0.9997529910530587,"greedy_valid":0.6846169760142126,"greedy_valid_ci95":[0.656964563697037,0.7034120641352818],"alpha_valid":0.7051261370834527,"alpha_valid_ci95":[0.6882810986926144,0.7169444111974532],"tokens_per_call":{"1":1.7119553246441914,"2":2.2188463399822393,"3":2.579745241431733,"4":2.836705316002687,"5":3.0196647757288058,"6":3.1499374191704237,"7":3.242697416327927,"8":3.3087481093873166},"speedup":{"1":1.3439153748641524,"2":1.4336290795605895,"3":1.4162211633697683,"4":1.3537606962178808,"5":1.274506027985563,"6":1.191741137376374,"7":1.1116566619534527,"8":1.0369481811625507},"tokens_per_call_ci95":{"1":[1.7060653965165566,1.7187037139746981],"2":[2.2045937406841816,2.235238742564321],"3":[2.5565873536971124,2.6064743861658157],"4":[2.8051178636546545,2.873282822150297],"5":[2.9805966567277222,3.0650390361734066],"6":[3.1044961603509624,3.2028549395144426],"7":[3.1919773125149558,3.301903741215396],"8":[3.2537447269136264,3.3730904829674087]},"speedup_ci95":{"1":[1.3392916765388367,1.3492129805003992],"2":[1.424420266654156,1.4442204506718148],"3":[1.403508012403548,1.4308948527880978],"4":[1.33868621835753,1.3712165771333078],"5":[1.2580165972483643,1.2936570837308397],"6":[1.174548980751376,1.2117618481048196],"7":[1.094268872079888,1.1319536854005146],"8":[1.0197103451205554,1.0571128189801793]},"tpc_valid_ci95":{"1":[1.6882810986926142,1.716944411197453],"2":[2.162011969640492,2.230953700597048],"3":[2.488071974160553,2.5994697888711342],"4":[2.7124929124965944,2.86367533980172],"5":[2.8669576026741543,3.0530960339106388],"6":[2.973272729485033,3.1889001427792922],"7":[3.0464474218948596,3.2862641403921167],"8":[3.096812179680178,3.3560687149530026]}},"by_kind":{"text":{"n_tokens":2149,"n_runs":14,"alpha":0.6276326415029997,"alpha_ci95":[0.6018315624022902,0.6599639904563732],"greedy":0.5980695029317894,"greedy_ci95":[0.5514963665559943,0.6484443906244225],"greedy_corpus":0.38723608202569926,"greedy_corpus_ci95":[0.33289818327968773,0.4599957326900628],"coverage":0.9994822235620533,"greedy_valid":0.5792142435640475,"greedy_valid_ci95":[0.5363003264790211,0.6278916309569845],"alpha_valid":0.6210010320888443,"alpha_valid_ci95":[0.5986809073019624,0.6464385113942677],"tokens_per_call":{"1":1.6276326415029996,"2":2.0217836980773054,"3":2.2694535309585437,"4":2.425171367473649,"5":2.5231339069113443,"6":2.5847990463713346,"7":2.62363902606463,"8":2.64811722393049},"speedup":{"1":1.2777205690231777,"2":1.3063040238146337,"3":1.2458781077328291,"4":1.157364376326951,"5":1.0649358828239934,"6":0.9779277952205179,"7":0.8994319936235659,"8":0.8299091523677264},"tokens_per_call_ci95":{"1":[1.60183156240229,1.6599639904563732],"2":[1.9640327919177876,2.09551646277392],"3":[2.1820169237954725,2.382965415235161],"4":[2.313206654475274,2.572671377684679],"5":[2.3921607750718237,2.6978704858401663],"6":[2.4396778568363597,2.7804973922960547],"7":[2.468275136401152,2.8350281779216777],"8":[2.4854858818460492,2.8710165349022376]},"speedup_ci95":{"1":[1.2574662630886833,1.303101252924825],"2":[1.2689903185122997,1.3539438417149174],"3":[1.1978774092418163,1.3081935372662143],"4":[1.1039314635159359,1.2277557967501358],"5":[1.009656221526713,1.138686725945515],"6":[0.9230229293583347,1.0519679231086532],"7":[0.8461703781235336,0.9719001054317548],"8":[0.7789411521455176,0.8997648885716343]},"tpc_valid_ci95":{"1":[1.5986809073019626,1.646438511394268],"2":[1.9570997361227254,2.0643212605276977],"3":[2.1716782458185393,2.334456762969741],"4":[2.3001423027475685,2.509082755193312],"5":[2.377051280950898,2.6219677217865254],"6":[2.4230952178352836,2.69494091165445],"7":[2.4506608437693704,2.7421135919655204],"8":[2.4671638577309705,2.772607829265497]}},"thinking":{"n_tokens":18273,"n_runs":13,"alpha":0.643343326326918,"alpha_ci95":[0.6350077978544963,0.6523788573812551],"greedy":0.6024165373901444,"greedy_ci95":[0.5907617455783141,0.6151699253802015],"greedy_corpus":0.419728047998233,"greedy_corpus_ci95":[0.40337749380995275,0.4371070116127227],"coverage":0.9995620564667069,"greedy_valid":0.5991817606784781,"greedy_valid_ci95":[0.5879494317285655,0.6123625710932985],"alpha_valid":0.6409874760879462,"alpha_valid_ci95":[0.6325700241872793,0.6503994851158189],"tokens_per_call":{"1":1.643343326326918,"2":2.0572530380120417,"3":2.3235636290235724,"4":2.494916448803069,"5":2.6051754509414864,"6":2.676126175479871,"7":2.721784458592219,"8":2.751167873906936},"speedup":{"1":1.2900537360051472,"2":1.3292212832241626,"3":1.2755833145885176,"4":1.1906488170214597,"5":1.099563051790596,"6":1.0124803219008067,"7":0.9330780635159421,"8":0.8622048063516378},"tokens_per_call_ci95":{"1":[1.6350077978544963,1.6523788573812552],"2":[2.038242701402781,2.077977031064955],"3":[2.294300009792611,2.3556282814801976],"4":[2.456898397574532,2.536762087136988],"5":[2.5601496419519867,2.654929952444028],"6":[2.6257149874423735,2.732020169506017],"7":[2.667349493229249,2.782312197317285],"8":[2.6937877291611145,2.815121653024]},"speedup_ci95":{"1":[1.2835101979171812,1.297146788568601],"2":[1.316938426640462,1.3426113583050292],"3":[1.259518256610668,1.2931860757743339],"4":[1.1725054648693525,1.210618808322648],"5":[1.080559066502794,1.1205628702453123],"6":[0.9934078520153182,1.0336271458370012],"7":[0.9144167503806956,0.9538280920717652],"8":[0.8442221026939571,0.8822476602472737]},"tpc_valid_ci95":{"1":[1.6325700241872794,1.6503994851158192],"2":[2.032714859687783,2.073418976749957],"3":[2.285834487959016,2.348550638117968],"4":[2.445950377337075,2.527496130782012],"5":[2.5472348893539336,2.6438821886107795],"6":[2.6113044355704016,2.719579621946481],"7":[2.6518329099703477,2.768813194587764],"8":[2.6774700080018885,2.800834685624141]}},"tool_result":{"n_tokens":12634,"n_runs":13,"alpha":0.7198316064705366,"alpha_ci95":[0.7050683331260383,0.7329903393833369],"greedy":0.6842574475730602,"greedy_ci95":[0.6637609694313117,0.706651033561046],"greedy_corpus":0.565976211443455,"greedy_corpus_ci95":[0.5411112956076515,0.5905024840189552],"coverage":1.0,"greedy_valid":0.6856946809959847,"greedy_valid_ci95":[0.6644661806031985,0.706997542483552],"alpha_valid":0.7185439456755666,"alpha_valid_ci95":[0.7027480304654805,0.7322139053265182],"tokens_per_call":{"1":1.7198316064705366,"2":2.23804221989076,"3":2.6111429412462095,"4":2.8797950045984626,"5":3.0732582598801073,"6":3.212590279506586,"7":3.3129472205287094,"8":3.385238822724128},"speedup":{"1":1.3500983961678124,"2":1.4460318183842378,"3":1.433457782801893,"4":1.3743243855458327,"5":1.2971261609091245,"6":1.2154450975193551,"7":1.1357395944057687,"8":1.0609200591805243},"tokens_per_call_ci95":{"1":[1.7050683331260381,1.732990339383337],"2":[2.2021896875156353,2.270265177743148],"3":[2.552694212234045,2.664082444925668],"4":[2.7998238532489803,2.952746698430522],"5":[2.9740671373228937,3.164334808726022],"6":[3.096920559198698,3.319426850555835],"7":[3.183540616592533,3.4331078198602496],"8":[3.244613676085206,3.5164348729257284]},"speedup_ci95":{"1":[1.3385089640457344,1.3604282355162303],"2":[1.4228669727332866,1.4668515428393383],"3":[1.4013707667393487,1.4625203601002352],"4":[1.3361597581107996,1.409139117719539],"5":[1.2552606913914655,1.3355667227374568],"6":[1.1716828426882628,1.2558654360063197],"7":[1.0913766166447025,1.1769328707437412],"8":[1.0168487109811144,1.1020363669605842]},"tpc_valid_ci95":{"1":[1.702748030465481,1.7322139053265182],"2":[2.196602827674172,2.2683511086006556],"3":[2.543658317803236,2.660918224177505],"4":[2.7875508842268513,2.9483613251708305],"5":[2.9589459089292487,3.1588311608977597],"6":[3.079393428486026,3.3129401014422024],"7":[3.1640376885929977,3.4257808107963443],"8":[3.2235212781029148,3.508404347402518]}},"tool_use:Bash":{"n_tokens":22279,"n_runs":13,"alpha":0.7643614106661357,"alpha_ci95":[0.755879682702107,0.773354422285809],"greedy":0.7801231107770051,"greedy_ci95":[0.7708455078923272,0.7897636960751325],"greedy_corpus":0.7288676027006984,"greedy_corpus_ci95":[0.7159793651180538,0.7415127294164292],"coverage":0.9997771291712927,"greedy_valid":0.7786241318840851,"greedy_valid_ci95":[0.7688059977049776,0.7882531626293429],"alpha_valid":0.7628964283998674,"alpha_valid_ci95":[0.7539375081492489,0.771705837539884],"tokens_per_call":{"1":1.7643614106661358,"2":2.348630286971115,"3":2.7952542281671664,"4":3.1366722941122687,"5":3.39767574493846,"6":3.597211742507848,"7":3.7497615001188658,"8":3.866393318761733},"speedup":{"1":1.38505508436912,"2":1.517484386307469,"3":1.5345306704518353,"4":1.4969139179632502,"5":1.4340526315604885,"6":1.360962026518283,"7":1.2854876102082053,"8":1.2117119185272203},"tokens_per_call_ci95":{"1":[1.755879682702107,1.773354422285809],"2":[2.3272337777028165,2.371431485620697],"3":[2.7591087301640616,2.8339570287578475],"4":[3.0855542326753937,3.1916532045170394],"5":[3.332307756014844,3.468279125477639],"6":[3.5188237313973545,3.6822090063207127],"7":[3.659807368101097,3.8476526271677463],"8":[3.7663740350428627,3.9755991842863256]},"speedup_ci95":{"1":[1.3783967770802648,1.3921147584202074],"2":[1.5036597886616647,1.5322165743115919],"3":[1.514687618351029,1.5557776232120941],"4":[1.472518848778691,1.5231524543132786],"5":[1.4064628485520965,1.4638521095742767],"6":[1.331304860331758,1.3931197243930873],"7":[1.254649669663931,1.3190464994780067],"8":[1.180366282381872,1.2459365919941217]},"tpc_valid_ci95":{"1":[1.7539375081492488,1.7717058375398842],"2":[2.3223592743977206,2.367235737658017],"3":[2.750913764502571,2.8268096386647272],"4":[3.074017068970768,3.1814655016120046],"5":[3.3176167693091654,3.4551555021476292],"6":[3.5012757204557063,3.6663636739911514],"7":[3.639743092511602,3.8293542538371175],"8":[3.7441388380290315,3.9551350363969093]}},"tool_use:Edit":{"n_tokens":1348,"n_runs":4,"alpha":0.8166142276453451,"alpha_ci95":[0.7707801748853211,0.8321928982673487],"greedy":0.8267077782991393,"greedy_ci95":[0.777262180974478,0.8477508650519031],"greedy_corpus":0.8125984521250071,"greedy_corpus_ci95":[0.7944111776447106,0.870242214532872],"coverage":1.0,"greedy_valid":0.8170222611908332,"greedy_valid_ci95":[0.7672877846790891,0.8562781109445278],"alpha_valid":0.8036746514098111,"alpha_valid_ci95":[0.7617994097066043,0.8308606020367573],"tokens_per_call":{"1":1.816614227645345,"2":2.483705499735784,"3":3.0288354390456944,"4":3.4744470947970485,"5":3.8388234865402606,"6":4.136864975632374,"7":4.380720058801293,"8":4.5802974981664235},"speedup":{"1":1.4260744749498395,"2":1.604758457277527,"3":1.6627614154490102,"4":1.6581101644539629,"5":1.6202472914521686,"6":1.565133370976941,"7":1.50179187641697,"8":1.4354465806665861},"tokens_per_call_ci95":{"1":[1.770780174885321,1.832192898267349],"2":[2.3648822528815674,2.524737918193958],"3":[2.8228043564592467,3.1010689655073023],"4":[3.175761635538705,3.5806875701324503],"5":[3.447814108834616,3.979822766778394],"6":[3.657506761779623,4.31198024287569],"7":[3.8191337014887425,4.588399335590267],"8":[3.943712542343917,4.818433341492841]},"speedup_ci95":{"1":[1.3900939284310023,1.438304008435596],"2":[1.5279850192307198,1.6312701031028112],"3":[1.549655061075439,1.7024159701844168],"4":[1.515568521867867,1.7088112996918097],"5":[1.4552144663219517,1.679758676308375],"6":[1.3837739257032515,1.6313861372974474],"7":[1.309269684175048,1.5729881744217267],"8":[1.235943448281624,1.5100773840530257]},"tpc_valid_ci95":{"1":[1.7617994097066043,1.8308606020367573],"2":[2.3421377503359353,2.52118994205364],"3":[2.7842391556574695,3.0947573931037047],"4":[3.1210317452618748,3.5713119907918496],"5":[3.377600141216069,3.9672624307304063],"6":[3.573053793803345,4.296242051634474],"7":[3.721950270969331,4.569578257516652],"8":[3.8353795193817724,4.796682542094363]}},"tool_use:Read":{"n_tokens":307,"n_runs":5,"alpha":0.7738909607981079,"alpha_ci95":[0.6622245371842584,0.8300352668762208],"greedy":0.8141464947575233,"greedy_ci95":[0.7089201877934272,0.8866666666666667],"greedy_corpus":0.8319328523385257,"greedy_corpus_ci95":[0.813953488372093,0.8495145631067961],"coverage":1.0,"greedy_valid":0.7837904476801412,"greedy_valid_ci95":[0.6256830601092895,0.8881159420289855],"alpha_valid":0.7302192739470665,"alpha_valid_ci95":[0.5923930349804106,0.8155476582410017],"tokens_per_call":{"1":1.7738909607981077,"2":2.3746484007088235,"3":2.8423052069267714,"4":3.20727186693027,"5":3.4927564135359677,"6":3.7165433816910998,"7":3.8923115967656767,"8":4.0306170143775395},"speedup":{"1":1.392535950694039,"2":1.534295070209984,"3":1.5603605821835973,"4":1.5306061475761104,"5":1.4741832070622587,"6":1.406109724545468,"7":1.334356420415908,"8":1.2631789558606046},"tokens_per_call_ci95":{"1":[1.6622245371842583,1.830035266876221],"2":[2.1007658748331637,2.5189938111345],"3":[2.3911787091938757,3.0908537002845735],"4":[2.583497214020767,3.5655175759910596],"5":[2.710855246871723,3.959505332739595],"6":[2.7951948612331456,4.286529065558329],"7":[2.8510466233199376,4.557970296903385],"8":[2.8880330306187885,4.783276091804088]},"speedup_ci95":{"1":[1.3048758223072563,1.4366102294227419],"2":[1.3573355636395585,1.6275587515016838],"3":[1.3127024478898284,1.6968080231028537],"4":[1.2329222099312327,1.7015717243596038],"5":[1.1441671873330466,1.6711833230559932],"6":[1.0575285346436771,1.6217569888517784],"7":[0.9773915248443247,1.5625565370439112],"8":[0.9050977890221026,1.4990592451938]},"tpc_valid_ci95":{"1":[1.5923930349804105,1.815547658241002],"2":[1.9433225428737126,2.480665737249436],"3":[2.151210739118808,3.0231013860432974],"4":[2.2743622586290426,3.465483701453158],"5":[2.34731636103416,3.8262677721700764],"6":[2.3905338631721995,4.12050458982664],"7":[2.416135610428025,4.360468946769791],"8":[2.4313019071857047,4.556171515543473]}}}},"dpo":{"target":"qwen2.5-1.5b-instruct","split":"heldout","n_blocks":700,"n_runs":14,"n_tokens":56990,"max_tokens":256,"ctx_budget":1024,"bootstrap":{"n_resamples":1000,"seed":0,"unit":"runs (with replacement)"},"overall":{"n_tokens":56990,"n_runs":14,"alpha":0.7134852706768697,"alpha_ci95":[0.7077544247823708,0.7200690497397886],"greedy":0.6983847108234288,"greedy_ci95":[0.6896355959689505,0.7085801146057011],"greedy_corpus":0.582658975249723,"greedy_corpus_ci95":[0.5727483766530856,0.5943206876838237],"coverage":0.9997529910530587,"greedy_valid":0.6864168310792765,"greedy_valid_ci95":[0.6585379777165,0.7056751759765897],"alpha_valid":0.7067104821850885,"alpha_valid_ci95":[0.6900778289984095,0.7183414802600979],"tokens_per_call":{"1":1.7134852706768697,"2":2.222556525761306,"3":2.5857856773434778,"4":2.8449596478272636,"5":3.0298910399836267,"6":3.16184987531976,"7":3.2560117059289477,"8":3.323204067217583},"speedup":{"1":1.3540562662300497,"2":1.4518046534162694,"3":1.4394758046459168,"4":1.379853027543069,"5":1.3019298271116413,"6":1.2195310469463885,"7":1.139213307749968,"8":1.0639124570358094},"tokens_per_call_ci95":{"1":[1.7077544247823708,1.7200690497397884],"2":[2.2086707506898775,2.2385684861491915],"3":[2.563196496950195,2.6119238826385156],"4":[2.8141136627282135,2.8807655482290686],"5":[2.991701397215276,3.0743501109255758],"6":[3.1173899022216482,3.2137443630525753],"7":[3.206346497899754,3.3141178497397177],"8":[3.2693059221999285,3.386393690933446]},"speedup_ci95":{"1":[1.349527550444136,1.3592589997744693],"2":[1.4427342731441244,1.4622638873354095],"3":[1.4269006794498031,1.4540266293444783],"4":[1.36489220869315,1.3972194882979936],"5":[1.2855199185206303,1.3210336792902029],"6":[1.2023827572812869,1.2395468419559938],"7":[1.1218364458007304,1.1595434841343852],"8":[1.0466571495868697,1.0841423997257267]},"tpc_valid_ci95":{"1":[1.6900778289984097,1.718341480260098],"2":[2.1662852392476504,2.2343559629471073],"3":[2.49490541530565,2.6050305708864125],"4":[2.7216789132134998,2.871301518105996],"5":[2.8781702765529125,3.0625749851118975],"6":[2.9861614970205173,3.199974651101076],"7":[3.0606838441550983,3.2986745310426464],"8":[3.112110063809146,3.369574749310097]}},"by_kind":{"text":{"n_tokens":2149,"n_runs":14,"alpha":0.6311233088911604,"alpha_ci95":[0.6059977564449807,0.662529121495991],"greedy":0.6021032141996941,"greedy_ci95":[0.5558846361549885,0.6522144585977174],"greedy_corpus":0.3874933739635741,"greedy_corpus_ci95":[0.3318145754370086,0.4590785957041624],"coverage":0.9994822235620533,"greedy_valid":0.5787876585105244,"greedy_valid_ci95":[0.5321539153335447,0.6279180445785142],"alpha_valid":0.6245134653114554,"alpha_valid_ci95":[0.6024409827026004,0.6489496659274416],"tokens_per_call":{"1":1.6311233088911603,"2":2.0296548277847797,"3":2.28144960583195,"4":2.440621985373422,"5":2.541298227463939,"6":2.605010916037536,"7":2.6453536997729135,"8":2.67091295113127},"speedup":{"1":1.288970950141581,"2":1.3257985970896289,"3":1.2700555718476976,"4":1.1837424963751777,"5":1.0919838100643795,"6":1.0047572829246822,"7":0.9255562972944295,"8":0.8550836791512575},"tokens_per_call_ci95":{"1":[1.6059977564449806,1.6625291214959907],"2":[1.973231037302371,2.1014739590641742],"3":[2.1957735816436177,2.3922876976613496],"4":[2.330633864273182,2.58496026938508],"5":[2.412358893016763,2.712611459924303],"6":[2.4618840771084445,2.797184091761424],"7":[2.4918962275756487,2.8532159238006174],"8":[2.510083523439089,2.8903386446945447]},"speedup_ci95":{"1":[1.2691158557824667,1.3137889267424645],"2":[1.2889417969874644,1.3727118466682977],"3":[1.2223607590339516,1.3317578052615113],"4":[1.1303964174563268,1.2537489790105165],"5":[1.0365791888455225,1.1655962945318947],"6":[0.9495530099173591,1.0788788141254206],"7":[0.8718646001231759,0.9982831279012336],"8":[0.8035946859631518,0.9253320671688944]},"tpc_valid_ci95":{"1":[1.6024409827026003,1.6489496659274416],"2":[1.9653761203438966,2.070085334841397],"3":[2.1840231213237766,2.3433811865018277],"4":[2.3157450354608375,2.5207364381445805],"5":[2.395099714858454,2.6358310694558567],"6":[2.4429062258978718,2.7105216920012207],"7":[2.471706827388813,2.7589921465545166],"8":[2.489057490053985,2.79044703184769]}},"thinking":{"n_tokens":18273,"n_runs":13,"alpha":0.6471807193201664,"alpha_ci95":[0.6391782189380361,0.6559414588011091],"greedy":0.6054579417719492,"greedy_ci95":[0.5940024635701882,0.6175141001690242],"greedy_corpus":0.4189582351769569,"greedy_corpus_ci95":[0.4018512770420954,0.4374140491917973],"coverage":0.9995620564667069,"greedy_valid":0.6024771605933974,"greedy_valid_ci95":[0.5918573105628822,0.6149895594004687],"alpha_valid":0.6447955668653965,"alpha_valid_ci95":[0.6365648000810695,0.6540041226146562],"tokens_per_call":{"1":1.6471807193201664,"2":2.066041532194626,"3":2.3371433758423192,"4":2.512617780859204,"5":2.6262008611979026,"6":2.6997253564467227,"7":2.7473211963913435,"8":2.7781335208677},"speedup":{"1":1.3016600800587785,"2":1.3495668955209152,"3":1.3010596241563963,"4":1.218661661730405,"5":1.1284660695911708,"6":1.041288808843975,"7":0.9612326828842426,"8":0.8894099791574362},"tokens_per_call_ci95":{"1":[1.6391782189380362,1.6559414588011092],"2":[2.0477270149827116,2.0862006562285513],"3":[2.308862507401215,2.368425501921647],"4":[2.475774626934816,2.5535484789843177],"5":[2.582461218718569,2.6749783146766033],"6":[2.650652964839796,2.754629178291936],"7":[2.6942396439788543,2.806875482005396],"8":[2.722099300149303,2.841145998709901]},"speedup_ci95":{"1":[1.2953362230794587,1.3085831242156118],"2":[1.337603599647384,1.3627351140755473],"3":[1.2853160046398837,1.3184740077240775],"4":[1.200792115662984,1.2385137351229123],"5":[1.1096713524150206,1.1494255102893465],"6":[1.022361501263786,1.0624653093030547],"7":[0.9426605104334427,0.9820695350925839],"8":[0.8714708143523959,0.9095832091995929]},"tpc_valid_ci95":{"1":[1.6365648000810693,1.654004122614656],"2":[2.041779544895928,2.081725515017584],"3":[2.299724988062254,2.361457068987379],"4":[2.463923977660125,2.544402658516679],"5":[2.5684472747631744,2.664049828289681],"6":[2.634983126579822,2.7422995705858537],"7":[2.677337507860311,2.7934752246453654],"8":[2.704298816165114,2.8269443133810537]}},"tool_result":{"n_tokens":12634,"n_runs":13,"alpha":0.7205968754961638,"alpha_ci95":[0.7060116095755292,0.7336151190169511],"greedy":0.6861187710441516,"greedy_ci95":[0.6654813256063669,0.7085833985738273],"greedy_corpus":0.5668152286718052,"greedy_corpus_ci95":[0.5424343461120897,0.5912046888723029],"coverage":1.0,"greedy_valid":0.6874894550507044,"greedy_valid_ci95":[0.6662476796900401,0.708753858551461],"alpha_valid":0.7192849965777443,"alpha_valid_ci95":[0.7035690297902961,0.7328246333429582],"tokens_per_call":{"1":1.7205968754961636,"2":2.2399091981121337,"3":2.614199587444024,"4":2.883993678540075,"5":3.0784848159998814,"6":3.218705066134001,"7":3.319808314861684,"8":3.392714244026947},"speedup":{"1":1.3596761062329423,"2":1.4631396589273256,"3":1.45529348685504,"4":1.398785185507916,"5":1.3228103424742947,"6":1.241460193842085,"7":1.1615344639525553,"8":1.0861659032589182},"tokens_per_call_ci95":{"1":[1.7060116095755293,1.733615119016951],"2":[2.204464002749493,2.2718062636205194],"3":[2.55637717960076,2.6666314267956963],"4":[2.8048319684966097,2.9562811387011774],"5":[2.9802439343599993,3.1687725493443937],"6":[3.104086819113603,3.324659463394494],"7":[3.19152133384809,3.43902046263587],"8":[3.2532511166042513,3.522917422620454]},"speedup_ci95":{"1":[1.3481503166318065,1.3699635796745246],"2":[1.439986367223708,1.4839752631509509],"3":[1.4231044474516439,1.4844816615754417],"4":[1.3603904317009918,1.433845813110446],"5":[1.2805967010063828,1.36160655379421],"6":[1.1972517347754417,1.2823269908485673],"7":[1.1166494177115875,1.2032444077290205],"8":[1.0415172582882801,1.127850007168695]},"tpc_valid_ci95":{"1":[1.7035690297902961,1.7328246333429582],"2":[2.198578409523,2.269856576577728],"3":[2.546851678632712,2.66340681347298],"4":[2.791885964760302,2.9518101215286294],"5":[2.9642844997897813,3.163159170010239],"6":[3.0855787698822317,3.318040958971921],"7":[3.170917661864747,3.4315421491799367],"8":[3.2309594633449548,3.5147186172786893]}},"tool_use:Bash":{"n_tokens":22279,"n_runs":13,"alpha":0.7643802996267969,"alpha_ci95":[0.7559024672197435,0.7735147461558912],"greedy":0.780841186008442,"greedy_ci95":[0.7710364356291656,0.7907989729156423],"greedy_corpus":0.7282436311423532,"greedy_corpus_ci95":[0.7149305096325,0.7412494424964038],"coverage":0.9997771291712927,"greedy_valid":0.7792932055307309,"greedy_valid_ci95":[0.7690967559023353,0.7893852634389577],"alpha_valid":0.7629190241232453,"alpha_valid_ci95":[0.7538013112084011,0.7717971399799272],"tokens_per_call":{"1":1.7643802996267968,"2":2.3486780862340324,"3":2.7953352149585164,"4":3.1367871466889166,"5":3.3978229967825784,"6":3.597388751892093,"7":3.7499650825290023,"8":3.866620147094738},"speedup":{"1":1.3942753063636049,"2":1.5341890005714918,"3":1.5561295133869608,"4":1.5213942469878172,"5":1.4600284460331636,"6":1.3875191561473021,"7":1.3120377048509204,"8":1.2378852631111632},"tokens_per_call_ci95":{"1":[1.7559024672197432,1.7735147461558913],"2":[2.327291007198371,2.3718398086775703],"3":[2.759205014354251,2.8346530675343837],"4":[3.0856898780411095,3.192645947978507],"5":[3.332480592063271,3.469558720022833],"6":[3.5190303017280766,3.6837548325999325],"7":[3.660043687566851,3.849438684249358],"8":[3.7666360538722103,3.977597586701692]},"speedup_ci95":{"1":[1.387575825316835,1.401493666960529],"2":[1.5202186648310685,1.5493185579235356],"3":[1.5360162650065785,1.578017289303386],"4":[1.4966112167973173,1.5484867001736589],"5":[1.4319511242560325,1.4908529464932256],"6":[1.3572961643754504,1.4208307050758027],"7":[1.2805760090573244,1.3468415265458178],"8":[1.2058756963998591,1.2734143122034247]},"tpc_valid_ci95":{"1":[1.7538013112084014,1.771797139979927],"2":[2.3220177279932863,2.367467965264142],"3":[2.7503400080239824,2.827205004592643],"4":[3.0732099043400645,3.1820287366946154],"5":[3.316589655542065,3.455880678333475],"6":[3.5000496311283924,3.6672388236736535],"7":[3.638342001287483,3.8303644357636264],"8":[3.7425869712504323,3.9562643166366107]}},"tool_use:Edit":{"n_tokens":1348,"n_runs":4,"alpha":0.8164570018282697,"alpha_ci95":[0.7711859046866041,0.8314936977736689],"greedy":0.830579853724664,"greedy_ci95":[0.7865429234338747,0.8522088353413655],"greedy_corpus":0.8149362071432258,"greedy_corpus_ci95":[0.7954091816367266,0.8754325259515571],"coverage":1.0,"greedy_valid":0.8229458447973815,"greedy_valid_ci95":[0.7747093486223922,0.8638305847076462],"alpha_valid":0.8037963474431818,"alpha_valid_ci95":[0.7618227286879065,0.8306385892002025],"tokens_per_call":{"1":1.8164570018282695,"2":2.483281669437141,"3":3.0280730765501187,"4":3.4733031696873664,"5":3.8372768998067825,"6":4.134910943878196,"7":4.378365633229709,"8":4.577558027508921},"speedup":{"1":1.4354281462200253,"2":1.6221139222532202,"3":1.685691883355042,"4":1.6846101483120384,"5":1.648859706444076,"6":1.594842409115926,"7":1.5319024764216387,"8":1.46548960273409},"tokens_per_call_ci95":{"1":[1.771185904686604,1.8314936977736687],"2":[2.3659136042738997,2.5228754672109983],"3":[2.8245592233223116,3.0977550512537455],"4":[3.178260259978709,3.575763802364038],"5":[3.451029513921162,3.9732250663929087],"6":[3.6613853177934628,4.30371160254207],"7":[3.823608748708801,4.578509074549149],"8":[3.9487131720406112,4.80700144068717]},"speedup_ci95":{"1":[1.3996533346048856,1.4473106716882604],"2":[1.545447479266763,1.6479771384136366],"3":[1.5723981675616063,1.7244830010739975],"4":[1.5415094008102141,1.7343052694048304],"5":[1.4828910344052602,1.7072759896317962],"6":[1.4122027439494516,1.6599491194605387],"7":[1.3378041492377002,1.6019286594954891],"8":[1.2641670652843269,1.5389451295472651]},"tpc_valid_ci95":{"1":[1.7618227286879062,1.8306385892002024],"2":[2.3421965986333944,2.5205990550687054],"3":[2.784338603894425,3.0937068430416326],"4":[3.1211724328099275,3.569752287503114],"5":[3.37778009946873,3.965174003885782],"6":[3.573269652284976,4.293626540521005],"7":[3.7221980368414274,4.566451892170913],"8":[3.835655065143305,4.793071157363443]}},"tool_use:Read":{"n_tokens":307,"n_runs":5,"alpha":0.7744123127833636,"alpha_ci95":[0.6647791822417444,0.8302326583862305],"greedy":0.8141464947575233,"greedy_ci95":[0.7089201877934272,0.8866666666666667],"greedy_corpus":0.8319328523385257,"greedy_corpus_ci95":[0.813953488372093,0.8495145631067961],"coverage":1.0,"greedy_valid":0.7837904476801412,"greedy_valid_ci95":[0.6256830601092895,0.8881159420289855],"alpha_valid":0.7315249196847892,"alpha_valid_ci95":[0.5954446065993536,0.8157892596595007],"tokens_per_call":{"1":1.7744123127833635,"2":2.375929519741063,"3":2.844422028743777,"4":3.2102105121609625,"5":3.496456710046411,"6":3.720924574067447,"7":3.8972871698367557,"8":4.036102888704245},"speedup":{"1":1.4022029556466276,"2":1.5519900137377234,"3":1.583455552582767,"4":1.557005807670718,"5":1.5024108853890206,"6":1.4351671396043462,"7":1.363582753685016,"8":1.2921445852604367},"tokens_per_call_ci95":{"1":[1.6647791822417444,1.8302326583862305],"2":[2.106710543383747,2.5195189254372976],"3":[2.400497312250708,3.0917868953202263],"4":[2.595800640211531,3.566902453265422],"5":[2.725634226862418,3.9613589059789183],"6":[2.8119448924237074,4.288849535332847],"7":[2.869322426094282,4.560742951137939],"8":[2.9074658160068547,4.786477744539513]},"speedup_ci95":{"1":[1.315567003802317,1.4463141540561892],"2":[1.3761324559509156,1.6457845989169075],"3":[1.33632799902134,1.721160600404859],"4":[1.2590067402290248,1.7300073668348661],"5":[1.1711921157956016,1.7021771567125936],"6":[1.0845720808506858,1.6542167940503647],"7":[1.003918470587759,1.5957126486069768],"8":[0.9308152726971693,1.5323745381679064]},"tpc_valid_ci95":{"1":[1.5954446065993535,1.815789259659501],"2":[1.9499988861276125,2.4813014986996893],"3":[2.161116319619434,3.024219436205578],"4":[2.286825056751237,3.46712630444451],"5":[2.3616776462787845,3.8284452384525123],"6":[2.4062482170029584,4.123205617641535],"7":[2.4327875229537224,4.36366823726322],"8":[2.4485902095449954,4.559835315205661]}}}},"tiny":{"target":"qwen2.5-1.5b-instruct","split":"heldout","n_blocks":700,"n_runs":14,"n_tokens":56990,"max_tokens":256,"ctx_budget":1024,"bootstrap":{"n_resamples":1000,"seed":0,"unit":"runs (with replacement)"},"overall":{"n_tokens":56990,"n_runs":14,"alpha":0.011431486292284255,"alpha_ci95":[0.010874165725538603,0.012014273193847305],"greedy":0.13834779932707034,"greedy_ci95":[0.1260590977818676,0.1516096974149861],"greedy_corpus":0.13037314735225822,"greedy_corpus_ci95":[0.11805678373137873,0.1438604757526249],"coverage":0.9997529910530587,"greedy_valid":0.13163783369697282,"greedy_valid_ci95":[0.11537458962773083,0.14648388018975872],"alpha_valid":0.012320248311363656,"alpha_valid_ci95":[0.011088339465835635,0.014279432352507443],"tokens_per_call":{"1":1.0114314862922842,"2":1.0115622457693376,"3":1.0115637423899662,"4":1.011563759530275,"5":1.011563759726699,"6":1.0115637597289515,"7":1.0115637597289773,"8":1.0115637597289777},"speedup":{"1":0.9405917993995854,"2":0.8791393289657924,"3":0.8251319050085602,"4":0.7773750166125104,"5":0.7348438010147679,"6":0.6967250515749497,"7":0.6623659643787779,"8":0.6312364558599204},"tokens_per_call_ci95":{"1":[1.0108741657255385,1.012014273193847],"2":[1.0109924132059318,1.0121586159543225],"3":[1.010993699048634,1.012160350127683],"4":[1.0109937130311006,1.0121603709625155],"5":[1.0109937131831483,1.0121603712128309],"6":[1.0109937131848017,1.0121603712158382],"7":[1.0109937131848195,1.0121603712158742],"8":[1.0109937131848197,1.0121603712158747]},"speedup_ci95":{"1":[0.940073513028415,0.9411337684680163],"2":[0.8786440927907466,0.8796576287406561],"3":[0.8246669210155003,0.8256185575629529],"4":[0.776936942489559,0.7778335055784106],"5":[0.7344296944744828,0.7352772054818443],"6":[0.6963324260938044,0.6971359739364653],"7":[0.6619927012746677,0.6627566220502071],"8":[0.6308807351683192,0.6316087536186056]},"tpc_valid_ci95":{"1":[1.0110883394658357,1.0142794323525075],"2":[1.0112112907413038,1.0144833345408648],"3":[1.0112126540668587,1.0144862461483715],"4":[1.0112126691838763,1.0144862877244738],"5":[1.011212669351499,1.0144862883181571],"6":[1.0112126693533576,1.0144862883266343],"7":[1.0112126693533783,1.0144862883267554],"8":[1.0112126693533785,1.0144862883267571]}},"by_kind":{"text":{"n_tokens":2149,"n_runs":14,"alpha":0.01912194914772695,"alpha_ci95":[0.015882715946445344,0.02169430201659077],"greedy":0.07716248426427014,"greedy_ci95":[0.05828483799937087,0.09863345533057742],"greedy_corpus":0.05470942383202837,"greedy_corpus_ci95":[0.03610379372478358,0.07603078866182164],"coverage":0.9994822235620533,"greedy_valid":0.07348025341207219,"greedy_valid_ci95":[0.05417539301244082,0.09582369893391952],"alpha_valid":0.018866715343182305,"alpha_valid_ci95":[0.01700573226780824,0.020826994388279174],"tokens_per_call":{"1":1.019121949147727,"2":1.0194898845154832,"3":1.0194970062394884,"4":1.0194971448666141,"5":1.0194971475795773,"6":1.019497147632943,"7":1.0194971476339978,"8":1.0194971476340189},"speedup":{"1":0.9477436296455801,"2":0.8860291659844417,"3":0.8316030633140427,"4":0.7834717312283975,"5":0.7406069581352922,"6":0.7021892549367365,"7":0.6675606998365882,"8":0.6361870520195553},"tokens_per_call_ci95":{"1":[1.0158827159464454,1.0216943020165905],"2":[1.0161349766310637,1.022164944756674],"3":[1.0161389832164582,1.0221751550224236],"4":[1.01613904685193,1.0221753765270127],"5":[1.0161390478626344,1.0221753813324002],"6":[1.0161390478786871,1.0221753814366494],"7":[1.0161390478789423,1.0221753814389114],"8":[1.0161390478789463,1.0221753814389603]},"speedup_ci95":{"1":[0.9447312692170579,0.9501358173976984],"2":[0.8831134467802246,0.8883540359321087],"3":[0.8288639260575915,0.8337876275827646],"4":[0.7808910718528941,0.7855299211960386],"5":[0.7381674888122202,0.7425525433265433],"6":[0.6998763288339819,0.7040339163009566],"7":[0.665361835987083,0.6693143914847534],"8":[0.6340915291546556,0.6378583246395317]},"tpc_valid_ci95":{"1":[1.0170057322678083,1.020826994388279],"2":[1.0172949272309366,1.0212607580859827],"3":[1.0172998452041795,1.021269792080183],"4":[1.0172999288379445,1.0212699802311327],"5":[1.0172999302601986,1.0212699841497515],"6":[1.0172999302843853,1.0212699842313646],"7":[1.0172999302847965,1.0212699842330644],"8":[1.0172999302848036,1.0212699842330997]}},"thinking":{"n_tokens":18273,"n_runs":13,"alpha":0.01615645693465084,"alpha_ci95":[0.015402953287721663,0.017036519738140004],"greedy":0.09750761886363711,"greedy_ci95":[0.08418682161884006,0.11039220021417093],"greedy_corpus":0.08731160456730876,"greedy_corpus_ci95":[0.07356988558146553,0.10032982063328001],"coverage":0.9995620564667069,"greedy_valid":0.09230150206232685,"greedy_valid_ci95":[0.08014025450802828,0.1062534974723095],"alpha_valid":0.016360630101816587,"alpha_valid_ci95":[0.01558705656280183,0.01722102505098267],"tokens_per_call":{"1":1.0161564569346508,"2":1.0164176717567168,"3":1.016421898028053,"4":1.0164219664549972,"5":1.0164219675636854,"6":1.0164219675816621,"7":1.016421967581954,"8":1.0164219675819584},"speedup":{"1":0.9449858376501699,"2":0.8833591344817062,"3":0.8290946994905092,"4":0.781108492286418,"5":0.7383730139573268,"6":0.7000711927194769,"7":0.6655470901345597,"8":0.6342680768303752},"tokens_per_call_ci95":{"1":[1.0154029532877216,1.01703651973814],"2":[1.0156402042579475,1.017326762745094],"3":[1.0156438586235668,1.0173317074758845],"4":[1.01564391491159,1.0173317917168903],"5":[1.0156439157785917,1.017331793152064],"6":[1.015643915791946,1.0173317931765142],"7":[1.0156439157921517,1.0173317931769308],"8":[1.0156439157921548,1.017331793176938]},"speedup_ci95":{"1":[0.9442851086727505,0.9458042617027501],"2":[0.8826834447176611,0.8841492169950002],"3":[0.8284600532402603,0.8298368304817905],"4":[0.7805105686995129,0.7818076824476068],"5":[0.7378078033854305,0.7390339507368384],"6":[0.6995353014638795,0.700697844552741],"7":[0.665037626426383,0.6661428385506506],"8":[0.6337825566152158,0.6348358265925645]},"tpc_valid_ci95":{"1":[1.0155870565628018,1.0172210250509828],"2":[1.0158300128958087,1.0175175887573729],"3":[1.0158337998699356,1.0175226958884787],"4":[1.0158338588977163,1.0175227838385137],"5":[1.0158338598177856,1.0175227853531035],"6":[1.0158338598321266,1.0175227853791864],"7":[1.0158338598323502,1.0175227853796356],"8":[1.0158338598323537,1.0175227853796434]}},"tool_result":{"n_tokens":12634,"n_runs":13,"alpha":0.012197036006695592,"alpha_ci95":[0.011416043983428869,0.012907069732168635],"greedy":0.1052809889884423,"greedy_ci95":[0.0832186136794605,0.13041066753288938],"greedy_corpus":0.09993216150273408,"greedy_corpus_ci95":[0.07661601204798678,0.12510958375672723],"coverage":1.0,"greedy_valid":0.10865280342118828,"greedy_valid_ci95":[0.08633321217952325,0.132462783406322],"alpha_valid":0.012074121454827629,"alpha_valid_ci95":[0.011107943006830524,0.013057765168951053],"tokens_per_call":{"1":1.0121970360066956,"2":1.012345945017109,"3":1.0123477647064334,"4":1.0123477869641384,"5":1.012347787236639,"6":1.0123477872399782,"7":1.0123477872400193,"8":1.0123477872400197},"speedup":{"1":0.9413037307495254,"2":0.8798204346848703,"3":0.8257714315163287,"4":0.7779775326019197,"5":0.7354133526124449,"6":0.6972650586706989,"7":0.6628793409538639,"8":0.6317257050472376},"tokens_per_call_ci95":{"1":[1.0114160439834288,1.0129070697321685],"2":[1.0115463700437546,1.0130736621879477],"3":[1.011547857851794,1.0130758124085644],"4":[1.0115478748366757,1.0130758401616153],"5":[1.011547875030576,1.013075840519826],"6":[1.0115478750327895,1.0130758405244493],"7":[1.0115478750328146,1.013075840524509],"8":[1.011547875032815,1.0130758405245097]},"speedup_ci95":{"1":[0.9405774386551652,0.9419640343969087],"2":[0.87912553152051,0.8804528869021447],"3":[0.8251189480008176,0.8263652995665783],"4":[0.7773628094097174,0.7785370330398816],"5":[0.7348322616823582,0.7359422421034076],"6":[0.6967141108253962,0.6977665129362296],"7":[0.6623555631737215,0.6633560659366069],"8":[0.631226543484935,0.6321800251734367]},"tpc_valid_ci95":{"1":[1.0111079430068304,1.0130577651689512],"2":[1.0112313294047022,1.0132282704014304],"3":[1.0112326999737784,1.0132304968187495],"4":[1.0112327151979814,1.0132305258907848],"5":[1.011232715367091,1.0132305262704004],"6":[1.0112327153689695,1.0132305262753574],"7":[1.0112327153689904,1.0132305262754222],"8":[1.0112327153689906,1.013230526275423]}},"tool_use:Bash":{"n_tokens":22279,"n_runs":13,"alpha":0.0067919091107881595,"alpha_ci95":[0.006293704694370764,0.007307287095732287],"greedy":0.19767504555744822,"greedy_ci95":[0.18356875295968844,0.21111423056393047],"greedy_corpus":0.19074870849931652,"greedy_corpus_ci95":[0.1773022018306149,0.20381649345686856],"coverage":0.9997771291712927,"greedy_valid":0.1968960788948457,"greedy_valid_ci95":[0.18256827550496574,0.21048875898581615],"alpha_valid":0.006892309360455615,"alpha_valid_ci95":[0.006394009441949893,0.007478920020983423],"tokens_per_call":{"1":1.006791909110788,"2":1.0068381071669732,"3":1.006838421866353,"4":1.0068384240132382,"5":1.0068384240279058,"6":1.0068384240280064,"7":1.0068384240280068,"8":1.0068384240280068},"speedup":{"1":0.9362771737341395,"2":0.8750336240937552,"3":0.8212774640454893,"4":0.7737436510743045,"5":0.7314111121579526,"6":0.6934704275057441,"7":0.6592718425219162,"8":0.6282877498273632},"tokens_per_call_ci95":{"1":[1.0062937046943705,1.0073072870957323],"2":[1.0063333154131828,1.0073606835405773],"3":[1.00633356471135,1.0073610737237317],"4":[1.006333566280359,1.007361076574912],"5":[1.006333566290234,1.0073610765957464],"6":[1.006333566290296,1.0073610765958987],"7":[1.0063335662902964,1.0073610765958998],"8":[1.0063335662902964,1.0073610765958998]},"speedup_ci95":{"1":[0.9358138630750807,0.9367564551415305],"2":[0.8745949142807399,0.8754877903542548],"3":[0.8208656523833983,0.821703790835056],"4":[0.7733556737621614,0.7741453035059834],"5":[0.7310443615944264,0.731790789658129],"6":[0.6931227014924689,0.6938304098932292],"7":[0.658941264662541,0.6596140723308016],"8":[0.6279727082830208,0.62861389570935]},"tpc_valid_ci95":{"1":[1.0063940094419501,1.0074789200209833],"2":[1.006434892806369,1.0075348542683473],"3":[1.0064351542150851,1.00753527259615],"4":[1.0064351558865359,1.0075352757247904],"5":[1.006435155897223,1.0075352757481892],"6":[1.0064351558972915,1.0075352757483642],"7":[1.006435155897292,1.0075352757483655],"8":[1.006435155897292,1.0075352757483655]}},"tool_use:Edit":{"n_tokens":1348,"n_runs":4,"alpha":0.006283106295016248,"alpha_ci95":[0.005931027088210992,0.006991421475129969],"greedy":0.13452674053641045,"greedy_ci95":[0.11180124223602485,0.20069204152249134],"greedy_corpus":0.1358564744860226,"greedy_corpus_ci95":[0.11126326600479287,0.2093425605536332],"coverage":1.0,"greedy_valid":0.15493740149156193,"greedy_valid_ci95":[0.11226309921962097,0.22222638680659673],"alpha_valid":0.006571654913985032,"alpha_valid_ci95":[0.006115660040010261,0.007009894636847218],"tokens_per_call":{"1":1.0062831062950162,"2":1.0063226792361515,"3":1.0063229291064861,"4":1.0063229306883332,"5":1.0063229306983745,"6":1.0063229306984383,"7":1.0063229306984387,"8":1.0063229306984387},"speedup":{"1":0.9358040069774067,"2":0.8745856704783175,"3":0.820856976927236,"4":0.773347500433046,"5":0.7310366354390823,"6":0.693115376118053,"7":0.6589343005399241,"8":0.6279660714563793},"tokens_per_call_ci95":{"1":[1.005931027088211,1.00699142147513],"2":[1.0059662041705322,1.007040301449373],"3":[1.0059664128067602,1.0070406431898744],"4":[1.0059664140441873,1.0070406455791263],"5":[1.0059664140515265,1.0070406455958307],"6":[1.00596641405157,1.0070406455959475],"7":[1.0059664140515703,1.0070406455959482],"8":[1.0059664140515703,1.0070406455959482]},"speedup_ci95":{"1":[0.9354765870590549,0.9364627124446921],"2":[0.8742758613179891,0.8752093492619226],"3":[0.8205661668069916,0.8214424158512119],"4":[0.7730735215270657,0.7738990560022299],"5":[0.7307776462795996,0.7315580146781272],"6":[0.6928698215726399,0.6936097097121284],"7":[0.658700855549088,0.6594042560080453],"8":[0.62774359778385,0.6284139402178531]},"tpc_valid_ci95":{"1":[1.0061156600400103,1.007009894636847],"2":[1.0061530613377352,1.007059033259667],"3":[1.0061532900713572,1.0070593777162353],"4":[1.0061532914702143,1.0070593801308396],"5":[1.0061532914787692,1.0070593801477659],"6":[1.0061532914788216,1.0070593801478847],"7":[1.0061532914788218,1.0070593801478853],"8":[1.0061532914788218,1.0070593801478853]}},"tool_use:Read":{"n_tokens":307,"n_runs":5,"alpha":0.004577397789147002,"alpha_ci95":[0.0033199163595042623,0.007152924141963991],"greedy":0.1769388524897556,"greedy_ci95":[0.1349206349206349,0.19718309859154928],"greedy_corpus":0.18079280671231573,"greedy_corpus_ci95":[0.15526583448913545,0.2184873949579832],"coverage":1.0,"greedy_valid":0.167232018463836,"greedy_valid_ci95":[0.10254658385093167,0.2146457845433255],"alpha_valid":0.005554969813837579,"alpha_valid_ci95":[0.0038114376248701344,0.007932750146129371],"tokens_per_call":{"1":1.004577397789147,"2":1.0045991862605008,"3":1.0045992946804416,"4":1.0045992952472718,"5":1.0045992952503975,"6":1.0045992952504157,"7":1.004599295250416,"8":1.004599295250416},"speedup":{"1":0.9342177646520189,"2":0.8730877987809228,"3":0.8194510094158459,"4":0.7720229065881151,"5":0.7297845118710163,"6":0.6919282042913936,"7":0.6578056742474271,"8":0.6268904877169064},"tokens_per_call_ci95":{"1":[1.0033199163595043,1.007152924141964],"2":[1.0033309382041384,1.0072040884657447],"3":[1.0033309747957406,1.0072044544402714],"4":[1.0033309749172217,1.0072044570580596],"5":[1.0033309749176251,1.0072044570767844],"6":[1.0033309749176265,1.0072044570769183],"7":[1.0033309749176265,1.0072044570769192],"8":[1.0033309749176265,1.0072044570769192]},"speedup_ci95":{"1":[0.933048356010258,0.9366129036203316],"2":[0.8719855762040168,0.8753516950328019],"3":[0.8184164416879133,0.8215760365845752],"4":[0.7710482171250401,0.7740249432238284],"5":[0.7288631489561462,0.7316770144447325],"6":[0.6910546354819367,0.6937225365719583],"7":[0.6569751855982546,0.6595115187965014],"8":[0.626099029913021,0.6285161619292211]},"tpc_valid_ci95":{"1":[1.0038114376248701,1.0079327501461295],"2":[1.0038259646816383,1.0079956786710103],"3":[1.003826020050609,1.0079961778672752],"4":[1.0038260202616445,1.0079961818272745],"5":[1.003826020262449,1.0079961818586882],"6":[1.003826020262452,1.0079961818589374],"7":[1.003826020262452,1.0079961818589394],"8":[1.003826020262452,1.0079961818589394]}}}}},"corpus":{"by_split":[{"split":"test","blocks":566,"tokens":252500,"runs":3},{"split":"train","blocks":45257,"tokens":15697302,"runs":26},{"split":"val","blocks":667,"tokens":236985,"runs":3}],"by_kind":[{"split":"test","kind":"tool_use","blocks":179,"tokens":51271},{"split":"test","kind":"thinking","blocks":163,"tokens":77998},{"split":"test","kind":"tool_result","blocks":123,"tokens":113227},{"split":"test","kind":"text","blocks":97,"tokens":9779},{"split":"test","kind":"code","blocks":4,"tokens":225},{"split":"train","kind":"tool_use","blocks":14018,"tokens":2739505},{"split":"train","kind":"thinking","blocks":12872,"tokens":5079065},{"split":"train","kind":"tool_result","blocks":11955,"tokens":6848761},{"split":"train","kind":"text","blocks":6047,"tokens":957678},{"split":"train","kind":"code","blocks":365,"tokens":72293},{"split":"val","kind":"tool_use","blocks":210,"tokens":48476},{"split":"val","kind":"thinking","blocks":186,"tokens":89471},{"split":"val","kind":"tool_result","blocks":161,"tokens":86447},{"split":"val","kind":"text","blocks":101,"tokens":11720},{"split":"val","kind":"code","blocks":9,"tokens":871}],"length_percentiles":[{"kind":"tool_result","p50":243,"p90":1539,"p99":4096,"max_tokens":4096},{"kind":"thinking","p50":132,"p90":1039,"p99":4096,"max_tokens":4096},{"kind":"code","p50":101,"p90":332,"p99":1820,"max_tokens":4096},{"kind":"tool_use","p50":85,"p90":410,"p99":1932,"max_tokens":4096},{"kind":"text","p50":29,"p90":163,"p99":3098,"max_tokens":4096}]},"heldout":{"blocks":14974,"runs":14,"tokens":4238167,"kinds":{"text":1305,"thinking":4561,"tool_use":4884,"tool_result":4202,"code":22}},"sweep":{"target":"qwen2.5-0.5b-instruct","n_gen":32,"n_prompts_per_cell":115,"rows":3277,"cells":[{"drafter":"ngram","k":1,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":1215,"n_acc":597,"n_steps":3109,"n_emit":3680,"accept":0.49135802469135803,"tpc":1.1836603409456417},{"drafter":"ngram","k":1,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":1178,"n_acc":507,"n_steps":3192,"n_emit":3680,"accept":0.4303904923599321,"tpc":1.1528822055137844},{"drafter":"ngram","k":2,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":2079,"n_acc":764,"n_steps":2954,"n_emit":3680,"accept":0.36748436748436747,"tpc":1.2457684495599188},{"drafter":"ngram","k":2,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":1910,"n_acc":596,"n_steps":3108,"n_emit":3680,"accept":0.31204188481675393,"tpc":1.184041184041184},{"drafter":"ngram","k":4,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":3495,"n_acc":891,"n_steps":2827,"n_emit":3680,"accept":0.2549356223175966,"tpc":1.3017332861690838},{"drafter":"ngram","k":4,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":3510,"n_acc":919,"n_steps":2791,"n_emit":3680,"accept":0.2618233618233618,"tpc":1.3185238265854533},{"drafter":"ngram","k":8,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":6041,"n_acc":958,"n_steps":2768,"n_emit":3680,"accept":0.15858301605694422,"tpc":1.329479768786127},{"drafter":"ngram","k":8,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":5963,"n_acc":717,"n_steps":2990,"n_emit":3680,"accept":0.120241489183297,"tpc":1.2307692307692308},{"drafter":"tiny","k":1,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":3406,"n_acc":281,"n_steps":3406,"n_emit":3680,"accept":0.0825014679976512,"tpc":1.080446271285966},{"drafter":"tiny","k":1,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":3657,"n_acc":24,"n_steps":3657,"n_emit":3680,"accept":0.006562756357670222,"tpc":1.0062893081761006},{"drafter":"tiny","k":2,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":6634,"n_acc":323,"n_steps":3367,"n_emit":3680,"accept":0.04868857401266204,"tpc":1.092961092961093},{"drafter":"tiny","k":2,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":7191,"n_acc":28,"n_steps":3653,"n_emit":3680,"accept":0.0038937560839938812,"tpc":1.0073911853271285},{"drafter":"tiny","k":4,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":12811,"n_acc":337,"n_steps":3352,"n_emit":3680,"accept":0.026305518694871594,"tpc":1.097852028639618},{"drafter":"tiny","k":4,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":13972,"n_acc":15,"n_steps":3665,"n_emit":3680,"accept":0.00107357572287432,"tpc":1.004092769440655},{"drafter":"tiny","k":8,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":23982,"n_acc":342,"n_steps":3348,"n_emit":3680,"accept":0.01426069552164123,"tpc":1.099163679808841},{"drafter":"tiny","k":8,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":26111,"n_acc":19,"n_steps":3663,"n_emit":3680,"accept":0.0007276626709049826,"tpc":1.0046410046410046},{"drafter":"base","k":1,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":1840,"n_acc":1840,"n_steps":1840,"n_emit":3680,"accept":1.0,"tpc":2.0},{"drafter":"base","k":1,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":1840,"n_acc":1840,"n_steps":1840,"n_emit":3680,"accept":1.0,"tpc":2.0},{"drafter":"base","k":2,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":2530,"n_acc":2530,"n_steps":1265,"n_emit":3680,"accept":1.0,"tpc":2.909090909090909},{"drafter":"base","k":2,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":2530,"n_acc":2530,"n_steps":1265,"n_emit":3680,"accept":1.0,"tpc":2.909090909090909},{"drafter":"base","k":4,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":2990,"n_acc":2990,"n_steps":805,"n_emit":3680,"accept":1.0,"tpc":4.571428571428571},{"drafter":"base","k":4,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":2990,"n_acc":2990,"n_steps":805,"n_emit":3680,"accept":1.0,"tpc":4.571428571428571},{"drafter":"base","k":8,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":3335,"n_acc":3335,"n_steps":460,"n_emit":3680,"accept":1.0,"tpc":8.0},{"drafter":"base","k":8,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":3335,"n_acc":3335,"n_steps":460,"n_emit":3680,"accept":1.0,"tpc":8.0},{"drafter":"lora","k":1,"temp":0.0,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":2296,"n_acc":1438,"n_steps":2296,"n_emit":3680,"accept":0.6263066202090593,"tpc":1.602787456445993},{"drafter":"lora","k":1,"temp":0.7,"n_prompts":115,"n_runs":13,"status":"complete","n_draft":2249,"n_acc":1490,"n_steps":2249,"n_emit":3680,"accept":0.6625166740773677,"tpc":1.6362827923521566},{"drafter":"lora","k":2,"temp":0.0,"n_prompts":94,"n_runs":13,"status":"partial","n_draft":3057,"n_acc":1522,"n_steps":1544,"n_emit":3008,"accept":0.4978737324174027,"tpc":1.9481865284974094},{"drafter":"lora","k":2,"temp":0.7,"n_prompts":8,"n_runs":4,"status":"partial","n_draft":265,"n_acc":128,"n_steps":133,"n_emit":256,"accept":0.4830188679245283,"tpc":1.9248120300751879},{"drafter":"lora","k":4,"temp":0.0,"n_prompts":51,"n_runs":13,"status":"partial","n_draft":2491,"n_acc":1015,"n_steps":648,"n_emit":1632,"accept":0.4074668807707748,"tpc":2.5185185185185186},{"drafter":"lora","k":4,"temp":0.7,"n_prompts":51,"n_runs":13,"status":"partial","n_draft":2515,"n_acc":1012,"n_steps":653,"n_emit":1632,"accept":0.40238568588469187,"tpc":2.4992343032159265},{"drafter":"lora","k":8,"temp":0.0,"n_prompts":1,"n_runs":1,"status":"partial","n_draft":82,"n_acc":22,"n_steps":11,"n_emit":32,"accept":0.2682926829268293,"tpc":2.909090909090909},{"drafter":"lora","k":8,"temp":0.7,"n_prompts":20,"n_runs":10,"status":"partial","n_draft":1531,"n_acc":444,"n_steps":211,"n_emit":640,"accept":0.29000653167864143,"tpc":3.0331753554502368},{"drafter":"lora_dpo","k":1,"temp":0.0,"n_prompts":43,"n_runs":13,"status":"partial","n_draft":825,"n_acc":571,"n_steps":825,"n_emit":1376,"accept":0.6921212121212121,"tpc":1.667878787878788},{"drafter":"lora_dpo","k":1,"temp":0.7,"n_prompts":19,"n_runs":10,"status":"partial","n_draft":372,"n_acc":241,"n_steps":372,"n_emit":608,"accept":0.6478494623655914,"tpc":1.6344086021505377},{"drafter":"lora_dpo","k":2,"temp":0.0,"n_prompts":0,"n_runs":0,"status":"pending","n_draft":0,"n_acc":0,"n_steps":0,"n_emit":0,"accept":null,"tpc":null},{"drafter":"lora_dpo","k":2,"temp":0.7,"n_prompts":0,"n_runs":0,"status":"pending","n_draft":0,"n_acc":0,"n_steps":0,"n_emit":0,"accept":null,"tpc":null},{"drafter":"lora_dpo","k":4,"temp":0.0,"n_prompts":0,"n_runs":0,"status":"pending","n_draft":0,"n_acc":0,"n_steps":0,"n_emit":0,"accept":null,"tpc":null},{"drafter":"lora_dpo","k":4,"temp":0.7,"n_prompts":0,"n_runs":0,"status":"pending","n_draft":0,"n_acc":0,"n_steps":0,"n_emit":0,"accept":null,"tpc":null},{"drafter":"lora_dpo","k":8,"temp":0.0,"n_prompts":0,"n_runs":0,"status":"pending","n_draft":0,"n_acc":0,"n_steps":0,"n_emit":0,"accept":null,"tpc":null},{"drafter":"lora_dpo","k":8,"temp":0.7,"n_prompts":0,"n_runs":0,"status":"pending","n_draft":0,"n_acc":0,"n_steps":0,"n_emit":0,"accept":null,"tpc":null},{"drafter":"none","k":0,"temp":0.0,"n_prompts":0,"n_runs":0,"status":"pending","n_draft":0,"n_acc":0,"n_steps":0,"n_emit":0,"accept":null,"tpc":null},{"drafter":"none","k":0,"temp":0.7,"n_prompts":0,"n_runs":0,"status":"pending","n_draft":0,"n_acc":0,"n_steps":0,"n_emit":0,"accept":null,"tpc":null}],"statuses":{"complete":26,"partial":8,"pending":8}},"judge":{"meta":{"created":"2026-10-06T17:18:20+00:00","judge_model":"qwen-agent","rubric_version":"v1","calls_used":150,"cache_hits":0,"invalid":18,"wall_s":3749.9,"n_prompts":44,"n_gen":16,"k":4,"temperature":0.7,"seed":12345,"target":"qwen2.5-0.5b-instruct","drafter":"dpo adapter merged into qwen2.5-0.5b"},"spec_vs_drafter_alone":{"n_prompts":25,"n_verdicts":29,"wins":19,"ties":0,"losses":10,"win_rate":0.6551724137931034,"win_ci95":[0.47345103949857975,0.8005928156337425],"loss_rate":0.3448275862068966,"loss_ci95":[0.19940718436625743,0.5265489605014202],"order_bias_disagreement":0.04,"both_orders_valid":12,"both_orders_disagree":1},"spec_vs_target":{"n_prompts":44,"n_verdicts":79,"wins":34,"ties":2,"losses":43,"win_rate":0.43037974683544306,"win_ci95":[0.32693823359163393,0.5402780111661755],"loss_rate":0.5443037974683544,"loss_ci95":[0.43498332849961774,0.6495154248363216],"order_bias_disagreement":0.18181818181818182,"both_orders_valid":39,"both_orders_disagree":8}},"curves":{"distill_curve":{"cfg":{"n_layers":4,"hidden":256,"vocab":151936,"n_params_M":43.824896,"tie_word_embeddings":true},"steps":1200,"batch":16,"seed":0,"peak_lr":0.001,"val_tokens":25146,"val_kl_step0":3.7879278659820557,"val_kl_final":2.6291682720184326,"kl_drop_frac":0.30590856926553534,"curve":[{"step":0,"val_kl":3.7879278659820557},{"step":25,"train_loss":3.2964470386505127,"val_kl":3.440807342529297},{"step":50,"train_loss":3.3482024669647217,"val_kl":3.2190022468566895},{"step":75,"train_loss":3.1245064735412598,"val_kl":3.1247265338897705},{"step":100,"train_loss":3.0589778423309326,"val_kl":3.045606851577759},{"step":125,"train_loss":2.9507038593292236,"val_kl":3.007833957672119},{"step":150,"train_loss":2.8744351863861084,"val_kl":2.9722635746002197},{"step":175,"train_loss":2.971992254257202,"val_kl":2.941195249557495},{"step":200,"train_loss":3.359013795852661,"val_kl":2.9081735610961914},{"step":225,"train_loss":2.726900815963745,"val_kl":2.897998332977295},{"step":250,"train_loss":3.1082329750061035,"val_kl":2.865818738937378},{"step":275,"train_loss":2.8227930068969727,"val_kl":2.8509745597839355},{"step":300,"train_loss":2.8071019649505615,"val_kl":2.8392252922058105},{"step":325,"train_loss":2.6850948333740234,"val_kl":2.817128896713257},{"step":350,"train_loss":2.656618118286133,"val_kl":2.805037260055542},{"step":375,"train_loss":2.8315200805664062,"val_kl":2.7966787815093994},{"step":400,"train_loss":2.631049871444702,"val_kl":2.7760002613067627},{"step":425,"train_loss":2.715193510055542,"val_kl":2.7659549713134766},{"step":450,"train_loss":2.5417938232421875,"val_kl":2.764688491821289},{"step":475,"train_loss":2.6781835556030273,"val_kl":2.7575502395629883},{"step":500,"train_loss":2.808349370956421,"val_kl":2.7459137439727783},{"step":525,"train_loss":2.7859623432159424,"val_kl":2.735214948654175},{"step":550,"train_loss":2.6303582191467285,"val_kl":2.7211689949035645},{"step":575,"train_loss":2.8113672733306885,"val_kl":2.7160751819610596},{"step":600,"train_loss":2.6080615520477295,"val_kl":2.7082293033599854},{"step":625,"train_loss":2.3058998584747314,"val_kl":2.7031121253967285},{"step":650,"train_loss":2.8408682346343994,"val_kl":2.6961092948913574},{"step":675,"train_loss":2.595034122467041,"val_kl":2.6866953372955322},{"step":700,"train_loss":2.598254680633545,"val_kl":2.6834981441497803},{"step":725,"train_loss":2.4859395027160645,"val_kl":2.6800925731658936},{"step":750,"train_loss":2.462690591812134,"val_kl":2.6728360652923584},{"step":775,"train_loss":2.574542760848999,"val_kl":2.6684114933013916},{"step":800,"train_loss":2.3669815063476562,"val_kl":2.6618635654449463},{"step":825,"train_loss":2.6285853385925293,"val_kl":2.6570844650268555},{"step":850,"train_loss":2.6675031185150146,"val_kl":2.656144857406616},{"step":875,"train_loss":2.7212071418762207,"val_kl":2.6516149044036865},{"step":900,"train_loss":2.4483072757720947,"val_kl":2.6482009887695312},{"step":925,"train_loss":2.701512575149536,"val_kl":2.6456501483917236},{"step":950,"train_loss":2.5716919898986816,"val_kl":2.6415839195251465},{"step":975,"train_loss":2.327781915664673,"val_kl":2.637615203857422},{"step":1000,"train_loss":2.443568468093872,"val_kl":2.636082172393799},{"step":1025,"train_loss":2.4940853118896484,"val_kl":2.6339213848114014},{"step":1050,"train_loss":2.4401471614837646,"val_kl":2.6326797008514404},{"step":1075,"train_loss":2.3165555000305176,"val_kl":2.6315860748291016},{"step":1100,"train_loss":2.507467746734619,"val_kl":2.6310055255889893},{"step":1125,"train_loss":2.566948890686035,"val_kl":2.6302409172058105},{"step":1150,"train_loss":2.5282790660858154,"val_kl":2.629762649536133},{"step":1175,"train_loss":2.2921111583709717,"val_kl":2.629451274871826},{"step":1200,"train_loss":2.521691083908081,"val_kl":2.6291682720184326}]},"distill_curve_tinit":{"cfg":{"n_layers":4,"hidden":256,"vocab":151936,"n_params_M":43.824896,"tie_word_embeddings":true},"steps":1200,"batch":16,"seed":0,"peak_lr":0.001,"teacher_init":true,"val_tokens":25146,"val_kl_step0":3.7706851959228516,"val_kl_final":2.6378531455993652,"kl_drop_frac":0.30043135172047497,"curve":[{"step":0,"val_kl":3.7706851959228516},{"step":25,"train_loss":3.243398666381836,"val_kl":3.4384210109710693},{"step":50,"train_loss":3.382613182067871,"val_kl":3.24110746383667},{"step":75,"train_loss":3.120757579803467,"val_kl":3.137375593185425},{"step":100,"train_loss":3.068692922592163,"val_kl":3.0513899326324463},{"step":125,"train_loss":2.9330990314483643,"val_kl":2.9982526302337646},{"step":150,"train_loss":2.9006094932556152,"val_kl":2.963433027267456},{"step":175,"train_loss":2.9641361236572266,"val_kl":2.9223475456237793},{"step":200,"train_loss":3.346369981765747,"val_kl":2.894104242324829},{"step":225,"train_loss":2.7291622161865234,"val_kl":2.8813436031341553},{"step":250,"train_loss":3.086744546890259,"val_kl":2.853276491165161},{"step":275,"train_loss":2.8312275409698486,"val_kl":2.8355839252471924},{"step":300,"train_loss":2.8064303398132324,"val_kl":2.8269035816192627},{"step":325,"train_loss":2.678917646408081,"val_kl":2.813077449798584},{"step":350,"train_loss":2.6579458713531494,"val_kl":2.8102188110351562},{"step":375,"train_loss":2.8274669647216797,"val_kl":2.7959952354431152},{"step":400,"train_loss":2.6204588413238525,"val_kl":2.774482011795044},{"step":425,"train_loss":2.716571569442749,"val_kl":2.769601583480835},{"step":450,"train_loss":2.549912691116333,"val_kl":2.766221523284912},{"step":475,"train_loss":2.6650023460388184,"val_kl":2.7632107734680176},{"step":500,"train_loss":2.8133785724639893,"val_kl":2.74474835395813},{"step":525,"train_loss":2.7550694942474365,"val_kl":2.7409114837646484},{"step":550,"train_loss":2.6376726627349854,"val_kl":2.7255258560180664},{"step":575,"train_loss":2.821810483932495,"val_kl":2.7193875312805176},{"step":600,"train_loss":2.611309289932251,"val_kl":2.713186502456665},{"step":625,"train_loss":2.309516429901123,"val_kl":2.706911563873291},{"step":650,"train_loss":2.8207476139068604,"val_kl":2.705016613006592},{"step":675,"train_loss":2.5845160484313965,"val_kl":2.6979923248291016},{"step":700,"train_loss":2.5895473957061768,"val_kl":2.690488815307617},{"step":725,"train_loss":2.4982903003692627,"val_kl":2.6908764839172363},{"step":750,"train_loss":2.466019630432129,"val_kl":2.68265700340271},{"step":775,"train_loss":2.561558246612549,"val_kl":2.6750152111053467},{"step":800,"train_loss":2.358829975128174,"val_kl":2.6712138652801514},{"step":825,"train_loss":2.649077892303467,"val_kl":2.663294792175293},{"step":850,"train_loss":2.679588556289673,"val_kl":2.662080764770508},{"step":875,"train_loss":2.7410004138946533,"val_kl":2.6586904525756836},{"step":900,"train_loss":2.4484262466430664,"val_kl":2.65496563911438},{"step":925,"train_loss":2.722778081893921,"val_kl":2.65358829498291},{"step":950,"train_loss":2.597452163696289,"val_kl":2.64890193939209},{"step":975,"train_loss":2.344724178314209,"val_kl":2.64616060256958},{"step":1000,"train_loss":2.4528119564056396,"val_kl":2.644289970397949},{"step":1025,"train_loss":2.510200262069702,"val_kl":2.6415975093841553},{"step":1050,"train_loss":2.44018292427063,"val_kl":2.6414270401000977},{"step":1075,"train_loss":2.3308258056640625,"val_kl":2.6405937671661377},{"step":1100,"train_loss":2.532075881958008,"val_kl":2.6402740478515625},{"step":1125,"train_loss":2.5816421508789062,"val_kl":2.6392264366149902},{"step":1150,"train_loss":2.5210869312286377,"val_kl":2.638392686843872},{"step":1175,"train_loss":2.314781904220581,"val_kl":2.637984037399292},{"step":1200,"train_loss":2.54268741607666,"val_kl":2.6378531455993652}]}},"commits":{"alignment":{"sha":"7a848915e7019931254185bfee9cdec6e0966c8c","short":"7a84891","date":"2026-09-30","subject":"alpha: fix off-by-one logit alignment, known-answer tests, 1.5B default target","paths":["draftlab/eval/alpha.py","draftlab/tests/test_alpha_known.py"],"diff":"diff --git a/draftlab/eval/alpha.py b/draftlab/eval/alpha.py\nindex 316693f..89ae749 100644\n--- a/draftlab/eval/alpha.py\n+++ b/draftlab/eval/alpha.py\n@@ -7,8 +7,10 @@ At every position of a block, with context = the run's recent preceding blocks\n     greedy_corpus = [argmax q == actual token]  (what results/ngram.json measures)\n \"Batched\" = one teacher-forced forward per model per block (all positions at once),\n-top-k-free. Blocks are selected round-robin across runs; per-kind aggregation uses\n-bootstrap CIs over runs (units = runs, so no run leaks). Expected tokens per target\n-call for k drafts: (1 - a^(k+1))/(1 - a); speedup divides by 1 + k*c, c = measured\n-draft/target forward cost ratio.\n+top-k-free. Alignment: logits row i predicts token i+1, so ids[j] is scored by row\n+len(ctx)+j-1; with empty ctx there is no row for ids[0] and position 0 is invalid\n+(valid[0] = False; no fabricated BOS). Blocks are selected round-robin across runs;\n+per-kind aggregation uses bootstrap CIs over runs (units = runs, so no run leaks).\n+Expected tokens per target call for k drafts: (1 - a^(k+1))/(1 - a); speedup divides\n+by 1 + k*c, c = measured draft/target forward cost ratio.\n \n CLI: python -m draftlab.eval.alpha [--drafter ngram|jax|hf] [--draft-model PATH]\n@@ -29,5 +31,5 @@ import numpy as np\n ROOT = Path(__file__).resolve().parents[2]\n CORPUS_DIR = ROOT / \"data\" / \"corpus\"\n-TARGET_DIR = ROOT / \"models\" / \"qwen2.5-0.5b\"\n+TARGET_DIR = ROOT / \"models\" / \"qwen2.5-1.5b-instruct\"  # target T (TODO.md model roles)\n RESULTS_DIR = ROOT / \"results\" / \"alpha\"\n \n@@ -118,6 +120,16 @@ class JaxModel:\n         logits, _ = self.M.prefill(self.cfg, self.params, full, self.max_len)\n         n = len(ids)\n-        rows = np.asarray(logits)[len(ctx) : len(ctx) + n]  # slice before softmax\n-        return _softmax(rows), np.ones(n, dtype=bool)\n+        # Row i predicts token i+1: ids[j] is scored by row len(ctx)+j-1 (slice\n+        # before softmax/transfer). Empty ctx: no row predicts ids[0] -> dummy\n+        # row 0, position 0 invalid (we do not fabricate a BOS).\n+        L = np.asarray(logits)\n+        valid = np.ones(n, dtype=bool)\n+        if ctx:\n+            rows = L[len(ctx) - 1 : len(ctx) + n - 1]\n+        else:\n+            valid[0] = False\n+            rows = L[: n - 1]\n+            rows = np.vstack([L[:1], rows]) if n > 1 else L[:1]\n+        return _softmax(rows), valid\n \n \n@@ -146,6 +158,15 @@ class HFDrafter:\n         n = len(ids)\n         with self.torch.no_grad():\n-            logits = self.model(input_ids=self.torch.tensor([ctx + ids])).logits[0, -n:]\n-        return _softmax(logits.float().numpy()), np.ones(n, dtype=bool)\n+            logits = self.model(input_ids=self.torch.tensor([ctx + ids])).logits[0]\n+        # Same alignment as JaxModel: ids[j] is scored by row len(ctx)+j-1;\n+        # empty ctx -> dummy row 0, position 0 invalid.\n+        valid = np.ones(n, dtype=bool)\n+        if ctx:\n+            rows = logits[len(ctx) - 1 : len(ctx) + n - 1]\n+        else:\n+            valid[0] = False\n+            rows = logits[: n - 1]\n+            rows = self.torch.cat([logits[:1], rows]) if n > 1 else logits[:1]\n+        return _softmax(rows.float().numpy()), valid\n \n \ndiff --git a/draftlab/tests/test_alpha_known.py b/draftlab/tests/test_alpha_known.py\nnew file mode 100644\nindex 0000000..e62812a\n--- /dev/null\n+++ b/draftlab/tests/test_alpha_known.py\n@@ -0,0 +1,97 @@\n+\"\"\"Known answers for the alpha metric (CLAUDE.md \"Known answers\"), on the real\n+0.5B checkpoint. Skipped only if models/ has not been fetched.\"\"\"\n+\n+from pathlib import Path\n+\n+import numpy as np\n+import pytest\n+\n+from draftlab.eval.alpha import JaxModel, NGramDrafter, alpha_and_greedy, load_blocks\n+\n+MODEL = Path(__file__).resolve().parents[2] / \"models\" / \"qwen2.5-0.5b\"\n+HAS_MODEL = (MODEL / \"config.json\").exists()\n+needs_model = pytest.mark.skipif(not HAS_MODEL, reason=\"models/ not fetched\")\n+\n+\n+@pytest.fixture(scope=\"module\")\n+def tok():\n+    from transformers import AutoTokenizer\n+\n+    return AutoTokenizer.from_pretrained(MODEL)\n+\n+\n+@pytest.fixture(scope=\"module\")\n+def target():\n+    return JaxModel.from_path(MODEL, max_len=512)\n+\n+\n+@needs_model\n+def test_per_position_ce_matches_hf_labels_loss(target, tok):\n+    \"\"\"JaxModel rows must be aligned (row i predicts token i+1): per-position CE\n+    from dists == HF per-token NLL, and their mean == model(ids, labels=ids).loss.\"\"\"\n+    import torch\n+    from torch.nn import functional as F\n+    from transformers import AutoModelForCausalLM\n+\n+    hf = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.float32).eval()\n+    for b in load_blocks(\"val\", max_tokens=256, n_blocks=2):\n+        ids = torch.tensor([tok.encode(b[\"text\"], add_special_tokens=False)])\n+        with torch.no_grad():\n+            out = hf(ids, labels=ids)\n+            nll = F.cross_entropy(hf(ids).logits[0, :-1], ids[0][1:], reduction=\"none\").numpy()\n+        p, valid = target.dists([], ids[0].tolist())\n+        assert not valid[0] and valid[1:].all()\n+        arr = ids[0].numpy()[1:]  # ce_j = -log p[j, ids[j]] for j >= 1\n+        ce = -np.log(np.maximum(p.astype(np.float64)[np.arange(1, len(arr) + 1), arr], 1e-30))\n+        assert np.max(np.abs(ce - nll)) \u003c 1e-3, np.max(np.abs(ce - nll))\n+        assert abs(float(ce.mean()) - out.loss.item()) \u003c 1e-3\n+\n+\n+@needs_model\n+def test_target_top1_on_val_in_sanity_range(target, tok):\n+    \"\"\"Qwen2.5-0.5B top-1 next-token accuracy on corpus val blocks is 40-70%\n+    (57.5% measured on 372 tok); under 20% means an alignment bug.\"\"\"\n+    hits = tot = 0\n+    for b in load_blocks(\"val\", max_tokens=256, n_blocks=40):\n+        ids = np.asarray(tok.encode(b[\"text\"], add_special_tokens=False))\n+        p, valid = target.dists([], ids.tolist())\n+        ok = valid & (p.argmax(axis=-1) == ids)\n+        hits += int(ok.sum())\n+        tot += int(valid.sum())\n+    acc = hits / tot\n+    assert 0.40 \u003c= acc \u003c= 0.70, f\"top-1 {acc:.3f} outside sanity range 0.40-0.70\"\n+\n+\n+@needs_model\n+def test_uniform_q_alpha_exact(target, tok):\n+    \"\"\"q uniform -> alpha = sum_x min(p, 1/V) = 1 - TV(p, u) = 1 - 0.5*sum|p - 1/V|\n+    (checked via the independent total-variation identity, not the same min-sum).\"\"\"\n+    ids = np.asarray(tok.encode(load_blocks(\"val\", 256, 1)[0][\"text\"], add_special_tokens=False))\n+    p, valid = target.dists([], ids.tolist())\n+    v = p.shape[1]\n+    a, _ = alpha_and_greedy(p, np.full_like(p, 1.0 / v))\n+    expected = 1.0 - 0.5 * np.abs(p.astype(np.float64) - 1.0 / v).sum(axis=-1)\n+    assert np.max(np.abs(a - expected)) \u003c 1e-6\n+    assert (a >= 1.0 / v - 1e-6).all() and (a \u003c= 1.0).all()  # TV bounds\n+\n+\n+def test_ngram_repeated_block_planted_effect():\n+    \"\"\"Planted effect: a block repeated verbatim after itself -> ngram acceptance\n+    on the copy ~= 1 (all but the first position, which has no prior context).\"\"\"\n+    x = list(range(1, 65))  # no internal n-gram repeats, so every proposal is exact\n+    d = NGramDrafter(vocab_size=1000)\n+    q, valid = d.dists(x, x)\n+    match = valid & (q.argmax(axis=-1) == np.asarray(x))\n+    assert match.sum() == len(x) - 1 and match[1:].all()\n+\n+\n+def test_ngram_shuffled_block_null_control():\n+    \"\"\"Null control: the same tokens shuffled within the block -> ngram acceptance\n+    collapses (proposals were learned on the original order).\"\"\"\n+    x = list(range(1, 65))\n+    rng = np.random.default_rng(0)\n+    y = [x[i] for i in rng.permutation(len(x))]\n+    d = NGramDrafter(vocab_size=1000)\n+    q, valid = d.dists(x, y)\n+    acc = float((valid & (q.argmax(axis=-1) == np.asarray(y))).mean())\n+    assert acc \u003c 0.5, f\"shuffled acceptance {acc:.3f} did not collapse\""},"cache":{"sha":"ae371e3c9a650410f432695c29db78d3341c5f73","short":"ae371e3","date":"2026-09-30","subject":"spec: fix ModelDrafter propose double-decode and full-accept rollback; regression tests","paths":["draftlab/jaxlm/spec.py","draftlab/tests/test_spec.py"],"diff":"diff --git a/draftlab/jaxlm/spec.py b/draftlab/jaxlm/spec.py\nindex 29a8fa1..255289a 100644\n--- a/draftlab/jaxlm/spec.py\n+++ b/draftlab/jaxlm/spec.py\n@@ -143,5 +143,11 @@ class StepStats:\n \n class SpecDecoder:\n-    \"\"\"Speculative sampling over one jitted target model.\"\"\"\n+    \"\"\"Speculative sampling over one jitted target model.\n+\n+    temperature > 0 scales the TARGET sampling distribution (softmax(lg/T));\n+    draft probs stay unscaled. Accept/reject + residual max(0, p - q) is\n+    lossless for any draft q, so output still follows exactly the target's\n+    T-scaled distribution.\n+    \"\"\"\n \n     def __init__(\n@@ -167,4 +173,5 @@ class SpecDecoder:\n         cfg, params, drafter = self.cfg, self.params, self.drafter\n         T0 = self.temperature == 0.0\n+        T = self.temperature if not T0 else 1.0\n         rng = self.rng\n         ctx = list(prompt)\n@@ -187,5 +194,5 @@ class SpecDecoder:\n                     tok = int(np.argmax(np.asarray(prev_lg)))\n                 else:\n-                    tok = _categorical(step_rng, jax.nn.softmax(prev_lg))\n+                    tok = _categorical(step_rng, jax.nn.softmax(prev_lg / T))\n                 lg, cache = M.decode(cfg, params, cache, [tok])\n                 prev_lg = lg[-1]\n@@ -203,5 +210,5 @@ class SpecDecoder:\n                     bonus = int(np.argmax(np.asarray(Pd[A] if A \u003c m else scored[m - 1])))\n                 else:\n-                    P = [np.asarray(jax.nn.softmax(row), np.float32) for row in Pd]\n+                    P = [np.asarray(jax.nn.softmax(row / T), np.float32) for row in Pd]\n                     A = 0\n                     for i, q in enumerate(dprobs):\n@@ -213,5 +220,5 @@ class SpecDecoder:\n                             break\n                     if A == m:\n-                        bonus = _categorical(step_rng, jax.nn.softmax(scored[m - 1]))\n+                        bonus = _categorical(step_rng, jax.nn.softmax(scored[m - 1] / T))\n                     else:\n                         resid = np.maximum(0.0, P[A] - dprobs[A])\ndiff --git a/draftlab/tests/test_spec.py b/draftlab/tests/test_spec.py\nindex ce6d277..b8ad5f0 100644\n--- a/draftlab/tests/test_spec.py\n+++ b/draftlab/tests/test_spec.py\n@@ -78,4 +78,30 @@ def test_lossless_chi_square():\n \n \n+def test_temperature_scaling_chi_square():\n+    \"\"\"T=0.7: first-token distribution == softmax(logits/0.7), chi2 p > 0.01,\n+    and distinguishable from the unscaled softmax (so the test fails if the\n+    temperature is ignored).\"\"\"\n+    from scipy.special import logsumexp\n+\n+    cfg, params = _tiny16(1)\n+    prompt = [i % 7 for i in range(32)]\n+    lg, _ = M.prefill(cfg, params, prompt, len(prompt) + 8)\n+    lg = np.asarray(lg[-1], np.float64)\n+    pT = np.exp(lg / 0.7 - logsumexp(lg / 0.7))\n+    p1 = np.exp(lg - logsumexp(lg))\n+    assert float(np.abs(pT - p1).sum()) > 0.05, \"guard: T=0.7 must change the distribution\"\n+    N = 8000\n+    obs = np.zeros(cfg.vocab_size, np.int64)\n+    for i in range(N):\n+        dec = SpecDecoder(cfg, params, NGramDrafter(cfg.vocab_size, max_n=4),\n+                          max_len=len(prompt) + 8, k=2, temperature=0.7, seed=i)\n+        out, _ = dec.generate(prompt, 1)\n+        obs[out[0]] += 1\n+    exp = N * pT\n+    chi2 = float(((obs - exp) ** 2 / exp).sum())\n+    pval = float(stats.chi2.sf(chi2, df=cfg.vocab_size - 1))\n+    assert pval > 0.01, f\"chi2={chi2:.1f}, p={pval:.4g} (not sampling softmax(lg/0.7))\"\n+\n+\n @pytest.mark.parametrize(\"kind\", [\"ngram\", \"model\"])\n def test_rollback_cache_equals_fresh_prefill(kind):"},"sampling":{"sha":"1e9f5d3911dcbf6a4fc7860ff877aa63d89c1952","short":"1e9f5d3","date":"2026-10-06","subject":"spec: lossless T>0 model drafting, one target forward per step","paths":["draftlab/jaxlm/spec.py","draftlab/tests/test_spec.py"],"diff":"diff --git a/draftlab/jaxlm/spec.py b/draftlab/jaxlm/spec.py\nindex 255289a..a24a63e 100644\n--- a/draftlab/jaxlm/spec.py\n+++ b/draftlab/jaxlm/spec.py\n@@ -4,9 +4,8 @@ Drafter protocol: ``propose(ctx, k) -> (tokens, probs)`` \u2014 up to k draft\n tokens plus a (V,) float32 draft distribution for each; ``rollback(n_kept)``\n undoes draft-side state (KV cache) added for the drafted suffix beyond the\n-n_kept drafts that were committed. The engine scores all m drafted tokens plus\n-the bonus position with one target call; accepts while u \u003c p/q, then either\n-resamples from max(0, p - q) at the first rejection or draws the bonus from p.\n-Target-cache rollback is truncating the positional cache's `len`; a\n-fully-accepted step decodes the bonus in.\n+n_kept drafts that were committed. The engine scores the pending token plus all m drafted\n+tokens with one target call; accepts while u \u003c p/q, then either resamples\n+from max(0, p - q) at the first rejection or draws the bonus from p.\n+Target-cache rollback is truncating the positional cache's `len`.\n \"\"\"\n \n@@ -84,36 +83,65 @@ class NGramDrafter:\n \n \n-class ModelDrafter:\n-    \"\"\"A second JAX Qwen2 proposes its own greedy continuation.\n+class NullDrafter:\n+    \"\"\"No drafting: plain autoregressive target decoding.\"\"\"\n+\n+    def propose(self, ctx: list[int], k: int) -> tuple[list[int], list[np.ndarray]]:\n+        return [], []\n \n-    Keeps its own KV cache, extended from committed tokens; a partial\n-    acceptance restores the pre-draft snapshot so the committed suffix is\n-    re-decoded at the next propose.\n+    def rollback(self, n_kept: int) -> None:\n+        pass\n+\n+\n+class ModelDrafter:\n+    \"\"\"A second JAX Qwen2 proposes its own continuation.\n+\n+    T=0: argmax drafts with one-hot q. T>0: drafts sampled from\n+    softmax(logits/T) and q is exactly that distribution, so accept/reject is\n+    lossless (SpecDecoder sets T and the seed via `configure`). Keeps its own\n+    KV cache; rollback truncates it to the committed drafts (no re-decode).\n+    The last draft is never decoded here: on full acceptance the next propose\n+    decodes it together with the bonus token.\n     \"\"\"\n \n-    def __init__(self, dcfg: M.Qwen2Cfg, dparams: dict, max_len: int) -> None:\n+    def __init__(self, dcfg: M.Qwen2Cfg, dparams: dict, max_len: int,\n+                 temperature: float = 0.0, seed: int = 0) -> None:\n         self.dcfg = dcfg\n         self.dparams = dparams\n         self.cache = M.empty_cache(dcfg, max_len)\n         self._last = 0\n-        self._undo: tuple[dict, int] | None = None\n+        self._undo: tuple[int, int] | None = None\n+        self.configure(temperature, seed)\n+\n+    def configure(self, temperature: float, seed: int) -> None:\n+        self.temperature = temperature\n+        self._rng = np.random.default_rng(seed)\n+\n+    def _pick(self, row) -> tuple[int, np.ndarray]:\n+        if self.temperature == 0.0:\n+            t = int(np.argmax(np.asarray(row)))\n+            q = np.zeros(row.shape[-1], np.float32)\n+            q[t] = 1.0\n+            return t, q\n+        q = np.asarray(jax.nn.softmax(row.astype(jnp.float32) / self.temperature), np.float64)\n+        q /= q.sum()\n+        t = int(self._rng.choice(q.size, p=q))\n+        return t, q.astype(np.float32)\n \n     def propose(self, ctx: list[int], k: int) -> tuple[list[int], list[np.ndarray]]:\n-        # ctx grows by >=1 committed token between proposes (first call: prompt),\n-        # so extending the cache to len(ctx) gives the dist over the FIRST draft\n-        # token on its last row; each draft then decodes the previous one.\n+        # extending the cache to len(ctx) gives the dist over the FIRST draft on\n+        # its last row; drafts 1..k-1 each decode the previous draft\n         if len(ctx) \u003c= self._last:\n             raise RuntimeError(\"propose called without new committed tokens\")\n         lg, self.cache = M.decode(self.dcfg, self.dparams, self.cache, ctx[self._last :])\n         self._last = len(ctx)\n-        self._undo = (self.cache, self._last, k)\n         toks: list[int] = []\n         probs: list[np.ndarray] = []\n-        for _ in range(k):\n-            p = np.asarray(jax.nn.softmax(lg[-1]), np.float32)\n-            t = int(np.argmax(p))\n+        for j in range(k):\n+            t, q = self._pick(lg[-1])\n             toks.append(t)\n-            probs.append(p)\n-            lg, self.cache = M.decode(self.dcfg, self.dparams, self.cache, [t])\n+            probs.append(q)\n+            if j \u003c k - 1:\n+                lg, self.cache = M.decode(self.dcfg, self.dparams, self.cache, [t])\n+        self._undo = (self._last, len(toks))\n         return toks, probs\n \n@@ -121,10 +149,9 @@ class ModelDrafter:\n         if self._undo is None:\n             return\n-        cache, last, m = self._undo\n+        last, m = self._undo\n         self._undo = None\n-        if n_kept == m:  # all drafts committed; the cache already holds them\n-            self._last = last + m  # only the bonus is new; extend from after the drafts\n-            return\n-        self.cache, self._last = cache, last\n+        keep = min(n_kept, m - 1)  # cached drafts that were committed\n+        self._last = last + keep\n+        self.cache = {\"k\": self.cache[\"k\"], \"v\": self.cache[\"v\"], \"len\": jnp.int32(self._last)}\n \n \n@@ -145,8 +172,9 @@ class SpecDecoder:\n     \"\"\"Speculative sampling over one jitted target model.\n \n-    temperature > 0 scales the TARGET sampling distribution (softmax(lg/T));\n-    draft probs stay unscaled. Accept/reject + residual max(0, p - q) is\n-    lossless for any draft q, so output still follows exactly the target's\n-    T-scaled distribution.\n+    temperature > 0 samples the target's softmax(lg/T); the drafter is\n+    configured to draft at the same T and report its true q, so accept/reject\n+    + residual max(0, p - q) is lossless. One target forward per step: the\n+    cache holds every committed token but the newest (`pend`), and each step\n+    decodes [pend] + drafts, so row j scores draft j and row m is the bonus.\n     \"\"\"\n \n@@ -168,4 +196,6 @@ class SpecDecoder:\n         self.temperature = temperature\n         self.rng = jax.random.PRNGKey(seed)\n+        if hasattr(drafter, \"configure\"):\n+            drafter.configure(temperature, seed + 1)\n \n     def _steps(self, prompt: list[int], n_tokens: int):\n@@ -176,8 +206,7 @@ class SpecDecoder:\n         rng = self.rng\n         ctx = list(prompt)\n-        pre_lg, cache = M.prefill(cfg, params, ctx, self.max_len)\n-        prev_lg = pre_lg[-1]  # dist over position L (first generated token)\n+        _, cache = M.prefill(cfg, params, ctx, self.max_len)\n+        cache = {\"k\": cache[\"k\"], \"v\": cache[\"v\"], \"len\": jnp.int32(len(ctx) - 1)}\n         out: list[int] = []\n-        stats: list[StepStats] = []\n         while len(out) \u003c n_tokens:\n             rem = n_tokens - len(out)\n@@ -190,51 +219,30 @@ class SpecDecoder:\n                 dtoks, dprobs = drafter.propose(ctx, k)\n             m = len(dtoks)\n-            if m == 0:  # no draft: draw from the held last row, then one decode\n-                if T0:\n-                    tok = int(np.argmax(np.asarray(prev_lg)))\n-                else:\n-                    tok = _categorical(step_rng, jax.nn.softmax(prev_lg / T))\n-                lg, cache = M.decode(cfg, params, cache, [tok])\n-                prev_lg = lg[-1]\n-                emitted, A = [tok], 0\n+            rows, cache2 = M.decode(cfg, params, cache, [ctx[-1]] + dtoks)\n+            if T0:\n+                A = 0\n+                while A \u003c m and dtoks[A] == int(np.argmax(np.asarray(rows[A]))):\n+                    A += 1\n+                bonus = int(np.argmax(np.asarray(rows[A])))\n             else:\n-                # one target call over the m drafts: row r is the dist over the\n-                # token at position L+r+1, i.e. draft r+1... so draft j's dist is\n-                # prev (j=0) or row j-1, and row m-1 is the bonus dist\n-                scored, cache2 = M.decode(cfg, params, cache, dtoks)\n-                Pd = [prev_lg if j == 0 else scored[j - 1] for j in range(m)]\n-                if T0:\n-                    A = 0\n-                    while A \u003c m and dtoks[A] == int(np.argmax(np.asarray(Pd[A]))):\n+                A = 0\n+                for i, q in enumerate(dprobs):\n+                    p_i = np.asarray(jax.nn.softmax(rows[i] / T), np.float32)\n+                    u = float(jax.random.uniform(step_rng))\n+                    step_rng = jax.random.fold_in(step_rng, i + 1)\n+                    if u * q[dtoks[i]] \u003c p_i[dtoks[i]]:\n                         A += 1\n-                    bonus = int(np.argmax(np.asarray(Pd[A] if A \u003c m else scored[m - 1])))\n-                else:\n-                    P = [np.asarray(jax.nn.softmax(row / T), np.float32) for row in Pd]\n-                    A = 0\n-                    for i, q in enumerate(dprobs):\n-                        u = float(jax.random.uniform(step_rng))\n-                        step_rng = jax.random.fold_in(step_rng, i + 1)\n-                        if u * q[dtoks[i]] \u003c P[i][dtoks[i]]:\n-                            A += 1\n-                        else:\n-                            break\n-                    if A == m:\n-                        bonus = _categorical(step_rng, jax.nn.softmax(scored[m - 1] / T))\n                     else:\n-                        resid = np.maximum(0.0, P[A] - dprobs[A])\n-                        bonus = _categorical(step_rng, resid / resid.sum())\n-                emitted = dtoks[:A] + [bonus]\n-                E = len(emitted) if A \u003c m else min(rem, m + 1)\n-                emitted = emitted[:E]\n-                # Roll back the target cache to the A committed drafts, then\n-                # commit the bonus (its key/value only exists after this decode;\n-                # it was not part of the scoring call).\n-                trunc = {\"k\": cache2[\"k\"], \"v\": cache2[\"v\"], \"len\": jnp.int32(len(ctx) + A)}\n-                prev2, cache_b = M.decode(cfg, params, trunc, [bonus])\n-                if A == m and rem >= m + 1:\n-                    cache, prev_lg = cache_b, prev2[-1]\n-                else:\n-                    cache = {\"k\": cache_b[\"k\"], \"v\": cache_b[\"v\"], \"len\": jnp.int32(len(ctx) + E)}\n-                    prev_lg = prev2[-1] if len(out) + E \u003c n_tokens else None\n+                        resid = np.maximum(0.0, p_i - q)\n+                        break\n+                if A == m:\n+                    bonus = _categorical(step_rng, jax.nn.softmax(rows[m] / T))\n+                elif resid.sum() > 0:\n+                    bonus = _categorical(step_rng, resid / resid.sum())\n+                else:  # p == q on the draft's support: residual is p itself\n+                    bonus = _categorical(step_rng, p_i)\n+            emitted = (dtoks[:A] + [bonus])[:rem]\n+            # keep [pend] + the A accepted drafts; the bonus becomes the new pend\n+            cache = {\"k\": cache2[\"k\"], \"v\": cache2[\"v\"], \"len\": jnp.int32(len(ctx) + A)}\n             drafter.rollback(A)\n             ctx.extend(emitted)\n@@ -242,9 +250,8 @@ class SpecDecoder:\n             dt_ms = (time.perf_counter() - t0) * 1e3\n             st = StepStats(m, A, [x \u003c A for x in range(m)], dt_ms)\n-            stats.append(st)\n             yield emitted, st\n         self.rng = rng\n         self.last_ctx = ctx  # prompt + generated\n-        self.last_cache = cache\n+        self.last_cache = cache  # holds last_ctx[:-1] (the newest token is pending)\n \n     def generate(self, prompt: list[int], n_tokens: int) -> tuple[list[int], list[StepStats]]:\ndiff --git a/draftlab/tests/test_spec.py b/draftlab/tests/test_spec.py\nindex b8ad5f0..f9f510b 100644\n--- a/draftlab/tests/test_spec.py\n+++ b/draftlab/tests/test_spec.py\n@@ -118,6 +118,6 @@ def test_rollback_cache_equals_fresh_prefill(kind):\n     _, ref_cache = M.prefill(cfg, params, full, max_len)\n     c = dec.last_cache\n-    assert int(c[\"len\"]) == len(full)\n-    n = len(full)\n+    n = len(full) - 1  # the newest token is pending (decoded by the next step)\n+    assert int(c[\"len\"]) == n\n     for key in (\"k\", \"v\"):\n         diff = float(abs(np.asarray(c[key])[:, :, :n] - np.asarray(ref_cache[key])[:, :, :n]).max())\n@@ -167,11 +167,11 @@ def test_model_drafter_full_accept_steps_keep_cache_in_sync():\n     assert out == _greedy(cfg, params, prompt, 20)\n     assert all(s.drafted == 1 and s.accepted == 1 for s in st)  # strict, no rejections\n-    # The drafter ends one token behind the final bonus: its cache must equal a\n-    # fresh prefill of that prefix (a desynced rollback re-decodes already\n-    # cached drafts and the KV diverges from the first duplicated slot on).\n+    # The drafter's cache must equal a fresh prefill of the committed prefix it\n+    # claims to hold (a desynced rollback re-decodes cached drafts and the KV\n+    # diverges from the first duplicated slot on).\n     full = prompt + out\n-    n = len(full) - 1\n     d = dec.drafter\n-    assert int(d.cache[\"len\"]) == n and d._last == n\n+    n = d._last\n+    assert int(d.cache[\"len\"]) == n and 0 \u003c n \u003c len(full)\n     _, ref = M.prefill(d.dcfg, d.dparams, full[:n], 84)\n     for key in (\"k\", \"v\"):\n@@ -198,2 +198,78 @@ def test_model_drafter_first_proposal_position():\n                     jax.numpy.asarray(prompt + [toks[0]], jax.numpy.int32))\n     assert toks[1] == int(np.argmax(np.asarray(lg2[-1])))\n+\n+\n+def _probs(cfg, params, ids, T=1.0):\n+    lg, _ = M.step(cfg, params, M.empty_cache(cfg, len(ids) + 1),\n+                   jax.numpy.asarray(ids, jax.numpy.int32))\n+    return np.asarray(jax.nn.softmax(lg[-1].astype(np.float64) / T), np.float64)\n+\n+\n+def test_model_drafter_lossless_second_token_chi_square():\n+    \"\"\"T=0.8, model drafter != target, k=3: the marginal of the SECOND emitted\n+    token equals sum_x p(x) p(y|x) exactly (covers draft rows, rejection +\n+    residual, and the pending-token cache layout across steps).\"\"\"\n+    cfg, params = _tiny16(1)\n+    dcfg, dparams = _tiny16(42)\n+    T = 0.8\n+    prompt = [i % 7 for i in range(20)]\n+    p1 = _probs(cfg, params, prompt, T)\n+    exp_p = sum(p1[x] * _probs(cfg, params, prompt + [x], T) for x in range(cfg.vocab_size))\n+    N = 6000\n+    obs = np.zeros(cfg.vocab_size, np.int64)\n+    acc = drafted = 0\n+    for i in range(N):\n+        dec = SpecDecoder(cfg, params, ModelDrafter(dcfg, dparams, len(prompt) + 8),\n+                          max_len=len(prompt) + 8, k=3, temperature=T, seed=i)\n+        out, st = dec.generate(prompt, 2)\n+        obs[out[1]] += 1\n+        acc += sum(s.accepted for s in st)\n+        drafted += sum(s.drafted for s in st)\n+    assert 0 \u003c acc \u003c drafted, \"need both accepts and rejects\"\n+    exp = N * exp_p\n+    chi2 = float(((obs - exp) ** 2 / exp).sum())\n+    pval = float(stats.chi2.sf(chi2, df=cfg.vocab_size - 1))\n+    assert pval > 0.01, f\"chi2={chi2:.1f}, p={pval:.4g} (second-token marginal shifted)\"\n+\n+\n+def test_identity_drafter_at_temperature_chi_square():\n+    \"\"\"Regression: argmax drafts + soft q made drafter==target accept the argmax\n+    at T\u003c1 every time (output collapsed to greedy). The first token must follow\n+    softmax(lg/0.7) when the drafter is the target itself.\"\"\"\n+    cfg, params = _tiny16(1)\n+    prompt = [i % 7 for i in range(32)]\n+    pT = _probs(cfg, params, prompt, 0.7)\n+    N = 4000\n+    obs = np.zeros(cfg.vocab_size, np.int64)\n+    for i in range(N):\n+        dec = SpecDecoder(cfg, params, ModelDrafter(cfg, params, len(prompt) + 8),\n+                          max_len=len(prompt) + 8, k=2, temperature=0.7, seed=i)\n+        out, st = dec.generate(prompt, 1)\n+        obs[out[0]] += 1\n+    exp = N * pT\n+    chi2 = float(((obs - exp) ** 2 / exp).sum())\n+    pval = float(stats.chi2.sf(chi2, df=cfg.vocab_size - 1))\n+    assert pval > 0.01, f\"chi2={chi2:.1f}, p={pval:.4g} (identity drafter not sampling at T)\"\n+\n+\n+@pytest.mark.parametrize(\"kind\", [\"ngram\", \"model\", \"none\"])\n+def test_one_target_forward_per_step(kind, monkeypatch):\n+    from draftlab.jaxlm import spec as S\n+\n+    cfg, params = _tiny16(2)\n+    calls = []\n+    real = M.decode\n+\n+    def counting(c, p, cache, ids):\n+        if p is params:\n+            calls.append(len(ids))\n+        return real(c, p, cache, ids)\n+\n+    monkeypatch.setattr(S.M, \"decode\", counting)\n+    prompt = [i % 7 for i in range(30)]\n+    d = S.NullDrafter() if kind == \"none\" else _make_drafter(kind, cfg, 80)\n+    dec = SpecDecoder(cfg, params, d, max_len=80, k=3, temperature=1.0, seed=1)\n+    out, st = dec.generate(prompt, 25)\n+    assert len(out) == 25\n+    assert len(calls) == len(st)\n+    assert [c - 1 for c in calls] == [s.drafted for s in st]"},"bootstrap":{"sha":"0158d68f3e36e6b04490b52dc384aceeb967406c","short":"0158d68","date":"2026-10-06","subject":"alpha: fix run bootstrap (CIs were ~100x too narrow)","paths":["draftlab/eval/alpha.py","draftlab/tests/test_alpha.py"],"diff":"diff --git a/draftlab/eval/alpha.py b/draftlab/eval/alpha.py\nindex b0c260f..139486b 100644\n--- a/draftlab/eval/alpha.py\n+++ b/draftlab/eval/alpha.py\n@@ -271,5 +271,6 @@ def _bootstrap(\n         sums_v = np.array([np.concatenate([r[3] for r in by_run[rn]]).sum() for rn in runs])\n         cnts = np.array([sum(r[0].size for r in by_run[rn]) for rn in runs])\n-        w = rng.multinomial(n_boot, np.full(len(runs), 1.0 / len(runs)), size=len(runs))\n+        # (n_boot, n_runs): each replicate draws len(runs) runs with replacement\n+        w = rng.multinomial(len(runs), np.full(len(runs), 1.0 / len(runs)), size=n_boot)\n         # per-run rates over the drafter's valid (proposed) positions: this is\n         # what the sweep's accept = n_acc/n_draft measures (spec.py only scores\n@@ -282,6 +283,6 @@ def _bootstrap(\n             w @ sums_gc / (w @ cnts),\n             w @ sums_v / (w @ cnts),\n-            w @ gv / n_boot,\n-            w @ av / n_boot,\n+            w @ gv / len(runs),\n+            w @ av / len(runs),\n             int(cnts.sum()),\n             len(runs),\ndiff --git a/draftlab/tests/test_alpha.py b/draftlab/tests/test_alpha.py\nindex 5d9f914..fb475a6 100644\n--- a/draftlab/tests/test_alpha.py\n+++ b/draftlab/tests/test_alpha.py\n@@ -99,2 +99,21 @@ def test_identical_models_alpha_one():\n     assert valid2.all()\n     assert a2.min() >= 1.0 - 1e-6 and g2.all()\n+\n+\n+def test_bootstrap_resamples_runs():\n+    \"\"\"Regression: weights were (n_runs, n_runs) with n_boot draws per row, so\n+    every 'replicate' was ~equal weights and CIs collapsed to the point\n+    estimate. Two runs at alpha 0.2 / 0.8: n_boot replicates, CI spans both.\"\"\"\n+    from draftlab.eval.alpha import _bootstrap\n+\n+    def rec(run, a):\n+        x = np.full(100, a)\n+        return (run, \"text\", x, x > 0.5, x > 0.5, np.ones(100, bool))\n+\n+    out = _bootstrap([rec(\"r1\", 0.2), rec(\"r2\", 0.8)], n_boot=2000, seed=0)\n+    b_alpha = out[\"overall\"][0]\n+    assert b_alpha.shape == (2000,)\n+    lo, hi = np.percentile(b_alpha, [2.5, 97.5])\n+    assert lo == pytest.approx(0.2) and hi == pytest.approx(0.8)\n+    assert out[\"overall\"][5].shape == (2000,)  # alpha over valid positions, per replicate\n+    assert np.mean(out[\"overall\"][5]) == pytest.approx(0.5, abs=0.02)"},"heldout":{"sha":"5f9841408c775654b3f22368bffe32685b0fa44a","short":"5f98414","date":"2026-10-06","subject":"eval: temporal hold-out split, head-to-head speed bench, 1.5B sweep","paths":["draftlab/data/corpus.py","tools/bench_speedup.py"],"diff":"diff --git a/draftlab/data/corpus.py b/draftlab/data/corpus.py\nindex 600aeb7..7438c29 100644\n--- a/draftlab/data/corpus.py\n+++ b/draftlab/data/corpus.py\n@@ -193,4 +193,46 @@ def build(raw_path: Path = RAW_PATH, out_dir: Path = OUT_DIR, tok=None) -> dict[\n \n \n+HELDOUT_RAW = ROOT / \"data/raw/heldout_turns.jsonl\"\n+\n+\n+def build_heldout(raw_path: Path = HELDOUT_RAW, out_dir: Path = OUT_DIR, tok=None) -> int:\n+    \"\"\"Temporal hold-out: runs exported after the corpus (tools/export_traces.py\n+    --since) -> data/corpus/heldout.parquet. Drops runs already in a split and\n+    blocks whose (kind, text) occurs in any split, so no drafter saw them.\"\"\"\n+    import pyarrow as pa\n+    import pyarrow.parquet as pq\n+\n+    if tok is None:\n+        from transformers import AutoTokenizer\n+\n+        tok = AutoTokenizer.from_pretrained(TOKENIZER_DIR)\n+    old_runs: set[str] = set()\n+    seen: set[tuple[str, str]] = set()\n+    for name in (\"train\", \"val\", \"test\"):\n+        t = pq.read_table(out_dir / f\"{name}.parquet\", columns=[\"run\", \"kind\", \"text\"]).to_pydict()\n+        old_runs.update(t[\"run\"])\n+        seen.update(zip(t[\"kind\"], t[\"text\"], strict=True))\n+    rs = []\n+    for r in expand(load_raw(raw_path)):\n+        if r[\"run\"] in old_runs:\n+            continue\n+        ids = tok.encode(r[\"text\"])\n+        if len(ids) \u003c MIN_TOKENS:\n+            continue\n+        text = tok.decode(ids[:MAX_TOKENS]) if len(ids) > MAX_TOKENS else r[\"text\"]\n+        key = (r[\"kind\"], text)\n+        if key in seen:\n+            continue\n+        seen.add(key)\n+        rs.append({**r, \"text\": text, \"n_tokens\": min(len(ids), MAX_TOKENS)})\n+    if not rs:\n+        raise RuntimeError(f\"no new blocks in {raw_path}\")\n+    cols = [\"run\", \"project\", \"seq\", \"role\", \"kind\", \"tool\", \"text\", \"n_tokens\"]\n+    pq.write_table(pa.table({c: [r[c] for r in rs] for c in cols}), out_dir / \"heldout.parquet\")\n+    print(f\"heldout: {len(rs)} blocks, {len({r['run'] for r in rs})} runs \"\n+          f\"-> {out_dir / 'heldout.parquet'}\")\n+    return len(rs)\n+\n+\n STATS_SQL = Path(__file__).resolve().parent / \"stats.sql\"\n RESULTS_DIR = ROOT / \"results\"\ndiff --git a/tools/bench_speedup.py b/tools/bench_speedup.py\nnew file mode 100644\nindex 0000000..d8176db\n--- /dev/null\n+++ b/tools/bench_speedup.py\n@@ -0,0 +1,113 @@\n+\"\"\"Head-to-head CPU speedup: each config vs `none` on the same prompts, one process,\n+nothing else running. A warm-up pass compiles every shape; then `--reps` passes\n+run the configs interleaved (none, A, B, ..., none, A, B, ...), so drift hits all\n+configs alike. Speedup per rep = none wall / config wall over the same prompts.\n+\n+    python tools/bench_speedup.py            # -> results/speedup_bench.json\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import argparse\n+import json\n+import sys\n+import time\n+from pathlib import Path\n+\n+import numpy as np\n+\n+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))\n+from draftlab.jaxlm import model as M  # noqa: E402\n+from draftlab.jaxlm import spec as S  # noqa: E402\n+from draftlab.sweep import (  # noqa: E402\n+    CKPT,\n+    LORA_BASE,\n+    MODELS,\n+    RESULTS,\n+    load_prompts,\n+    merge_adapter,\n+)\n+\n+# (target, drafter, k, temperature)\n+CONFIGS = [\n+    (\"qwen2.5-0.5b-instruct\", \"ngram\", 8, 0.0),\n+    (\"qwen2.5-1.5b-instruct\", \"ngram\", 8, 0.0),\n+    (\"qwen2.5-1.5b-instruct\", \"lora\", 2, 0.0),\n+    (\"qwen2.5-1.5b-instruct\", \"lora\", 4, 0.0),\n+    (\"qwen2.5-1.5b-instruct\", \"lora\", 4, 0.7),\n+]\n+\n+\n+def _drafter(name, tcfg, max_len, cache):\n+    if name == \"none\":\n+        return S.NullDrafter()\n+    if name == \"ngram\":\n+        return S.NGramDrafter(tcfg.vocab_size)\n+    if name not in cache:\n+        dcfg, dp = M.load_safetensors(LORA_BASE)\n+        cache[name] = (dcfg, merge_adapter(dp, dcfg, CKPT / (\"lora\" if name == \"lora\" else \"dpo\")))\n+    dcfg, dp = cache[name]\n+    return S.ModelDrafter(dcfg, dp, max_len)\n+\n+\n+def _pass(tcfg, tp, prompts, drafter, k, temp, n_gen, max_len, cache):\n+    wall = drafted = acc = steps = toks = 0\n+    for p in prompts:\n+        dec = S.SpecDecoder(tcfg, tp, _drafter(drafter, tcfg, max_len, cache), max_len,\n+                            k=k if drafter != \"none\" else 0, temperature=temp, seed=p[\"i\"])\n+        t0 = time.perf_counter()\n+        out, st = dec.generate(p[\"ids\"], n_gen)\n+        wall += time.perf_counter() - t0\n+        toks += len(out)\n+        steps += len(st)\n+        drafted += sum(s.drafted for s in st)\n+        acc += sum(s.accepted for s in st)\n+    return {\"wall_s\": wall, \"tok_s\": toks / wall, \"tokens_per_step\": toks / steps,\n+            \"accept\": acc / drafted if drafted else None}\n+\n+\n+def main() -> None:\n+    ap = argparse.ArgumentParser()\n+    ap.add_argument(\"--n-per-kind\", type=int, default=3)\n+    ap.add_argument(\"--n-gen\", type=int, default=32)\n+    ap.add_argument(\"--reps\", type=int, default=3)\n+    ap.add_argument(\"--out\", default=str(RESULTS / \"speedup_bench.json\"))\n+    a = ap.parse_args()\n+    out: dict = {\"n_gen\": a.n_gen, \"reps\": a.reps, \"split\": \"heldout\", \"targets\": {}}\n+    for target in dict.fromkeys(c[0] for c in CONFIGS):\n+        tdir = MODELS / target\n+        prompts = load_prompts(\"heldout\", a.n_per_kind, 128, tdir)\n+        max_len = max(len(p[\"ids\"]) for p in prompts) + a.n_gen + 16\n+        tcfg, tp = M.load_safetensors(tdir)\n+        cache: dict = {}\n+        cfgs = [c for c in CONFIGS if c[0] == target]\n+        temps = sorted({c[3] for c in cfgs})\n+        runs = [(\"none\", 0, t) for t in temps] + [(d, k, t) for _, d, k, t in cfgs]\n+        print(f\"{target}: {len(prompts)} prompts, {len(runs)} configs\", flush=True)\n+        for d, k, t in runs:  # warm-up: compile every prompt/step shape\n+            _pass(tcfg, tp, prompts, d, k, t, a.n_gen, max_len, cache)\n+        reps = {f\"{d}-k{k}-T{t}\": [] for d, k, t in runs}\n+        for r in range(a.reps):\n+            for d, k, t in runs:\n+                res = _pass(tcfg, tp, prompts, d, k, t, a.n_gen, max_len, cache)\n+                reps[f\"{d}-k{k}-T{t}\"].append(res)\n+                print(f\"  rep {r} {d} k={k} T={t}: {res['tok_s']:.2f} tok/s\", flush=True)\n+        rows = {}\n+        for d, k, t in runs:\n+            key = f\"{d}-k{k}-T{t}\"\n+            base = reps[f\"none-k0-T{t}\"]\n+            sp = [b[\"wall_s\"] / x[\"wall_s\"] for b, x in zip(base, reps[key], strict=True)]\n+            rows[key] = {\n+                \"drafter\": d, \"k\": k, \"temp\": t,\n+                \"tok_s\": float(np.mean([x[\"tok_s\"] for x in reps[key]])),\n+                \"speedup\": float(np.mean(sp)), \"speedup_range\": [float(min(sp)), float(max(sp))],\n+                \"accept\": reps[key][0][\"accept\"],\n+                \"tokens_per_step\": reps[key][0][\"tokens_per_step\"],\n+            }\n+        out[\"targets\"][target] = {\"n_prompts\": len(prompts), \"configs\": rows}\n+    Path(a.out).write_text(json.dumps(out, indent=1) + \"\\n\")\n+    print(json.dumps(out, indent=1))\n+\n+\n+if __name__ == \"__main__\":\n+    main()"},"serving":{"sha":"48cf15bd6c819347b6b874db09abdca5183b1471","short":"48cf15b","date":"2026-10-06","subject":"serve: stop at EOS, real finish_reason, '+none' drafter","paths":["draftlab/serve/app.py","draftlab/tests/test_serve.py"],"diff":"diff --git a/draftlab/serve/app.py b/draftlab/serve/app.py\nindex de10174..21cd299 100644\n--- a/draftlab/serve/app.py\n+++ b/draftlab/serve/app.py\n@@ -2,5 +2,7 @@\n \n Model names select base + drafter + k: ``\u003cbase>`` (default ngram, k=5) or\n-``\u003cbase>+\u003cdrafter>-k\u003cK>`` (e.g. ``qwen0.5b+ngram-k4``). ``usage`` adds\n+``\u003cbase>+\u003cdrafter>-k\u003cK>`` (e.g. ``qwen0.5b+ngram-k4``); ``\u003cbase>+none`` decodes\n+without drafting. Generation stops at the tokenizer's EOS (finish_reason\n+\"stop\") or at max_tokens (\"length\"). ``usage`` adds\n ``draft_tokens`` (drafts proposed) and ``accepted_tokens`` (drafts accepted).\n \"\"\"\n@@ -21,5 +23,5 @@ from pydantic import BaseModel, Field\n \n from draftlab.jaxlm import model as M\n-from draftlab.jaxlm.spec import ModelDrafter, NGramDrafter, SpecDecoder\n+from draftlab.jaxlm.spec import ModelDrafter, NGramDrafter, NullDrafter, SpecDecoder\n \n DEFAULT_K = 5\n@@ -43,4 +45,8 @@ class HFChatTokenizer:\n \n         self.tok = AutoTokenizer.from_pretrained(path)\n+        ids = {self.tok.eos_token_id}\n+        ids |= {self.tok.convert_tokens_to_ids(t) for t in (\"\u003c|im_end|>\", \"\u003c|endoftext|>\")\n+                if t in self.tok.get_vocab()}\n+        self.eos_ids = frozenset(i for i in ids if i is not None)\n \n     def apply_chat_template(self, messages: list[dict]) -> str:\n@@ -59,4 +65,5 @@ class ToyTokenizer:\n \n     vocab_size = 512\n+    eos_ids: frozenset[int] = frozenset()\n \n     def apply_chat_template(self, messages: list[dict]) -> str:\n@@ -99,4 +106,6 @@ def parse_engine_spec(model: str) -> tuple[str, str, int]:\n \n def make_drafter(kind: str, engine: Engine, max_len: int):\n+    if kind == \"none\":\n+        return NullDrafter()\n     if kind == \"ngram\":\n         return NGramDrafter(vocab_size=engine.cfg.vocab_size, max_n=4)\n@@ -157,5 +166,5 @@ def create_app(engines: dict[str, Engine]) -> FastAPI:\n         except ValueError:\n             return _err(request, 400, f\"bad model name {body.model!r}\", \"invalid_model_name\")\n-        if drafter not in (\"ngram\", \"model\"):\n+        if drafter not in (\"ngram\", \"model\", \"none\"):\n             return _err(request, 400, f\"unknown drafter {drafter!r}\", \"invalid_drafter\")\n         engine = engines.get(base)\n@@ -182,14 +191,31 @@ def create_app(engines: dict[str, Engine]) -> FastAPI:\n         created = int(time.time())\n \n+        eos = getattr(engine.tokenizer, \"eos_ids\", frozenset())\n+\n+        def tokens():\n+            \"\"\"(token, new_step_stats_or_None) until EOS (excluded) or max_tokens.\"\"\"\n+            last_st = None\n+            for tok, st in decoder.generate_stream(prompt_ids, body.max_tokens):\n+                if tok in eos:\n+                    return\n+                yield tok, (st if st is not last_st else None)\n+                last_st = st\n+\n         if not body.stream:\n-            toks, stats = decoder.generate(prompt_ids, body.max_tokens)\n-            content = engine.tokenizer.decode(toks)\n-            usage = _usage(len(prompt_ids), len(toks),\n-                           sum(s.drafted for s in stats), sum(s.accepted for s in stats))\n+            toks: list[int] = []\n+            draft_tokens = accepted_tokens = 0\n+            for tok, st in tokens():\n+                toks.append(tok)\n+                if st is not None:\n+                    draft_tokens += st.drafted\n+                    accepted_tokens += st.accepted\n+            finish = \"length\" if len(toks) == body.max_tokens else \"stop\"\n+            usage = _usage(len(prompt_ids), len(toks), draft_tokens, accepted_tokens)\n             return {\n                 \"id\": cid, \"object\": \"chat.completion\", \"created\": created,\n                 \"model\": body.model,\n-                \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": content},\n-                             \"finish_reason\": \"stop\", \"logprobs\": None}],\n+                \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\",\n+                                                     \"content\": engine.tokenizer.decode(toks)},\n+                             \"finish_reason\": finish, \"logprobs\": None}],\n                 \"usage\": usage,\n             }\n@@ -198,10 +224,8 @@ def create_app(engines: dict[str, Engine]) -> FastAPI:\n             draft_tokens = accepted_tokens = 0\n             completion_tokens = 0\n-            last_st = None\n-            for tok, st in decoder.generate_stream(prompt_ids, body.max_tokens):\n-                if st is not last_st:  # one StepStats per target-call step\n+            for tok, st in tokens():\n+                if st is not None:  # one StepStats per target-call step\n                     draft_tokens += st.drafted\n                     accepted_tokens += st.accepted\n-                    last_st = st\n                 completion_tokens += 1\n                 delta: dict[str, Any] = {\"role\": \"assistant\",\n@@ -215,8 +239,9 @@ def create_app(engines: dict[str, Engine]) -> FastAPI:\n             usage = _usage(len(prompt_ids), completion_tokens,\n                            draft_tokens, accepted_tokens)\n+            finish = \"length\" if completion_tokens == body.max_tokens else \"stop\"\n             final = {\n                 \"id\": cid, \"object\": \"chat.completion.chunk\", \"created\": created,\n                 \"model\": body.model,\n-                \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}],\n+                \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": finish}],\n                 \"usage\": usage,\n             }\ndiff --git a/draftlab/tests/test_serve.py b/draftlab/tests/test_serve.py\nindex 8e83c87..621f121 100644\n--- a/draftlab/tests/test_serve.py\n+++ b/draftlab/tests/test_serve.py\n@@ -44,5 +44,5 @@ def test_nonstream_usage(client):\n     assert j[\"object\"] == \"chat.completion\"\n     ch = j[\"choices\"][0]\n-    assert ch[\"finish_reason\"] == \"stop\"\n+    assert ch[\"finish_reason\"] == \"length\"  # toy tokenizer has no EOS\n     assert ch[\"message\"][\"role\"] == \"assistant\"\n     assert len(ch[\"message\"][\"content\"]) == 32\n@@ -65,5 +65,5 @@ def test_stream_chunks_and_usage(client):\n     assert all(c[\"choices\"][0][\"finish_reason\"] is None for c in chunks[:-1])\n     last = chunks[-1]\n-    assert last[\"choices\"][0][\"finish_reason\"] == \"stop\"\n+    assert last[\"choices\"][0][\"finish_reason\"] == \"length\"\n     assert last[\"choices\"][0][\"delta\"] == {}\n     # deterministic (seed 0, T=0): stream must equal the non-stream result\n@@ -98,2 +98,35 @@ def test_missing_fields_400(client):\n def test_unknown_model_404(client):\n     assert client.post(\"/v1/chat/completions\", json=_body(model=\"nope\")).status_code == 404\n+\n+\n+def test_eos_stops_generation():\n+    \"\"\"A token in tokenizer.eos_ids ends the completion (excluded, finish 'stop').\"\"\"\n+    cfg = M.Qwen2Cfg.from_hf(tiny_config())\n+    params = M.init_params(cfg, jax.random.PRNGKey(0))\n+    tok = ToyTokenizer()\n+    c = TestClient(create_app({\"tiny\": Engine(\"tiny\", cfg, params, tok)}))\n+    full = c.post(\"/v1/chat/completions\", json=_body(model=\"tiny+none\")).json()\n+    text = full[\"choices\"][0][\"message\"][\"content\"]\n+    # ToyTokenizer.decode maps id -> chr(33 + id % 90); mark the 4th token's ids as EOS\n+    stop_ch = text[3]\n+    tok.eos_ids = frozenset(i for i in range(tok.vocab_size) if chr(33 + i % 90) == stop_ch)\n+    cut = text.index(stop_ch)\n+    for stream in (False, True):\n+        r = c.post(\"/v1/chat/completions\", json=_body(model=\"tiny+none\", stream=stream))\n+        if stream:\n+            chunks = _sse_chunks(r)\n+            content = \"\".join(ch[\"choices\"][0][\"delta\"].get(\"content\", \"\") for ch in chunks)\n+            fin, usage = chunks[-1][\"choices\"][0][\"finish_reason\"], chunks[-1][\"usage\"]\n+        else:\n+            j = r.json()\n+            content, fin, usage = j[\"choices\"][0][\"message\"][\"content\"], j[\"choices\"][0][\n+                \"finish_reason\"], j[\"usage\"]\n+        assert content == text[:cut] and fin == \"stop\"\n+        assert usage[\"completion_tokens\"] == cut\n+\n+\n+def test_none_drafter_drafts_nothing(client):\n+    j = client.post(\"/v1/chat/completions\", json=_body(model=\"tiny+none\")).json()\n+    assert j[\"usage\"][\"draft_tokens\"] == 0 and j[\"usage\"][\"completion_tokens\"] == 32\n+    spec = client.post(\"/v1/chat/completions\", json=_body()).json()\n+    assert j[\"choices\"][0][\"message\"][\"content\"] == spec[\"choices\"][0][\"message\"][\"content\"]"}},"sources":[{"path":"results/alpha/ngram.json","purpose":"Held-out distribution overlap","sha256":"ae8577962fed1941ab7d85e796a4f498f9d5d849e3cf1379d3451457ac524e38","bytes":20555},{"path":"results/alpha/lora.json","purpose":"Held-out distribution overlap","sha256":"7f6295f6f5ac22a1f7a195195e4cfa1ef8add371ebe8b9d164482cd857c32a54","bytes":20407},{"path":"results/alpha/dpo.json","purpose":"Held-out distribution overlap","sha256":"e3edb088d1d4ef0313b8361b944b21defd34049d8d7089949331779038d19424","bytes":20401},{"path":"results/alpha/tiny.json","purpose":"Held-out distribution overlap","sha256":"ebff23ea1323e9afe4b91b09cda1115b90cc9f6e961c42b3840dcd855b478d16","bytes":20648},{"path":"results/corpus_stats.json","purpose":"Original corpus composition","sha256":"04367bcad7bfd723340637cff2f2f8f0cbe2ba304492e89138ac5dd99f06ce48","bytes":4126},{"path":"data/corpus/heldout.parquet","purpose":"Temporal hold-out size; aggregate columns only","sha256":"0f6018450ec3a74b12b81730399444a9d6e146eff0bc55763b33954c042113a4","bytes":6859939},{"path":"results/sweep.parquet","purpose":"Corrected 0.5B decoding sweep, completed cells","sha256":"1d8b94049b5e64d2b4ac62ef69c812bb81dd4fd43414f0e5ab70aeafccf416de","bytes":45960},{"path":"results/sweep.parquet.part-lora-2-0.0.jsonl","purpose":"Recoverable partial sweep cell; read without modifying it","sha256":"09db99171a10f2ef02a8fc68816f46a4de091345d87c2c8c6811a525b62d892c","bytes":27979},{"path":"results/sweep.parquet.part-lora-2-0.7.jsonl","purpose":"Recoverable partial sweep cell; read without modifying it","sha256":"7bd28a066b299d99092d6b55fd4bee99d05225c90992b9a5856cfeb6884cb108","bytes":2345},{"path":"results/sweep.parquet.part-lora-4-0.0.jsonl","purpose":"Recoverable partial sweep cell; read without modifying it","sha256":"713d3f1129da5dc848cc71b0a30bee74fa52743909d791a8f9f92062fa624bbb","bytes":15002},{"path":"results/sweep.parquet.part-lora-4-0.7.jsonl","purpose":"Recoverable partial sweep cell; read without modifying it","sha256":"1123dcb21ea3c64d05a35d622f79dfe1377e7df1d8c7b058e433484b33085632","bytes":15008},{"path":"results/sweep.parquet.part-lora-8-0.0.jsonl","purpose":"Recoverable partial sweep cell; read without modifying it","sha256":"67d1af2b1ed2e2e48cf74961ee9cfa2d50aebc43ed904cde8ebb5adba0f5fbcb","bytes":293},{"path":"results/sweep.parquet.part-lora-8-0.7.jsonl","purpose":"Recoverable partial sweep cell; read without modifying it","sha256":"0db87d2ca6730fecf24a7cdde5e4ba794c1ecc74676a084e271d60fde0d4d55f","bytes":5829},{"path":"results/sweep.parquet.part-lora_dpo-1-0.0.jsonl","purpose":"Recoverable partial sweep cell; read without modifying it","sha256":"54f4914cebd96222c77e9eced86308a5a8bef6eb8f7a8d03ac5bfc2adcdc77f7","bytes":12814},{"path":"results/sweep.parquet.part-lora_dpo-1-0.7.jsonl","purpose":"Recoverable partial sweep cell; read without modifying it","sha256":"cc0712dd30427ce0516ca8e8353192aec3647282684a43d754b769829347c182","bytes":5619},{"path":"results/judge.json","purpose":"Corrected held-out judge rerun","sha256":"5e6f7777e66187453aab1ceb51847caac564bcd86be0b8fc3bbb13870e6850b4","bytes":9914},{"path":"results/distill_curve.json","purpose":"Recorded distillation training curve","sha256":"90784039de12c6d4e711aefc394f8f0cd8e48e31f7437f4e56fc617ec117e0e2","bytes":4878},{"path":"results/distill_curve_tinit.json","purpose":"Recorded distillation training curve","sha256":"8443bb1b739233877a69f8d7185554b673fe5cdf804a2b3675f39e370db2b6e9","bytes":4875},{"path":"ckpt/lora/adapter_config.json","purpose":"Implementation or outstanding work used for this report","sha256":"6439004389d3ba3281a21d141335bf0006e9138e30bd4fddedc569d09d6868fe","bytes":1283},{"path":"ckpt/dpo/adapter_config.json","purpose":"Implementation or outstanding work used for this report","sha256":"4cf534ec2e637fdbbcb02f6a6ad79d4c8df4aa42ac2dd2217480ce59dc3c5345","bytes":1283},{"path":"draftlab/jaxlm/spec.py","purpose":"Implementation or outstanding work used for this report","sha256":"1171a5da035e1c0ffe647b8de97532f95629657a45baf2d5df1ef9f3a12ac903","bytes":10548},{"path":"draftlab/jaxlm/model.py","purpose":"Implementation or outstanding work used for this report","sha256":"9a4218ae0d7bbb596bdbd3fed38e5a927357f66207d09e9a4a8ce2b0af829035","bytes":9242},{"path":"draftlab/eval/alpha.py","purpose":"Implementation or outstanding work used for this report","sha256":"8d4a9e9146a2e35d8f66ab32125569eecd6da2f5d4bfae4ddaa40da8a6e37970","bytes":20387},{"path":"draftlab/data/corpus.py","purpose":"Implementation or outstanding work used for this report","sha256":"eb1ea87d59692e2b8780a17f9736c3607bae2d962e151f1027e692030faba79e","bytes":9715},{"path":"draftlab/drafters/dpo.py","purpose":"Implementation or outstanding work used for this report","sha256":"96fbc3e9d5f695839f13eb804fcf527c690b28b542585501137b555193454a12","bytes":14145},{"path":"draftlab/drafters/lora.py","purpose":"Implementation or outstanding work used for this report","sha256":"8f6d2ee01ff835eb2eeacf98cd1b724936d8bd4a33c155e419e1b38ef37444c0","bytes":9635},{"path":"draftlab/drafters/distill.py","purpose":"Implementation or outstanding work used for this report","sha256":"4b1716c2da983c8ab5021bac9930fc30c8af1d49e2c28178940b07de6d78b43a","bytes":18128},{"path":"draftlab/kernels/verify.py","purpose":"Implementation or outstanding work used for this report","sha256":"036c1e6e5d465b24f271b3a2f4c614c1addb07704567bf94b2cbd8bf6f48b241","bytes":6945},{"path":"draftlab/eval/judge.py","purpose":"Implementation or outstanding work used for this report","sha256":"85fca070967976cd7b9184a799ff0a0e7848d3114e9058866460cf6c1fbd5d8c","bytes":18340},{"path":"draftlab/serve/app.py","purpose":"Implementation or outstanding work used for this report","sha256":"d51e2d288b0eb2d34b69d57dc62e378218291719eb4499b9eeae6f28390aa799","bytes":10971},{"path":"draftlab/sweep.py","purpose":"Implementation or outstanding work used for this report","sha256":"ff6886f87297f38cd411c3bd6eba1da8cd537e23675929f7365ef40cde6c4dc9","bytes":13186},{"path":"gateway/metrics.go","purpose":"Implementation or outstanding work used for this report","sha256":"2812c9aa9afc89f0b5393cd4e2d199019942c2f123bcf9a40cb78ae69353d225","bytes":3187},{"path":"TODO.md","purpose":"Implementation or outstanding work used for this report","sha256":"feed15cd0c0677a739f109d1daf2c7852b021d97253071cbd61cc5c22ffd1c38","bytes":25995}],"missing":["results/speedup_bench.json","results/sweep_1.5b.parquet"],"notes":["This is a frozen snapshot, not a live dashboard.","No raw corpus text, prompts, token sequences or run identifiers are exported.","Alpha entries are saved bootstrap means and run-level 95% intervals.","Sweep metrics are pooled counts; partial cells use different prompt subsets.","Sweep wall time is intentionally omitted; speedup needs the dedicated benchmark."]}
