{ "protocolVersion": "osai-bench/1.3", "clientVersion": "0.10.0", "scoringVersion": "osai-bench-derive/1.3", "createdAt": "2026-08-07T07:48:52.551Z", "qualityOverride": false, "cohortEligible": true, "qualityConditions": [], "runtime": { "name": "ollama", "version": "0.32.5", "endpoint": "loopback", "layerAssignment": null, "offloadPlacement": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 9953733507, "vramResidentBytes": 8916178042, "hostResidentBytes": 1037555465, "vramResidentFraction": 0.8957621816707937 }, "kvCacheMetadata": { "source": "ollama.show.model_info", "architecture": "qwen2", "blockCount": 48, "kvHeadCount": 8, "attentionHeadCount": 40, "embeddingLength": 5120, "headDimension": 128, "resolvedElementType": null, "projectedBytes": null, "calculationAvailable": false, "missingInputs": [ "resolvedKvCacheElementType" ] }, "environment": { "source": "client-process-env", "authoritative": false, "note": "Read from the benchmark client's own process environment. The Ollama server may have been started with a different environment; Ollama exposes no endpoint reporting its resolved configuration.", "declared": { "OLLAMA_KV_CACHE_TYPE": null, "OLLAMA_FLASH_ATTENTION": null, "OLLAMA_CONTEXT_LENGTH": null, "OLLAMA_NUM_PARALLEL": null, "OLLAMA_GPU_OVERHEAD": null, "OLLAMA_SCHED_SPREAD": null, "OLLAMA_LLM_LIBRARY": null, "LLAMA_ARG_FIT": null, "LLAMA_ARG_FIT_TARGET": null, "OLLAMA_MAX_LOADED_MODELS": null, "OLLAMA_KEEP_ALIVE": null, "OLLAMA_DEBUG": null, "OLLAMA_MODELS": false, "OLLAMA_HOST": false }, "declaredNonDefault": [] } }, "model": { "identifier": "deepseek-r1:14b", "digest": "c333b7232bdb521236694ffbb5f5a6b11cc45d98e9142c73123b670fca400b09", "family": "qwen2", "parameterSize": "14.8B", "quantization": "Q4_K_M", "weightsBytes": 8988112209, "weightsSource": "ollama.tags.size" }, "system": { "cpu": { "model": "12th Gen Intel(R) Core(TM) i9-12900K" }, "gpu": { "present": true, "model": "NVIDIA GeForce RTX 3080", "totalVramBytes": 10737418240, "freeVramBytesAtCheck": 10326376448, "freeVramBytesAtLoad": 1290797056, "utilizationPercentAtCheck": 0, "driverVersion": "595.84", "provider": "nvidia-smi" }, "memory": { "totalBytes": 32817020928 }, "os": { "platform": "linux", "version": "Ubuntu 26.04 LTS", "architecture": "x64" }, "power": { "present": false, "onBattery": false } }, "configuration": { "memoryBandwidthGBps": 760, "memoryBandwidthSource": "manufacturer-table", "memoryBandwidthTableVersion": "osai-gpu-memory-bandwidth/2", "memoryBandwidthEntryId": "nvidia-geforce-rtx-3080-10gb", "fixedOptions": { "temperature": 0, "seed": 42, "stream": true }, "workloads": { "w1": { "numPredict": 1, "numCtx": 512, "keepAlive": "5m", "promptTokenRange": null, "warmups": 0, "repetitions": 1, "varyPromptPerCall": false }, "w2": { "numPredict": 128, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 20, "max": 64 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": false }, "w3": { "numPredict": 1, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 2000, "max": 4095 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": true }, "w4": { "numPredict": 512, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 20, "max": 64 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": false } }, "resolved": { "fixedOptions": { "temperature": 0, "seed": 42, "stream": true }, "workloads": { "w1": { "numPredict": 1, "numCtx": 512, "keepAlive": "5m" }, "w2": { "numPredict": 128, "numCtx": 4096, "keepAlive": "5m" }, "w3": { "numPredict": 1, "numCtx": 4096, "keepAlive": "5m" }, "w4": { "numPredict": 512, "numCtx": 4096, "keepAlive": "5m" } }, "runtimeModelParameters": "stop \"<｜begin▁of▁sentence｜>\"\nstop \"<｜end▁of▁sentence｜>\"\nstop \"<｜User｜>\"\nstop \"<｜Assistant｜>\"", "unresolvedRuntimeDefaults": true } }, "rawMeasurements": { "workloads": { "w1": { "warmup": null, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 2729326873, "load_duration": 2680092646, "prompt_eval_count": 8, "prompt_eval_duration": 45739000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2729326873, "load_duration": 2680092646, "prompt_eval_count": 8, "prompt_eval_duration": 45739000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w2": { "warmup": { "total_duration": 8077748143, "load_duration": 3813326275, "prompt_eval_count": 42, "prompt_eval_duration": 143686000, "eval_count": 128, "eval_duration": 4116946000, "timeToFirstTokenMs": 4037.9155499999997 }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 4291917394, "load_duration": 125353525, "prompt_eval_count": 42, "prompt_eval_duration": 32524000, "eval_count": 128, "eval_duration": 4116754000, "timeToFirstTokenMs": 240.73479200000293 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4291917394, "load_duration": 125353525, "prompt_eval_count": 42, "prompt_eval_duration": 32524000, "eval_count": 128, "eval_duration": 4116754000, "timeToFirstTokenMs": 240.73479200000293 }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 4268195256, "load_duration": 117337278, "prompt_eval_count": 42, "prompt_eval_duration": 32636000, "eval_count": 128, "eval_duration": 4099792000, "timeToFirstTokenMs": 233.42573099999936 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4268195256, "load_duration": 117337278, "prompt_eval_count": 42, "prompt_eval_duration": 32636000, "eval_count": 128, "eval_duration": 4099792000, "timeToFirstTokenMs": 233.42573099999936 }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 4259204350, "load_duration": 120543300, "prompt_eval_count": 42, "prompt_eval_duration": 32680999, "eval_count": 128, "eval_duration": 4101602000, "timeToFirstTokenMs": 222.7228099999993 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4259204350, "load_duration": 120543300, "prompt_eval_count": 42, "prompt_eval_duration": 32680999, "eval_count": 128, "eval_duration": 4101602000, "timeToFirstTokenMs": 222.7228099999993 }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 4269543180, "load_duration": 123640923, "prompt_eval_count": 42, "prompt_eval_duration": 32534999, "eval_count": 128, "eval_duration": 4109215000, "timeToFirstTokenMs": 225.6240800000014 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4269543180, "load_duration": 123640923, "prompt_eval_count": 42, "prompt_eval_duration": 32534999, "eval_count": 128, "eval_duration": 4109215000, "timeToFirstTokenMs": 225.6240800000014 }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 4263980153, "load_duration": 119275132, "prompt_eval_count": 42, "prompt_eval_duration": 32670000, "eval_count": 128, "eval_duration": 4107294000, "timeToFirstTokenMs": 222.27502000000095 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4263980153, "load_duration": 119275132, "prompt_eval_count": 42, "prompt_eval_duration": 32670000, "eval_count": 128, "eval_duration": 4107294000, "timeToFirstTokenMs": 222.27502000000095 }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w3": { "warmup": { "total_duration": 1916996040, "load_duration": 125280051, "prompt_eval_count": 2789, "prompt_eval_duration": 1780028000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 2138942207, "load_duration": 121986629, "prompt_eval_count": 2789, "prompt_eval_duration": 1767885000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2138942207, "load_duration": 121986629, "prompt_eval_count": 2789, "prompt_eval_duration": 1767885000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 2135237200, "load_duration": 114778645, "prompt_eval_count": 2789, "prompt_eval_duration": 1768610000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2135237200, "load_duration": 114778645, "prompt_eval_count": 2789, "prompt_eval_duration": 1768610000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 2135496057, "load_duration": 117840610, "prompt_eval_count": 2789, "prompt_eval_duration": 1768063000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2135496057, "load_duration": 117840610, "prompt_eval_count": 2789, "prompt_eval_duration": 1768063000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 2131155500, "load_duration": 117635433, "prompt_eval_count": 2789, "prompt_eval_duration": 1765872000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2131155500, "load_duration": 117635433, "prompt_eval_count": 2789, "prompt_eval_duration": 1765872000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 2133606466, "load_duration": 121958495, "prompt_eval_count": 2789, "prompt_eval_duration": 1766795000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2133606466, "load_duration": 121958495, "prompt_eval_count": 2789, "prompt_eval_duration": 1766795000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w4": { "warmup": { "total_duration": 17245523117, "load_duration": 118454750, "prompt_eval_count": 42, "prompt_eval_duration": 112394000, "eval_count": 512, "eval_duration": 16778706999, "timeToFirstTokenMs": 535.0367540000007 }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 16968158623, "load_duration": 120741728, "prompt_eval_count": 42, "prompt_eval_duration": 33929000, "eval_count": 512, "eval_duration": 16759887999, "timeToFirstTokenMs": 274.7875150000036 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 16968158623, "load_duration": 120741728, "prompt_eval_count": 42, "prompt_eval_duration": 33929000, "eval_count": 512, "eval_duration": 16759887999, "timeToFirstTokenMs": 274.7875150000036 }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 16993110411, "load_duration": 114871785, "prompt_eval_count": 42, "prompt_eval_duration": 33881000, "eval_count": 512, "eval_duration": 16791841000, "timeToFirstTokenMs": 269.24547199999506 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 16993110411, "load_duration": 114871785, "prompt_eval_count": 42, "prompt_eval_duration": 33881000, "eval_count": 512, "eval_duration": 16791841000, "timeToFirstTokenMs": 269.24547199999506 }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 16931116165, "load_duration": 117682820, "prompt_eval_count": 42, "prompt_eval_duration": 34138000, "eval_count": 512, "eval_duration": 16775277999, "timeToFirstTokenMs": 222.4673480000056 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 16931116165, "load_duration": 117682820, "prompt_eval_count": 42, "prompt_eval_duration": 34138000, "eval_count": 512, "eval_duration": 16775277999, "timeToFirstTokenMs": 222.4673480000056 }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 16938917788, "load_duration": 120869974, "prompt_eval_count": 42, "prompt_eval_duration": 34126000, "eval_count": 512, "eval_duration": 16780990000, "timeToFirstTokenMs": 224.49799399999029 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 16938917788, "load_duration": 120869974, "prompt_eval_count": 42, "prompt_eval_duration": 34126000, "eval_count": 512, "eval_duration": 16780990000, "timeToFirstTokenMs": 224.49799399999029 }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 16936308314, "load_duration": 113514434, "prompt_eval_count": 42, "prompt_eval_duration": 34094000, "eval_count": 512, "eval_duration": 16785177000, "timeToFirstTokenMs": 217.59099699999206 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 16936308314, "load_duration": 113514434, "prompt_eval_count": 42, "prompt_eval_duration": 34094000, "eval_count": 512, "eval_duration": 16785177000, "timeToFirstTokenMs": 217.59099699999206 }, "validity": { "valid": true, "reasons": [] } } ], "failed": false } } }, "derived": { "generationTokensPerSecond": { "median": 30.510714802881118, "coefficientOfVariation": 0.0007213417651565949, "samples": 5 }, "prefillTokensPerSecond": { "median": 1577.5913026016965, "coefficientOfVariation": 0.0006218620040799351, "samples": 5 }, "timeToFirstTokenMs": { "median": 225.6240800000014, "coefficientOfVariation": 0.034756956700548014, "samples": 5 }, "coldLoad": { "seconds": 2.680092646, "samples": 1 }, "passFailureRate": { "failedMeasuredPasses": 0, "totalMeasuredPasses": 16, "percent": 0 }, "attemptFailureRate": { "failedAttempts": 0, "totalAttempts": 16, "percent": 0 }, "roofline": { "modelWeightsGB": 8.988112209, "memoryBandwidthGBps": 760, "theoreticalMaxTokensPerSecond": 84.55613173576035, "utilization": 0.3608338529277537 }, "diagnostics": [ { "id": "partial-cpu-offload", "status": "detected", "message": "10% of the model's resident bytes are on the host, not in VRAM", "evidence": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 9953733507, "vramResidentBytes": 8916178042, "hostResidentBytes": 1037555465, "vramResidentFraction": 0.8957621816707937 } }, { "id": "context-vram-headroom", "status": "unavailable", "message": "Ollama /api/show does not expose the resolved KV-cache element type actually in use, so v1.1 does not calculate or threshold projected KV-cache VRAM", "evidence": { "configuredContext": 4096, "availableArchitectureMetadata": { "source": "ollama.show.model_info", "architecture": "qwen2", "blockCount": 48, "kvHeadCount": 8, "attentionHeadCount": 40, "embeddingLength": 5120, "headDimension": 128, "resolvedElementType": null, "projectedBytes": null, "calculationAvailable": false, "missingInputs": [ "resolvedKvCacheElementType" ] }, "missingInput": "resolvedKvCacheElementType" } }, { "id": "cpu-only-with-gpu", "status": "not-detected", "message": "90% of the model's resident bytes are in VRAM", "evidence": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 9953733507, "vramResidentBytes": 8916178042, "hostResidentBytes": 1037555465, "vramResidentFraction": 0.8957621816707937, "gpuModel": "NVIDIA GeForce RTX 3080" } }, { "id": "weights-exceed-vram", "status": "not-detected", "message": "Quantized on-disk model weight size does not exceed total VRAM", "evidence": { "weightsBytes": 8988112209, "totalVramBytes": 10737418240 } } ] } }