{ "protocolVersion": "osai-bench/1.3", "clientVersion": "0.10.0", "scoringVersion": "osai-bench-derive/1.3", "createdAt": "2026-08-07T07:19:10.994Z", "qualityOverride": false, "cohortEligible": true, "qualityConditions": [], "runtime": { "name": "ollama", "version": "0.32.5", "endpoint": "loopback", "layerAssignment": null, "offloadPlacement": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 9672073196, "vramResidentBytes": 8822015917, "hostResidentBytes": 850057279, "vramResidentFraction": 0.9121121954131249 }, "kvCacheMetadata": { "source": "ollama.show.model_info", "architecture": "phi3", "blockCount": 40, "kvHeadCount": 10, "attentionHeadCount": 40, "embeddingLength": 5120, "headDimension": 128, "resolvedElementType": null, "projectedBytes": null, "calculationAvailable": false, "missingInputs": [ "resolvedKvCacheElementType" ] }, "environment": { "source": "client-process-env", "authoritative": false, "note": "Read from the benchmark client's own process environment. The Ollama server may have been started with a different environment; Ollama exposes no endpoint reporting its resolved configuration.", "declared": { "OLLAMA_KV_CACHE_TYPE": null, "OLLAMA_FLASH_ATTENTION": null, "OLLAMA_CONTEXT_LENGTH": null, "OLLAMA_NUM_PARALLEL": null, "OLLAMA_GPU_OVERHEAD": null, "OLLAMA_SCHED_SPREAD": null, "OLLAMA_LLM_LIBRARY": null, "LLAMA_ARG_FIT": null, "LLAMA_ARG_FIT_TARGET": null, "OLLAMA_MAX_LOADED_MODELS": null, "OLLAMA_KEEP_ALIVE": null, "OLLAMA_DEBUG": null, "OLLAMA_MODELS": false, "OLLAMA_HOST": false }, "declaredNonDefault": [] } }, "model": { "identifier": "phi3:14b-medium-128k-instruct-q4_K_M", "digest": "1372226c9b0b61f3c693ef9dc54ec646d4cd75347eca849a76a04de22e350517", "family": "phi3", "parameterSize": "14.0B", "quantization": "Q4_K_M", "weightsBytes": 8566823460, "weightsSource": "ollama.tags.size" }, "system": { "cpu": { "model": "12th Gen Intel(R) Core(TM) i9-12900K" }, "gpu": { "present": true, "model": "NVIDIA GeForce RTX 3080", "totalVramBytes": 10737418240, "freeVramBytesAtCheck": 10326376448, "freeVramBytesAtLoad": 1347420160, "utilizationPercentAtCheck": 0, "driverVersion": "595.84", "provider": "nvidia-smi" }, "memory": { "totalBytes": 32817020928 }, "os": { "platform": "linux", "version": "Ubuntu 26.04 LTS", "architecture": "x64" }, "power": { "present": false, "onBattery": false } }, "configuration": { "memoryBandwidthGBps": 760, "memoryBandwidthSource": "manufacturer-table", "memoryBandwidthTableVersion": "osai-gpu-memory-bandwidth/2", "memoryBandwidthEntryId": "nvidia-geforce-rtx-3080-10gb", "fixedOptions": { "temperature": 0, "seed": 42, "stream": true }, "workloads": { "w1": { "numPredict": 1, "numCtx": 512, "keepAlive": "5m", "promptTokenRange": null, "warmups": 0, "repetitions": 1, "varyPromptPerCall": false }, "w2": { "numPredict": 128, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 20, "max": 64 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": false }, "w3": { "numPredict": 1, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 2000, "max": 4095 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": true }, "w4": { "numPredict": 512, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 20, "max": 64 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": false } }, "resolved": { "fixedOptions": { "temperature": 0, "seed": 42, "stream": true }, "workloads": { "w1": { "numPredict": 1, "numCtx": 512, "keepAlive": "5m" }, "w2": { "numPredict": 128, "numCtx": 4096, "keepAlive": "5m" }, "w3": { "numPredict": 1, "numCtx": 4096, "keepAlive": "5m" }, "w4": { "numPredict": 512, "numCtx": 4096, "keepAlive": "5m" } }, "runtimeModelParameters": "stop \"<|end|>\"\nstop \"<|user|>\"\nstop \"<|assistant|>\"", "unresolvedRuntimeDefaults": true } }, "rawMeasurements": { "workloads": { "w1": { "warmup": null, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 2453750799, "load_duration": 2390294903, "prompt_eval_count": 15, "prompt_eval_duration": 61405000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2454.364603 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2453750799, "load_duration": 2390294903, "prompt_eval_count": 15, "prompt_eval_duration": 61405000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2454.364603 }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w2": { "warmup": { "total_duration": 7534953348, "load_duration": 2858083822, "prompt_eval_count": 51, "prompt_eval_duration": 178147000, "eval_count": 128, "eval_duration": 4497023000, "timeToFirstTokenMs": 3039.6400599999997 }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 4573744638, "load_duration": 22142156, "prompt_eval_count": 51, "prompt_eval_duration": 35434000, "eval_count": 128, "eval_duration": 4498539000, "timeToFirstTokenMs": 76.69656700000269 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4573744638, "load_duration": 22142156, "prompt_eval_count": 51, "prompt_eval_duration": 35434000, "eval_count": 128, "eval_duration": 4498539000, "timeToFirstTokenMs": 76.69656700000269 }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 4559264026, "load_duration": 21346140, "prompt_eval_count": 51, "prompt_eval_duration": 36288000, "eval_count": 128, "eval_duration": 4499816000, "timeToFirstTokenMs": 60.36970899999869 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4559264026, "load_duration": 21346140, "prompt_eval_count": 51, "prompt_eval_duration": 36288000, "eval_count": 128, "eval_duration": 4499816000, "timeToFirstTokenMs": 60.36970899999869 }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 4552288794, "load_duration": 20994011, "prompt_eval_count": 51, "prompt_eval_duration": 35461000, "eval_count": 128, "eval_duration": 4493943000, "timeToFirstTokenMs": 59.17236099999718 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4552288794, "load_duration": 20994011, "prompt_eval_count": 51, "prompt_eval_duration": 35461000, "eval_count": 128, "eval_duration": 4493943000, "timeToFirstTokenMs": 59.17236099999718 }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 4553464976, "load_duration": 23398921, "prompt_eval_count": 51, "prompt_eval_duration": 35602000, "eval_count": 128, "eval_duration": 4492785000, "timeToFirstTokenMs": 61.80017800000496 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4553464976, "load_duration": 23398921, "prompt_eval_count": 51, "prompt_eval_duration": 35602000, "eval_count": 128, "eval_duration": 4492785000, "timeToFirstTokenMs": 61.80017800000496 }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 4559052392, "load_duration": 25293023, "prompt_eval_count": 51, "prompt_eval_duration": 35450000, "eval_count": 128, "eval_duration": 4496689000, "timeToFirstTokenMs": 63.49745699999767 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 4559052392, "load_duration": 25293023, "prompt_eval_count": 51, "prompt_eval_duration": 35450000, "eval_count": 128, "eval_duration": 4496689000, "timeToFirstTokenMs": 63.49745699999767 }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w3": { "warmup": { "total_duration": 2175052682, "load_duration": 20106006, "prompt_eval_count": 3276, "prompt_eval_duration": 2134911999, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2175.460635999996 }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 2445542885, "load_duration": 21519218, "prompt_eval_count": 3276, "prompt_eval_duration": 2118891999, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2446.1444319999937 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2445542885, "load_duration": 21519218, "prompt_eval_count": 3276, "prompt_eval_duration": 2118891999, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2446.1444319999937 }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 2442511869, "load_duration": 21847384, "prompt_eval_count": 3276, "prompt_eval_duration": 2117572999, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2442.922101999997 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2442511869, "load_duration": 21847384, "prompt_eval_count": 3276, "prompt_eval_duration": 2117572999, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2442.922101999997 }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 2444231219, "load_duration": 22417383, "prompt_eval_count": 3276, "prompt_eval_duration": 2118036999, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2444.5338769999944 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2444231219, "load_duration": 22417383, "prompt_eval_count": 3276, "prompt_eval_duration": 2118036999, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2444.5338769999944 }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 2452497508, "load_duration": 32060123, "prompt_eval_count": 3276, "prompt_eval_duration": 2120034999, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2452.9217100000023 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2452497508, "load_duration": 32060123, "prompt_eval_count": 3276, "prompt_eval_duration": 2120034999, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2452.9217100000023 }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 2449353313, "load_duration": 25517596, "prompt_eval_count": 3276, "prompt_eval_duration": 2123917000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2449.829297000004 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2449353313, "load_duration": 25517596, "prompt_eval_count": 3276, "prompt_eval_duration": 2123917000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2449.829297000004 }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w4": { "warmup": { "total_duration": 6089397417, "load_duration": 26847023, "prompt_eval_count": 51, "prompt_eval_duration": 36501000, "eval_count": 163, "eval_duration": 5733437000, "timeToFirstTokenMs": 357.0938859999951 }, "measuredPasses": [ { "index": 1, "valid": false, "attempts": [ { "measurement": { "total_duration": 5822962369, "load_duration": 26596156, "prompt_eval_count": 51, "prompt_eval_duration": 35545000, "eval_count": 163, "eval_duration": 5740605000, "timeToFirstTokenMs": 83.10216100000252 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5793374003, "load_duration": 20733198, "prompt_eval_count": 51, "prompt_eval_duration": 35457000, "eval_count": 163, "eval_duration": 5735717000, "timeToFirstTokenMs": 58.550984000001336 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5804520758, "load_duration": 27212378, "prompt_eval_count": 51, "prompt_eval_duration": 35548000, "eval_count": 163, "eval_duration": 5740193000, "timeToFirstTokenMs": 65.24198900000192 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } } ], "measurement": { "total_duration": 5804520758, "load_duration": 27212378, "prompt_eval_count": 51, "prompt_eval_duration": 35548000, "eval_count": 163, "eval_duration": 5740193000, "timeToFirstTokenMs": 65.24198900000192 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "index": 2, "valid": false, "attempts": [ { "measurement": { "total_duration": 5807895698, "load_duration": 24917752, "prompt_eval_count": 51, "prompt_eval_duration": 35487000, "eval_count": 163, "eval_duration": 5746056000, "timeToFirstTokenMs": 62.72781200001191 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5788998801, "load_duration": 20429559, "prompt_eval_count": 51, "prompt_eval_duration": 35432000, "eval_count": 163, "eval_duration": 5731502000, "timeToFirstTokenMs": 58.51107300000149 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5794582163, "load_duration": 19974173, "prompt_eval_count": 51, "prompt_eval_duration": 35404000, "eval_count": 163, "eval_duration": 5737664000, "timeToFirstTokenMs": 57.7952959999966 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } } ], "measurement": { "total_duration": 5794582163, "load_duration": 19974173, "prompt_eval_count": 51, "prompt_eval_duration": 35404000, "eval_count": 163, "eval_duration": 5737664000, "timeToFirstTokenMs": 57.7952959999966 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "index": 3, "valid": false, "attempts": [ { "measurement": { "total_duration": 5800842781, "load_duration": 25662901, "prompt_eval_count": 51, "prompt_eval_duration": 35827000, "eval_count": 163, "eval_duration": 5737998000, "timeToFirstTokenMs": 63.44842800000333 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5792974858, "load_duration": 19852349, "prompt_eval_count": 51, "prompt_eval_duration": 35626000, "eval_count": 163, "eval_duration": 5735795000, "timeToFirstTokenMs": 58.05603400000837 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5800422992, "load_duration": 23913629, "prompt_eval_count": 51, "prompt_eval_duration": 35486000, "eval_count": 163, "eval_duration": 5739494000, "timeToFirstTokenMs": 61.61620999999286 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } } ], "measurement": { "total_duration": 5800422992, "load_duration": 23913629, "prompt_eval_count": 51, "prompt_eval_duration": 35486000, "eval_count": 163, "eval_duration": 5739494000, "timeToFirstTokenMs": 61.61620999999286 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "index": 4, "valid": false, "attempts": [ { "measurement": { "total_duration": 5803246170, "load_duration": 27303610, "prompt_eval_count": 51, "prompt_eval_duration": 35392000, "eval_count": 163, "eval_duration": 5738928000, "timeToFirstTokenMs": 65.52275400000508 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5790672277, "load_duration": 20034808, "prompt_eval_count": 51, "prompt_eval_duration": 35515000, "eval_count": 163, "eval_duration": 5733969000, "timeToFirstTokenMs": 57.38356999999087 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5792428797, "load_duration": 19824422, "prompt_eval_count": 51, "prompt_eval_duration": 35614000, "eval_count": 163, "eval_duration": 5735629000, "timeToFirstTokenMs": 57.651968999998644 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } } ], "measurement": { "total_duration": 5792428797, "load_duration": 19824422, "prompt_eval_count": 51, "prompt_eval_duration": 35614000, "eval_count": 163, "eval_duration": 5735629000, "timeToFirstTokenMs": 57.651968999998644 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "index": 5, "valid": false, "attempts": [ { "measurement": { "total_duration": 5793089816, "load_duration": 23292605, "prompt_eval_count": 51, "prompt_eval_duration": 35770000, "eval_count": 163, "eval_duration": 5732717000, "timeToFirstTokenMs": 60.91126700001769 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5795913447, "load_duration": 19741523, "prompt_eval_count": 51, "prompt_eval_duration": 35608000, "eval_count": 163, "eval_duration": 5738943000, "timeToFirstTokenMs": 57.98154800001066 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } }, { "measurement": { "total_duration": 5796970208, "load_duration": 20481013, "prompt_eval_count": 51, "prompt_eval_duration": 35552000, "eval_count": 163, "eval_duration": 5739403000, "timeToFirstTokenMs": 58.680652000010014 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } } ], "measurement": { "total_duration": 5796970208, "load_duration": 20481013, "prompt_eval_count": 51, "prompt_eval_duration": 35552000, "eval_count": 163, "eval_duration": 5739403000, "timeToFirstTokenMs": 58.680652000010014 }, "validity": { "valid": false, "reasons": [ { "code": "eval-count-mismatch", "expected": 512, "actual": 163, "message": "eval_count must equal num_predict (512)" } ] } } ], "failed": true } } }, "derived": { "generationTokensPerSecond": { "median": null, "coefficientOfVariation": null, "samples": 0 }, "prefillTokensPerSecond": { "median": 1546.091070968266, "coefficientOfVariation": 0.001197676172147206, "samples": 5 }, "timeToFirstTokenMs": { "median": 61.80017800000496, "coefficientOfVariation": 0.11059008832924301, "samples": 5 }, "coldLoad": { "seconds": 2.390294903, "samples": 1 }, "passFailureRate": { "failedMeasuredPasses": 5, "totalMeasuredPasses": 16, "percent": 31.25 }, "attemptFailureRate": { "failedAttempts": 15, "totalAttempts": 26, "percent": 57.692307692307686 }, "roofline": { "modelWeightsGB": 8.56682346, "memoryBandwidthGBps": 760, "theoreticalMaxTokensPerSecond": 88.71432959352707, "utilization": null }, "diagnostics": [ { "id": "partial-cpu-offload", "status": "detected", "message": "9% of the model's resident bytes are on the host, not in VRAM", "evidence": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 9672073196, "vramResidentBytes": 8822015917, "hostResidentBytes": 850057279, "vramResidentFraction": 0.9121121954131249 } }, { "id": "context-vram-headroom", "status": "unavailable", "message": "Ollama /api/show does not expose the resolved KV-cache element type actually in use, so v1.1 does not calculate or threshold projected KV-cache VRAM", "evidence": { "configuredContext": 4096, "availableArchitectureMetadata": { "source": "ollama.show.model_info", "architecture": "phi3", "blockCount": 40, "kvHeadCount": 10, "attentionHeadCount": 40, "embeddingLength": 5120, "headDimension": 128, "resolvedElementType": null, "projectedBytes": null, "calculationAvailable": false, "missingInputs": [ "resolvedKvCacheElementType" ] }, "missingInput": "resolvedKvCacheElementType" } }, { "id": "cpu-only-with-gpu", "status": "not-detected", "message": "91% of the model's resident bytes are in VRAM", "evidence": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 9672073196, "vramResidentBytes": 8822015917, "hostResidentBytes": 850057279, "vramResidentFraction": 0.9121121954131249, "gpuModel": "NVIDIA GeForce RTX 3080" } }, { "id": "weights-exceed-vram", "status": "not-detected", "message": "Quantized on-disk model weight size does not exceed total VRAM", "evidence": { "weightsBytes": 8566823460, "totalVramBytes": 10737418240 } } ] } }