{ "protocolVersion": "osai-bench/1.3", "clientVersion": "0.12.0", "scoringVersion": "osai-bench-derive/1.4", "createdAt": "2026-10-20T07:25:33.761Z", "qualityOverride": false, "cohortEligible": true, "qualityConditions": [], "runtime": { "name": "ollama", "version": "0.35.1", "endpoint": "loopback", "layerAssignment": null, "offloadPlacement": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 5576117451, "vramResidentBytes": 0, "hostResidentBytes": 5576117451, "vramResidentFraction": 0 }, "kvCacheMetadata": { "source": "ollama.show.model_info", "architecture": "llama", "blockCount": 32, "kvHeadCount": 8, "attentionHeadCount": 32, "embeddingLength": 4096, "headDimension": 128, "resolvedElementType": null, "projectedBytes": null, "calculationAvailable": false, "missingInputs": [ "resolvedKvCacheElementType" ] }, "environment": { "source": "client-process-env", "authoritative": false, "note": "Read from the benchmark client's own process environment. The Ollama server may have been started with a different environment; Ollama exposes no endpoint reporting its resolved configuration.", "declared": { "OLLAMA_KV_CACHE_TYPE": null, "OLLAMA_FLASH_ATTENTION": null, "OLLAMA_CONTEXT_LENGTH": null, "OLLAMA_NUM_PARALLEL": null, "OLLAMA_GPU_OVERHEAD": null, "OLLAMA_SCHED_SPREAD": null, "OLLAMA_LLM_LIBRARY": null, "LLAMA_ARG_FIT": null, "LLAMA_ARG_FIT_TARGET": null, "OLLAMA_MAX_LOADED_MODELS": null, "OLLAMA_KEEP_ALIVE": null, "OLLAMA_DEBUG": null, "OLLAMA_MODELS": false, "OLLAMA_HOST": false }, "declaredNonDefault": [] } }, "model": { "identifier": "llama3.1:8b", "digest": "46e0c10c039e019119339687c3c1757cc81b9da49709a3b3924863ba87ca666e", "family": "llama", "parameterSize": "8.0B", "quantization": "Q4_K_M", "weightsBytes": 4920753328, "weightsSource": "ollama.tags.size" }, "system": { "cpu": { "model": "Intel(R) Core(TM) i7-14700K" }, "gpu": { "present": true, "model": "NVIDIA GeForce RTX 4070 Ti", "totalVramBytes": 12878610432, "freeVramBytesAtCheck": 10566500352, "freeVramBytesAtLoad": 10568597504, "utilizationPercentAtCheck": 10, "driverVersion": "617.42", "provider": "nvidia-smi" }, "memory": { "totalBytes": 34018738176 }, "os": { "platform": "win32", "version": "10.0.26200", "architecture": "x64" }, "power": { "present": false, "onBattery": false } }, "configuration": { "memoryBandwidthGBps": 504, "memoryBandwidthSource": "manufacturer-table", "memoryBandwidthTableVersion": "osai-gpu-memory-bandwidth/2", "memoryBandwidthEntryId": "nvidia-geforce-rtx-4070-ti-12gb", "fixedOptions": { "temperature": 0, "seed": 42, "stream": true }, "workloads": { "w1": { "numPredict": 1, "numCtx": 512, "keepAlive": "5m", "promptTokenRange": null, "warmups": 0, "repetitions": 1, "varyPromptPerCall": false }, "w2": { "numPredict": 128, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 20, "max": 64 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": false }, "w3": { "numPredict": 1, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 2000, "max": 4095 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": true }, "w4": { "numPredict": 512, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 20, "max": 64 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": false } }, "resolved": { "fixedOptions": { "temperature": 0, "seed": 42, "stream": true }, "workloads": { "w1": { "numPredict": 1, "numCtx": 512, "keepAlive": "5m" }, "w2": { "numPredict": 128, "numCtx": 4096, "keepAlive": "5m" }, "w3": { "numPredict": 1, "numCtx": 4096, "keepAlive": "5m" }, "w4": { "numPredict": 512, "numCtx": 4096, "keepAlive": "5m" } }, "runtimeModelParameters": "stop \"<|start_header_id|>\"\nstop \"<|end_header_id|>\"\nstop \"<|eot_id|>\"", "unresolvedRuntimeDefaults": true } }, "rawMeasurements": { "workloads": { "w1": { "warmup": null, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 3967689600, "load_duration": 3743178000, "prompt_eval_count": 15, "prompt_eval_duration": 220686000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 3968.2974999999997, "timeToFirstVisibleTokenMs": 3968.2974999999997 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 3967689600, "load_duration": 3743178000, "prompt_eval_count": 15, "prompt_eval_duration": 220686000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 3968.2974999999997, "timeToFirstVisibleTokenMs": 3968.2974999999997 }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w2": { "warmup": { "total_duration": 14986919800, "load_duration": 3945314100, "prompt_eval_count": 45, "prompt_eval_duration": 471741000, "eval_count": 128, "eval_duration": 10566180000, "timeToFirstTokenMs": 4421.5038, "timeToFirstVisibleTokenMs": 4421.5038 }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 10024809200, "load_duration": 13803400, "prompt_eval_count": 45, "prompt_eval_duration": 86888000, "eval_count": 128, "eval_duration": 9912361000, "timeToFirstTokenMs": 114.03379999999743, "timeToFirstVisibleTokenMs": 114.03379999999743 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 10024809200, "load_duration": 13803400, "prompt_eval_count": 45, "prompt_eval_duration": 86888000, "eval_count": 128, "eval_duration": 9912361000, "timeToFirstTokenMs": 114.03379999999743, "timeToFirstVisibleTokenMs": 114.03379999999743 }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 10479287600, "load_duration": 3195400, "prompt_eval_count": 45, "prompt_eval_duration": 81696000, "eval_count": 128, "eval_duration": 10385792000, "timeToFirstTokenMs": 94.10360000000219, "timeToFirstVisibleTokenMs": 94.10360000000219 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 10479287600, "load_duration": 3195400, "prompt_eval_count": 45, "prompt_eval_duration": 81696000, "eval_count": 128, "eval_duration": 10385792000, "timeToFirstTokenMs": 94.10360000000219, "timeToFirstVisibleTokenMs": 94.10360000000219 }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 10713324700, "load_duration": 3184000, "prompt_eval_count": 45, "prompt_eval_duration": 85200000, "eval_count": 128, "eval_duration": 10622560000, "timeToFirstTokenMs": 91.37250000000495, "timeToFirstVisibleTokenMs": 91.37250000000495 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 10713324700, "load_duration": 3184000, "prompt_eval_count": 45, "prompt_eval_duration": 85200000, "eval_count": 128, "eval_duration": 10622560000, "timeToFirstTokenMs": 91.37250000000495, "timeToFirstVisibleTokenMs": 91.37250000000495 }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 10110155900, "load_duration": 3675900, "prompt_eval_count": 45, "prompt_eval_duration": 89207000, "eval_count": 128, "eval_duration": 10014407000, "timeToFirstTokenMs": 95.86240000000544, "timeToFirstVisibleTokenMs": 95.86240000000544 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 10110155900, "load_duration": 3675900, "prompt_eval_count": 45, "prompt_eval_duration": 89207000, "eval_count": 128, "eval_duration": 10014407000, "timeToFirstTokenMs": 95.86240000000544, "timeToFirstVisibleTokenMs": 95.86240000000544 }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 10883984300, "load_duration": 3723800, "prompt_eval_count": 45, "prompt_eval_duration": 80562000, "eval_count": 128, "eval_duration": 10797449000, "timeToFirstTokenMs": 86.8353999999963, "timeToFirstVisibleTokenMs": 86.8353999999963 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 10883984300, "load_duration": 3723800, "prompt_eval_count": 45, "prompt_eval_duration": 80562000, "eval_count": 128, "eval_duration": 10797449000, "timeToFirstTokenMs": 86.8353999999963, "timeToFirstVisibleTokenMs": 86.8353999999963 }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w3": { "warmup": { "total_duration": 30421871100, "load_duration": 3098900, "prompt_eval_count": 2664, "prompt_eval_duration": 30407696000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 30422.23109999999, "timeToFirstVisibleTokenMs": 30422.23109999999 }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 29701644000, "load_duration": 3776500, "prompt_eval_count": 2664, "prompt_eval_duration": 29600929000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 29701.6342, "timeToFirstVisibleTokenMs": 29701.6342 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 29701644000, "load_duration": 3776500, "prompt_eval_count": 2664, "prompt_eval_duration": 29600929000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 29701.6342, "timeToFirstVisibleTokenMs": 29701.6342 }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 29916281200, "load_duration": 3662200, "prompt_eval_count": 2664, "prompt_eval_duration": 29802148000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 29916.409599999984, "timeToFirstVisibleTokenMs": 29916.409599999984 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 29916281200, "load_duration": 3662200, "prompt_eval_count": 2664, "prompt_eval_duration": 29802148000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 29916.409599999984, "timeToFirstVisibleTokenMs": 29916.409599999984 }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 30253346700, "load_duration": 3603100, "prompt_eval_count": 2664, "prompt_eval_duration": 30139759000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 30253.316900000005, "timeToFirstVisibleTokenMs": 30253.316900000005 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 30253346700, "load_duration": 3603100, "prompt_eval_count": 2664, "prompt_eval_duration": 30139759000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 30253.316900000005, "timeToFirstVisibleTokenMs": 30253.316900000005 }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 31447139000, "load_duration": 3368700, "prompt_eval_count": 2664, "prompt_eval_duration": 31356299000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 31447.55230000001, "timeToFirstVisibleTokenMs": 31447.55230000001 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 31447139000, "load_duration": 3368700, "prompt_eval_count": 2664, "prompt_eval_duration": 31356299000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 31447.55230000001, "timeToFirstVisibleTokenMs": 31447.55230000001 }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 32882615800, "load_duration": 8993500, "prompt_eval_count": 2664, "prompt_eval_duration": 32764898000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 32882.94869999998, "timeToFirstVisibleTokenMs": 32882.94869999998 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 32882615800, "load_duration": 8993500, "prompt_eval_count": 2664, "prompt_eval_duration": 32764898000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 32882.94869999998, "timeToFirstVisibleTokenMs": 32882.94869999998 }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w4": { "warmup": { "total_duration": 44968269200, "load_duration": 16767100, "prompt_eval_count": 45, "prompt_eval_duration": 80727000, "eval_count": 512, "eval_duration": 44774892000, "timeToFirstTokenMs": 194.1130000000121, "timeToFirstVisibleTokenMs": 194.1130000000121 }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 49034233200, "load_duration": 17897600, "prompt_eval_count": 45, "prompt_eval_duration": 78768000, "eval_count": 512, "eval_duration": 48907572000, "timeToFirstTokenMs": 127.02750000002561, "timeToFirstVisibleTokenMs": 127.02750000002561 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 49034233200, "load_duration": 17897600, "prompt_eval_count": 45, "prompt_eval_duration": 78768000, "eval_count": 512, "eval_duration": 48907572000, "timeToFirstTokenMs": 127.02750000002561, "timeToFirstVisibleTokenMs": 127.02750000002561 }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 43408247600, "load_duration": 22034000, "prompt_eval_count": 45, "prompt_eval_duration": 99908000, "eval_count": 512, "eval_duration": 43282986000, "timeToFirstTokenMs": 125.82110000000102, "timeToFirstVisibleTokenMs": 125.82110000000102 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 43408247600, "load_duration": 22034000, "prompt_eval_count": 45, "prompt_eval_duration": 99908000, "eval_count": 512, "eval_duration": 43282986000, "timeToFirstTokenMs": 125.82110000000102, "timeToFirstVisibleTokenMs": 125.82110000000102 }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 41194208000, "load_duration": 6731600, "prompt_eval_count": 45, "prompt_eval_duration": 76423000, "eval_count": 512, "eval_duration": 41108768000, "timeToFirstTokenMs": 85.7451000000001, "timeToFirstVisibleTokenMs": 85.7451000000001 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 41194208000, "load_duration": 6731600, "prompt_eval_count": 45, "prompt_eval_duration": 76423000, "eval_count": 512, "eval_duration": 41108768000, "timeToFirstTokenMs": 85.7451000000001, "timeToFirstVisibleTokenMs": 85.7451000000001 }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 42677078900, "load_duration": 3134800, "prompt_eval_count": 45, "prompt_eval_duration": 85730000, "eval_count": 512, "eval_duration": 42585694000, "timeToFirstTokenMs": 91.87640000000829, "timeToFirstVisibleTokenMs": 91.87640000000829 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 42677078900, "load_duration": 3134800, "prompt_eval_count": 45, "prompt_eval_duration": 85730000, "eval_count": 512, "eval_duration": 42585694000, "timeToFirstTokenMs": 91.87640000000829, "timeToFirstVisibleTokenMs": 91.87640000000829 }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 42648960100, "load_duration": 2110400, "prompt_eval_count": 45, "prompt_eval_duration": 81551000, "eval_count": 512, "eval_duration": 42562980000, "timeToFirstTokenMs": 86.25559999997495, "timeToFirstVisibleTokenMs": 86.25559999997495 }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 42648960100, "load_duration": 2110400, "prompt_eval_count": 45, "prompt_eval_duration": 81551000, "eval_count": 512, "eval_duration": 42562980000, "timeToFirstTokenMs": 86.25559999997495, "timeToFirstVisibleTokenMs": 86.25559999997495 }, "validity": { "valid": true, "reasons": [] } } ], "failed": false } } }, "derived": { "generationTokensPerSecond": { "median": 12.022816864273716, "coefficientOfVariation": 0.06442487853052463, "samples": 5 }, "prefillTokensPerSecond": { "median": 88.38823163781767, "coefficientOfVariation": 0.041939039488486596, "samples": 5 }, "timeToFirstTokenMs": { "median": 94.10360000000219, "coefficientOfVariation": 0.10791484795598816, "samples": 5, "reasoningWithheldPasses": 0 }, "timeToFirstVisibleTokenMs": { "median": 91.87640000000829, "coefficientOfVariation": 0.205229615527255, "samples": 5 }, "coldLoad": { "seconds": 3.743178, "samples": 1 }, "passFailureRate": { "failedMeasuredPasses": 0, "totalMeasuredPasses": 16, "percent": 0 }, "attemptFailureRate": { "failedAttempts": 0, "totalAttempts": 16, "percent": 0 }, "roofline": { "modelWeightsGB": 4.920753328, "memoryBandwidthGBps": 504, "theoreticalMaxTokensPerSecond": 102.42334179446598, "utilization": 0.11738356368414567 }, "diagnostics": [ { "id": "partial-cpu-offload", "status": "not-detected", "message": "The whole resident footprint is on one side; no split between VRAM and host memory", "evidence": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 5576117451, "vramResidentBytes": 0, "hostResidentBytes": 5576117451, "vramResidentFraction": 0 } }, { "id": "context-vram-headroom", "status": "unavailable", "message": "Ollama /api/show does not expose the resolved KV-cache element type actually in use, so v1.1 does not calculate or threshold projected KV-cache VRAM", "evidence": { "configuredContext": 4096, "availableArchitectureMetadata": { "source": "ollama.show.model_info", "architecture": "llama", "blockCount": 32, "kvHeadCount": 8, "attentionHeadCount": 32, "embeddingLength": 4096, "headDimension": 128, "resolvedElementType": null, "projectedBytes": null, "calculationAvailable": false, "missingInputs": [ "resolvedKvCacheElementType" ] }, "missingInput": "resolvedKvCacheElementType" } }, { "id": "cpu-only-with-gpu", "status": "detected", "message": "A GPU was detected but none of the model is resident in VRAM; it is executing on the CPU", "evidence": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 5576117451, "vramResidentBytes": 0, "hostResidentBytes": 5576117451, "vramResidentFraction": 0, "gpuModel": "NVIDIA GeForce RTX 4070 Ti" } }, { "id": "weights-exceed-vram", "status": "not-detected", "message": "Quantized on-disk model weight size does not exceed total VRAM", "evidence": { "weightsBytes": 4920753328, "totalVramBytes": 12878610432 } } ] } }