{ "protocolVersion": "osai-bench/1.3", "clientVersion": "0.12.0", "scoringVersion": "osai-bench-derive/1.4", "createdAt": "2026-10-20T07:07:09.272Z", "qualityOverride": true, "cohortEligible": false, "qualityConditions": [ { "code": "gpu-utilization", "detected": true } ], "runtime": { "name": "ollama", "version": "0.35.1", "endpoint": "loopback", "layerAssignment": null, "offloadPlacement": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 3184001022, "vramResidentBytes": 0, "hostResidentBytes": 3184001022, "vramResidentFraction": 0 }, "kvCacheMetadata": { "source": "ollama.show.model_info", "architecture": "qwen3", "blockCount": 36, "kvHeadCount": 8, "attentionHeadCount": 32, "embeddingLength": 2560, "headDimension": 128, "resolvedElementType": null, "projectedBytes": null, "calculationAvailable": false, "missingInputs": [ "resolvedKvCacheElementType" ] }, "environment": { "source": "client-process-env", "authoritative": false, "note": "Read from the benchmark client's own process environment. The Ollama server may have been started with a different environment; Ollama exposes no endpoint reporting its resolved configuration.", "declared": { "OLLAMA_KV_CACHE_TYPE": null, "OLLAMA_FLASH_ATTENTION": null, "OLLAMA_CONTEXT_LENGTH": null, "OLLAMA_NUM_PARALLEL": null, "OLLAMA_GPU_OVERHEAD": null, "OLLAMA_SCHED_SPREAD": null, "OLLAMA_LLM_LIBRARY": null, "LLAMA_ARG_FIT": null, "LLAMA_ARG_FIT_TARGET": null, "OLLAMA_MAX_LOADED_MODELS": null, "OLLAMA_KEEP_ALIVE": null, "OLLAMA_DEBUG": null, "OLLAMA_MODELS": false, "OLLAMA_HOST": false }, "declaredNonDefault": [] } }, "model": { "identifier": "qwen3:4b", "digest": "359d7dd4bcdab3d86b87d73ac27966f4dbb9f5efdfcc75d34a8764a09474fae7", "family": "qwen3", "parameterSize": "4.0B", "quantization": "Q4_K_M", "weightsBytes": 2497293931, "weightsSource": "ollama.tags.size" }, "system": { "cpu": { "model": "Intel(R) Core(TM) i7-14700K" }, "gpu": { "present": true, "model": "NVIDIA GeForce RTX 4070 Ti", "totalVramBytes": 12878610432, "freeVramBytesAtCheck": 10847518720, "freeVramBytesAtLoad": 10782507008, "utilizationPercentAtCheck": 31, "driverVersion": "617.42", "provider": "nvidia-smi" }, "memory": { "totalBytes": 34018738176 }, "os": { "platform": "win32", "version": "10.0.26200", "architecture": "x64" }, "power": { "present": false, "onBattery": false } }, "configuration": { "memoryBandwidthGBps": 504, "memoryBandwidthSource": "manufacturer-table", "memoryBandwidthTableVersion": "osai-gpu-memory-bandwidth/2", "memoryBandwidthEntryId": "nvidia-geforce-rtx-4070-ti-12gb", "fixedOptions": { "temperature": 0, "seed": 42, "stream": true }, "workloads": { "w1": { "numPredict": 1, "numCtx": 512, "keepAlive": "5m", "promptTokenRange": null, "warmups": 0, "repetitions": 1, "varyPromptPerCall": false }, "w2": { "numPredict": 128, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 20, "max": 64 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": false }, "w3": { "numPredict": 1, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 2000, "max": 4095 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": true }, "w4": { "numPredict": 512, "numCtx": 4096, "keepAlive": "5m", "promptTokenRange": { "min": 20, "max": 64 }, "warmups": 1, "repetitions": 5, "varyPromptPerCall": false } }, "resolved": { "fixedOptions": { "temperature": 0, "seed": 42, "stream": true }, "workloads": { "w1": { "numPredict": 1, "numCtx": 512, "keepAlive": "5m" }, "w2": { "numPredict": 128, "numCtx": 4096, "keepAlive": "5m" }, "w3": { "numPredict": 1, "numCtx": 4096, "keepAlive": "5m" }, "w4": { "numPredict": 512, "numCtx": 4096, "keepAlive": "5m" } }, "runtimeModelParameters": "temperature 0.6\ntop_k 20\ntop_p 0.95\nrepeat_penalty 1\nstop \"<|im_start|>\"\nstop \"<|im_end|>\"", "unresolvedRuntimeDefaults": true } }, "rawMeasurements": { "workloads": { "w1": { "warmup": null, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 2273073200, "load_duration": 2138397100, "prompt_eval_count": 15, "prompt_eval_duration": 131615000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2273.6176, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 2273073200, "load_duration": 2138397100, "prompt_eval_count": 15, "prompt_eval_duration": 131615000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 2273.6176, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w2": { "warmup": { "total_duration": 8705429500, "load_duration": 2145098700, "prompt_eval_count": 49, "prompt_eval_duration": 287334000, "eval_count": 128, "eval_duration": 6270053000, "timeToFirstTokenMs": 2436.5290000000005, "timeToFirstVisibleTokenMs": null }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 6497953400, "load_duration": 4201900, "prompt_eval_count": 49, "prompt_eval_duration": 53602000, "eval_count": 128, "eval_duration": 6426194000, "timeToFirstTokenMs": 72.35099999999875, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 6497953400, "load_duration": 4201900, "prompt_eval_count": 49, "prompt_eval_duration": 53602000, "eval_count": 128, "eval_duration": 6426194000, "timeToFirstTokenMs": 72.35099999999875, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 6111619500, "load_duration": 3808900, "prompt_eval_count": 49, "prompt_eval_duration": 47280000, "eval_count": 128, "eval_duration": 6058187000, "timeToFirstTokenMs": 53.71310000000085, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 6111619500, "load_duration": 3808900, "prompt_eval_count": 49, "prompt_eval_duration": 47280000, "eval_count": 128, "eval_duration": 6058187000, "timeToFirstTokenMs": 53.71310000000085, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 6204801400, "load_duration": 4319200, "prompt_eval_count": 49, "prompt_eval_duration": 52371000, "eval_count": 128, "eval_duration": 6145346000, "timeToFirstTokenMs": 59.92120000000068, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 6204801400, "load_duration": 4319200, "prompt_eval_count": 49, "prompt_eval_duration": 52371000, "eval_count": 128, "eval_duration": 6145346000, "timeToFirstTokenMs": 59.92120000000068, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 6334496600, "load_duration": 4126700, "prompt_eval_count": 49, "prompt_eval_duration": 48614000, "eval_count": 128, "eval_duration": 6279574000, "timeToFirstTokenMs": 55.450000000004366, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 6334496600, "load_duration": 4126700, "prompt_eval_count": 49, "prompt_eval_duration": 48614000, "eval_count": 128, "eval_duration": 6279574000, "timeToFirstTokenMs": 55.450000000004366, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 6061793700, "load_duration": 2190700, "prompt_eval_count": 49, "prompt_eval_duration": 52110000, "eval_count": 128, "eval_duration": 6005176000, "timeToFirstTokenMs": 56.855200000005425, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 6061793700, "load_duration": 2190700, "prompt_eval_count": 49, "prompt_eval_duration": 52110000, "eval_count": 128, "eval_duration": 6005176000, "timeToFirstTokenMs": 56.855200000005425, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w3": { "warmup": { "total_duration": 19327573200, "load_duration": 3678600, "prompt_eval_count": 2796, "prompt_eval_duration": 19303341000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 19327.380100000002, "timeToFirstVisibleTokenMs": null }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 19954192700, "load_duration": 3854200, "prompt_eval_count": 2796, "prompt_eval_duration": 19837196000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 19954.145900000003, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 19954192700, "load_duration": 3854200, "prompt_eval_count": 2796, "prompt_eval_duration": 19837196000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 19954.145900000003, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 19559333900, "load_duration": 7144200, "prompt_eval_count": 2796, "prompt_eval_duration": 19414651000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 19560.482099999994, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 19559333900, "load_duration": 7144200, "prompt_eval_count": 2796, "prompt_eval_duration": 19414651000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 19560.482099999994, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 19583234400, "load_duration": 2969100, "prompt_eval_count": 2796, "prompt_eval_duration": 19424679000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 19583.627499999988, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 19583234400, "load_duration": 2969100, "prompt_eval_count": 2796, "prompt_eval_duration": 19424679000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 19583.627499999988, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 19826392200, "load_duration": 3807600, "prompt_eval_count": 2796, "prompt_eval_duration": 19679498000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 19827.032399999982, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 19826392200, "load_duration": 3807600, "prompt_eval_count": 2796, "prompt_eval_duration": 19679498000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 19827.032399999982, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 20845633000, "load_duration": 3636400, "prompt_eval_count": 2796, "prompt_eval_duration": 20672675000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 20846.33619999999, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 20845633000, "load_duration": 3636400, "prompt_eval_count": 2796, "prompt_eval_duration": 20672675000, "eval_count": 1, "eval_duration": 1000, "timeToFirstTokenMs": 20846.33619999999, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "failed": false }, "w4": { "warmup": { "total_duration": 25672604500, "load_duration": 3836400, "prompt_eval_count": 49, "prompt_eval_duration": 50818000, "eval_count": 512, "eval_duration": 25509678000, "timeToFirstTokenMs": 163.58289999997942, "timeToFirstVisibleTokenMs": null }, "measuredPasses": [ { "index": 1, "valid": true, "attempts": [ { "measurement": { "total_duration": 24602423400, "load_duration": 2769500, "prompt_eval_count": 49, "prompt_eval_duration": 49486000, "eval_count": 512, "eval_duration": 24528744000, "timeToFirstTokenMs": 73.62940000000526, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 24602423400, "load_duration": 2769500, "prompt_eval_count": 49, "prompt_eval_duration": 49486000, "eval_count": 512, "eval_duration": 24528744000, "timeToFirstTokenMs": 73.62940000000526, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 2, "valid": true, "attempts": [ { "measurement": { "total_duration": 24093501300, "load_duration": 3112000, "prompt_eval_count": 49, "prompt_eval_duration": 42861000, "eval_count": 512, "eval_duration": 24045346000, "timeToFirstTokenMs": 48.57220000002417, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 24093501300, "load_duration": 3112000, "prompt_eval_count": 49, "prompt_eval_duration": 42861000, "eval_count": 512, "eval_duration": 24045346000, "timeToFirstTokenMs": 48.57220000002417, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 3, "valid": true, "attempts": [ { "measurement": { "total_duration": 23827133100, "load_duration": 3084200, "prompt_eval_count": 49, "prompt_eval_duration": 43453000, "eval_count": 512, "eval_duration": 23778393000, "timeToFirstTokenMs": 49.275599999993574, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 23827133100, "load_duration": 3084200, "prompt_eval_count": 49, "prompt_eval_duration": 43453000, "eval_count": 512, "eval_duration": 23778393000, "timeToFirstTokenMs": 49.275599999993574, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 4, "valid": true, "attempts": [ { "measurement": { "total_duration": 23338200200, "load_duration": 2158700, "prompt_eval_count": 49, "prompt_eval_duration": 44684000, "eval_count": 512, "eval_duration": 23289903000, "timeToFirstTokenMs": 49.399199999985285, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 23338200200, "load_duration": 2158700, "prompt_eval_count": 49, "prompt_eval_duration": 44684000, "eval_count": 512, "eval_duration": 23289903000, "timeToFirstTokenMs": 49.399199999985285, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } }, { "index": 5, "valid": true, "attempts": [ { "measurement": { "total_duration": 23449495200, "load_duration": 3113900, "prompt_eval_count": 49, "prompt_eval_duration": 45588000, "eval_count": 512, "eval_duration": 23398685000, "timeToFirstTokenMs": 51.36920000001555, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "measurement": { "total_duration": 23449495200, "load_duration": 3113900, "prompt_eval_count": 49, "prompt_eval_duration": 45588000, "eval_count": 512, "eval_duration": 23398685000, "timeToFirstTokenMs": 51.36920000001555, "timeToFirstVisibleTokenMs": null }, "validity": { "valid": true, "reasons": [] } } ], "failed": false } } }, "derived": { "generationTokensPerSecond": { "median": 21.53215316106517, "coefficientOfVariation": 0.020984739692020065, "samples": 5 }, "prefillTokensPerSecond": { "median": 142.07679484507176, "coefficientOfVariation": 0.025440030281688498, "samples": 5 }, "timeToFirstTokenMs": { "median": 56.855200000005425, "coefficientOfVariation": 0.1248997248750438, "samples": 5, "reasoningWithheldPasses": 0 }, "timeToFirstVisibleTokenMs": { "median": null, "coefficientOfVariation": null, "samples": 0 }, "coldLoad": { "seconds": 2.1383971, "samples": 1 }, "passFailureRate": { "failedMeasuredPasses": 0, "totalMeasuredPasses": 16, "percent": 0 }, "attemptFailureRate": { "failedAttempts": 0, "totalAttempts": 16, "percent": 0 }, "roofline": { "modelWeightsGB": 2.497293931, "memoryBandwidthGBps": 504, "theoreticalMaxTokensPerSecond": 201.8184538646524, "utilization": 0.10669070517954468 }, "diagnostics": [ { "id": "partial-cpu-offload", "status": "not-detected", "message": "The whole resident footprint is on one side; no split between VRAM and host memory", "evidence": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 3184001022, "vramResidentBytes": 0, "hostResidentBytes": 3184001022, "vramResidentFraction": 0 } }, { "id": "context-vram-headroom", "status": "unavailable", "message": "Ollama /api/show does not expose the resolved KV-cache element type actually in use, so v1.1 does not calculate or threshold projected KV-cache VRAM", "evidence": { "configuredContext": 4096, "availableArchitectureMetadata": { "source": "ollama.show.model_info", "architecture": "qwen3", "blockCount": 36, "kvHeadCount": 8, "attentionHeadCount": 32, "embeddingLength": 2560, "headDimension": 128, "resolvedElementType": null, "projectedBytes": null, "calculationAvailable": false, "missingInputs": [ "resolvedKvCacheElementType" ] }, "missingInput": "resolvedKvCacheElementType" } }, { "id": "cpu-only-with-gpu", "status": "detected", "message": "A GPU was detected but none of the model is resident in VRAM; it is executing on the CPU", "evidence": { "source": "ollama.ps.size_vram", "granularity": "bytes", "residentBytes": 3184001022, "vramResidentBytes": 0, "hostResidentBytes": 3184001022, "vramResidentFraction": 0, "gpuModel": "NVIDIA GeForce RTX 4070 Ti" } }, { "id": "weights-exceed-vram", "status": "not-detected", "message": "Quantized on-disk model weight size does not exceed total VRAM", "evidence": { "weightsBytes": 2497293931, "totalVramBytes": 12878610432 } } ] } }