Qwen3.5 0.8B — Text, Image
35 configurations testedQuality criteria
{ "version": "multimodal-v1", "scope": "Fixed English object, printed-text, counting, ordered-frame and capital-city checks. Modality support is scoped to each precision/device.", "references": "research/multimodal-llm-audit/fixtures.json", "nonEmpty": true, "nonRepetitive": true }
| Precision | Device | Result | |
|---|---|---|---|
Q8 | GPU · WebGPUAccelerated | Passed | Moderate |
FP16 | GPU · WebGPUAccelerated | Passed | Moderate |
Q4F16 | GPU · WebGPUAccelerated | Passed | Moderate |
FP32 | GPU · WebGPUAccelerated | Passed | Moderate |
Q4 | GPU · WebGPUAccelerated | Passed | Moderate |
FP16 | GPU · WebGPUAccelerated | Passed | Very slow |
FP16 | GPU · WebGPUAccelerated | Passed | Very slow |
FP32 | GPU · WebGPUAccelerated | Passed | Very slow |
Q8 | GPU · WebGPUAccelerated | Passed | Very slow |
FP16 | GPU · WebGPUAccelerated | Passed | Very slow |
Q4F16 | GPU · WebGPUAccelerated | Passed | Very slow |
Q4F16 | GPU · WebGPUAccelerated | Passed | Very slow |
FP16 | GPU · WebGPUAccelerated | Passed | Very slow |
Q8 | GPU · WebGPUAccelerated | Passed | Very slow |
FP32 | GPU · WebGPUAccelerated | Passed | Very slow |
Q8 | GPU · WebGPUAccelerated | Passed | Very slow |
Q4 | GPU · WebGPUAccelerated | Passed | Very slow |
Q4 | GPU · WebGPUAccelerated | Passed | Very slow |
Q4F16 | GPU · WebGPUAccelerated | Passed | Very slow |
Q4 | GPU · WebGPUAccelerated | Passed | Very slow |
FP32 | GPU · WebGPUAccelerated | Passed | Very slow |
FP16 | GPU · WebGPUAccelerated | Passed | Very slow |
Q8 | GPU · WebGPUAccelerated | Passed | Very slow |
FP32 | GPU · WebGPUAccelerated | Passed | Very slow |
FP16 | CPU · WASM | Failed | — |
FP32 | CPU · WASM | Failed | — |
Q4 | CPU · WASM | Failed | — |
Q4F16 | CPU · WASM | Failed | — |
Q8 | CPU · WASM | Failed | — |
FP32 | GPU · WebGPU | Failed | — |
Q4 | GPU · WebGPU | Failed | — |
Q4 | GPU · WebGPU | Failed | — |
Q4F16 | GPU · WebGPU | Failed | — |
Q4F16 | GPU · WebGPU | Failed | — |
Q8 | GPU · WebGPU | Failed | — |
Test setup & sample details
- Quality gate
- { "version": "multimodal-v1", "scope": "Fixed English object, printed-text, counting, ordered-frame and capital-city checks. Modality support is scoped to each precision/device.", "references": "research/multimodal-llm-audit/fixtures.json", "nonEmpty": true, "nonRepetitive": true }
- Tested
- 2026-11-05T15:03:40.192Z
- Environment
- Not recorded
- Runtime
- Transformers.js 4.2.0
- Artifact revision
- 8fe9dbe3fb39fe902e18df9be84b6a0ac77ae0f6
- Fixture
{ "file": "model-samples/multimodal/qwen3.5-0.8b-text-regression-fixture.json", "url": "https://assets.localai.tools/samples/v1/files/a1fb53a41e7676c395e3feb366f30d88db75fca4bd55c85c88c023d5a3a0d6bb/qwen3.5-0.8b-text-regression-fixture.json", "size": 424, "sha1": "b1328ae38cb093ad33065cfa8034413e69cf28e0", "sha256": "a1fb53a41e7676c395e3feb366f30d88db75fca4bd55c85c88c023d5a3a0d6bb" }- Rerun when
- artifact revision change; runtime version change; fixture or quality gate change; browser/backend change; Input modality preprocessing or modality-specific quality gates
Per-case quality & failure details
[
{
"precision": "fp16",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": null,
"grade": "failed"
},
"failure": "Error: [WorkerError:runtime_error] init: Error: Can't create a session. ERROR_CODE: 6, ERROR_MESSAGE: std::bad_alloc\n at Le (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:12:31247)\n at Id (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:22953)\n at async $2.loadModel (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:35835)\n at async gd.createInferenceSessionHandler (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:37556)\n at async u2.create (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:10:47888)\n at async Pc (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:54332)\n at async https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16886\n at async Promise.all (index 1)\n at async RT (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16743)\n at async Promise.all (index 0)"
},
{
"precision": "fp32",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": null,
"grade": "failed"
},
"failure": "Error: [WorkerError:runtime_error] init: Error: Can't create a session. ERROR_CODE: 6, ERROR_MESSAGE: std::bad_alloc\n at Le (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:12:31247)\n at Id (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:22953)\n at async $2.loadModel (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:35835)\n at async gd.createInferenceSessionHandler (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:37556)\n at async u2.create (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:10:47888)\n at async Pc (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:54332)\n at async https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16886\n at async Promise.all (index 1)\n at async RT (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16743)\n at async Promise.all (index 0)"
},
{
"precision": "q4",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": null,
"grade": "failed"
},
"failure": "Error: [WorkerError:runtime_error] init: Error: Can't create a session. ERROR_CODE: 9, ERROR_MESSAGE: Could not find an implementation for GatherBlockQuantized(1) node with name '/model/pos_embed/emb_00_Quant'\n at Le (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:12:31247)\n at Id (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:22953)\n at async $2.loadModel (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:35835)\n at async gd.createInferenceSessionHandler (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:37556)\n at async u2.create (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:10:47888)\n at async Pc (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:54332)\n at async https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16886\n at async Promise.all (index 2)\n at async RT (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16743)\n at async Promise.all (index 0)"
},
{
"precision": "q4f16",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": null,
"grade": "failed"
},
"failure": "Error: [WorkerError:runtime_error] init: Error: Can't create a session. ERROR_CODE: 9, ERROR_MESSAGE: Could not find an implementation for GatherBlockQuantized(1) node with name '/model/pos_embed/emb_00_Quant'\n at Le (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:12:31247)\n at Id (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:22953)\n at async $2.loadModel (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:35835)\n at async gd.createInferenceSessionHandler (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:37556)\n at async u2.create (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:10:47888)\n at async Pc (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:54332)\n at async https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16886\n at async Promise.all (index 2)\n at async RT (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16743)\n at async Promise.all (index 0)"
},
{
"precision": "q8",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": null,
"grade": "failed"
},
"failure": "Error: [WorkerError:runtime_error] init: Error: Can't create a session. ERROR_CODE: 9, ERROR_MESSAGE: Could not find an implementation for GatherBlockQuantized(1) node with name '/model/pos_embed/emb_00_Quant'\n at Le (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:12:31247)\n at Id (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:22953)\n at async $2.loadModel (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:35835)\n at async gd.createInferenceSessionHandler (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:37556)\n at async u2.create (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:10:47888)\n at async Pc (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:13:54332)\n at async https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16886\n at async Promise.all (index 2)\n at async RT (https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0:37:16743)\n at async Promise.all (index 0)"
},
{
"precision": "fp16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 87471.94999998808,
"inferenceMs": 29256.37999999523,
"totalMs": 116728.53500002623,
"wallMs": 29256.585000038147,
"charactersPerSecond": 0.1367223149275697,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.034180578731892425
}
},
{
"precision": "fp16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 87471.94999998808,
"inferenceMs": 26417.769999980927,
"totalMs": 143146.39499998093,
"wallMs": 26417.849999964237,
"charactersPerSecond": 0.4920930116360838,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.03785330858739106
}
},
{
"precision": "fp16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 87471.94999998808,
"inferenceMs": 24532.42500001192,
"totalMs": 167678.88499999046,
"wallMs": 24532.474999964237,
"charactersPerSecond": 1.7120199083449594,
"inputModality": "image",
"grade": "slow",
"imagesPerSecond": 0.04076237877011808
}
},
{
"precision": "fp16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 87471.94999998808,
"inferenceMs": 10604.660000026226,
"totalMs": 178283.60000002384,
"wallMs": 10604.710000038147,
"charactersPerSecond": 0.7543853362559683,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.3771926681279841
}
},
{
"precision": "fp16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 87471.94999998808,
"inferenceMs": 11329.125,
"totalMs": 189612.74000000954,
"wallMs": 11329.139999985695,
"charactersPerSecond": 0.7944126311608355,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.3530722805159269
}
},
{
"precision": "fp16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 87471.94999998808,
"inferenceMs": 763.4300000071526,
"totalMs": 190376.24500000477,
"wallMs": 763.5049999952316,
"charactersPerSecond": 6.549388941950349,
"inputModality": "text",
"grade": "good"
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 167816.94499999285,
"inferenceMs": 30783.55000001192,
"totalMs": 198600.75499999523,
"wallMs": 30783.805000007153,
"charactersPerSecond": 0.1299395293914591,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.03248488234786478
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 167816.94499999285,
"inferenceMs": 26696.064999997616,
"totalMs": 225296.91000002623,
"wallMs": 26696.150000035763,
"charactersPerSecond": 0.48696315355844244,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.03745870411988019
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 167816.94499999285,
"inferenceMs": 28828.725000023842,
"totalMs": 254125.69499999285,
"wallMs": 28828.774999976158,
"charactersPerSecond": 1.4568802470440598,
"inputModality": "image",
"grade": "slow",
"imagesPerSecond": 0.03468762492962047
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 167816.94499999285,
"inferenceMs": 11565.399999976158,
"totalMs": 265691.14500004053,
"wallMs": 11565.445000052452,
"charactersPerSecond": 0.6917184014402002,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.3458592007201001
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "fail",
"quality": {
"passed": false,
"nonEmpty": true,
"gibberish": false,
"grade": "failed",
"score": 0,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 167816.94499999285,
"inferenceMs": 11957.264999985695,
"totalMs": 277648.4250000119,
"wallMs": 11957.274999976158,
"charactersPerSecond": 0.8363116481914521,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.3345246592765808
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 167816.94499999285,
"inferenceMs": 861.2700000405312,
"totalMs": 278509.77000004053,
"wallMs": 861.3400000333786,
"charactersPerSecond": 5.805380426306154,
"inputModality": "text",
"grade": "good"
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 31029.585000038147,
"inferenceMs": 28692.430000007153,
"totalMs": 59722.21500003338,
"wallMs": 28692.625,
"charactersPerSecond": 0.20911439010214555,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.03485239835035759
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 31029.585000038147,
"inferenceMs": 27612.464999973774,
"totalMs": 87334.79000002146,
"wallMs": 27612.569999992847,
"charactersPerSecond": 0.47080186430339876,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.03621552802333836
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 31029.585000038147,
"inferenceMs": 28245.37000000477,
"totalMs": 115580.22500002384,
"wallMs": 28245.42500001192,
"charactersPerSecond": 1.3453532384243359,
"inputModality": "image",
"grade": "slow",
"imagesPerSecond": 0.035404032590114105
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "fail",
"quality": {
"passed": false,
"nonEmpty": true,
"gibberish": false,
"grade": "failed",
"score": 0,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 31029.585000038147,
"inferenceMs": 11748.785000026226,
"totalMs": 127329.06000000238,
"wallMs": 11748.82999998331,
"charactersPerSecond": 0.9362670267585496,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.3404607370031089
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "fail",
"quality": {
"passed": false,
"nonEmpty": true,
"gibberish": false,
"grade": "failed",
"score": 0,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 31029.585000038147,
"inferenceMs": 13568.830000042915,
"totalMs": 140897.905000031,
"wallMs": 13568.84500002861,
"charactersPerSecond": 0.8106815399680893,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.29479328726112336
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 31029.585000038147,
"inferenceMs": 902.3249999880791,
"totalMs": 141800.31499999762,
"wallMs": 902.4099999666214,
"charactersPerSecond": 5.541240683862308,
"inputModality": "text",
"grade": "good"
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 28071.735000014305,
"inferenceMs": 28396.799999952316,
"totalMs": 56468.75,
"wallMs": 28397.014999985695,
"charactersPerSecond": 0.21129141311732572,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.035215235519554286
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 28071.735000014305,
"inferenceMs": 25716.375,
"totalMs": 82185.20500004292,
"wallMs": 25716.455000042915,
"charactersPerSecond": 0.5055144825038521,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.03888572942337324
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 28071.735000014305,
"inferenceMs": 25627.680000007153,
"totalMs": 107812.95000004768,
"wallMs": 25627.740000009537,
"charactersPerSecond": 1.4827717530416094,
"inputModality": "image",
"grade": "slow",
"imagesPerSecond": 0.03902030929056867
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "fail",
"quality": {
"passed": false,
"nonEmpty": true,
"gibberish": false,
"grade": "failed",
"score": 0,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 28071.735000014305,
"inferenceMs": 11670.004999995232,
"totalMs": 119483.00499999523,
"wallMs": 11670.049999952316,
"charactersPerSecond": 0.9425874281977167,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.3427590647991697
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "fail",
"quality": {
"passed": false,
"nonEmpty": true,
"gibberish": false,
"grade": "failed",
"score": 0,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 28071.735000014305,
"inferenceMs": 12865.444999992847,
"totalMs": 132348.46000003815,
"wallMs": 12865.449999988079,
"charactersPerSecond": 0.855003460821302,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.31091034938956436
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 28071.735000014305,
"inferenceMs": 832.1899999976158,
"totalMs": 133180.74000000954,
"wallMs": 832.2749999761581,
"charactersPerSecond": 6.008243309838288,
"inputModality": "text",
"grade": "good"
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 51517.049999952316,
"inferenceMs": 29423.225000023842,
"totalMs": 80940.49000000954,
"wallMs": 29423.445000052452,
"charactersPerSecond": 0.13594702824033594,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.033986757060083986
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 51517.049999952316,
"inferenceMs": 27504.399999976158,
"totalMs": 108444.97499996424,
"wallMs": 27504.474999964237,
"charactersPerSecond": 0.47265164846392826,
"inputModality": "image",
"grade": "very-slow",
"imagesPerSecond": 0.036357819112609865
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 51517.049999952316,
"inferenceMs": 26608.035000026226,
"totalMs": 135053.07499998808,
"wallMs": 26608.100000023842,
"charactersPerSecond": 1.4281400336388066,
"inputModality": "image",
"grade": "slow",
"imagesPerSecond": 0.03758263246417912
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 51517.049999952316,
"inferenceMs": 13060.870000004768,
"totalMs": 148114.0299999714,
"wallMs": 13060.939999997616,
"charactersPerSecond": 0.6125166240837768,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.30625831204188847
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "fail",
"quality": {
"passed": false,
"nonEmpty": true,
"gibberish": false,
"grade": "failed",
"score": 0,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 51517.049999952316,
"inferenceMs": 13550.0150000453,
"totalMs": 161664.06000000238,
"wallMs": 13550.030000030994,
"charactersPerSecond": 0.885607875707878,
"inputModality": "video",
"grade": "very-slow",
"framesPerSecond": 0.2952026252359593
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"nonEmpty": true,
"gibberish": false,
"grade": "fixture-pass",
"score": 1,
"scope": "Fixed English object/OCR/count, color-order or capital-city fixture; not broad model quality certification."
},
"runtime": {
"loadMs": 51517.049999952316,
"inferenceMs": 688.1349999904633,
"totalMs": 162352.25999999046,
"wallMs": 688.1949999928474,
"charactersPerSecond": 7.266016116124444,
"inputModality": "text",
"grade": "good"
}
}
]