LFM2-VL 450M — Text & Image Chat
18 configurations testedQuality criteria
Nonempty, non-repetitive caption with at least five words and a person label matching the independently specified portrait reference.
| Precision | Device | Result | |
|---|---|---|---|
Q4F16 | GPU · WebGPUAccelerated | Passed | Very fast |
Q4 | GPU · WebGPUAccelerated | Passed | Very fast |
Q4F16 | GPU · WebGPUAccelerated | Passed | Moderate |
Q4F16 | GPU · WebGPUAccelerated | Passed | Moderate |
Q4 | GPU · WebGPUAccelerated | Passed | Moderate |
Q4 | GPU · WebGPUAccelerated | Passed | Moderate |
Q4 | GPU · WebGPUAccelerated | Passed | Moderate |
Q4F16 | GPU · WebGPUAccelerated | Passed | Slow |
Q4F16 | GPU · WebGPUAccelerated | Passed | Slow |
Q4 | GPU · WebGPUAccelerated | Passed | Very slow |
Q4 | CPU · WASM | Failed | — |
Q4F16 | CPU · WASM | Failed | — |
Q8 | GPU · WebGPU | Failed | — |
Q8 | CPU · WASM | Failed | — |
FP16 | GPU · WebGPU | Failed | — |
FP16 | CPU · WASM | Failed | — |
FP32 | GPU · WebGPU | Failed | — |
FP32 | CPU · WASM | Failed | — |
Test setup & sample details
- Quality gate
- Nonempty, non-repetitive caption with at least five words and a person label matching the independently specified portrait reference.
- Tested
- 2026-10-31T18:06:16.798Z
- Environment
- Isolated Chrome 156.0.8078.79 module worker per case on macOS arm64; Transformers.js 4.2.0; cold model cache
- Runtime
- Transformers.js 4.2.0
- Artifact revision
- 308cf784b35ce25274540e1bf5592c33a0433469
- Fixture
{ "file": "model-samples/lfm2-vl-450m.json", "url": "https://f005.backblazeb2.com/file/hubters-webai-public/assets/benchmarks/lfm2-vl-450m.json", "size": 417, "sha1": "2c1a441093498eea1887511f171471d1bfc608a3" }- Rerun when
- artifact revision change; runtime version change; fixture or quality-gate change; browser/backend behavior change
Per-case quality & failure details
[
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"nonempty": true,
"person": true,
"repetitive": false,
"referenceLabelRecall": 1,
"passed": true
},
"runtime": {
"loadMs": 51274.90000000596,
"inferenceMs": 5750.800000011921,
"totalMs": 58365.40000000596,
"grade": "slow",
"imagesPerSecond": 0.1738888502465617
}
},
{
"precision": "q4",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 25943.09999999404,
"grade": "failed"
},
"failure": "Can't create a session. ERROR_CODE: 9, ERROR_MESSAGE: Could not find an implementation for GatherBlockQuantized(1) node with name '/model/embed_tokens/Gather_Quant'"
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "pass",
"quality": {
"nonempty": true,
"person": true,
"repetitive": false,
"referenceLabelRecall": 1,
"passed": true
},
"runtime": {
"loadMs": 45174.70000001788,
"inferenceMs": 4149,
"totalMs": 50679.19999998808,
"grade": "good",
"imagesPerSecond": 0.24102193299590263
}
},
{
"precision": "q4f16",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 23055.30000001192,
"grade": "failed"
},
"failure": "Can't create a session. ERROR_CODE: 9, ERROR_MESSAGE: Failed to find kernel for com.microsoft.GatherBlockQuantized(1) (node:'/model/embed_tokens/Gather_Quant' ep:'CPUExecutionProvider'). Kernel not found"
},
{
"precision": "q8",
"device": "webgpu",
"status": "fail",
"quality": {
"nonempty": true,
"person": false,
"repetitive": false,
"referenceLabelRecall": 0,
"passed": false
},
"runtime": {
"loadMs": 72033.90000000596,
"inferenceMs": 4667.5,
"totalMs": 78078.10000002384,
"grade": "good",
"imagesPerSecond": 0.21424745581146223
},
"failure": "Caption quality gate failed"
},
{
"precision": "q8",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 38305.30000001192,
"grade": "failed"
},
"failure": "Can't create a session. ERROR_CODE: 9, ERROR_MESSAGE: Could not find an implementation for GatherBlockQuantized(1) node with name '/model/embed_tokens/Gather_Quant'"
},
{
"precision": "fp16",
"device": "webgpu",
"status": "fail",
"quality": {
"nonempty": true,
"person": false,
"repetitive": false,
"referenceLabelRecall": 0,
"passed": false
},
"runtime": {
"loadMs": 357256.90000000596,
"inferenceMs": 3265.9000000059605,
"totalMs": 361915.5,
"grade": "good",
"imagesPerSecond": 0.30619431090914445
},
"failure": "Caption quality gate failed"
},
{
"precision": "fp16",
"device": "wasm",
"status": "fail",
"quality": {
"nonempty": true,
"person": false,
"repetitive": false,
"referenceLabelRecall": 0,
"passed": false
},
"runtime": {
"loadMs": 5706.700000017881,
"inferenceMs": 63730.5,
"totalMs": 71155.09999999404,
"grade": "very-slow",
"imagesPerSecond": 0.01569107413247974
},
"failure": "Caption quality gate failed"
},
{
"precision": "fp32",
"device": "webgpu",
"status": "fail",
"quality": {
"nonempty": true,
"person": false,
"repetitive": false,
"referenceLabelRecall": 0,
"passed": false
},
"runtime": {
"loadMs": 2779,
"inferenceMs": 4625.4000000059605,
"totalMs": 8756.699999988079,
"grade": "good",
"imagesPerSecond": 0.21619751805221415
},
"failure": "Caption quality gate failed"
},
{
"precision": "fp32",
"device": "wasm",
"status": "fail",
"quality": {
"nonempty": true,
"person": false,
"repetitive": false,
"referenceLabelRecall": 0,
"passed": false
},
"runtime": {
"loadMs": 3455.7000000178814,
"inferenceMs": 62562.40000000596,
"totalMs": 67572.09999999404,
"grade": "very-slow",
"imagesPerSecond": 0.015984041532931998
},
"failure": "Caption quality gate failed"
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"scope": "Fixed authored/retained fixture; not broad task-quality certification."
},
"runtime": {
"loadMs": 12807.475000023842,
"inferenceMs": 1263.2950000166893,
"totalMs": 14070.770000040531,
"wallMs": 1263.5149999856949,
"charactersPerSecond": 3.166322988650439,
"version": "4.2.0",
"artifactRevision": "308cf784b35ce25274540e1bf5592c33a0433469",
"grade": "slow",
"totalTimingMethod": "Sum of measured initialization and this inference; excludes preceding fixture calls.",
"imagesPerSecond": 0.7915807471626097
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"scope": "Fixed authored/retained fixture; not broad task-quality certification."
},
"runtime": {
"loadMs": 12807.475000023842,
"inferenceMs": 886.0600000023842,
"totalMs": 13693.535000026226,
"wallMs": 886.1299999952316,
"charactersPerSecond": 14.671692661857008,
"version": "4.2.0",
"artifactRevision": "308cf784b35ce25274540e1bf5592c33a0433469",
"grade": "good",
"totalTimingMethod": "Sum of measured initialization and this inference; excludes preceding fixture calls.",
"imagesPerSecond": 1.1285917432197698
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"scope": "Fixed authored/retained fixture; not broad task-quality certification."
},
"runtime": {
"loadMs": 12807.475000023842,
"inferenceMs": 999.2749999761581,
"totalMs": 13806.75,
"wallMs": 999.3400000333786,
"charactersPerSecond": 1.000725526030231,
"version": "4.2.0",
"artifactRevision": "308cf784b35ce25274540e1bf5592c33a0433469",
"grade": "slow",
"totalTimingMethod": "Sum of measured initialization and this inference; excludes preceding fixture calls.",
"imagesPerSecond": 1.000725526030231
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"scope": "Fixed authored/retained fixture; not broad task-quality certification."
},
"runtime": {
"loadMs": 12807.475000023842,
"inferenceMs": 83.9549999833107,
"totalMs": 12891.430000007153,
"wallMs": 84.06999999284744,
"charactersPerSecond": 59.55571438263288,
"version": "4.2.0",
"artifactRevision": "308cf784b35ce25274540e1bf5592c33a0433469",
"grade": "fast",
"totalTimingMethod": "Sum of measured initialization and this inference; excludes preceding fixture calls."
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"scope": "Fixed authored/retained fixture; not broad task-quality certification."
},
"runtime": {
"loadMs": 13256.759999990463,
"inferenceMs": 2540.3799999952316,
"totalMs": 15797.139999985695,
"wallMs": 2540.564999997616,
"charactersPerSecond": 1.574567584380096,
"version": "4.2.0",
"artifactRevision": "308cf784b35ce25274540e1bf5592c33a0433469",
"grade": "slow",
"totalTimingMethod": "Sum of measured initialization and this inference; excludes preceding fixture calls.",
"imagesPerSecond": 0.393641896095024
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"scope": "Fixed authored/retained fixture; not broad task-quality certification."
},
"runtime": {
"loadMs": 13256.759999990463,
"inferenceMs": 757.9150000214577,
"totalMs": 14014.675000011921,
"wallMs": 757.9850000143051,
"charactersPerSecond": 17.152319190980457,
"version": "4.2.0",
"artifactRevision": "308cf784b35ce25274540e1bf5592c33a0433469",
"grade": "good",
"totalTimingMethod": "Sum of measured initialization and this inference; excludes preceding fixture calls.",
"imagesPerSecond": 1.319409168536958
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"scope": "Fixed authored/retained fixture; not broad task-quality certification."
},
"runtime": {
"loadMs": 13256.759999990463,
"inferenceMs": 531.5099999904633,
"totalMs": 13788.269999980927,
"wallMs": 531.5749999880791,
"charactersPerSecond": 1.8814321461834074,
"version": "4.2.0",
"artifactRevision": "308cf784b35ce25274540e1bf5592c33a0433469",
"grade": "slow",
"totalTimingMethod": "Sum of measured initialization and this inference; excludes preceding fixture calls.",
"imagesPerSecond": 1.8814321461834072
}
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"scope": "Fixed authored/retained fixture; not broad task-quality certification."
},
"runtime": {
"loadMs": 13256.759999990463,
"inferenceMs": 31.514999985694885,
"totalMs": 13288.274999976158,
"wallMs": 31.584999978542328,
"charactersPerSecond": 158.65460898840462,
"version": "4.2.0",
"artifactRevision": "308cf784b35ce25274540e1bf5592c33a0433469",
"grade": "fast",
"totalTimingMethod": "Sum of measured initialization and this inference; excludes preceding fixture calls."
}
}
]