Per-case quality & failure details
[
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "```json\n{\n \"name\": \"Maya\",\n \"date\": \"2026-04-17\"\n}\n```"
},
"runtime": {
"loadMs": 1725.199999988079,
"inferenceMs": 508.4000000357628,
"totalMs": 3098.699999988079,
"grade": "excellent",
"outputTokens": 27,
"tokensPerSecond": 53.10778913867175
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "To find the sum of 7 and 5, you simply add them together:\n\n7 + 5 = 12\n\nSo, the answer is \\(\\boxed{12}\\)."
},
"runtime": {
"loadMs": 1725.199999988079,
"inferenceMs": 535,
"totalMs": 3098.699999988079,
"grade": "excellent",
"outputTokens": 39,
"tokensPerSecond": 72.89719626168224
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "The team meeting is on Friday at 10 am in room B."
},
"runtime": {
"loadMs": 1725.199999988079,
"inferenceMs": 211.60000002384186,
"totalMs": 3098.699999988079,
"grade": "excellent",
"outputTokens": 15,
"tokensPerSecond": 70.88846880108642
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "```json\n{\n \"name\": \"Maya\",\n \"date\": \"2026-04-17\"\n}\n```"
},
"runtime": {
"loadMs": 2035.4000000357628,
"inferenceMs": 897.7999999523163,
"totalMs": 4611,
"grade": "excellent",
"outputTokens": 27,
"tokensPerSecond": 30.073513033452905
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "To find the sum of 7 and 5, you simply add them together:\n\n7 + 5 = 12\n\nSo, the answer is \\(\\boxed{12}\\)."
},
"runtime": {
"loadMs": 2035.4000000357628,
"inferenceMs": 904.2999999523163,
"totalMs": 4611,
"grade": "excellent",
"outputTokens": 39,
"tokensPerSecond": 43.12728077193019
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "The team meeting is on Friday at 10 am in room B."
},
"runtime": {
"loadMs": 2035.4000000357628,
"inferenceMs": 577.1999999880791,
"totalMs": 4611,
"grade": "good",
"outputTokens": 15,
"tokensPerSecond": 25.98752598806271
}
},
{
"precision": "fp16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "```json\n{\n \"name\": \"Maya\",\n \"date\": \"2026-04-17\"\n}\n```"
},
"runtime": {
"loadMs": 1338.300000011921,
"inferenceMs": 399.4000000357628,
"totalMs": 2435.599999964237,
"grade": "excellent",
"outputTokens": 27,
"tokensPerSecond": 67.60140209710161
}
},
{
"precision": "fp16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "To find the sum of 7 and 5, you simply add them together:\n\n7 + 5 = 12\n\nSo, the answer is \\(\\boxed{12}\\)."
},
"runtime": {
"loadMs": 1338.300000011921,
"inferenceMs": 438.4000000357628,
"totalMs": 2435.599999964237,
"grade": "excellent",
"outputTokens": 39,
"tokensPerSecond": 88.95985400734158
}
},
{
"precision": "fp16",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "The team meeting is on Friday at 10 am in room B."
},
"runtime": {
"loadMs": 1338.300000011921,
"inferenceMs": 162.70000004768372,
"totalMs": 2435.599999964237,
"grade": "excellent",
"outputTokens": 15,
"tokensPerSecond": 92.19422246837023
}
},
{
"precision": "fp16",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "```json\n{\n \"name\": \"Maya\",\n \"date\": \"2026-04-17\"\n}\n```"
},
"runtime": {
"loadMs": 1639.300000011921,
"inferenceMs": 936.8000000119209,
"totalMs": 4253.200000047684,
"grade": "good",
"outputTokens": 27,
"tokensPerSecond": 28.82152006795092
}
},
{
"precision": "fp16",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "To find the sum of 7 and 5, you simply add them together:\n\n7 + 5 = 12\n\nSo, the answer is \\(\\boxed{12}\\)."
},
"runtime": {
"loadMs": 1639.300000011921,
"inferenceMs": 981,
"totalMs": 4253.200000047684,
"grade": "excellent",
"outputTokens": 39,
"tokensPerSecond": 39.75535168195719
}
},
{
"precision": "fp16",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "The team meeting is on Friday at 10 am in room B."
},
"runtime": {
"loadMs": 1639.300000011921,
"inferenceMs": 594.8999999761581,
"totalMs": 4253.200000047684,
"grade": "good",
"outputTokens": 15,
"tokensPerSecond": 25.21432173575585
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "{\"name\": \"Maya\", \"date\": \"2026-04-17\"}"
},
"runtime": {
"loadMs": 860.5,
"inferenceMs": 205.5,
"totalMs": 1426.699999988079,
"grade": "excellent",
"outputTokens": 20,
"tokensPerSecond": 97.32360097323601
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "To find the sum of 7 and 5, you add the numbers together:\n\n7 + 5 = 12\n\nSo, the answer is 12."
},
"runtime": {
"loadMs": 860.5,
"inferenceMs": 170.5999999642372,
"totalMs": 1426.699999988079,
"grade": "excellent",
"outputTokens": 35,
"tokensPerSecond": 205.15826499025223
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "The team meeting is scheduled for Friday at 10 am in room B."
},
"runtime": {
"loadMs": 860.5,
"inferenceMs": 87.10000002384186,
"totalMs": 1426.699999988079,
"grade": "excellent",
"outputTokens": 16,
"tokensPerSecond": 183.69690006452726
}
},
{
"precision": "q4",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 961.5999999642372,
"grade": "failed"
},
"failure": "Can't create a session. ERROR_CODE: 9, ERROR_MESSAGE: Could not find an implementation for GatherBlockQuantized(1) node with name '/model/embed_tokens/GatherBlockQuantized'"
},
{
"precision": "q4f32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "{\"name\": \"Maya\", \"date\": \"2026-04-17\"}"
},
"runtime": {
"loadMs": 1351.5,
"inferenceMs": 245.19999998807907,
"totalMs": 2000.5,
"grade": "excellent",
"outputTokens": 20,
"tokensPerSecond": 81.56606851946307
}
},
{
"precision": "q4f32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "To find the sum of 7 and 5, you add the numbers together:\n\n7 + 5 = 12\n\nSo, the answer is 12."
},
"runtime": {
"loadMs": 1351.5,
"inferenceMs": 206.69999998807907,
"totalMs": 2000.5,
"grade": "excellent",
"outputTokens": 35,
"tokensPerSecond": 169.32752782785943
}
},
{
"precision": "q4f32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "The team meeting is scheduled for Friday at 10 am in room B."
},
"runtime": {
"loadMs": 1351.5,
"inferenceMs": 98.10000002384186,
"totalMs": 2000.5,
"grade": "excellent",
"outputTokens": 16,
"tokensPerSecond": 163.09887865557005
}
},
{
"precision": "q4f32",
"device": "wasm",
"status": "fail",
"quality": {
"passed": false,
"degenerate": false,
"answer": "{\"name\": \"Maya\", \"date\": \"2026-04-17\"}"
},
"runtime": {
"loadMs": 1442.5,
"inferenceMs": 4328.300000011921,
"totalMs": 15888.300000011921,
"grade": "slow",
"outputTokens": 20,
"tokensPerSecond": 4.620751796304535
},
"failure": "Frozen quality/repetition/interactive speed gate failed"
},
{
"precision": "q4f32",
"device": "wasm",
"status": "fail",
"quality": {
"passed": true,
"degenerate": false,
"answer": "To find the sum of 7 and 5, you add the numbers together:\n\n7 + 5 = 12\n\nSo, the answer is 12."
},
"runtime": {
"loadMs": 1442.5,
"inferenceMs": 6671.5,
"totalMs": 15888.300000011921,
"grade": "acceptable",
"outputTokens": 35,
"tokensPerSecond": 5.246196507532039
},
"failure": "Frozen quality/repetition/interactive speed gate failed"
},
{
"precision": "q4f32",
"device": "wasm",
"status": "fail",
"quality": {
"passed": false,
"degenerate": false,
"answer": "The team meeting is scheduled for Friday at 10 am in room B."
},
"runtime": {
"loadMs": 1442.5,
"inferenceMs": 3363.600000023842,
"totalMs": 15888.300000011921,
"grade": "slow",
"outputTokens": 16,
"tokensPerSecond": 4.756808181676355
},
"failure": "Frozen quality/repetition/interactive speed gate failed"
},
{
"precision": "uint8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "```json\n{\n \"name\": \"Maya\",\n \"date\": \"2026-04-17\"\n}\n```"
},
"runtime": {
"loadMs": 1918.5,
"inferenceMs": 321,
"totalMs": 3004.800000011921,
"grade": "excellent",
"outputTokens": 27,
"tokensPerSecond": 84.11214953271028
}
},
{
"precision": "uint8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "To find the sum of 7 and 5, you simply add them together:\n\n7 + 5 = 12\n\nSo, the answer is \\(\\boxed{12}\\)."
},
"runtime": {
"loadMs": 1918.5,
"inferenceMs": 224,
"totalMs": 3004.800000011921,
"grade": "excellent",
"outputTokens": 39,
"tokensPerSecond": 174.10714285714286
}
},
{
"precision": "uint8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"degenerate": false,
"answer": "The team meeting is on Friday at 10 am in room B."
},
"runtime": {
"loadMs": 1918.5,
"inferenceMs": 97.10000002384186,
"totalMs": 3004.800000011921,
"grade": "excellent",
"outputTokens": 15,
"tokensPerSecond": 154.47991757277973
}
},
{
"precision": "uint8",
"device": "wasm",
"status": "fail",
"quality": {
"passed": false,
"degenerate": false,
"answer": "```json\n{\n \"name\": \"Maya\",\n \"date\": \"2026-04-17\"\n}\n```"
},
"runtime": {
"loadMs": 1462.5,
"inferenceMs": 5896.400000035763,
"totalMs": 19024.30000001192,
"grade": "slow",
"outputTokens": 27,
"tokensPerSecond": 4.5790651922929655
},
"failure": "Frozen quality/repetition/interactive speed gate failed"
},
{
"precision": "uint8",
"device": "wasm",
"status": "fail",
"quality": {
"passed": false,
"degenerate": false,
"answer": "To find the sum of 7 and 5, you simply add them together:\n\n7 + 5 = 12\n\nSo, the answer is \\(\\boxed{12}\\)."
},
"runtime": {
"loadMs": 1462.5,
"inferenceMs": 8155.699999988079,
"totalMs": 19024.30000001192,
"grade": "slow",
"outputTokens": 39,
"tokensPerSecond": 4.781931655168411
},
"failure": "Frozen quality/repetition/interactive speed gate failed"
},
{
"precision": "uint8",
"device": "wasm",
"status": "fail",
"quality": {
"passed": false,
"degenerate": false,
"answer": "The team meeting is on Friday at 10 am in room B."
},
"runtime": {
"loadMs": 1462.5,
"inferenceMs": 3316.400000035763,
"totalMs": 19024.30000001192,
"grade": "slow",
"outputTokens": 15,
"tokensPerSecond": 4.522976721697698
},
"failure": "Frozen quality/repetition/interactive speed gate failed"
}
]