Qwen3 Reranker Desktop
Score and reorder candidate passages by relevance to a query using Qwen3 0.6B and the original yes/no causal-language-model scoring prompt. Runs locally after model downloads; returns complete passage scores and original indices.
Desktop app
Run Qwen3 Reranker Desktop on your computer
Download the desktop app to load this model, provide input and view results locally.
Public desktop downloads are coming soon.Device requirements
If your device falls short
MemoryAdvisory only: close other apps, reduce image/tile/token/batch settings or choose a smaller verified model/precision. CPU can need more RAM than GPU.
GPUIf the selected GPU runtime fails, existing CPU fallback may recover when enabled, but CPU has its own memory/runtime needs.
PlatformUse a compatible OS/architecture/runtime or another model. These estimates do not add hardware detection or block runs.
Benchmark evidence
Saved desktop measurements apply to the recorded hardware and runtime. Fallback cases report the device that actually ran; they do not prove GPU support on the requested device.
Qwen3 Reranker 0.6B Desktop
130 configurations testedQuality criteria
{ "version": 1, "expectedTop1Accuracy": 1, "minimumPositiveLogitMargin": 0.5, "maximumCpuScoreDifference": 0.08, "allScoresFinite": true, "minimumUsablePairsPerSecond": 0.25, "goodPairsPerSecond": 2, "scope": "Four short authored factual/code/negation/Chinese retrieval queries with hard negatives, plus all-irrelevant stability; not broad retrieval, multilingual or safety certification. All complete rankings retained; no calibrated relevance threshold or unknown rejection." }
| Precision | Device | Result | Platform | Inference / load | |
|---|---|---|---|---|---|
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 86 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 87 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 87 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 87 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 88 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 89 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 89 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 89 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 89 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 89 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 90 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 90 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 91 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 92 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 92 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 93 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 93 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 93 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 93 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 94 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 94 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Very fast | macos · aarch64 | 94 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 125 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 125 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 125 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 126 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 126 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 126 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 127 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 127 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 127 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 127 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 127 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 127 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 128 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 128 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 128 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 128 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 128 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 128 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 128 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 128 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 128 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 129 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 129 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 129 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 129 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 129 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 129 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 130 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 130 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 130 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 132 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 159 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 164 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 179 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 189 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 191 ms / 0 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 195 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 210 ms / 911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 223 ms / 1006 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 232 ms / 927 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 237 ms / 885 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 238 ms / 889 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 240 ms / 897 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 241 ms / 852 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 244 ms / 1024 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 271 ms / 1087 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 306 ms / 0 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 317 ms / 0 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 363 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 310 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 311 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 311 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 319 ms / 693 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 319 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 320 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 321 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 323 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 324 ms / 727 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 326 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 330 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 330 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 331 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 332 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 332 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 333 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 335 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 335 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 335 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 336 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 336 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 336 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 336 ms / 735 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 336 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 337 ms / 763 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 337 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 338 ms / 701 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 340 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 342 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 343 ms / 790 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 345 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 356 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 358 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 358 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 359 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 364 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 365 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 370 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 371 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 388 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 397 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 409 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 414 ms / 0 ms |
FP32 | cpu | Passed | Fast | macos · aarch64 | 430 ms / 0 ms |
FP32 | cpu | Failed | — | macos · aarch64 | 341 ms / 0 ms |
FP32 | cpu | Failed | — | macos · aarch64 | 320 ms / 0 ms |
FP32 | mps | Failed | — | macos · aarch64 | 193 ms / 0 ms |
FP16 | mps | Failed | — | macos · aarch64 | 163 ms / 0 ms |
BF16 | mps | Failed | — | macos · aarch64 | 171 ms / 0 ms |
Test setup & sample details
- Quality gate
- { "version": 1, "expectedTop1Accuracy": 1, "minimumPositiveLogitMargin": 0.5, "maximumCpuScoreDifference": 0.08, "allScoresFinite": true, "minimumUsablePairsPerSecond": 0.25, "goodPairsPerSecond": 2, "scope": "Four short authored factual/code/negation/Chinese retrieval queries with hard negatives, plus all-irrelevant stability; not broad retrieval, multilingual or safety certification. All complete rankings retained; no calibrated relevance threshold or unknown rejection." }
- Tested
- 2026-11-02T17:55:32.592776+00:00
- Environment
- { "hardware": "Apple M3 Max", "os": "macOS-27.2-arm64-arm-64bit-Mach-O", "runtime": "Transformers5.22.1/Torch2.10.0", "interpreterScope": "Private isolated app-ownedCPython3.13.18 probe venv; normal managed packaged provisioning verified separately" }
- Runtime
- transformers 5.22.1
- Artifact revision
- e61197ed45024b0ed8a2d74b80b4d909f1255473
- Fixture
{ "file": "model-samples/qwen3-reranker-capital.json", "url": "https://assets.localai.tools/samples/v1/files/fcb30f2647fd37f102ebd90d61b0fd0bcb050ce208ff9caef38f79144fff9358/qwen3-reranker-capital.json", "size": 356, "sha1": "202d7afe4b252011e63de03681b2849227d4a46b", "sha256": "fcb30f2647fd37f102ebd90d61b0fd0bcb050ce208ff9caef38f79144fff9358" }- Rerun when
- Exact original/ONNX artifact revision or tokenizer; Pinned runtime or relevant browser/backend/OS behavior; Fixture bytes/expected ranking or quality gate; Prompt/truncation/yes-no scoring/preprocessing
Per-case quality & failure details
[
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 790.1897499978077,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 344.20116600813344,
"inferenceMs": 343.10774998448323,
"pairsPerSecond": 8.743608968715142,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 1134.390916005941,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 2.9982118606567383,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 410.04079100093804,
"inferenceMs": 409.24254099081736,
"pairsPerSecond": 7.330616198249327,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 410.04079100093804,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 3.133108139038086,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 388.9655000093626,
"inferenceMs": 388.1438320095185,
"pairsPerSecond": 7.729093579738839,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 388.9655000093626,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 2,
"topIndex": 2,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 13.028543472290039,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 335.80820899805985,
"inferenceMs": 335.0870410067728,
"pairsPerSecond": 8.952897703792024,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 335.80820899805985,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": null,
"topIndex": 2,
"top1Accuracy": null,
"reciprocalRank": null,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 330.9867499920074,
"inferenceMs": 330.3401259909151,
"pairsPerSecond": 9.081548876331496,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 330.9867499920074,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 342.1672920085257,
"inferenceMs": 341.5180840092944,
"pairsPerSecond": 8.78430789017414,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 342.1672920085257,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 345.26108400314115,
"inferenceMs": 344.632709005964,
"pairsPerSecond": 8.704919532022958,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 345.26108400314115,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 364.299500011839,
"inferenceMs": 363.53416700148955,
"pairsPerSecond": 8.25231923795407,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 364.299500011839,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 370.7476670097094,
"inferenceMs": 370.0802489911439,
"pairsPerSecond": 8.106349928638831,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 370.7476670097094,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1785.344250005437,
"durationMs": 358.45133299881127,
"inferenceMs": 357.6907920069061,
"pairsPerSecond": 8.387132313828413,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 358.45133299881127,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2847031296
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 762.6786249893485,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.096458003507,
"durationMs": 337.4808749940712,
"inferenceMs": 336.6192909743404,
"pairsPerSecond": 8.912145205096644,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 1100.1594999834197,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846408704
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.096458003507,
"durationMs": 371.2950840126723,
"inferenceMs": 370.6117919937242,
"pairsPerSecond": 8.094723548490872,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 371.2950840126723,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846408704
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.096458003507,
"durationMs": 358.63533298834227,
"inferenceMs": 357.9156680061715,
"pairsPerSecond": 8.38186273518563,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 358.63533298834227,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846408704
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.096458003507,
"durationMs": 359.3918330007,
"inferenceMs": 358.56437499751337,
"pairsPerSecond": 8.36669844855838,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 359.3918330007,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846408704
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.096458003507,
"durationMs": 356.6448329947889,
"inferenceMs": 355.94120700261556,
"pairsPerSecond": 8.42835822596386,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 356.6448329947889,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846408704
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.096458003507,
"durationMs": 366.2617919908371,
"inferenceMs": 365.46491700573824,
"pairsPerSecond": 8.208722261439108,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 366.2617919908371,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846408704
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 734.9930829950608,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.0631669904105,
"durationMs": 337.31083299790043,
"inferenceMs": 336.4197089977097,
"pairsPerSecond": 8.917432361313956,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 1072.3039159929613,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843230208
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.0631669904105,
"durationMs": 335.54416699917056,
"inferenceMs": 334.89312598248944,
"pairsPerSecond": 8.95808174980833,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 335.54416699917056,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843230208
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.0631669904105,
"durationMs": 334.0185830020346,
"inferenceMs": 333.35074901697226,
"pairsPerSecond": 8.999529801108254,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 334.0185830020346,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843230208
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.0631669904105,
"durationMs": 336.244500009343,
"inferenceMs": 335.6231239886256,
"pairsPerSecond": 8.938597449267743,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 336.244500009343,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843230208
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.0631669904105,
"durationMs": 337.152749998495,
"inferenceMs": 336.4817079855129,
"pairsPerSecond": 8.915789265219622,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 337.152749998495,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843230208
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1752.0631669904105,
"durationMs": 336.27708301355597,
"inferenceMs": 335.6357080047019,
"pairsPerSecond": 8.93826231372847,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 336.27708301355597,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843230208
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 693.4515829925658,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 319.6760419959901,
"inferenceMs": 318.7767080089543,
"pairsPerSecond": 9.410976161770675,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 1013.1276249885559,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 2.9982118606567383,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 332.74058399547357,
"inferenceMs": 331.983832991682,
"pairsPerSecond": 9.036584622104675,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 332.74058399547357,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 3.133108139038086,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 331.0189170006197,
"inferenceMs": 330.2942919981433,
"pairsPerSecond": 9.082809096854946,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 331.0189170006197,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 2,
"topIndex": 2,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 13.028543472290039,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 323.7972089991672,
"inferenceMs": 323.0990840093,
"pairsPerSecond": 9.285077391038499,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 323.7972089991672,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": null,
"topIndex": 2,
"top1Accuracy": null,
"reciprocalRank": null,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 336.9600000005448,
"inferenceMs": 336.2661239807494,
"pairsPerSecond": 8.921505278276989,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 336.9600000005448,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 430.9472500026459,
"inferenceMs": 430.1967909996165,
"pairsPerSecond": 6.973552715326216,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 430.9472500026459,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 397.7445830096258,
"inferenceMs": 397.0674999873154,
"pairsPerSecond": 7.555390456523985,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 397.7445830096258,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 414.71899999305606,
"inferenceMs": 413.89545900165103,
"pairsPerSecond": 7.248207088901724,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 414.71899999305606,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 332.2473330044886,
"inferenceMs": 331.5907929936657,
"pairsPerSecond": 9.047295833866258,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 332.2473330044886,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1717.2544999921229,
"durationMs": 341.1412089917576,
"inferenceMs": 340.40454198839143,
"pairsPerSecond": 8.813043393828472,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 341.1412089917576,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846736384
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 701.4084159891354,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1791.4164170069853,
"durationMs": 338.5878330009291,
"inferenceMs": 337.68066800257657,
"pairsPerSecond": 8.884133100498099,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 1039.9962489900645,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843770880
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1791.4164170069853,
"durationMs": 338.06824999919627,
"inferenceMs": 337.32641499955207,
"pairsPerSecond": 8.893463027507003,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 338.06824999919627,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843770880
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1791.4164170069853,
"durationMs": 335.9474579920061,
"inferenceMs": 335.2452069957508,
"pairsPerSecond": 8.948673798751804,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 335.9474579920061,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843770880
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1791.4164170069853,
"durationMs": 326.2949159980053,
"inferenceMs": 325.6662920175586,
"pairsPerSecond": 9.211883678272274,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 326.2949159980053,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843770880
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1791.4164170069853,
"durationMs": 321.2924159888644,
"inferenceMs": 320.6639569980325,
"pairsPerSecond": 9.355588411261348,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 321.2924159888644,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843770880
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1791.4164170069853,
"durationMs": 331.20170899201185,
"inferenceMs": 330.5130420048954,
"pairsPerSecond": 9.076797641030957,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 331.20170899201185,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2843770880
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 726.561916002538,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1764.4653750030557,
"durationMs": 324.75966701167636,
"inferenceMs": 323.8248320121784,
"pairsPerSecond": 9.264267911014237,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 1051.3215830142144,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2844262400
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1764.4653750030557,
"durationMs": 310.755833008443,
"inferenceMs": 310.14545798825566,
"pairsPerSecond": 9.672880652386022,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 310.755833008443,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2844262400
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1764.4653750030557,
"durationMs": 311.73291699087713,
"inferenceMs": 311.02883401035797,
"pairsPerSecond": 9.6454079878012,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 311.73291699087713,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2844262400
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1764.4653750030557,
"durationMs": 320.9387090028031,
"inferenceMs": 320.2900419855723,
"pairsPerSecond": 9.366510371044058,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 320.9387090028031,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2844262400
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1764.4653750030557,
"durationMs": 311.8038329994306,
"inferenceMs": 311.19612499605864,
"pairsPerSecond": 9.640222866007717,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 311.8038329994306,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2844262400
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.749804496765137,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1764.4653750030557,
"durationMs": 319.9561670044204,
"inferenceMs": 319.32049998431467,
"pairsPerSecond": 9.394949588727824,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 319.9561670044204,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2844262400
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1086.7796660022577,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 312.09895900974516,
"inferenceMs": 270.8504170150263,
"pairsPerSecond": 11.076224408521274,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843770880,
"totalMs": 1398.8786250120029,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 2.9982147216796875,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 321.22041599359363,
"inferenceMs": 305.82816698006354,
"pairsPerSecond": 9.809430013016314,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2847637504,
"totalMs": 321.22041599359363,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 3.1331214904785156,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 209.58787499694154,
"inferenceMs": 194.6449569950346,
"pairsPerSecond": 15.412677761163524,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2850160640,
"totalMs": 209.58787499694154,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 2,
"topIndex": 2,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 13.02855110168457,
"cpuScoreDifference": 1.3271346688270569e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 191.65416700707283,
"inferenceMs": 178.86045899649616,
"pairsPerSecond": 16.772851958625296,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 191.65416700707283,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": null,
"topIndex": 2,
"top1Accuracy": null,
"reciprocalRank": null,
"positiveLogitMargin": null,
"cpuScoreDifference": 3.1650415621697903e-10,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 132.0984999911161,
"inferenceMs": 126.27700000302866,
"pairsPerSecond": 23.757295468913952,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 132.0984999911161,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 133.61429200449493,
"inferenceMs": 128.2153749925783,
"pairsPerSecond": 23.39812990582178,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 133.61429200449493,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 133.66450001194607,
"inferenceMs": 128.39262500347104,
"pairsPerSecond": 23.365828059975378,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 133.66450001194607,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 134.22370800981298,
"inferenceMs": 128.73558299907017,
"pairsPerSecond": 23.30358033195584,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 134.22370800981298,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 133.08574999973644,
"inferenceMs": 127.50654100091197,
"pairsPerSecond": 23.528204721501645,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 133.08574999973644,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 130.78937500540633,
"inferenceMs": 125.57183299213648,
"pairsPerSecond": 23.890708039500108,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 130.78937500540633,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 134.38558299094439,
"inferenceMs": 128.3038330002455,
"pairsPerSecond": 23.381998260287826,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 134.38558299094439,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 135.2674579975428,
"inferenceMs": 129.12874901667237,
"pairsPerSecond": 23.232626528525085,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 135.2674579975428,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 135.25800000934396,
"inferenceMs": 129.75408401689492,
"pairsPerSecond": 23.12065953630699,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 135.25800000934396,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 134.0503339888528,
"inferenceMs": 128.58887501352,
"pairsPerSecond": 23.330167556754624,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 134.0503339888528,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 134.8714580090018,
"inferenceMs": 128.90841699845623,
"pairsPerSecond": 23.272336049522096,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 134.8714580090018,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 138.85650000884198,
"inferenceMs": 131.68979100009892,
"pairsPerSecond": 22.78080918207051,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 138.85650000884198,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 130.22641598945484,
"inferenceMs": 125.02641600440256,
"pairsPerSecond": 23.99492919875717,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 130.22641598945484,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 136.1791249946691,
"inferenceMs": 129.20549900445621,
"pairsPerSecond": 23.218826002881904,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 136.1791249946691,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 135.56104199960828,
"inferenceMs": 128.92699999792967,
"pairsPerSecond": 23.268981672172426,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 135.56104199960828,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 133.2028330070898,
"inferenceMs": 127.06679200346116,
"pairsPerSecond": 23.609630436867278,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 133.2028330070898,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 132.0138750015758,
"inferenceMs": 126.11745898902882,
"pairsPerSecond": 23.787348905126414,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 132.0138750015758,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 130.7239580055466,
"inferenceMs": 124.97074998100288,
"pairsPerSecond": 24.005617318100736,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 130.7239580055466,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 130.31783398764674,
"inferenceMs": 124.7032079991186,
"pairsPerSecond": 24.05711968549521,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 130.31783398764674,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 135.70370800152887,
"inferenceMs": 130.4672499973094,
"pairsPerSecond": 22.994276341854896,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 135.70370800152887,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2167.3515000002226,
"durationMs": 136.54787500854582,
"inferenceMs": 130.10516700160224,
"pairsPerSecond": 23.058269468752574,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2852290560,
"totalMs": 136.54787500854582,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1175715840
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 897.1162079978967,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1884.6156250074273,
"durationMs": 268.32925000053365,
"inferenceMs": 239.7008330008248,
"pairsPerSecond": 12.515601061718785,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843770880,
"totalMs": 1165.4454579984304,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1093484544
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1884.6156250074273,
"durationMs": 132.21504099783488,
"inferenceMs": 127.06308400083799,
"pairsPerSecond": 23.610319421966924,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843770880,
"totalMs": 132.21504099783488,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1093484544
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1884.6156250074273,
"durationMs": 133.41770799888764,
"inferenceMs": 128.08099898393266,
"pairsPerSecond": 23.42267802249372,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843770880,
"totalMs": 133.41770799888764,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1093484544
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1884.6156250074273,
"durationMs": 132.75141699705273,
"inferenceMs": 127.8770409990102,
"pairsPerSecond": 23.460036114091977,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843770880,
"totalMs": 132.75141699705273,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1093484544
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1884.6156250074273,
"durationMs": 132.77650000236463,
"inferenceMs": 127.46349997178186,
"pairsPerSecond": 23.536149569595583,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843754496,
"totalMs": 132.77650000236463,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1093484544
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1884.6156250074273,
"durationMs": 132.6017910032533,
"inferenceMs": 127.53229201189242,
"pairsPerSecond": 23.5234539634891,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843754496,
"totalMs": 132.6017910032533,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1093484544
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1023.5523750016,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2203.2024169893702,
"durationMs": 278.24937499826774,
"inferenceMs": 243.96233199513517,
"pairsPerSecond": 12.296980338996853,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843770880,
"totalMs": 1301.8017499998678,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1100660736
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2203.2024169893702,
"durationMs": 132.31404199905228,
"inferenceMs": 127.22045899135992,
"pairsPerSecond": 23.58111284761001,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843770880,
"totalMs": 132.31404199905228,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1100660736
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2203.2024169893702,
"durationMs": 132.49754199932795,
"inferenceMs": 127.36712500918657,
"pairsPerSecond": 23.55395868269477,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843770880,
"totalMs": 132.49754199932795,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1100660736
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2203.2024169893702,
"durationMs": 132.9312919988297,
"inferenceMs": 127.94666600530036,
"pairsPerSecond": 23.447269816907312,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843754496,
"totalMs": 132.9312919988297,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1100660736
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2203.2024169893702,
"durationMs": 132.63483399350662,
"inferenceMs": 127.41141699370928,
"pairsPerSecond": 23.54577062860952,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843754496,
"totalMs": 132.63483399350662,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1100660736
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.74978256225586,
"cpuScoreDifference": 8.940696716308594e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2203.2024169893702,
"durationMs": 133.5231249977369,
"inferenceMs": 128.28979201731272,
"pairsPerSecond": 23.384557358976384,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2843754496,
"totalMs": 133.5231249977369,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1100660736
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 852.1234999934677,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 272.61666700360365,
"inferenceMs": 241.09091798891313,
"pairsPerSecond": 12.443438454773974,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 1124.7401669970714,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 2.9921875,
"cpuScoreDifference": 0.00036454200744628906,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 379.26929199602455,
"inferenceMs": 362.9621649888577,
"pairsPerSecond": 8.26532429376515,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1251704832,
"totalMs": 379.26929199602455,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 3.125,
"cpuScoreDifference": 0.00009971857070922852,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 203.83341700653546,
"inferenceMs": 191.4110840007197,
"pairsPerSecond": 15.67307356134465,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1254227968,
"totalMs": 203.83341700653546,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 2,
"topIndex": 2,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 13.0390625,
"cpuScoreDifference": 0.000021128682419657707,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 200.32370800618082,
"inferenceMs": 188.9518340030918,
"pairsPerSecond": 15.877062087425474,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 200.32370800618082,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": null,
"topIndex": 2,
"top1Accuracy": null,
"reciprocalRank": null,
"positiveLogitMargin": null,
"cpuScoreDifference": 5.740148480981588e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 93.98254100233316,
"inferenceMs": 87.67854099278338,
"pairsPerSecond": 34.2158978243824,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 93.98254100233316,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 99.62658300355542,
"inferenceMs": 93.66420899459627,
"pairsPerSecond": 32.02931015168321,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 99.62658300355542,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 96.29716699419077,
"inferenceMs": 90.31408399459906,
"pairsPerSecond": 33.21741047807566,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 96.29716699419077,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 94.23608300858177,
"inferenceMs": 88.42850002110936,
"pairsPerSecond": 33.925713986823816,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 94.23608300858177,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 96.8782080017263,
"inferenceMs": 89.41650198539719,
"pairsPerSecond": 33.55085396306307,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 96.8782080017263,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1876.5556250000373,
"durationMs": 98.14962500240654,
"inferenceMs": 90.55987499596085,
"pairsPerSecond": 33.1272542076036,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 98.14962500240654,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1066844160
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 885.2396669972222,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1948.0131669988623,
"durationMs": 270.30237499275245,
"inferenceMs": 236.5778740058886,
"pairsPerSecond": 12.680813929054617,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 1155.5420419899747,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989200384
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1948.0131669988623,
"durationMs": 93.30325000337325,
"inferenceMs": 87.9385000152979,
"pairsPerSecond": 34.11475064366706,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 93.30325000337325,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989200384
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1948.0131669988623,
"durationMs": 94.79987500526477,
"inferenceMs": 89.14729101525154,
"pairsPerSecond": 33.6521723300235,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 94.79987500526477,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989200384
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1948.0131669988623,
"durationMs": 94.62220899877138,
"inferenceMs": 89.01599998353049,
"pairsPerSecond": 33.70180642305936,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 94.62220899877138,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989200384
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1948.0131669988623,
"durationMs": 92.76595800474752,
"inferenceMs": 87.21374999731779,
"pairsPerSecond": 34.39824569052774,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 92.76595800474752,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989200384
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1948.0131669988623,
"durationMs": 94.16720799345057,
"inferenceMs": 88.71883300889749,
"pairsPerSecond": 33.814691855776935,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 94.16720799345057,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989200384
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 910.7412910088897,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1912.8601249976782,
"durationMs": 237.75233299238607,
"inferenceMs": 209.6786660113139,
"pairsPerSecond": 14.307607240491148,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 1148.4936240012757,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989003776
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1912.8601249976782,
"durationMs": 93.25275001174305,
"inferenceMs": 88.01091799978167,
"pairsPerSecond": 34.086680018579536,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 93.25275001174305,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989003776
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1912.8601249976782,
"durationMs": 93.11512499698438,
"inferenceMs": 88.10604200698435,
"pairsPerSecond": 34.04987821110139,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 93.11512499698438,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989003776
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1912.8601249976782,
"durationMs": 92.80475000559818,
"inferenceMs": 87.7583340043202,
"pairsPerSecond": 34.18478750807091,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 92.80475000559818,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989003776
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1912.8601249976782,
"durationMs": 93.58745800273027,
"inferenceMs": 88.1794579763664,
"pairsPerSecond": 34.02152915029316,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 93.58745800273027,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989003776
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.7578125,
"cpuScoreDifference": 0.0001227259635925293,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1912.8601249976782,
"durationMs": 92.50587499991525,
"inferenceMs": 87.33887500420678,
"pairsPerSecond": 34.34896545044233,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 92.50587499991525,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 989003776
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 889.0625420026481,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 271.2446670047939,
"inferenceMs": 238.18600100639742,
"pairsPerSecond": 12.595198657033682,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 1160.307209007442,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 3.0625,
"cpuScoreDifference": 0.0006129741668701172,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 330.6220419908641,
"inferenceMs": 317.49020800634753,
"pairsPerSecond": 9.449110316939354,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1251704832,
"totalMs": 330.6220419908641,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 3.0625,
"cpuScoreDifference": 0.0003275275230407715,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 174.25329200341366,
"inferenceMs": 163.79312401113566,
"pairsPerSecond": 18.315787174289696,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1254244352,
"totalMs": 174.25329200341366,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 2,
"topIndex": 2,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 13.0625,
"cpuScoreDifference": 0.00008779787458479404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 170.5465830018511,
"inferenceMs": 158.96354199503548,
"pairsPerSecond": 18.872251853155685,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 170.5465830018511,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": null,
"topIndex": 2,
"top1Accuracy": null,
"reciprocalRank": null,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000004864945367444307,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 91.00133299943991,
"inferenceMs": 85.9462510125013,
"pairsPerSecond": 34.90553647957996,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 91.00133299943991,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 92.59629200096242,
"inferenceMs": 87.44029200170189,
"pairsPerSecond": 34.309126048453834,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 92.59629200096242,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 93.27762499742676,
"inferenceMs": 87.86575100384653,
"pairsPerSecond": 34.14299617001701,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 93.27762499742676,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 93.01920799771324,
"inferenceMs": 88.05762598058209,
"pairsPerSecond": 34.06859958570245,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 93.01920799771324,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 92.91829100402538,
"inferenceMs": 87.81187600106932,
"pairsPerSecond": 34.16394383788666,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 92.91829100402538,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1892.586999994819,
"durationMs": 92.8946670028381,
"inferenceMs": 87.5779159978265,
"pairsPerSecond": 34.25521109767506,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1256357888,
"totalMs": 92.8946670028381,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 656343040
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 926.9398329925025,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2009.4444999995176,
"durationMs": 261.7421250033658,
"inferenceMs": 232.3797499993816,
"pairsPerSecond": 12.909902863773558,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 1188.6819579958683,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 576241664
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2009.4444999995176,
"durationMs": 98.08283300662879,
"inferenceMs": 92.5849159975769,
"pairsPerSecond": 32.40268641685126,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 98.08283300662879,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 576241664
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2009.4444999995176,
"durationMs": 98.77370799949858,
"inferenceMs": 93.09933298209216,
"pairsPerSecond": 32.223646549401764,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 98.77370799949858,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 576241664
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2009.4444999995176,
"durationMs": 94.10858398769051,
"inferenceMs": 89.01870898262132,
"pairsPerSecond": 33.700780816599746,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 94.10858398769051,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 576241664
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2009.4444999995176,
"durationMs": 95.5479580006795,
"inferenceMs": 89.89516699512023,
"pairsPerSecond": 33.372205651087434,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 95.5479580006795,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 576241664
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2009.4444999995176,
"durationMs": 99.95254200475756,
"inferenceMs": 92.46883299783804,
"pairsPerSecond": 32.44336391776612,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239433216,
"totalMs": 99.95254200475756,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 576241664
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1005.765000008978,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2038.3346250018803,
"durationMs": 266.4597910043085,
"inferenceMs": 222.70520900201518,
"pairsPerSecond": 13.470722186712992,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 1272.2247910132864,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 577699840
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2038.3346250018803,
"durationMs": 98.48429200064857,
"inferenceMs": 93.16737500193994,
"pairsPerSecond": 32.20011296805919,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 98.48429200064857,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 577699840
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2038.3346250018803,
"durationMs": 98.93179200298619,
"inferenceMs": 93.74533299705945,
"pairsPerSecond": 32.00159308297622,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 98.93179200298619,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 577699840
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2038.3346250018803,
"durationMs": 98.09812500316184,
"inferenceMs": 92.95624900551047,
"pairsPerSecond": 32.27324716837659,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239449600,
"totalMs": 98.09812500316184,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 577699840
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2038.3346250018803,
"durationMs": 99.3212500034133,
"inferenceMs": 94.0596670116065,
"pairsPerSecond": 31.894648315412542,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239433216,
"totalMs": 99.3212500034133,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 577699840
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 0,
"topIndex": 0,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 10.8125,
"cpuScoreDifference": 0.0009261965751647949,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2038.3346250018803,
"durationMs": 97.5374169938732,
"inferenceMs": 92.23449901037384,
"pairsPerSecond": 32.52579059016283,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1239433216,
"totalMs": 97.5374169938732,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 577699840
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "fail",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 0.25395774841308594,
"cpuScoreDifference": 0,
"passed": false
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2738.862332989811,
"durationMs": 341.36333300557453,
"inferenceMs": 340.6493330257945,
"pairsPerSecond": 8.806710329806622,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
91
],
[
1,
93
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 341.36333300557453,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2846588928
},
"failure": "Default upstream instruction does not meet frozen negation separation gate; use recorded explicit retrieval instruction. Complete diagnostic preserved."
},
{
"precision": "fp32",
"device": "cpu",
"status": "fail",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 0.25395774841308594,
"cpuScoreDifference": 0,
"passed": false
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"cpu/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1816.4006250008242,
"durationMs": 320.3262500028359,
"inferenceMs": 319.62691599619575,
"pairsPerSecond": 9.385942953676988,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
91
],
[
1,
93
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 320.3262500028359,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2845130752
},
"failure": "Default upstream instruction does not meet frozen negation separation gate; use recorded explicit retrieval instruction. Complete diagnostic preserved."
},
{
"precision": "fp32",
"device": "mps",
"status": "fail",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 0.25397396087646484,
"cpuScoreDifference": 5.529727786779404e-10,
"passed": false
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float32": 310
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1950.1326250028796,
"durationMs": 205.71108399599325,
"inferenceMs": 193.2224999909522,
"pairsPerSecond": 15.526142142558333,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
91
],
[
1,
93
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 2849193984,
"totalMs": 205.71108399599325,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1174142976
},
"failure": "Default upstream instruction does not meet frozen negation separation gate; use recorded explicit retrieval instruction. Complete diagnostic preserved."
},
{
"precision": "fp16",
"device": "mps",
"status": "fail",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 0.25,
"cpuScoreDifference": 0.0000020265579223632812,
"passed": false
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.float16": 310
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 1823.2833340007346,
"durationMs": 172.81445799744688,
"inferenceMs": 162.80250099953264,
"pairsPerSecond": 18.427235340866243,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
91
],
[
1,
93
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1244872704,
"totalMs": 172.81445799744688,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 1069285376
},
"failure": "Default upstream instruction does not meet frozen negation separation gate; use recorded explicit retrieval instruction. Complete diagnostic preserved."
},
{
"precision": "bf16",
"device": "mps",
"status": "fail",
"quality": {
"finite": true,
"completeDocuments": true,
"expectedIndex": 1,
"topIndex": 1,
"top1Accuracy": 1,
"reciprocalRank": 1,
"positiveLogitMargin": 0.25,
"cpuScoreDifference": 0.000008966802852228284,
"passed": false
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 0,
"placement": {
"mps:0/torch.bfloat16": 310
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2210.3267909988062,
"durationMs": 181.25033300020732,
"inferenceMs": 170.57141700934153,
"pairsPerSecond": 17.587940890681008,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
91
],
[
1,
93
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 1244872704,
"totalMs": 181.25033300020732,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "e61197ed45024b0ed8a2d74b80b4d909f1255473",
"fallback": false,
"grade": "excellent",
"observedResidentBytesAtCaseEnd": 653557760
},
"failure": "Default upstream instruction does not meet frozen negation separation gate; use recorded explicit retrieval instruction. Complete diagnostic preserved."
}
]Qwen3 Reranker 4B Desktop
450 configurations testedQuality criteria
{ "version": 1, "expectedTop1Accuracy": 1, "minimumPositiveLogitMargin": 0.5, "maximumCpuScoreDifference": 0.08, "allScoresFinite": true, "minimumUsablePairsPerSecond": 0.25, "goodPairsPerSecond": 2, "scope": "Four short authored factual/code/negation/Chinese retrieval queries with hard negatives, plus all-irrelevant stability; not broad retrieval, multilingual or safety certification. All complete rankings retained; no calibrated relevance threshold or unknown rejection." }
| Precision | Device | Result | Platform | Inference / load | |
|---|---|---|---|---|---|
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 507 ms / 1612 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 538 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 539 ms / 5115 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 551 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 555 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 561 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 561 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 562 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 564 ms / 1991 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 571 ms / 5115 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 572 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 573 ms / 1612 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 573 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 574 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 578 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 579 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 580 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 582 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 582 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 584 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 584 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 585 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 590 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 591 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 591 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 597 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 598 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 598 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 603 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 603 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 603 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 604 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 606 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 608 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 609 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 610 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 611 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 612 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 613 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 613 ms / 1991 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 614 ms / 5115 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 618 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 619 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 620 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 620 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 621 ms / 1612 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 624 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 626 ms / 1612 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 631 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 632 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 633 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 634 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 635 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 635 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 639 ms / 1912 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 640 ms / 1702 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 641 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 641 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 641 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 641 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 642 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 643 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 644 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 649 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 649 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 649 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 650 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 651 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 651 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 653 ms / 1912 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 654 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 656 ms / 1912 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 656 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 656 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 656 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 657 ms / 1612 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 659 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 659 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 660 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 662 ms / 1911 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 665 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 665 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 666 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 667 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 669 ms / 1612 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 670 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 670 ms / 1702 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 672 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 673 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 676 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 677 ms / 1702 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 678 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 678 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 679 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 680 ms / 1612 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 680 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 683 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 683 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 685 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 687 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 687 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 688 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 688 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 689 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 689 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 689 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 692 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 693 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 693 ms / 1702 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 693 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 695 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 695 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 695 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 696 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 696 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 698 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 700 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 705 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 705 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 707 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 709 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 711 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 712 ms / 1912 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 713 ms / 1612 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 713 ms / 1912 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 717 ms / 1702 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 719 ms / 1912 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 720 ms / 1912 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 722 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 725 ms / 1702 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 729 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 731 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 731 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 733 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 736 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 738 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 743 ms / 1911 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 743 ms / 5115 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 745 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 747 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 747 ms / 1612 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 748 ms / 1911 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 755 ms / 1702 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 758 ms / 1912 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 758 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 763 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 766 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 769 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 773 ms / 1612 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 781 ms / 1911 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 782 ms / 5115 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 785 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 787 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 788 ms / 1702 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 799 ms / 5115 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 805 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 806 ms / 1612 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 806 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 806 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 807 ms / 1635 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 815 ms / 1912 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 816 ms / 5115 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 832 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 836 ms / 1702 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 838 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 838 ms / 5115 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 841 ms / 1911 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 848 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 849 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 850 ms / 5115 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 852 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 856 ms / 1612 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 857 ms / 5115 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 858 ms / 1912 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 861 ms / 1635 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 861 ms / 1702 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 863 ms / 5115 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 866 ms / 1702 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 870 ms / 5115 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 879 ms / 1911 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 882 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 893 ms / 5115 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 896 ms / 1702 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 898 ms / 1911 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 899 ms / 5115 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 900 ms / 1991 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 902 ms / 1912 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 908 ms / 1612 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 925 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 928 ms / 5115 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 932 ms / 1912 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 950 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 953 ms / 5115 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 967 ms / 1702 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 971 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 972 ms / 5115 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 974 ms / 1991 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 987 ms / 1702 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 990 ms / 1911 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1002 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1007 ms / 5115 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1008 ms / 1702 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1011 ms / 1612 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1037 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1055 ms / 3675 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1063 ms / 1612 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1074 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1075 ms / 3675 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1083 ms / 1635 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1085 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1092 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1108 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1113 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1114 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1120 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1124 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1130 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1130 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1134 ms / 2836 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1137 ms / 1702 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1139 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1140 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1142 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1145 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1146 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1148 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1152 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1154 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1163 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1170 ms / 2836 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1171 ms / 1702 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1173 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1174 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1176 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1177 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1187 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1187 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1190 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1194 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1197 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1198 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1198 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1201 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1215 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1218 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1222 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1228 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1235 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1242 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1252 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1257 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1278 ms / 5115 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1334 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1367 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1370 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1399 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1405 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1419 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1437 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1448 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1460 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1465 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 1530 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 1553 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 1588 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 1682 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 1762 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 1791 ms / 2836 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 1978 ms / 3675 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 1982 ms / 2836 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1723 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1724 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1725 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1732 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1733 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1735 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1736 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1737 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1738 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1738 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1739 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1740 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1740 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1741 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1741 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1741 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1742 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1742 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1743 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1745 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1745 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1745 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1746 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1746 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1747 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1748 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1748 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1748 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1749 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1750 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1752 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1753 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1753 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1754 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1755 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1760 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1762 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1762 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1763 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1764 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1764 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1764 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1766 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1768 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1786 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1787 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1788 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1790 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1792 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1792 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1793 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1794 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1796 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1797 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1798 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1800 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1800 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1800 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1801 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1801 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1801 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1802 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1802 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1803 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1805 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1805 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1805 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1806 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1806 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1807 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1808 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1809 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1810 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1811 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1811 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1812 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1813 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1815 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1815 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1816 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1818 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1820 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1822 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1823 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1824 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1824 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1824 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1825 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1825 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1825 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1826 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1827 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1827 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1828 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1829 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1830 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1836 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1840 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1841 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1842 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1842 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1843 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1844 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1844 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1849 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1849 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1850 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1850 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1851 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1852 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1856 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1863 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1876 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1879 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1879 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1883 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1888 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1889 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1889 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1890 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1890 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1892 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1893 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1894 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1895 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1895 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1896 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1897 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1897 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1898 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1899 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1899 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1900 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1901 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1901 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1902 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1903 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1903 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1903 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1905 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1908 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1912 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1912 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1913 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1913 ms / 2741 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1913 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1914 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1915 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1919 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1920 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1925 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1925 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1926 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1930 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1932 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1949 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1950 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1969 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1971 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1977 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1977 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1978 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1979 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1980 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1981 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1984 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1984 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1985 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1986 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1988 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 1999 ms / 1366 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 2006 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 2013 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 2015 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 2017 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 2023 ms / 1391 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 2088 ms / 1383 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 2107 ms / 1600 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 2659 ms / 2903 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3654 ms / 2741 ms |
Test setup & sample details
- Quality gate
- { "version": 1, "expectedTop1Accuracy": 1, "minimumPositiveLogitMargin": 0.5, "maximumCpuScoreDifference": 0.08, "allScoresFinite": true, "minimumUsablePairsPerSecond": 0.25, "goodPairsPerSecond": 2, "scope": "Four short authored factual/code/negation/Chinese retrieval queries with hard negatives, plus all-irrelevant stability; not broad retrieval, multilingual or safety certification. All complete rankings retained; no calibrated relevance threshold or unknown rejection." }
- Tested
- 2026-11-06T01:27:11.910501+00:00
- Environment
- { "hardware": "Apple M3 Max", "os": "macOS-27.2-arm64-arm-64bit-Mach-O", "runtime": "Transformers5.22.1/Torch2.10.0", "interpreterScope": "Existing app-private pinned CPython venv, managed public runtime tested separately" }
- Runtime
- transformers 5.22.1
- Artifact revision
- 22e683669bc0f0bd69640a1354a6d0aebcfeede5
- Fixture
{ "file": "model-samples/qwen3-reranker-family-capital.json", "size": 356, "sha1": "202d7afe4b252011e63de03681b2849227d4a46b", "sha256": "fcb30f2647fd37f102ebd90d61b0fd0bcb050ce208ff9caef38f79144fff9358", "url": "https://assets.localai.tools/samples/v1/files/fcb30f2647fd37f102ebd90d61b0fd0bcb050ce208ff9caef38f79144fff9358/qwen3-reranker-family-capital.json" }- Rerun when
- Exact original/ONNX artifact revision or tokenizer; Pinned runtime or relevant browser/backend/OS behavior; Fixture bytes/expected ranking or quality gate; Prompt/truncation/yes-no scoring/preprocessing
Per-case quality & failure details
[
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 3658.7025829940103,
"inferenceMs": 3653.7501660059206,
"pairsPerSecond": 0.8210742014907482,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6399.960541981272,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602513408,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1741.766249993816,
"inferenceMs": 1741.0152080119587,
"pairsPerSecond": 1.7231325643764241,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4483.024208981078,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602546176,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1755.552416027058,
"inferenceMs": 1754.8843330005184,
"pairsPerSecond": 1.7095143785747808,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4496.81037501432,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16606724096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1762.988124974072,
"inferenceMs": 1762.282875017263,
"pairsPerSecond": 1.7023373730341744,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4504.246083961334,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16606740480,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1842.934417014476,
"inferenceMs": 1842.2289179870859,
"pairsPerSecond": 1.6284621149460374,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4584.192376001738,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16606937088,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1745.7076670252718,
"inferenceMs": 1744.9517079512589,
"pairsPerSecond": 1.7192452870356445,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4486.965626012534,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16607019008,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1899.1225420031697,
"inferenceMs": 1898.195710091386,
"pairsPerSecond": 1.5804482035498695,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4640.380500990432,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619012096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1883.3219169755466,
"inferenceMs": 1882.617167022545,
"pairsPerSecond": 1.5935263167416311,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4624.579875962809,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618995712,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1897.8946250281297,
"inferenceMs": 1897.1101660863496,
"pairsPerSecond": 1.5813525506475257,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4639.152584015392,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619077632,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1890.203583985567,
"inferenceMs": 1889.44887497928,
"pairsPerSecond": 1.5877645803106997,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4631.461542972829,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619077632,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1913.5617910069413,
"inferenceMs": 1912.8351680119522,
"pairsPerSecond": 1.5683525952306492,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4654.819749994203,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619126784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1879.9320829566568,
"inferenceMs": 1879.1612080531195,
"pairsPerSecond": 1.596456965556516,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4621.190041943919,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619143168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1807.0442500174977,
"inferenceMs": 1806.1720849946141,
"pairsPerSecond": 1.6609713021940242,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4548.30220900476,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619143168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1815.9829590003937,
"inferenceMs": 1815.295459004119,
"pairsPerSecond": 1.6526235358104273,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4557.240917987656,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619143168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1811.6987500106916,
"inferenceMs": 1810.8971680048853,
"pairsPerSecond": 1.6566374132139052,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4552.956708997954,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619143168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1850.9528329595923,
"inferenceMs": 1850.1886659651063,
"pairsPerSecond": 1.6214562629131242,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4592.210791946854,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619143168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1826.462083030492,
"inferenceMs": 1825.7574589806609,
"pairsPerSecond": 1.6431536320683748,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4567.720042017754,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619143168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1809.6227499772795,
"inferenceMs": 1808.8390419725329,
"pairsPerSecond": 1.6585223617953924,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4550.880708964542,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619143168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1725.488415977452,
"inferenceMs": 1724.729958048556,
"pairsPerSecond": 1.7394027314248934,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4466.746374964714,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1724.0902920020744,
"inferenceMs": 1723.3972489484586,
"pairsPerSecond": 1.7407478176203823,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4465.3482509893365,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1724.7572090127505,
"inferenceMs": 1724.0772080258466,
"pairsPerSecond": 1.7400612838186915,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4466.015168000013,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1742.5367910182104,
"inferenceMs": 1741.7880410212092,
"pairsPerSecond": 1.7223680088198918,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4483.794750005472,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1748.6139590037055,
"inferenceMs": 1747.9200420202687,
"pairsPerSecond": 1.7163256487021918,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4489.871917990968,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1735.726708022412,
"inferenceMs": 1734.9849999300204,
"pairsPerSecond": 1.7291215774897208,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4476.984667009674,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1736.5672920132056,
"inferenceMs": 1735.8045830042101,
"pairsPerSecond": 1.7283051498849071,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4477.825251000468,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1732.181874976959,
"inferenceMs": 1731.5277499728836,
"pairsPerSecond": 1.7325740231694127,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4473.439833964221,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1739.0936659649014,
"inferenceMs": 1738.3356669452041,
"pairsPerSecond": 1.7257886707645664,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4480.351624952164,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1793.0313749820925,
"inferenceMs": 1792.327291041147,
"pairsPerSecond": 1.6738014396116943,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4534.289333969355,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1767.055000003893,
"inferenceMs": 1766.3164169644006,
"pairsPerSecond": 1.6984499329717002,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4508.312958991155,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619159552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2741.257958987262,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4675.408333016094,
"durationMs": 1844.237667042762,
"inferenceMs": 1843.5688330209814,
"pairsPerSecond": 1.627278540548997,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4585.495626030024,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618455040,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 2107.7669170335867,
"inferenceMs": 2106.824707996566,
"pairsPerSecond": 1.4239438091898864,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3707.4167919927277,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16603578368,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1824.8316250392236,
"inferenceMs": 1824.004750000313,
"pairsPerSecond": 1.6447325589472754,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3424.4814999983646,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16603561984,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1754.542208975181,
"inferenceMs": 1753.8225829484873,
"pairsPerSecond": 1.710549304797106,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3354.192083934322,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16603529216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1746.8674159608781,
"inferenceMs": 1746.1821250617504,
"pairsPerSecond": 1.718033850503372,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3346.517290920019,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16603545600,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1751.167708018329,
"inferenceMs": 1750.3915420384146,
"pairsPerSecond": 1.7139022486970867,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3350.81758297747,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16603709440,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1753.769749950152,
"inferenceMs": 1752.974751056172,
"pairsPerSecond": 1.7113766174855012,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3353.419624909293,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16603725824,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1903.6447500111535,
"inferenceMs": 1902.774250076618,
"pairsPerSecond": 1.576645258826264,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3503.2946249702945,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615849984,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1925.975207996089,
"inferenceMs": 1925.244125071913,
"pairsPerSecond": 1.558243944719448,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3525.62508295523,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615849984,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1969.1781669971533,
"inferenceMs": 1968.5027499799617,
"pairsPerSecond": 1.5240009189880677,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3568.8280419562943,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615882752,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1901.5661670127884,
"inferenceMs": 1900.7457500556484,
"pairsPerSecond": 1.5783278746840121,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3501.2160419719294,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615882752,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1896.0943330312148,
"inferenceMs": 1895.3910419950262,
"pairsPerSecond": 1.5827868410954917,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3495.744207990356,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615882752,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1893.910791957751,
"inferenceMs": 1893.190500966739,
"pairsPerSecond": 1.5846265858972353,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3493.560666916892,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615882752,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1844.058166956529,
"inferenceMs": 1843.2250420446508,
"pairsPerSecond": 1.627582054045969,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3443.70804191567,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615882752,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1823.0255840462632,
"inferenceMs": 1822.1778339357115,
"pairsPerSecond": 1.646381568323832,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3422.6754590054043,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615882752,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1827.3837500019,
"inferenceMs": 1826.5856670332141,
"pairsPerSecond": 1.6424085955260312,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3427.033624961041,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615882752,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1821.157374943141,
"inferenceMs": 1820.405793027021,
"pairsPerSecond": 1.6479842085162326,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3420.807249902282,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1824.312207987532,
"inferenceMs": 1823.6161679960787,
"pairsPerSecond": 1.645083023856175,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3423.962082946673,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1827.5800409610383,
"inferenceMs": 1826.866707939189,
"pairsPerSecond": 1.6421559312250937,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3427.2299159201793,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1764.8554590414278,
"inferenceMs": 1764.0471659251489,
"pairsPerSecond": 1.7006348004457463,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3364.505334000569,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1740.4409589944407,
"inferenceMs": 1739.7264589671977,
"pairsPerSecond": 1.7244090210486156,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3340.0908339535818,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1746.619916986674,
"inferenceMs": 1745.895957981702,
"pairsPerSecond": 1.7183154507488938,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3346.269791945815,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1738.6940420255996,
"inferenceMs": 1738.0132079706527,
"pairsPerSecond": 1.7261088616828604,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3338.3439169847406,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1740.0557500077412,
"inferenceMs": 1739.3320829723962,
"pairsPerSecond": 1.7248000133897439,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3339.705624966882,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1747.317583009135,
"inferenceMs": 1746.5732910204679,
"pairsPerSecond": 1.7176490762933827,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3346.967457968276,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1787.7712909830734,
"inferenceMs": 1787.010584026575,
"pairsPerSecond": 1.6787813272153436,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3387.4211659422144,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1745.2670829952694,
"inferenceMs": 1744.578832120169,
"pairsPerSecond": 1.7196127482265335,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3344.9169579544105,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1743.5549999936484,
"inferenceMs": 1742.8777909954078,
"pairsPerSecond": 1.7212910827709917,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3343.2048749527894,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1742.0909579959698,
"inferenceMs": 1741.4291250752285,
"pairsPerSecond": 1.722722996188778,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3341.740832955111,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1748.247833980713,
"inferenceMs": 1747.582168027293,
"pairsPerSecond": 1.716657479623097,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3347.897708939854,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1599.649874959141,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3181.8459590431303,
"durationMs": 1752.3223340394907,
"inferenceMs": 1751.5651660505682,
"pairsPerSecond": 1.7127538604597878,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3351.9722089986317,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615948288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 2660.1578340050764,
"inferenceMs": 2659.203833027277,
"pairsPerSecond": 1.1281572186156017,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5562.66516802134,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16603365376,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1768.8838330213912,
"inferenceMs": 1768.1611250154674,
"pairsPerSecond": 1.6966779540375636,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4671.391167037655,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16603348992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1798.9684580243193,
"inferenceMs": 1798.2082080561668,
"pairsPerSecond": 1.6683273864281547,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4701.475792040583,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16607576064,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1800.8037499967031,
"inferenceMs": 1800.0770409707911,
"pairsPerSecond": 1.6665953354874656,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4703.311084012967,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16607739904,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1749.9403330148198,
"inferenceMs": 1749.2581240367144,
"pairsPerSecond": 1.7150127581382806,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4652.447667031083,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16607772672,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1749.023667012807,
"inferenceMs": 1748.2072910061106,
"pairsPerSecond": 1.71604363820807,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4651.5310010290705,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16607772672,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1901.1011670227163,
"inferenceMs": 1900.2526669646613,
"pairsPerSecond": 1.578737423793228,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4803.60850103898,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619749376,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1903.207833005581,
"inferenceMs": 1902.5162910693325,
"pairsPerSecond": 1.5768590335243928,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4805.7151670218445,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619782144,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1930.9937080251984,
"inferenceMs": 1930.182789976243,
"pairsPerSecond": 1.5542569416635015,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4833.501042041462,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619864064,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1905.9667909750715,
"inferenceMs": 1905.2127490285784,
"pairsPerSecond": 1.5746272963634256,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4808.474124991335,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619913216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1895.632416009903,
"inferenceMs": 1894.8662489419803,
"pairsPerSecond": 1.5832252021350233,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4798.139750026166,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619896832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1894.6987909730524,
"inferenceMs": 1893.9603330218233,
"pairsPerSecond": 1.5839824877501445,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4797.206124989316,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619896832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1825.4395840340294,
"inferenceMs": 1824.630375020206,
"pairsPerSecond": 1.6441686168722134,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4727.946918050293,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619896832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1842.0049169799313,
"inferenceMs": 1841.2829580483958,
"pairsPerSecond": 1.6292987380819222,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4744.512250996195,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619896832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1844.9985419865698,
"inferenceMs": 1844.1612500464544,
"pairsPerSecond": 1.6267557947681799,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4747.505876002833,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619962368,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1863.6976249981672,
"inferenceMs": 1863.0044999881648,
"pairsPerSecond": 1.610302068523752,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4766.204959014431,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620011520,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1840.8704579924233,
"inferenceMs": 1840.0873329374008,
"pairsPerSecond": 1.630357400053935,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4743.377792008687,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620011520,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1897.5230000214651,
"inferenceMs": 1896.7483330052346,
"pairsPerSecond": 1.5816542172712802,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4800.030334037729,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620142592,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1742.0743749826215,
"inferenceMs": 1741.3302510394715,
"pairsPerSecond": 1.7228208136906695,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4644.581708998885,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620175360,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1762.7929170266725,
"inferenceMs": 1762.0219170348719,
"pairsPerSecond": 1.7025894916496815,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4665.300251042936,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620175360,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1765.1138749788515,
"inferenceMs": 1764.429500035476,
"pairsPerSecond": 1.7002662900046057,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4667.621208995115,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1764.733750023879,
"inferenceMs": 1764.045584073756,
"pairsPerSecond": 1.7006363254355497,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4667.241084040143,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1754.0212079766206,
"inferenceMs": 1753.3155419514515,
"pairsPerSecond": 1.7110439782339355,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4656.528541992884,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1742.9932079976425,
"inferenceMs": 1742.2470420133322,
"pairsPerSecond": 1.721914245027625,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4645.500542013906,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1737.7320840023458,
"inferenceMs": 1736.9035429437645,
"pairsPerSecond": 1.7272116302529361,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4640.239418018609,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1764.1822089790367,
"inferenceMs": 1763.4828329319134,
"pairsPerSecond": 1.7011790214097464,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4666.6895429953,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1740.5422919546254,
"inferenceMs": 1739.8584999609739,
"pairsPerSecond": 1.7242781525436073,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4643.049625970889,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1760.8638749807142,
"inferenceMs": 1760.14562608907,
"pairsPerSecond": 1.704404428550498,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4663.371208996978,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1745.87350001093,
"inferenceMs": 1745.0696249725297,
"pairsPerSecond": 1.7191291150043513,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4648.3808340271935,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2902.5073340162635,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4414.796874974854,
"durationMs": 1733.4445000160486,
"inferenceMs": 1732.759458012879,
"pairsPerSecond": 1.7313424469432053,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 4635.951834032312,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620158976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 1277.5611669640057,
"inferenceMs": 1234.5113760093227,
"pairsPerSecond": 2.4301112636951063,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 6392.9169999901205,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4231905280,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 751.2416670215316,
"inferenceMs": 743.4352489653975,
"pairsPerSecond": 4.035321171783223,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 5866.5975000476465,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4232167424,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 622.804582992103,
"inferenceMs": 613.5107490699738,
"pairsPerSecond": 4.889889874868086,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 5738.160416018218,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4232396800,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 545.3317090286873,
"inferenceMs": 537.5756680150516,
"pairsPerSecond": 5.580609723422234,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 5660.687542054802,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4232560640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 546.438334044069,
"inferenceMs": 539.4013340119272,
"pairsPerSecond": 5.561721506483453,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 5661.794167070184,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4232740864,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 579.1477920138277,
"inferenceMs": 571.2602089624852,
"pairsPerSecond": 5.251547286040731,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 5694.503625039943,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4232904704,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 1027.4404580122791,
"inferenceMs": 1006.5900839981623,
"pairsPerSecond": 2.980359182641697,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 6142.796291038394,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4265934848,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 1045.7803750177845,
"inferenceMs": 1036.529792996589,
"pairsPerSecond": 2.8942728132560993,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 6161.136208043899,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4266147840,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 793.1270829867572,
"inferenceMs": 782.0106650469825,
"pairsPerSecond": 3.8362648159277506,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 5908.482916012872,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4266295296,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 858.1630000262521,
"inferenceMs": 847.6542499265634,
"pairsPerSecond": 3.5391788577240137,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 5973.518833052367,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4266491904,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 824.6694170520641,
"inferenceMs": 816.0837920149788,
"pairsPerSecond": 3.6760931038622275,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 5940.025250078179,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4266672128,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 1013.2064999779686,
"inferenceMs": 1002.4553330149502,
"pairsPerSecond": 2.992652042637454,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 6128.5623330040835,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4266786816,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 987.7989589585923,
"inferenceMs": 972.4585840594955,
"pairsPerSecond": 3.0849642845216105,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 6103.154791984707,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4289183744,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 891.2520409794524,
"inferenceMs": 881.8821670138277,
"pairsPerSecond": 3.4018150181655287,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 6006.607874005567,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4289380352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 861.6189159802161,
"inferenceMs": 849.5145009947009,
"pairsPerSecond": 3.5314288296283163,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 5976.974749006331,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4289544192,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 847.5665419828147,
"inferenceMs": 838.14704103861,
"pairsPerSecond": 3.579324215333956,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 5962.9223750089295,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4289708032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 808.4268340026028,
"inferenceMs": 798.7709169974551,
"pairsPerSecond": 3.755770191629997,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 5923.782667028718,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4289888256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 865.8011670340784,
"inferenceMs": 856.9872929947451,
"pairsPerSecond": 3.500635335579469,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 5981.157000060193,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4290052096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 1160.9050419647247,
"inferenceMs": 1139.2111250315793,
"pairsPerSecond": 2.633401249410059,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 6276.2608749908395,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4311695360,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 874.7707500006072,
"inferenceMs": 862.5861670006998,
"pairsPerSecond": 3.4779134129072653,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 5990.126583026722,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4311891968,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 858.7497090338729,
"inferenceMs": 848.659833020065,
"pairsPerSecond": 3.534985259434413,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 5974.105542059988,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4312072192,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 847.7529169758782,
"inferenceMs": 837.9449179628864,
"pairsPerSecond": 3.580187594303035,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 5963.108750001993,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4312236032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 876.9387080101296,
"inferenceMs": 869.8719160747714,
"pairsPerSecond": 3.448783602001159,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 5992.2945410362445,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4312416256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 907.8599580097944,
"inferenceMs": 898.8432490732521,
"pairsPerSecond": 3.3376231095834954,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 6023.215791035909,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4312580096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 981.0999999754131,
"inferenceMs": 970.6332490313798,
"pairsPerSecond": 3.090765748024579,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 6096.455833001528,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4312793088,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 961.348083044868,
"inferenceMs": 952.5963760097511,
"pairsPerSecond": 3.1492876474781917,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 6076.703916070983,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4312956928,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 933.403457980603,
"inferenceMs": 924.5242920005694,
"pairsPerSecond": 3.244912033093612,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 6048.759291006718,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4313137152,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 935.4351669899188,
"inferenceMs": 928.0889169313014,
"pairsPerSecond": 3.23244890146885,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 6050.791000016034,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4313317376,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 902.8690829873085,
"inferenceMs": 893.4222919633612,
"pairsPerSecond": 3.357874576206599,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 6018.224916013423,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4313497600,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 5115.355833026115,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 6610.69875000976,
"durationMs": 1286.3848750130273,
"inferenceMs": 1277.7802089694887,
"pairsPerSecond": 2.3478216198225956,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 6401.740708039142,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4313661440,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1233.3836250472814,
"inferenceMs": 1197.9685409460217,
"pairsPerSecond": 2.5042393831401744,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4908.458500052802,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4257988608,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 961.6926250164397,
"inferenceMs": 949.9752929550596,
"pairsPerSecond": 3.1579768676593583,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4636.767500021961,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4258201600,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1087.8774169832468,
"inferenceMs": 1075.416166044306,
"pairsPerSecond": 2.7896177263494875,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4762.952291988768,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4258381824,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1206.8012079689652,
"inferenceMs": 1194.0483329817653,
"pairsPerSecond": 2.5124611099355003,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4881.876082974486,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4258562048,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1409.1322500025854,
"inferenceMs": 1399.114208004903,
"pairsPerSecond": 2.1442138053032243,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 5084.207125008106,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4258758656,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1095.7236669491976,
"inferenceMs": 1085.2143749943934,
"pairsPerSecond": 2.7644307605264618,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4770.7985419547185,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4258922496,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1223.6634170403704,
"inferenceMs": 1201.364333042875,
"pairsPerSecond": 2.4971608674293266,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 4898.738292045891,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4291575808,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1141.015209024772,
"inferenceMs": 1130.108458979521,
"pairsPerSecond": 2.6546124632223145,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 4816.090084030293,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4291788800,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1186.5871250047348,
"inferenceMs": 1173.5192500054836,
"pairsPerSecond": 2.556413113790832,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 4861.662000010256,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4291969024,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1382.4789589853026,
"inferenceMs": 1367.3399160034023,
"pairsPerSecond": 2.1940411194669864,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 5057.5538339908235,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4292165632,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1433.1288340035826,
"inferenceMs": 1419.1414170200005,
"pairsPerSecond": 2.1139542289587903,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 5108.2037090091035,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4292313088,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1212.416458001826,
"inferenceMs": 1197.0737510127947,
"pairsPerSecond": 2.5061112546004987,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 4887.491333007347,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4292493312,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1699.8331249924377,
"inferenceMs": 1681.6454179934226,
"pairsPerSecond": 1.7839670407925043,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 5374.9079999979585,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 4314628096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1127.0484999986365,
"inferenceMs": 1113.875209062826,
"pairsPerSecond": 2.693299909712589,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 4802.123375004157,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4314824704,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1209.0464170323685,
"inferenceMs": 1197.70520797465,
"pairsPerSecond": 2.5047899767197945,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 4884.121292037889,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4314988544,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1198.700791981537,
"inferenceMs": 1186.5125420154072,
"pairsPerSecond": 2.5284182794260293,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 4873.775666987058,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4315152384,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1187.5047500361688,
"inferenceMs": 1175.9404169861227,
"pairsPerSecond": 2.5511496642736815,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 4862.57962504169,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4315348992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1239.9077080190182,
"inferenceMs": 1228.1470830203034,
"pairsPerSecond": 2.442704169131186,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 4914.982583024539,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4315529216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1211.7454590043053,
"inferenceMs": 1190.149875008501,
"pairsPerSecond": 2.5206909339704557,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4886.820334009826,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4336762880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1198.5871249926277,
"inferenceMs": 1187.2842489392497,
"pairsPerSecond": 2.5267748668276173,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4873.6619999981485,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4336943104,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1773.1340000173077,
"inferenceMs": 1762.2433329815976,
"pairsPerSecond": 1.7023755708720436,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 5448.208875022829,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 4337139712,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1073.141041037161,
"inferenceMs": 1054.627875040751,
"pairsPerSecond": 2.8446052593518636,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4748.215916042682,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4337352704,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1253.6542089655995,
"inferenceMs": 1242.4765000469051,
"pairsPerSecond": 2.4145325886539872,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4928.72908397112,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4337532928,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1169.2392500117421,
"inferenceMs": 1154.2994159972295,
"pairsPerSecond": 2.5989790503430354,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4844.314125017263,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4337696768,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1268.61337502487,
"inferenceMs": 1257.2857909835875,
"pairsPerSecond": 2.3860923439316606,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4943.688250030391,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4337876992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1228.0209579621442,
"inferenceMs": 1217.7061660331674,
"pairsPerSecond": 2.4636485251387708,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4903.095832967665,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4338057216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1417.839500005357,
"inferenceMs": 1405.4885420482606,
"pairsPerSecond": 2.1344891190845354,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 5092.914375010878,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4338237440,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1987.9064169945195,
"inferenceMs": 1977.511249948293,
"pairsPerSecond": 1.5170583732853316,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 5662.98129200004,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 4338434048,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1544.9788749683648,
"inferenceMs": 1530.4740410065278,
"pairsPerSecond": 1.96017699067083,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 5220.053749973886,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 4338597888,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3675.074875005521,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 5190.658207982779,
"durationMs": 1464.2511249985546,
"inferenceMs": 1448.2637929031625,
"pairsPerSecond": 2.0714458337636517,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 5139.3260000040755,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4338778112,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1408.1698749796487,
"inferenceMs": 1370.311416045297,
"pairsPerSecond": 2.189283373744317,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4244.592207949609,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4261756928,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1993.1326669757254,
"inferenceMs": 1981.602832034696,
"pairsPerSecond": 1.5139259752266407,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4829.554999945685,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 4262019072,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1472.6806660182774,
"inferenceMs": 1459.9148749839514,
"pairsPerSecond": 2.0549143319284138,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4309.102998988237,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4262232064,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1602.1813750267029,
"inferenceMs": 1588.3738339180127,
"pairsPerSecond": 1.888724137818334,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4438.603707996663,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 4262395904,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1157.0647499756888,
"inferenceMs": 1145.2530829119496,
"pairsPerSecond": 2.6195083381675985,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 3993.4870829456486,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4262592512,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.275016784667969,
"cpuScoreDifference": 1.1920928955078125e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1263.9533340116031,
"inferenceMs": 1252.2545409738086,
"pairsPerSecond": 2.3956790746928074,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16629563392,
"totalMs": 4100.375666981563,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4262756352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1353.5523749887943,
"inferenceMs": 1334.1055419878103,
"pairsPerSecond": 2.248697652158776,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 4189.974707958754,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4295540736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1156.713500036858,
"inferenceMs": 1145.6390419625677,
"pairsPerSecond": 2.6186258412254957,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 3993.135833006818,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4295737344,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1451.8372919992544,
"inferenceMs": 1436.7418340407312,
"pairsPerSecond": 2.088057804764214,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 4288.259624969214,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4295901184,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1481.6928750369698,
"inferenceMs": 1465.3864580322988,
"pairsPerSecond": 2.0472415201846204,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 4318.11520800693,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4296130560,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1157.719500013627,
"inferenceMs": 1147.8095419588499,
"pairsPerSecond": 2.613674037663256,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 3994.141832983587,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4296310784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3968048095703125,
"cpuScoreDifference": 0.0000019073486328125,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1184.9470000015572,
"inferenceMs": 1172.5460410816595,
"pairsPerSecond": 2.5585349273215203,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16645505024,
"totalMs": 4021.369332971517,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4296491008,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1239.576958003454,
"inferenceMs": 1222.2789999796078,
"pairsPerSecond": 2.4544314350897394,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 4075.9992909734137,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4318674944,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1176.9086249987595,
"inferenceMs": 1163.0575830931775,
"pairsPerSecond": 2.5794079705163293,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 4013.3309579687193,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4318953472,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1153.9132919861004,
"inferenceMs": 1139.7470839438029,
"pairsPerSecond": 2.6321629090019414,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 3990.33562495606,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4319117312,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1144.051290990319,
"inferenceMs": 1130.394248990342,
"pairsPerSecond": 2.6539413153238995,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 3980.473623960279,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4319281152,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1117.99862497719,
"inferenceMs": 1107.8407929162495,
"pairsPerSecond": 2.707970332183637,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 3954.42095794715,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4319477760,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.22020149230957,
"cpuScoreDifference": 2.384185791015625e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1806.4584159874357,
"inferenceMs": 1791.2117909872904,
"pairsPerSecond": 1.67484382086746,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16650452992,
"totalMs": 4642.880748957396,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 4319641600,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1199.016499973368,
"inferenceMs": 1177.3445430444553,
"pairsPerSecond": 2.548107109107077,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4035.4388329433277,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4341071872,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1141.745584027376,
"inferenceMs": 1133.777207985986,
"pairsPerSecond": 2.6460224979554194,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 3978.167916997336,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4341235712,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1154.09450000152,
"inferenceMs": 1142.146666010376,
"pairsPerSecond": 2.626632891622647,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 3990.5168329714797,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4341432320,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1132.1752920048311,
"inferenceMs": 1120.2886250102893,
"pairsPerSecond": 2.6778813361355396,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 3968.597624974791,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4341596160,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1183.3223749999888,
"inferenceMs": 1170.2412499580532,
"pairsPerSecond": 2.56357396400745,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4019.7447079699486,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 1819951104,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.067499160766602,
"cpuScoreDifference": 4.284083843231201e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1086.720458988566,
"inferenceMs": 1073.5130419488996,
"pairsPerSecond": 2.794563161108576,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 3923.142791958526,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 1820114944,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1133.3009580266662,
"inferenceMs": 1123.8245419226587,
"pairsPerSecond": 2.6694558519495826,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 3969.723290996626,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 1820311552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1565.0250829639845,
"inferenceMs": 1552.783249935601,
"pairsPerSecond": 1.9320146582753388,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4401.447415933944,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 1820442624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1229.6989159658551,
"inferenceMs": 1215.0917919934727,
"pairsPerSecond": 2.4689492759046763,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4066.121248935815,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 1820590080,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1164.7915409994312,
"inferenceMs": 1151.5466659329832,
"pairsPerSecond": 2.6051918595668893,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 4001.213873969391,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 1820819456,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1101.5672920038924,
"inferenceMs": 1091.7263750452548,
"pairsPerSecond": 2.747941305233779,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 3937.989624973852,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 1820983296,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.979060471057892e-9,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2836.42233296996,
"placement": {
"mps:0/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3932.582082983572,
"durationMs": 1121.8069590395316,
"inferenceMs": 1113.178583967965,
"pairsPerSecond": 2.6949853718047576,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16654598144,
"totalMs": 3958.2292920094915,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 1821163520,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 705.1658749696799,
"inferenceMs": 673.4108760720119,
"pairsPerSecond": 4.4549325034649305,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2407.4724999954924,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2398535680,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 799.4027090026066,
"inferenceMs": 787.611165898852,
"pairsPerSecond": 3.8089861214401206,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2501.709334028419,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2399305728,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 1179.6052919817157,
"inferenceMs": 1170.64229096286,
"pairsPerSecond": 2.5626957296515256,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2881.9119170075282,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2399944704,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 699.0680000162683,
"inferenceMs": 691.7445820290595,
"pairsPerSecond": 4.3368608557803965,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2401.374625042081,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2400239616,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 811.1712499521673,
"inferenceMs": 804.7322910861112,
"pairsPerSecond": 3.7279478321306505,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2513.47787497798,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2400518144,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 707.4299170053564,
"inferenceMs": 695.9176249802113,
"pairsPerSecond": 4.31085503846135,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2409.736542031169,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2400780288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 1026.0128329973668,
"inferenceMs": 1007.5636679539457,
"pairsPerSecond": 2.9774793349705475,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2728.3194580231793,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2434318336,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 703.7572919507511,
"inferenceMs": 692.8240000270307,
"pairsPerSecond": 4.330104037797412,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2406.0639169765636,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2434596864,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 743.7670000363141,
"inferenceMs": 735.9162510256283,
"pairsPerSecond": 4.076550824661059,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2446.0736250621267,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2434875392,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 752.911249990575,
"inferenceMs": 744.6471260045655,
"pairsPerSecond": 4.028753882522346,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2455.2178750163876,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2435137536,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 779.5263330335729,
"inferenceMs": 769.4374999846332,
"pairsPerSecond": 3.8989521566857794,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2481.8329580593854,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2435416064,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 725.0204579904675,
"inferenceMs": 717.3796250717714,
"pairsPerSecond": 4.181886263775417,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2427.32708301628,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2435678208,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 910.6545000104234,
"inferenceMs": 896.0508329910226,
"pairsPerSecond": 3.3480243414159703,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2612.961125036236,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2457616384,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 681.3600840396248,
"inferenceMs": 670.195417071227,
"pairsPerSecond": 4.476306348244047,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2383.6667090654373,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2457911296,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 1001.4849159633741,
"inferenceMs": 986.629207036458,
"pairsPerSecond": 3.0406559816033742,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2703.7915409891866,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2458206208,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 974.0939170005731,
"inferenceMs": 967.103750037495,
"pairsPerSecond": 3.102045669747107,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2676.4005420263857,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2458550272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 839.4836670486256,
"inferenceMs": 832.2164589772001,
"pairsPerSecond": 3.6048313724617045,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2541.790292074438,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2458812416,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 791.5351669653319,
"inferenceMs": 785.2043329039589,
"pairsPerSecond": 3.8206615453903012,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2493.8417919911444,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2459074560,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 879.3344589648768,
"inferenceMs": 865.7300419872627,
"pairsPerSecond": 3.465283465401722,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2581.6410839906894,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2480373760,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 762.2477500117384,
"inferenceMs": 754.8286260571331,
"pairsPerSecond": 3.9744120671079712,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2464.554375037551,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2480668672,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 813.9367920230143,
"inferenceMs": 805.7657500030473,
"pairsPerSecond": 3.723166441349306,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2516.243417048827,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2480947200,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 686.0823750030249,
"inferenceMs": 676.8654159968719,
"pairsPerSecond": 4.432195720299387,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2388.3890000288375,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2481225728,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 868.9226669957861,
"inferenceMs": 861.3343329634517,
"pairsPerSecond": 3.4829680940249905,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2571.2292920215987,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2481487872,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 766.6215000208467,
"inferenceMs": 758.2401250256225,
"pairsPerSecond": 3.9565302613055775,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2468.9281250466593,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2481766400,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 770.1511250343174,
"inferenceMs": 763.4335820330307,
"pairsPerSecond": 3.9296149273535654,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2472.45775006013,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2482028544,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 731.1227499740198,
"inferenceMs": 724.6568760601804,
"pairsPerSecond": 4.1398903385977945,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2433.4293749998324,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2482290688,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 842.5407920149155,
"inferenceMs": 835.6067499844357,
"pairsPerSecond": 3.59020556027806,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2544.847417040728,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2482585600,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 1144.710541004315,
"inferenceMs": 1136.7563749081455,
"pairsPerSecond": 2.639087905042461,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2847.0171660301276,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2482864128,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 814.6609999821521,
"inferenceMs": 805.7539989822544,
"pairsPerSecond": 3.7232207395672767,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2516.9676250079647,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2483142656,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1702.3066250258125,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2768.0760420043953,
"durationMs": 647.6260839845054,
"inferenceMs": 639.6923330030404,
"pairsPerSecond": 4.689754504194974,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2349.932709010318,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2483404800,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 604.8435000120662,
"inferenceMs": 573.01708398154,
"pairsPerSecond": 5.235445999541343,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2216.706083039753,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2401828864,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 513.7198329903185,
"inferenceMs": 506.5130000584759,
"pairsPerSecond": 5.922848968641785,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2125.5824160180055,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2402172928,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 812.4687079689465,
"inferenceMs": 805.5345409666188,
"pairsPerSecond": 3.724235085437906,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2424.3312909966335,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2402746368,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 676.1356249917299,
"inferenceMs": 667.455792077817,
"pairsPerSecond": 4.49467970104938,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2287.998208019417,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2403041280,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 721.9110840233043,
"inferenceMs": 712.8453340264969,
"pairsPerSecond": 4.2084865493255625,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2333.773667050991,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2403319808,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 703.389625006821,
"inferenceMs": 694.6500000194646,
"pairsPerSecond": 4.318721658268103,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2315.252208034508,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2403598336,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 926.5647500287741,
"inferenceMs": 907.8585409442894,
"pairsPerSecond": 3.304479568898052,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2538.427333056461,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2437431296,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 1020.9517500479706,
"inferenceMs": 1011.2471670727246,
"pairsPerSecond": 2.9666337742968953,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2632.8143330756575,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2437890048,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 666.0770000307821,
"inferenceMs": 656.8387089646421,
"pairsPerSecond": 4.56733130836461,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2277.939583058469,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2438184960,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 730.2484170068055,
"inferenceMs": 722.4259169888683,
"pairsPerSecond": 4.152674937942773,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2342.1110000344925,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2438447104,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 754.6519999741577,
"inferenceMs": 747.0695420051925,
"pairsPerSecond": 4.01569041611276,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2366.5145830018446,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2438742016,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 694.5955830160528,
"inferenceMs": 686.5029180189595,
"pairsPerSecond": 4.36997414178092,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2306.4581660437398,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2438987776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 780.8392499573529,
"inferenceMs": 766.3196659414098,
"pairsPerSecond": 3.914815361438695,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2392.70183298504,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2461089792,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 781.4807909890078,
"inferenceMs": 772.9986670310609,
"pairsPerSecond": 3.8809898748240053,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2393.3433740166947,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2461368320,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 684.4006250030361,
"inferenceMs": 675.7849180139601,
"pairsPerSecond": 4.439282262789456,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2296.263208030723,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2461630464,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 793.9532920136116,
"inferenceMs": 786.8751250789501,
"pairsPerSecond": 3.812549036543758,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2405.8158750412986,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2461908992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 696.3986670016311,
"inferenceMs": 684.5491659478284,
"pairsPerSecond": 4.382446359197871,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2308.261250029318,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2462187520,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 678.616208024323,
"inferenceMs": 669.4922499591485,
"pairsPerSecond": 4.481007808802949,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2290.47879105201,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2462482432,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 874.0274999872781,
"inferenceMs": 856.0288750450127,
"pairsPerSecond": 3.504554679703124,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2485.890083014965,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2483634176,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 691.3707500207238,
"inferenceMs": 682.9809159971774,
"pairsPerSecond": 4.392509263044179,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2303.2333330484107,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2483912704,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 1073.1523330323398,
"inferenceMs": 1062.9411660484038,
"pairsPerSecond": 2.8223575262898297,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2685.0149160600267,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2484305920,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 718.3305840007961,
"inferenceMs": 709.2845829902217,
"pairsPerSecond": 4.229613996898842,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2330.193167028483,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2484568064,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 648.5860829707235,
"inferenceMs": 640.8210010267794,
"pairsPerSecond": 4.681494512809564,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2260.4486659984104,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2484846592,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 688.2107919664122,
"inferenceMs": 679.5522930915467,
"pairsPerSecond": 4.414671292406118,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2300.073374994099,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2485108736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 631.1237920308486,
"inferenceMs": 621.3044990436174,
"pairsPerSecond": 4.828550259362263,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2242.9863750585355,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2485403648,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 690.2582090115175,
"inferenceMs": 679.4371249852702,
"pairsPerSecond": 4.41541960201989,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2302.1207920392044,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2485665792,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 627.0102909766138,
"inferenceMs": 620.0697499443777,
"pairsPerSecond": 4.8381653842476755,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2238.8728740043007,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2485944320,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 688.5318330023438,
"inferenceMs": 678.1423339853063,
"pairsPerSecond": 4.423850052789954,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2300.3944160300307,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2486206464,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 632.9933329834603,
"inferenceMs": 626.0430000256747,
"pairsPerSecond": 4.792003105021487,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2244.855916011147,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2486517760,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1611.862583027687,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2662.39879099885,
"durationMs": 703.9203749736771,
"inferenceMs": 694.9464569916017,
"pairsPerSecond": 4.316879336268426,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2315.782958001364,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2486796288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 630.1333750016056,
"inferenceMs": 598.1889989925548,
"pairsPerSecond": 5.015137364699912,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2265.1585830026306,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2402795520,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 753.6857089726254,
"inferenceMs": 738.4323760052212,
"pairsPerSecond": 4.062660437817516,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2388.7109169736505,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2403188736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 591.7161669931374,
"inferenceMs": 584.9332920042798,
"pairsPerSecond": 5.128789968032884,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2226.7413749941625,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2403483648,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 814.2408339772373,
"inferenceMs": 807.2519999695942,
"pairsPerSecond": 3.716311635168445,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2449.2660419782624,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2403844096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 754.6601250069216,
"inferenceMs": 746.6299179941416,
"pairsPerSecond": 4.018054899353149,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2389.6853330079466,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2404089856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.26171875,
"cpuScoreDifference": 0.000041544437408447266,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 627.9050000011921,
"inferenceMs": 620.0237919692881,
"pairsPerSecond": 4.838524003202445,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2262.930208002217,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2404384768,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 748.5865830094554,
"inferenceMs": 731.2633759574965,
"pairsPerSecond": 4.102489060212925,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2383.6117910104804,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2438283264,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 713.485749962274,
"inferenceMs": 704.5687500503846,
"pairsPerSecond": 4.2579237296366985,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2348.510957963299,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2438594560,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 715.5490830191411,
"inferenceMs": 706.5923739573918,
"pairsPerSecond": 4.245729377459857,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2350.574291020166,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2438873088,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 696.1355839739554,
"inferenceMs": 688.5616260115057,
"pairsPerSecond": 4.356908498339509,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2331.1607919749804,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2439135232,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 710.1997080026194,
"inferenceMs": 700.4682919941843,
"pairsPerSecond": 4.282849108643033,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2345.2249160036445,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2439413760,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.390625,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 719.4691250333562,
"inferenceMs": 710.606665990781,
"pairsPerSecond": 4.22174480423312,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2354.4943330343813,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2439692288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 707.8402499901131,
"inferenceMs": 692.7253329777159,
"pairsPerSecond": 4.3307207881431795,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2342.865457991138,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2462089216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 697.7931670262478,
"inferenceMs": 688.7740000383928,
"pairsPerSecond": 4.35556510529256,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2332.818375027273,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2462416896,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 1091.4280000142753,
"inferenceMs": 1083.1844999920577,
"pairsPerSecond": 2.769611271230337,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2726.4532080153003,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2462826496,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 649.1614170372486,
"inferenceMs": 640.8889170270413,
"pairsPerSecond": 4.680998407518755,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2284.1866250382736,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2463105024,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 660.9516249736771,
"inferenceMs": 653.9998750085942,
"pairsPerSecond": 4.587156839992633,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2295.976832974702,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2463367168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.234375,
"cpuScoreDifference": 0.00011104345321655273,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 706.9154169876128,
"inferenceMs": 696.1891250102781,
"pairsPerSecond": 4.309173890005407,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2341.940624988638,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2463645696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 671.4224580209702,
"inferenceMs": 656.005917058792,
"pairsPerSecond": 4.57312948250608,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2306.4476660219952,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2484551680,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 698.3160829986446,
"inferenceMs": 688.684333988931,
"pairsPerSecond": 4.356132195753153,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2333.3412909996696,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2484797440,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 612.0737079763785,
"inferenceMs": 603.8249989505857,
"pairsPerSecond": 4.968326924545743,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2247.0989159774035,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2485092352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 694.4288340164348,
"inferenceMs": 686.961334082298,
"pairsPerSecond": 4.367058015001176,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2329.45404201746,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2485354496,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 616.1896659759805,
"inferenceMs": 608.8290420011617,
"pairsPerSecond": 4.927491615937526,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2251.2148739770055,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2485633024,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.0625,
"cpuScoreDifference": 0.000041682273149490356,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 698.3786249766126,
"inferenceMs": 688.3309590048157,
"pairsPerSecond": 4.358368544598633,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2333.4038329776376,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2485911552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 605.9933330398053,
"inferenceMs": 596.9029989792034,
"pairsPerSecond": 5.025942247116307,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2241.0185410408303,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2486190080,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 691.3608749746345,
"inferenceMs": 682.7725850744173,
"pairsPerSecond": 4.393849527032521,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2326.3860829756595,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2486468608,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 611.3106249831617,
"inferenceMs": 603.1395409954712,
"pairsPerSecond": 4.973973344623622,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2246.3358329841867,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2486714368,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 742.6302499952726,
"inferenceMs": 733.0121669801883,
"pairsPerSecond": 4.092701506387251,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2377.6554579962976,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2487058432,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 869.6943329996429,
"inferenceMs": 861.2165420199744,
"pairsPerSecond": 3.4834444691035906,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2504.719541000668,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2487369728,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 4.928369889967144e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1635.025208001025,
"placement": {
"mps:0/torch.float16": 398
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2697.294374986086,
"durationMs": 860.2336249896325,
"inferenceMs": 852.1947920089588,
"pairsPerSecond": 3.52032191246771,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2495.2588329906575,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2487631872,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 872.4376669852063,
"inferenceMs": 840.6242500059307,
"pairsPerSecond": 3.5687764182140054,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2783.0160000012256,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 581402624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 676.6381250345148,
"inferenceMs": 666.4690829929896,
"pairsPerSecond": 4.501334085187499,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2587.216458050534,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582025216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 631.7198330070823,
"inferenceMs": 624.1499159950763,
"pairsPerSecond": 4.806537537086949,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2542.2981660231017,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582303744,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 686.435166047886,
"inferenceMs": 678.118749987334,
"pairsPerSecond": 4.424003907952751,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2597.0134990639053,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582565888,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 600.3420409979299,
"inferenceMs": 591.4511239971034,
"pairsPerSecond": 5.072270350465497,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2510.920374013949,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582795264,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 704.0099170408212,
"inferenceMs": 694.5456250105053,
"pairsPerSecond": 4.319370667628385,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2614.5882500568405,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 583106560,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 896.4096669806167,
"inferenceMs": 879.3254999327473,
"pairsPerSecond": 3.4117059043885867,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2806.987999996636,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617201664,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 1006.6098750103265,
"inferenceMs": 990.3624579892494,
"pairsPerSecond": 3.0291939842822333,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2917.188208026346,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617627648,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 751.4847089769319,
"inferenceMs": 743.1706669158302,
"pairsPerSecond": 4.036757818295017,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2662.063041992951,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617857024,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 667.8534590173513,
"inferenceMs": 660.1772090070881,
"pairsPerSecond": 4.5442344253477405,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2578.4317920333706,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618119168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 680.4363750270568,
"inferenceMs": 672.0404180232435,
"pairsPerSecond": 4.464017222095473,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2591.014708043076,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618364928,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 674.9844159930944,
"inferenceMs": 662.0462909922935,
"pairsPerSecond": 4.531405191476137,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2585.5627490091138,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618627072,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 761.12808304606,
"inferenceMs": 748.1286249822006,
"pairsPerSecond": 4.010005632482484,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2671.7064160620794,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 640712704,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 649.446125025861,
"inferenceMs": 641.4492910262197,
"pairsPerSecond": 4.676909058860232,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2560.0244580418803,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641024000,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 665.9890420269221,
"inferenceMs": 656.1737919109873,
"pairsPerSecond": 4.571959497594445,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2576.5673750429414,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641269760,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 659.3850000062957,
"inferenceMs": 650.1770419999957,
"pairsPerSecond": 4.614127854732865,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2569.963333022315,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641531904,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 681.2140409601852,
"inferenceMs": 670.0511259841733,
"pairsPerSecond": 4.477270291268581,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2591.7923739762045,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641794048,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 611.1024169949815,
"inferenceMs": 602.6238330523483,
"pairsPerSecond": 4.978229926295327,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2521.680750011001,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642039808,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 719.7633330360986,
"inferenceMs": 704.9164589843713,
"pairsPerSecond": 4.255823455054996,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2630.341666052118,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663273472,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 627.3884170223027,
"inferenceMs": 617.8845419781283,
"pairsPerSecond": 4.855276020331632,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2537.966750038322,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663486464,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 657.0479160291143,
"inferenceMs": 649.1855840431526,
"pairsPerSecond": 4.621174705260529,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2567.6262490451336,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663764992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 787.9349169670604,
"inferenceMs": 781.2151669641025,
"pairsPerSecond": 3.840171218972061,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2698.5132499830797,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664027136,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 910.9667079756036,
"inferenceMs": 898.1938739889301,
"pairsPerSecond": 3.340036140167411,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2821.545040991623,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664387584,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 580.211499996949,
"inferenceMs": 572.4492920562625,
"pairsPerSecond": 5.240638850689938,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2490.7898330129683,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664682496,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 670.7817499991506,
"inferenceMs": 658.9255830040202,
"pairsPerSecond": 4.5528661769711505,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2581.36008301517,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664928256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 587.9893749952316,
"inferenceMs": 580.0709169707261,
"pairsPerSecond": 5.171781436081544,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2498.567708011251,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665206784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 658.3937500254251,
"inferenceMs": 650.9180000284687,
"pairsPerSecond": 4.608875464910774,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2568.9720830414444,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665452544,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 617.4155829939991,
"inferenceMs": 609.7805829485878,
"pairsPerSecond": 4.9198024402376515,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2527.9939160100184,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665698304,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 654.2845829972066,
"inferenceMs": 643.4537080349401,
"pairsPerSecond": 4.662340060424514,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2564.862916013226,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665960448,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1910.5783330160193,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3003.6309579736553,
"durationMs": 609.8572079790756,
"inferenceMs": 603.2035000389442,
"pairsPerSecond": 4.973445942880493,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2520.435540995095,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666206208,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 791.6076660039835,
"inferenceMs": 757.7533749863505,
"pairsPerSecond": 3.9590717758981135,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2703.348333016038,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582664192,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 864.5927499746904,
"inferenceMs": 857.903333962895,
"pairsPerSecond": 3.4968974722853243,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2776.333416986745,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 583417856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 822.2096250392497,
"inferenceMs": 814.941291930154,
"pairsPerSecond": 3.6812467716473503,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2733.950292051304,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 583712768,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 622.0977919874713,
"inferenceMs": 612.7997509902343,
"pairsPerSecond": 4.895563347002418,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2533.8384589995258,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 584024064,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 727.9897920088843,
"inferenceMs": 719.861084013246,
"pairsPerSecond": 4.167470733762846,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2639.7304590209387,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 584302592,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 598.240084014833,
"inferenceMs": 590.0527079938911,
"pairsPerSecond": 5.084291554562376,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2509.9807510268874,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 584581120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 734.3327919952571,
"inferenceMs": 719.2139569669962,
"pairsPerSecond": 4.1712204983497925,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2646.0734590073116,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618479616,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 705.6485000066459,
"inferenceMs": 693.238583044149,
"pairsPerSecond": 4.327514471030163,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2617.3891670187004,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618840064,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 614.7235420066863,
"inferenceMs": 605.8795410208404,
"pairsPerSecond": 4.951479290661193,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2526.464209018741,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 619134976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 722.7660000207834,
"inferenceMs": 712.929958011955,
"pairsPerSecond": 4.20798700669792,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2634.506667032838,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 619380736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 639.7140420158394,
"inferenceMs": 632.0778760127723,
"pairsPerSecond": 4.7462506027332925,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2551.454709027894,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 619642880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 720.9374580415897,
"inferenceMs": 711.8999580270611,
"pairsPerSecond": 4.2140752589930095,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2632.678125053644,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 619921408,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 667.7505420520902,
"inferenceMs": 653.182958078105,
"pairsPerSecond": 4.592893863653546,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2579.4912090641446,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642039808,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 671.0851249517873,
"inferenceMs": 659.4918739283457,
"pairsPerSecond": 4.54895673259797,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2582.8257919638418,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642351104,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 589.6725000347942,
"inferenceMs": 582.3442910914309,
"pairsPerSecond": 5.151591671616448,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2501.4131670468487,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642629632,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 912.3156669666059,
"inferenceMs": 901.8802079372108,
"pairsPerSecond": 3.326384117976853,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2824.0563339786604,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642908160,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 940.4610000201501,
"inferenceMs": 932.3673339677043,
"pairsPerSecond": 3.2176159446014174,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2852.2016670322046,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 643252224,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 619.5368330227211,
"inferenceMs": 612.2062919894233,
"pairsPerSecond": 4.900308995928825,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2531.2775000347756,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 643514368,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 644.3880419828929,
"inferenceMs": 630.9693749644794,
"pairsPerSecond": 4.754588921481151,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2556.1287089949474,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664666112,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 662.3211670084856,
"inferenceMs": 656.0490819974802,
"pairsPerSecond": 4.572828592131956,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2574.06183402054,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664944640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 590.6432080082595,
"inferenceMs": 584.0470010298304,
"pairsPerSecond": 5.136572903739255,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2502.383875020314,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665206784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 646.6839170316234,
"inferenceMs": 635.2969169965945,
"pairsPerSecond": 4.722201414391692,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2558.424584043678,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665485312,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 585.4903340223245,
"inferenceMs": 578.3804570673965,
"pairsPerSecond": 5.186897246167536,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2497.231001034379,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665763840,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 662.1349169872701,
"inferenceMs": 651.3887090259232,
"pairsPerSecond": 4.605544981714766,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2573.8755839993246,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666058752,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 568.6122910119593,
"inferenceMs": 562.0434579905123,
"pairsPerSecond": 5.337665544095066,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2480.352958024014,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666337280,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 648.5507920151576,
"inferenceMs": 639.4111249828711,
"pairsPerSecond": 4.691817021607758,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2560.291459027212,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666599424,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 579.7598329954781,
"inferenceMs": 573.3877921011299,
"pairsPerSecond": 5.232061165806757,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2491.5005000075325,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666877952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 654.5920000062324,
"inferenceMs": 644.3370830384083,
"pairsPerSecond": 4.655948072790299,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2566.332667018287,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 667140096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 588.1851660087705,
"inferenceMs": 582.0040409453213,
"pairsPerSecond": 5.15460338578963,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2499.925833020825,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 667435008,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1911.7406670120545,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2968.9640409778804,
"durationMs": 664.5061660092324,
"inferenceMs": 655.6771250325255,
"pairsPerSecond": 4.575422697339307,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2576.246833021287,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 667729920,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 762.4773749848828,
"inferenceMs": 728.7867079721764,
"pairsPerSecond": 4.116430729571613,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2753.622166987043,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 581615616,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 625.5034579662606,
"inferenceMs": 619.1497509134933,
"pairsPerSecond": 4.84535444869969,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2616.6482499684207,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582270976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 605.1447909558192,
"inferenceMs": 597.9844579705968,
"pairsPerSecond": 5.016852796109814,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2596.2895829579793,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582565888,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 650.5163330002688,
"inferenceMs": 640.6084580230527,
"pairsPerSecond": 4.683047753159768,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2641.661125002429,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582860800,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 571.1090000113472,
"inferenceMs": 564.0457510016859,
"pairsPerSecond": 5.318717488204309,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2562.2537920135073,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 583122944,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.09375,
"cpuScoreDifference": 0.004086017608642578,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 688.0897909868509,
"inferenceMs": 679.7043750411831,
"pairsPerSecond": 4.413683522072711,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8337424384,
"totalMs": 2679.234582989011,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 583401472,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 705.6299170362763,
"inferenceMs": 688.2554569747299,
"pairsPerSecond": 4.358846660201851,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2696.7747090384364,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617070592,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 678.129582956899,
"inferenceMs": 665.4011258506216,
"pairsPerSecond": 4.508558647484888,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2669.274374959059,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617447424,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 587.3470419901423,
"inferenceMs": 579.0491240331903,
"pairsPerSecond": 5.180907587087627,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2578.4918339923024,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617742336,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 738.4401250164956,
"inferenceMs": 730.6911669438705,
"pairsPerSecond": 4.1057017461255985,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2729.5849170186557,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618020864,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 598.1441249605268,
"inferenceMs": 590.9971250221133,
"pairsPerSecond": 5.076166825494708,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2589.288916962687,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618283008,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.3125,
"cpuScoreDifference": 0.0027905702590942383,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 985.8536250540055,
"inferenceMs": 974.3872089893557,
"pairsPerSecond": 3.0788581503565,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8344977408,
"totalMs": 2976.9984170561656,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618725376,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 714.1732499585487,
"inferenceMs": 698.4285839716904,
"pairsPerSecond": 4.295356846565718,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2705.318041960709,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 640843776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 621.8942079576664,
"inferenceMs": 610.8554999809712,
"pairsPerSecond": 4.911145107301896,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2613.0389999598265,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641122304,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 594.3745839758776,
"inferenceMs": 584.1052080504596,
"pairsPerSecond": 5.1360610360126016,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2585.5193759780377,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641400832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 657.2458749869838,
"inferenceMs": 648.5460829571821,
"pairsPerSecond": 4.625731430403326,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2648.390666989144,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641679360,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 580.7564580463804,
"inferenceMs": 573.7458749790676,
"pairsPerSecond": 5.22879576277468,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2571.9012500485405,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641941504,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.25,
"cpuScoreDifference": 0.0029804110527038574,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 673.1899160076864,
"inferenceMs": 664.5659179775976,
"pairsPerSecond": 4.514224877991907,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8349925376,
"totalMs": 2664.3347080098465,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642220032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 620.9933750214987,
"inferenceMs": 607.7257090364583,
"pairsPerSecond": 4.936437533235946,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2612.1381670236588,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663388160,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 642.6872499869205,
"inferenceMs": 632.7332090004347,
"pairsPerSecond": 4.74133482694748,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2633.8320419890806,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663666688,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 560.4592079762369,
"inferenceMs": 551.2619580258615,
"pairsPerSecond": 5.442058818539516,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2551.603999978397,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663961600,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 652.0114579470828,
"inferenceMs": 641.9675839715637,
"pairsPerSecond": 4.673133153297794,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2643.156249949243,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664223744,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 562.3955829651095,
"inferenceMs": 554.9628739827313,
"pairsPerSecond": 5.40576701729664,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2553.5403749672696,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664518656,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.1875,
"cpuScoreDifference": 0.002115819603204727,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 660.9337920090184,
"inferenceMs": 648.6434590187855,
"pairsPerSecond": 4.62503700343815,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2652.0785840111785,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664813568,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 569.3695420050062,
"inferenceMs": 561.3427489879541,
"pairsPerSecond": 5.344328407926718,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2560.5143340071663,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665075712,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 622.6378749706782,
"inferenceMs": 613.4778330451809,
"pairsPerSecond": 4.890152240886361,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2613.7826669728383,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665354240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 645.0807079672813,
"inferenceMs": 633.7417070171796,
"pairsPerSecond": 4.733789755009252,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2636.2254999694414,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665632768,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 911.256332998164,
"inferenceMs": 900.2219160320237,
"pairsPerSecond": 3.332511624715078,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2902.401125000324,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665976832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 568.021624989342,
"inferenceMs": 560.7856249553151,
"pairsPerSecond": 5.349637841089539,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2559.166416991502,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666255360,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.000005560690624406561,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1991.14479200216,
"placement": {
"mps:0/torch.bfloat16": 398
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3094.123291957658,
"durationMs": 642.4538749852218,
"inferenceMs": 634.9512910237536,
"pairsPerSecond": 4.724771872127384,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 8354070528,
"totalMs": 2633.598666987382,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666550272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1902.2980830050074,
"inferenceMs": 1901.3698739581741,
"pairsPerSecond": 1.577809789188862,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3285.3272080537863,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602775552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1857.1880829986185,
"inferenceMs": 1856.4967510174029,
"pairsPerSecond": 1.6159468086092428,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3240.2172080473974,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602906624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1950.3507079789415,
"inferenceMs": 1949.6311250259168,
"pairsPerSecond": 1.5387526191448757,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3333.3798330277205,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602873856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1913.7391669792123,
"inferenceMs": 1912.9229160025716,
"pairsPerSecond": 1.5682806530798896,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3296.7682920279913,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602873856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1850.3814169671386,
"inferenceMs": 1849.6649169828743,
"pairsPerSecond": 1.6219153925963643,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3233.4105420159176,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602873856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1896.704915969167,
"inferenceMs": 1896.0125839803368,
"pairsPerSecond": 1.5822679793095258,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3279.734041017946,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602890240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 2015.5131670180708,
"inferenceMs": 2014.6723330253735,
"pairsPerSecond": 1.4890758913113127,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3398.5422920668498,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16614899712,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 2006.2525829998776,
"inferenceMs": 2005.5052479729056,
"pairsPerSecond": 1.4958823982297202,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3389.2817080486566,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16614899712,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 2088.947082986124,
"inferenceMs": 2088.1955840159208,
"pairsPerSecond": 1.4366470377408516,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3471.976208034903,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16614916096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1977.5029169977643,
"inferenceMs": 1976.7100409953855,
"pairsPerSecond": 1.5176732741688963,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3360.532042046543,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16614948864,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1977.690082974732,
"inferenceMs": 1976.8837499432266,
"pairsPerSecond": 1.517539916085686,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3360.719208023511,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615047168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1988.4253750205971,
"inferenceMs": 1987.6820420031436,
"pairsPerSecond": 1.5092957206458755,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3371.454500069376,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615047168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1888.8417910202406,
"inferenceMs": 1888.0695420084521,
"pairsPerSecond": 1.5889245248926167,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3271.8709160690196,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615047168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 2013.5750000481494,
"inferenceMs": 2012.7490420127288,
"pairsPerSecond": 1.4904987841902189,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3396.6041250969283,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615047168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1890.6438749982044,
"inferenceMs": 1889.9341680225916,
"pairsPerSecond": 1.5873568776942388,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3273.6730000469834,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615047168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1892.727125028614,
"inferenceMs": 1891.931416990701,
"pairsPerSecond": 1.5856811579205068,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3275.756250077393,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615047168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1890.2170829824172,
"inferenceMs": 1889.49637504993,
"pairsPerSecond": 1.5877246654790353,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3273.246208031196,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615047168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1904.2097919736989,
"inferenceMs": 1903.4703749930486,
"pairsPerSecond": 1.5760686582847163,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3287.238917022478,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615047168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1842.7820830256678,
"inferenceMs": 1841.8616669951007,
"pairsPerSecond": 1.6287868159471175,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3225.8112080744468,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16615047168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1876.4921670081094,
"inferenceMs": 1875.7942499942146,
"pairsPerSecond": 1.5993225269825049,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3259.5212920568883,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619241472,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1816.5423329919577,
"inferenceMs": 1815.7660419819877,
"pairsPerSecond": 1.652195233657619,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3199.5714580407366,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619274240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1810.5947090080008,
"inferenceMs": 1809.8447910160758,
"pairsPerSecond": 1.657600704155273,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3193.6238340567797,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619274240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1806.0768749564886,
"inferenceMs": 1805.3690829547122,
"pairsPerSecond": 1.66171007819084,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3189.1060000052676,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16619257856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1879.6521250042133,
"inferenceMs": 1878.9533349336125,
"pairsPerSecond": 1.5966335854243003,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3262.6812500529923,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620732416,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1932.5350829749368,
"inferenceMs": 1931.6870420007035,
"pairsPerSecond": 1.5530466037049222,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3315.5642080237158,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620732416,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1790.7704169629142,
"inferenceMs": 1790.0064160348848,
"pairsPerSecond": 1.675971646317012,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3173.799542011693,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620732416,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1787.0412080083042,
"inferenceMs": 1786.3022910314612,
"pairsPerSecond": 1.6794469866954689,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3170.070333057083,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620716032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1792.2107499907725,
"inferenceMs": 1791.5181660209782,
"pairsPerSecond": 1.6745573988027709,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3175.2398750395514,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620716032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1801.5803749440238,
"inferenceMs": 1800.8308749995194,
"pairsPerSecond": 1.6658976929195533,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3184.6094999928027,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620716032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1383.029125048779,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2446.729374991264,
"durationMs": 1808.4608330391347,
"inferenceMs": 1807.763290940784,
"pairsPerSecond": 1.6595093035873962,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3191.4899580879137,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16620716032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1900.1550420070998,
"inferenceMs": 1899.2042919271626,
"pairsPerSecond": 1.579608898711911,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3291.5169579791836,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16595451904,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1836.65391698014,
"inferenceMs": 1835.9512510942295,
"pairsPerSecond": 1.634030314373541,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3228.0158329522237,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602251264,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1831.1029170290567,
"inferenceMs": 1830.378791957628,
"pairsPerSecond": 1.6390050044184779,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3222.4648330011405,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16606330880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1849.7364579816349,
"inferenceMs": 1849.0029160748236,
"pairsPerSecond": 1.622496089064361,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3241.0983739537187,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16606363648,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1849.610749981366,
"inferenceMs": 1848.8590829074383,
"pairsPerSecond": 1.6226223121787766,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3240.97266595345,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16606543872,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1823.9695419906639,
"inferenceMs": 1823.2465839246288,
"pairsPerSecond": 1.645416493002472,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3215.3314579627477,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16606543872,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1981.7089999560267,
"inferenceMs": 1980.813373986166,
"pairsPerSecond": 1.5145293541525493,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3373.0709159281105,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618553344,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1986.4502919954248,
"inferenceMs": 1985.680541023612,
"pairsPerSecond": 1.5108170413220192,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3377.8122079675086,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618586112,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1980.1934999995865,
"inferenceMs": 1979.4787920545787,
"pairsPerSecond": 1.5155504631025536,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3371.5554159716703,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618602496,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 2023.4733339748345,
"inferenceMs": 2022.7450839593075,
"pairsPerSecond": 1.4831330076095504,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3414.8352499469183,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618651648,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 2017.9057079949416,
"inferenceMs": 2017.1160830068402,
"pairsPerSecond": 1.4872718656469246,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3409.2676239670254,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1980.3505419986323,
"inferenceMs": 1979.6343329944648,
"pairsPerSecond": 1.5154313854832444,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3371.712457970716,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1915.287416020874,
"inferenceMs": 1914.4201669842005,
"pairsPerSecond": 1.5670541147327763,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3306.6493319929577,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1912.312375032343,
"inferenceMs": 1911.5842080209404,
"pairsPerSecond": 1.569378940991511,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3303.674291004427,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1950.0217079767026,
"inferenceMs": 1949.272625031881,
"pairsPerSecond": 1.5390356184532854,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3341.3836239487864,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1919.9732919805683,
"inferenceMs": 1919.1873340168968,
"pairsPerSecond": 1.5631616293136643,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3311.335207952652,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1920.4637500224635,
"inferenceMs": 1919.6050420287065,
"pairsPerSecond": 1.5628214837514147,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3311.8256659945473,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1899.825458007399,
"inferenceMs": 1899.0257910918444,
"pairsPerSecond": 1.5797573756358256,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3291.187373979483,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1807.4763329932466,
"inferenceMs": 1806.619959010277,
"pairsPerSecond": 1.66055953552262,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3198.8382489653304,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1801.424082950689,
"inferenceMs": 1800.7334590074606,
"pairsPerSecond": 1.6659878145728233,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3192.7859989227727,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1890.959250042215,
"inferenceMs": 1890.1583750266582,
"pairsPerSecond": 1.5871685884298923,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3282.321166014299,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622452736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1796.6475000139326,
"inferenceMs": 1795.920334989205,
"pairsPerSecond": 1.67045271527483,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3188.0094159860164,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622452736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1825.9475830127485,
"inferenceMs": 1825.2131270128302,
"pairsPerSecond": 1.6436436685669924,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3217.3094989848323,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622452736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1811.5702090435661,
"inferenceMs": 1810.8610420604236,
"pairsPerSecond": 1.656670462459426,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3202.93212501565,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1788.562625006307,
"inferenceMs": 1787.8252069931477,
"pairsPerSecond": 1.67801639011766,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3179.9245409783907,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1812.5702909892425,
"inferenceMs": 1811.8649999960326,
"pairsPerSecond": 1.6557524981202072,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3203.9322069613263,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1803.844833979383,
"inferenceMs": 1803.1357090221718,
"pairsPerSecond": 1.6637682815493013,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3195.206749951467,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1794.8984590475447,
"inferenceMs": 1794.2095000180416,
"pairsPerSecond": 1.6720455442744193,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3186.2603750196286,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1793.4399580117315,
"inferenceMs": 1792.7672920632176,
"pairsPerSecond": 1.6733906365211688,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3184.8018739838153,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1391.3619159720838,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2455.0319580012,
"durationMs": 1800.211916968692,
"inferenceMs": 1799.568500078749,
"pairsPerSecond": 1.667066299431625,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3191.573832940776,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16622436352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1913.6954590212554,
"inferenceMs": 1912.781541061122,
"pairsPerSecond": 1.568396565734182,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3279.5856260345317,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602398720,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1824.452165979892,
"inferenceMs": 1823.728039918933,
"pairsPerSecond": 1.6449821104540092,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3190.3423329931684,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602628096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1852.3381659761071,
"inferenceMs": 1851.5678330441006,
"pairsPerSecond": 1.6202484977651619,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3218.2283329893835,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602677248,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1825.3072910010815,
"inferenceMs": 1824.606457957998,
"pairsPerSecond": 1.6441901687432585,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3191.197458014358,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602742784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1818.471541046165,
"inferenceMs": 1817.6894999924116,
"pairsPerSecond": 1.6504468997661725,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3184.3617080594413,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602742784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 11.274993896484375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1851.5814590500668,
"inferenceMs": 1850.8270410238765,
"pairsPerSecond": 1.6208970009106856,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3217.471626063343,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16602742784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1985.0347499595955,
"inferenceMs": 1984.21133402735,
"pairsPerSecond": 1.5119357240596474,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3350.924916972872,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16614817792,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1971.492666983977,
"inferenceMs": 1970.7644179579802,
"pairsPerSecond": 1.522251961047921,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3337.3828339972533,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16614834176,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1986.148249998223,
"inferenceMs": 1985.4369590175338,
"pairsPerSecond": 1.5110023949007723,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3352.0384170114994,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1978.7740409956314,
"inferenceMs": 1978.0672080814838,
"pairsPerSecond": 1.516631986892742,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3344.6642080089077,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1999.3609589873813,
"inferenceMs": 1998.620291997213,
"pairsPerSecond": 1.5010354953426959,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3365.2511260006577,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.396820068359375,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1985.0642909877934,
"inferenceMs": 1984.3243340146728,
"pairsPerSecond": 1.5118496248697502,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3350.9544580010697,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1908.8229169719853,
"inferenceMs": 1908.0211680266075,
"pairsPerSecond": 1.5723096002665338,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3274.7130839852616,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1902.653958008159,
"inferenceMs": 1901.9332489697263,
"pairsPerSecond": 1.5773424233605962,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3268.5441250214353,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1912.4660000088625,
"inferenceMs": 1911.764333024621,
"pairsPerSecond": 1.5692310752830454,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3278.356167022139,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1926.6994160134345,
"inferenceMs": 1925.9286239976063,
"pairsPerSecond": 1.5576901254902003,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3292.589583026711,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1925.8026669849642,
"inferenceMs": 1925.0812079408206,
"pairsPerSecond": 1.5583758168877329,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3291.6928339982405,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.220200538635254,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1916.2744169589132,
"inferenceMs": 1915.4766670544632,
"pairsPerSecond": 1.5661897905617768,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3282.1645839721896,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1805.2983329980634,
"inferenceMs": 1804.5077500282787,
"pairsPerSecond": 1.6625032505141564,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3171.18850001134,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1806.0031670029275,
"inferenceMs": 1805.2465410437435,
"pairsPerSecond": 1.661822876705518,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3171.893334016204,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1801.874750002753,
"inferenceMs": 1801.1199170723557,
"pairsPerSecond": 1.6656303511852633,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3167.7649170160294,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1829.636333975941,
"inferenceMs": 1828.9134580409154,
"pairsPerSecond": 1.6403181828042985,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3195.5265009892173,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1803.1309999641962,
"inferenceMs": 1802.4444999173284,
"pairsPerSecond": 1.6644063105064257,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3169.0211669774726,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.06750774383545,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1802.8961250092834,
"inferenceMs": 1802.1975409938022,
"pairsPerSecond": 1.664634387607522,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3168.78629202256,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1800.9712910279632,
"inferenceMs": 1800.2164159552194,
"pairsPerSecond": 1.6664663056125721,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3166.8614580412395,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1815.9202499664389,
"inferenceMs": 1815.2019569533877,
"pairsPerSecond": 1.6527086633573063,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3181.8104169797152,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1813.3654170087539,
"inferenceMs": 1812.6363750197925,
"pairsPerSecond": 1.6550478856892865,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3179.2555840220302,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1829.0359170059673,
"inferenceMs": 1828.3033749903552,
"pairsPerSecond": 1.6408655374362178,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3194.9260840192437,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1806.581582990475,
"inferenceMs": 1805.8617909555323,
"pairsPerSecond": 1.6612567002774976,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3172.4717500037514,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 2,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 1365.8901670132764,
"placement": {
"cpu/torch.float32": 398
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 2391.9003749615513,
"durationMs": 1797.5247090216726,
"inferenceMs": 1796.8299590284005,
"pairsPerSecond": 1.6696070682293105,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 3163.414876034949,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 16618848256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
}
]Qwen3 Reranker 8B Desktop
450 configurations testedQuality criteria
{ "version": 1, "expectedTop1Accuracy": 1, "minimumPositiveLogitMargin": 0.5, "maximumCpuScoreDifference": 0.08, "allScoresFinite": true, "minimumUsablePairsPerSecond": 0.25, "goodPairsPerSecond": 2, "scope": "Four short authored factual/code/negation/Chinese retrieval queries with hard negatives, plus all-irrelevant stability; not broad retrieval, multilingual or safety certification. All complete rankings retained; no calibrated relevance threshold or unknown rejection." }
| Precision | Device | Result | Platform | Inference / load | |
|---|---|---|---|---|---|
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 772 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 773 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 787 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 788 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 791 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 792 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 793 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 793 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 795 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 799 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 802 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 804 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 805 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 806 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 806 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 807 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 809 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 810 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 811 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 813 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 818 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 821 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 822 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 823 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 824 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 826 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 827 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 829 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 829 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 830 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 832 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 833 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 835 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 835 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 836 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 837 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 838 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 840 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 841 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 842 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 843 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 843 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 844 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 845 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 847 ms / 3148 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 847 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 848 ms / 3148 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 849 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 849 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 849 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 851 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 851 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 852 ms / 3065 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 852 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 857 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 859 ms / 2630 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 859 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 859 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 861 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 862 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 863 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 864 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 864 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 865 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 866 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 866 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 866 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 866 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 866 ms / 2630 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 875 ms / 3065 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 875 ms / 3110 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 875 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 876 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 878 ms / 3065 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 879 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 879 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 881 ms / 3065 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 882 ms / 3148 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 883 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 884 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 887 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 888 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 888 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 889 ms / 3148 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 890 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 891 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 891 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 894 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 895 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 895 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 896 ms / 3065 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 896 ms / 3110 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 897 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 898 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 899 ms / 3148 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 901 ms / 3065 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 901 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 902 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 902 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 903 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 904 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 905 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 908 ms / 3148 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 909 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 910 ms / 2630 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 911 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 912 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 917 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 919 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 919 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 920 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 920 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 922 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 922 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 923 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 925 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 926 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 928 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 928 ms / 3132 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 928 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 932 ms / 2630 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 933 ms / 3148 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 937 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 940 ms / 3148 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 943 ms / 2649 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 943 ms / 2649 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 945 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 945 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 946 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 947 ms / 3148 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 948 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 948 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 949 ms / 3110 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 951 ms / 2649 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 954 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 954 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 956 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 959 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 960 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 960 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 961 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 961 ms / 11662 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 961 ms / 2630 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 961 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 962 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 964 ms / 3148 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 964 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 964 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 965 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 967 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 967 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 968 ms / 3132 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 969 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 969 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 970 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 972 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 972 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 974 ms / 3148 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 974 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 974 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 977 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 980 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 981 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 983 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 985 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 985 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 985 ms / 3148 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 987 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 988 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 990 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 991 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 994 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 996 ms / 12923 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 997 ms / 3132 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 999 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1001 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1002 ms / 3148 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1007 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1009 ms / 11662 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1015 ms / 2630 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1017 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1018 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1023 ms / 11662 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1025 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1028 ms / 3110 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1031 ms / 2649 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1031 ms / 11662 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1033 ms / 3065 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1036 ms / 12923 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1041 ms / 3110 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1041 ms / 3065 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1041 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1042 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1043 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1045 ms / 12923 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1046 ms / 3148 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1046 ms / 12923 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1047 ms / 3148 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1055 ms / 2630 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1061 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1063 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1063 ms / 12923 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1064 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1065 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1065 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1068 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1069 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1069 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1070 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1073 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1077 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1078 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1080 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1081 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1081 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1083 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1083 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1084 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1088 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1089 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1090 ms / 12923 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1090 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1094 ms / 11613 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1094 ms / 3132 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1095 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1096 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1100 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1100 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1105 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1107 ms / 11613 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1108 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1109 ms / 11613 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1110 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1111 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1111 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1112 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1114 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1114 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1115 ms / 11613 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1117 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1118 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1119 ms / 11613 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1119 ms / 3132 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1119 ms / 12923 ms |
FP16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1120 ms / 2649 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1127 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1129 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1130 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1133 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1136 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1141 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1142 ms / 12923 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1144 ms / 3148 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1169 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1177 ms / 11613 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1182 ms / 3148 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1183 ms / 3110 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1189 ms / 3132 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1197 ms / 12923 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1209 ms / 3110 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1221 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1241 ms / 11613 ms |
BF16 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1262 ms / 3148 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1274 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1296 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1319 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Fast | macos · aarch64 | 1370 ms / 11613 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 1835 ms / 11662 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 2056 ms / 12923 ms |
FP32 | mpsAccelerated | Passed | Moderate | macos · aarch64 | 2073 ms / 11613 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3202 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3203 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3206 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3207 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3207 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3207 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3208 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3210 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3213 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3216 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3216 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3217 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3218 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3222 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3223 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3224 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3224 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3226 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3226 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3228 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3229 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3229 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3231 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3233 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3234 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3234 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3236 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3236 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3238 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3240 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3242 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3242 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3242 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3242 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3242 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3243 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3244 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3246 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3247 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3251 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3253 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3254 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3258 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3258 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3267 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3278 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3289 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3292 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3309 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3312 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3325 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3330 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3333 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3337 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3338 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3339 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3341 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3342 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3343 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3343 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3345 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3347 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3349 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3349 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3349 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3350 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3350 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3350 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3351 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3353 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3353 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3354 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3355 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3356 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3356 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3356 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3356 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3357 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3358 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3359 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3359 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3359 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3362 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3362 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3363 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3364 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3366 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3367 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3370 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3371 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3372 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3377 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3379 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3379 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3380 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3382 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3383 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3383 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3385 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3385 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3390 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3396 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3401 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3401 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3403 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3405 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3408 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3415 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3416 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3420 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3430 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3431 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3451 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3452 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3454 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3455 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3461 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3462 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3467 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3482 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3483 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3483 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3484 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3485 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3486 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3490 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3491 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3492 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3500 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3502 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3505 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3505 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3505 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3506 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3509 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3510 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3514 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3514 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3514 ms / 7974 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3518 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3523 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3523 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3524 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3525 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3529 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3530 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3536 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3537 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3541 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3547 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3548 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3549 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3550 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3557 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3564 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3593 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3621 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3625 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3649 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3651 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3652 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3657 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3659 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3664 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3666 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3669 ms / 2380 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3670 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3679 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3680 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3682 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3685 ms / 2175 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3687 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3690 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3691 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3700 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3702 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3705 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3719 ms / 2156 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 3975 ms / 2845 ms |
FP32 | cpu | Passed | Moderate | macos · aarch64 | 7662 ms / 7974 ms |
Test setup & sample details
- Quality gate
- { "version": 1, "expectedTop1Accuracy": 1, "minimumPositiveLogitMargin": 0.5, "maximumCpuScoreDifference": 0.08, "allScoresFinite": true, "minimumUsablePairsPerSecond": 0.25, "goodPairsPerSecond": 2, "scope": "Four short authored factual/code/negation/Chinese retrieval queries with hard negatives, plus all-irrelevant stability; not broad retrieval, multilingual or safety certification. All complete rankings retained; no calibrated relevance threshold or unknown rejection." }
- Tested
- 2026-11-06T01:38:05.289787+00:00
- Environment
- { "hardware": "Apple M3 Max", "os": "macOS-27.2-arm64-arm-64bit-Mach-O", "runtime": "Transformers5.22.1/Torch2.10.0", "interpreterScope": "Existing app-private pinned CPython venv, managed public runtime tested separately" }
- Runtime
- transformers 5.22.1
- Artifact revision
- 77d193c791ed757ca307ee72715aa132723da912
- Fixture
{ "file": "model-samples/qwen3-reranker-family-capital.json", "size": 356, "sha1": "202d7afe4b252011e63de03681b2849227d4a46b", "sha256": "fcb30f2647fd37f102ebd90d61b0fd0bcb050ce208ff9caef38f79144fff9358", "url": "https://assets.localai.tools/samples/v1/files/fcb30f2647fd37f102ebd90d61b0fd0bcb050ce208ff9caef38f79144fff9358/qwen3-reranker-family-capital.json" }- Rerun when
- Exact original/ONNX artifact revision or tokenizer; Pinned runtime or relevant browser/backend/OS behavior; Fixture bytes/expected ranking or quality gate; Prompt/truncation/yes-no scoring/preprocessing
Per-case quality & failure details
[
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 7663.963375031017,
"inferenceMs": 7661.791000049561,
"pairsPerSecond": 0.39155335873565256,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 15638.045959058218,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30459887616,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3279.2047080001794,
"inferenceMs": 3278.466415940784,
"pairsPerSecond": 0.9150619891706667,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11253.28729202738,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30460854272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3224.2978750145994,
"inferenceMs": 3223.5941670369357,
"pairsPerSecond": 0.9306382393530451,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11198.3804590418,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30461018112,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3258.9015000266954,
"inferenceMs": 3258.159667951986,
"pairsPerSecond": 0.9207651882468179,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11232.984084053896,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30461083648,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3229.9247499904595,
"inferenceMs": 3229.2140839854255,
"pairsPerSecond": 0.9290186162874235,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11204.00733401766,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30461116416,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3229.913040995598,
"inferenceMs": 3229.21691701049,
"pairsPerSecond": 0.9290178012498795,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11203.995625022799,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30461165568,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3507.3007500031963,
"inferenceMs": 3506.2179579399526,
"pairsPerSecond": 0.8556227924183651,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11481.383334030397,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30478368768,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3491.8944999808446,
"inferenceMs": 3491.1009159404784,
"pairsPerSecond": 0.8593277800426519,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11465.977084008045,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30478417920,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3483.360999962315,
"inferenceMs": 3482.6725010061637,
"pairsPerSecond": 0.8614074390093479,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11457.443583989516,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30478450688,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3515.3395410161465,
"inferenceMs": 3514.4139169715345,
"pairsPerSecond": 0.853627395883175,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11489.422125043347,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30478467072,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3505.62287500361,
"inferenceMs": 3504.893417004496,
"pairsPerSecond": 0.855946142454737,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11479.70545903081,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30478630912,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3487.1458749985322,
"inferenceMs": 3486.427875061054,
"pairsPerSecond": 0.8604795818262738,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11461.228459025733,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30478630912,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3416.498083039187,
"inferenceMs": 3415.4547099606134,
"pairsPerSecond": 0.8783603516249219,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11390.580667066388,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30480039936,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3382.5579999829642,
"inferenceMs": 3381.7442919826135,
"pairsPerSecond": 0.8871161569230273,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11356.640584010165,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30480039936,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3358.5801249719225,
"inferenceMs": 3357.8077079728246,
"pairsPerSecond": 0.8934400838013323,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11332.662708999123,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30480039936,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3380.9233750216663,
"inferenceMs": 3380.145624978468,
"pairsPerSecond": 0.8875357256299011,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11355.005959048867,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30480039936,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3356.426625046879,
"inferenceMs": 3355.698543076869,
"pairsPerSecond": 0.8940016397448127,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11330.50920907408,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30480039936,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3350.7909170002677,
"inferenceMs": 3350.065708043985,
"pairsPerSecond": 0.8955048233222926,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11324.873501027469,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30480056320,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3243.508125015069,
"inferenceMs": 3242.472750018351,
"pairsPerSecond": 0.9252198033069118,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11217.59070904227,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30481760256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3223.785415990278,
"inferenceMs": 3223.0312510509975,
"pairsPerSecond": 0.9308007792421282,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11197.868000017479,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30481776640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3216.681666963268,
"inferenceMs": 3216.0024989861995,
"pairsPerSecond": 0.9328350960379256,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11190.764250990469,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30481776640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3217.7047500154004,
"inferenceMs": 3216.991541965399,
"pairsPerSecond": 0.9325483019974528,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11191.787334042601,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30481776640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3267.243958020117,
"inferenceMs": 3266.5164999780245,
"pairsPerSecond": 0.9184095656703961,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11241.326542047318,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30481776640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3210.4623750201426,
"inferenceMs": 3209.742000035476,
"pairsPerSecond": 0.934654561010462,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11184.544959047344,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30481776640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3325.306541984901,
"inferenceMs": 3324.505750031676,
"pairsPerSecond": 0.9023897762761925,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11299.389126012102,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30482219008,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3242.9789169691503,
"inferenceMs": 3242.1953749726526,
"pairsPerSecond": 0.9252989573539517,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11217.061500996351,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30482219008,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3228.6071669659577,
"inferenceMs": 3227.9042080626823,
"pairsPerSecond": 0.9293956098531606,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11202.689750993159,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30482251776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3259.1467080055736,
"inferenceMs": 3258.416706987191,
"pairsPerSecond": 0.920692554014637,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11233.229292032775,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30482251776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3289.587625011336,
"inferenceMs": 3288.7412079726346,
"pairsPerSecond": 0.9122031228019213,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11263.670209038537,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30482251776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 7974.082584027201,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 8995.273250038736,
"durationMs": 3313.099082966801,
"inferenceMs": 3312.395625980571,
"pairsPerSecond": 0.9056889148354397,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 11287.181666994002,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 30482251776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3669.860958994832,
"inferenceMs": 3668.9572510076687,
"pairsPerSecond": 0.8176710151572517,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6049.909792025574,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261699072,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3236.4434160408564,
"inferenceMs": 3235.7639580150135,
"pairsPerSecond": 0.9271380851402884,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5616.492249071598,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261699072,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3252.0506249857135,
"inferenceMs": 3251.288041996304,
"pairsPerSecond": 0.9227112335940522,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5632.099458016455,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261715456,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3244.9695830000564,
"inferenceMs": 3244.188376062084,
"pairsPerSecond": 0.9247305187750876,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5625.018416030798,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261715456,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3236.8303340044804,
"inferenceMs": 3236.1491250339895,
"pairsPerSecond": 0.9270277370077903,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5616.879167035222,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261764608,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3226.7315830104053,
"inferenceMs": 3226.0215820278972,
"pairsPerSecond": 0.9299379820373618,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5606.780416041147,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261780992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3506.1940000159666,
"inferenceMs": 3505.3678329568356,
"pairsPerSecond": 0.8558302988332755,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5886.242833046708,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276870656,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3483.1930000218563,
"inferenceMs": 3482.385124952998,
"pairsPerSecond": 0.8614785247339612,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5863.241833052598,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276968960,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3485.1805409998633,
"inferenceMs": 3484.4434590195306,
"pairsPerSecond": 0.8609696312432501,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5865.229374030605,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276968960,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3548.3739170012996,
"inferenceMs": 3547.6132499752566,
"pairsPerSecond": 0.8456389658655503,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5928.422750032041,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277100032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3483.704292040784,
"inferenceMs": 3483.012041077018,
"pairsPerSecond": 0.861323465041005,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5863.753125071526,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277149184,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3493.154667026829,
"inferenceMs": 3492.4514159793034,
"pairsPerSecond": 0.8589954855989838,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5873.203500057571,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277149184,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3383.650832984131,
"inferenceMs": 3382.8423339873552,
"pairsPerSecond": 0.8868282065229747,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5763.6996660148725,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277198336,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3359.2735420097597,
"inferenceMs": 3358.5033750277944,
"pairsPerSecond": 0.8932550201695636,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5739.322375040501,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3356.7702500149608,
"inferenceMs": 3356.0547909582965,
"pairsPerSecond": 0.8939067407607407,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5736.819083045702,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3372.545666003134,
"inferenceMs": 3371.8309590476565,
"pairsPerSecond": 0.8897243178665526,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5752.594499033876,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3355.8934169705026,
"inferenceMs": 3355.159167025704,
"pairsPerSecond": 0.8941453596252046,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5735.942250001244,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3364.5210419781506,
"inferenceMs": 3363.658916030545,
"pairsPerSecond": 0.891885912005698,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5744.569875008892,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3309.7484579775482,
"inferenceMs": 3308.9586249552667,
"pairsPerSecond": 0.9066296499976806,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5689.79729100829,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3213.39354198426,
"inferenceMs": 3212.697623996064,
"pairsPerSecond": 0.9337946956453675,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5593.442375015002,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3218.287333031185,
"inferenceMs": 3217.5875419634394,
"pairsPerSecond": 0.9323755642618311,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5598.336166061927,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3243.368625000585,
"inferenceMs": 3242.699248949066,
"pairsPerSecond": 0.9251551777341908,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5623.4174580313265,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3216.31345897913,
"inferenceMs": 3215.613666921854,
"pairsPerSecond": 0.9329478944750691,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5596.362292009871,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3202.635666995775,
"inferenceMs": 3201.963250001427,
"pairsPerSecond": 0.9369251817611158,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5582.684500026517,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3232.2030409704894,
"inferenceMs": 3231.4526260015555,
"pairsPerSecond": 0.9283750520929208,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5612.251874001231,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3207.745374995284,
"inferenceMs": 3207.0521249552257,
"pairsPerSecond": 0.9354384908982056,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5587.794208026025,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3207.81683298992,
"inferenceMs": 3207.112001022324,
"pairsPerSecond": 0.9354210264698261,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5587.865666020662,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3207.9775409656577,
"inferenceMs": 3207.2459990158677,
"pairsPerSecond": 0.9353819447964202,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5588.026373996399,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3254.477125010453,
"inferenceMs": 3253.7026669597253,
"pairsPerSecond": 0.9220264747802582,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5634.525958041195,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2380.0488330307417,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3928.5963750444353,
"durationMs": 3238.4809579816647,
"inferenceMs": 3237.7680010395125,
"pairsPerSecond": 0.9265642254283892,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5618.529791012406,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277247488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3685.902458964847,
"inferenceMs": 3684.907083050348,
"pairsPerSecond": 0.8141317901336645,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5861.369417980313,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261109248,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3222.307791991625,
"inferenceMs": 3221.5729579911567,
"pairsPerSecond": 0.9312221201008216,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5397.774751007091,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261387776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3235.165041987784,
"inferenceMs": 3234.4723329879344,
"pairsPerSecond": 0.9275083201063173,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5410.6320010032505,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261387776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3246.8027920112945,
"inferenceMs": 3246.091456967406,
"pairsPerSecond": 0.924188378476153,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5422.269751026761,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261420544,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3224.349165975582,
"inferenceMs": 3223.63929200219,
"pairsPerSecond": 0.9306252121454667,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5399.816124991048,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261420544,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3242.7439999883063,
"inferenceMs": 3242.064999998547,
"pairsPerSecond": 0.9253361669187213,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5418.210959003773,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33261486080,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3557.7354169799946,
"inferenceMs": 3556.9350830628537,
"pairsPerSecond": 0.8434227586230557,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5733.202375995461,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276723200,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3519.0601669601165,
"inferenceMs": 3518.273749097716,
"pairsPerSecond": 0.8526908972814777,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5694.527125975583,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276723200,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3502.3157080286182,
"inferenceMs": 3501.5955000417307,
"pairsPerSecond": 0.8567523004768104,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5677.7826670440845,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276870656,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3564.61954198312,
"inferenceMs": 3563.827375008259,
"pairsPerSecond": 0.8417916145540152,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5740.086500998586,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276985344,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3485.3485839557834,
"inferenceMs": 3484.611999010667,
"pairsPerSecond": 0.8609279887837568,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5660.81554297125,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277132800,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3500.9467090130784,
"inferenceMs": 3500.2391259768046,
"pairsPerSecond": 0.8570842996798957,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5676.413668028545,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277132800,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3367.0987079967745,
"inferenceMs": 3366.297292057425,
"pairsPerSecond": 0.891186885685444,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5542.565667012241,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277149184,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3342.8309579612687,
"inferenceMs": 3342.08516695071,
"pairsPerSecond": 0.8976431928385519,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5518.297916976735,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3359.774707991164,
"inferenceMs": 3358.9996250229888,
"pairsPerSecond": 0.8931230529623736,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5535.24166700663,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3379.339666047599,
"inferenceMs": 3378.578083065804,
"pairsPerSecond": 0.8879475111250728,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5554.806625063065,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3337.6085839699954,
"inferenceMs": 3336.853041080758,
"pairsPerSecond": 0.8990506813055045,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5513.075542985462,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3359.292707988061,
"inferenceMs": 3358.545041992329,
"pairsPerSecond": 0.8932439382204516,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5534.759667003527,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3243.260374991223,
"inferenceMs": 3242.4450420076028,
"pairsPerSecond": 0.9252277096861787,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5418.7273340066895,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3203.807790996507,
"inferenceMs": 3203.146207961254,
"pairsPerSecond": 0.9365791647423573,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5379.274750011973,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3206.2520000035875,
"inferenceMs": 3205.5365840205923,
"pairsPerSecond": 0.9358807554887442,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5381.718959019054,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3242.7407919894904,
"inferenceMs": 3242.052251007408,
"pairsPerSecond": 0.9253398056949284,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5418.207751004957,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3209.1225420008413,
"inferenceMs": 3208.4123319364153,
"pairsPerSecond": 0.9350419115829076,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5384.589501016308,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3240.3904579696245,
"inferenceMs": 3239.6682500257157,
"pairsPerSecond": 0.9260207430116298,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5415.857416985091,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3248.185791017022,
"inferenceMs": 3247.481499973219,
"pairsPerSecond": 0.9237927914369151,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5423.652750032488,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277181952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3292.8339579957537,
"inferenceMs": 3292.162375000771,
"pairsPerSecond": 0.911255174647726,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5468.30091701122,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277214720,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3226.3648330117576,
"inferenceMs": 3225.683082942851,
"pairsPerSecond": 0.9300355685478698,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5401.831792027224,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277214720,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3253.909124992788,
"inferenceMs": 3253.258293028921,
"pairsPerSecond": 0.9221524176018847,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5429.376084008254,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277214720,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3234.503625018988,
"inferenceMs": 3233.839833003003,
"pairsPerSecond": 0.927689729523229,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5409.970584034454,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277231104,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2175.4669590154663,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3634.2251659953035,
"durationMs": 3233.7005410226993,
"inferenceMs": 3232.9927919781767,
"pairsPerSecond": 0.9279327833466604,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5409.167500038166,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277231104,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1889.416124962736,
"inferenceMs": 1834.9094159784727,
"pairsPerSecond": 1.6349580932311245,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 13551.87995795859,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 268189696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 955.6196249905042,
"inferenceMs": 947.998000017833,
"pairsPerSecond": 3.164563638260383,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12618.083457986359,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 268550144,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 972.3527080495842,
"inferenceMs": 964.145831996575,
"pairsPerSecond": 3.111562484056517,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12634.816541045438,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 268713984,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 954.5563329593278,
"inferenceMs": 947.5111249485053,
"pairsPerSecond": 3.1661897375221235,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12617.020165955182,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 268959744,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 973.4326250036247,
"inferenceMs": 963.9569999999367,
"pairsPerSecond": 3.112172015971871,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12635.896457999479,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 269189120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 952.5442090234719,
"inferenceMs": 945.9956680075265,
"pairsPerSecond": 3.1712618793684912,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12615.008042019326,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 269303808,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1050.6777500268072,
"inferenceMs": 1031.273873988539,
"pairsPerSecond": 2.909023563641001,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12713.141583022662,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 303841280,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 994.8750410112552,
"inferenceMs": 988.4977489127778,
"pairsPerSecond": 3.0349082770290776,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12657.33887400711,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 304070656,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1019.3004159955308,
"inferenceMs": 1009.0584990684874,
"pairsPerSecond": 2.973068462105468,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12681.764248991385,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 304300032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1136.1391660175286,
"inferenceMs": 1127.1652919822372,
"pairsPerSecond": 2.6615439823596665,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12798.602999013383,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 304594944,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 992.7168750436977,
"inferenceMs": 984.6344169345684,
"pairsPerSecond": 3.046816105961242,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12655.180708039552,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 304775168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1004.4458750053309,
"inferenceMs": 994.0955829806626,
"pairsPerSecond": 3.017818458668634,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12666.909708001185,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 304939008,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1019.0901670139283,
"inferenceMs": 1006.5747090266086,
"pairsPerSecond": 2.980404706274709,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12681.554000009783,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 328368128,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 999.5285830227658,
"inferenceMs": 989.7953350446187,
"pairsPerSecond": 3.030929621288591,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12661.99241601862,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 328531968,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 996.721165953204,
"inferenceMs": 987.3095399234444,
"pairsPerSecond": 3.038560733680968,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12659.184998949058,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 328728576,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 977.6395829976536,
"inferenceMs": 968.5113760060631,
"pairsPerSecond": 3.097537183684272,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12640.103415993508,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 328876032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 992.0899579883553,
"inferenceMs": 982.7673749532551,
"pairsPerSecond": 3.0526043868140147,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12654.55379098421,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 329072640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 984.0540000004694,
"inferenceMs": 973.8073319313116,
"pairsPerSecond": 3.0806915306852583,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12646.517832996324,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 329236480,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1040.1078330469318,
"inferenceMs": 1023.3469989616424,
"pairsPerSecond": 2.9315569430935984,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12702.571666042786,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 352468992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1103.8880000123754,
"inferenceMs": 1096.3894160231575,
"pairsPerSecond": 2.7362540682686007,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12766.35183300823,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 352616448,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1026.814499986358,
"inferenceMs": 1018.235708004795,
"pairsPerSecond": 2.946272632569936,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12689.278332982212,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 352894976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 971.1387500283308,
"inferenceMs": 961.6632919642143,
"pairsPerSecond": 3.1195950028127277,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12633.602583024185,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 353026048,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 968.9437920460477,
"inferenceMs": 961.0376670025289,
"pairsPerSecond": 3.1216258248825803,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12631.407625041902,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 353206272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 967.0820839819498,
"inferenceMs": 960.7280000345781,
"pairsPerSecond": 3.1226320039511966,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12629.545916977804,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 353370112,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 975.3184170112945,
"inferenceMs": 965.4895010171458,
"pairsPerSecond": 3.1072321313069606,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12637.782250007149,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 353861632,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 963.2101249881089,
"inferenceMs": 953.7700840155594,
"pairsPerSecond": 3.145412138918649,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12625.673957983963,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 354041856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 977.4406669894233,
"inferenceMs": 966.5567079791799,
"pairsPerSecond": 3.103801334400983,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12639.904499985278,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 354205696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 969.3346670246683,
"inferenceMs": 961.2657490069978,
"pairsPerSecond": 3.120885148668873,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12631.798500020523,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 354402304,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 980.2093330072239,
"inferenceMs": 970.2768339775503,
"pairsPerSecond": 3.0919010894053898,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12642.673166003078,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 354533376,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11662.463832995854,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13108.873666962609,
"durationMs": 1097.1129999961704,
"inferenceMs": 1089.4134170375764,
"pairsPerSecond": 2.753775520920102,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12759.576832992025,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 354697216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 2096.5739999664947,
"inferenceMs": 2056.2214570236392,
"pairsPerSecond": 1.4589868176662601,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 15019.189958984498,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 335413248,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1004.2021669796668,
"inferenceMs": 995.7809579791501,
"pairsPerSecond": 3.01271075326469,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 13926.81812599767,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 335675392,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1026.182500005234,
"inferenceMs": 1017.2028749366291,
"pairsPerSecond": 2.949264177204471,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 13948.798459023237,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 335888384,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1044.236625020858,
"inferenceMs": 1035.9590839943849,
"pairsPerSecond": 2.895867265754157,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 13966.852584038861,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 336084992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1207.7936249552295,
"inferenceMs": 1197.4898320040666,
"pairsPerSecond": 2.5052404787265137,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 14130.409583973233,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 336248832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 998.1209999532439,
"inferenceMs": 991.1404580925591,
"pairsPerSecond": 3.0268162050144465,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 13920.736958971247,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 336412672,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1133.483082987368,
"inferenceMs": 1117.5700000021607,
"pairsPerSecond": 2.684395608323595,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 14056.099042005371,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 371179520,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1070.6289579975419,
"inferenceMs": 1062.9160009557381,
"pairsPerSecond": 2.8224243470815207,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 13993.244917015545,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 371408896,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1076.6682079993188,
"inferenceMs": 1067.8750009974465,
"pairsPerSecond": 2.8093175673162647,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 13999.284167017322,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 371556352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1091.1670409841463,
"inferenceMs": 1080.726207990665,
"pairsPerSecond": 2.7759112139768827,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 14013.78300000215,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 371834880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1122.7665840415284,
"inferenceMs": 1111.7505000438541,
"pairsPerSecond": 2.698447178464648,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 14045.382543059532,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 372031488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1087.1818750165403,
"inferenceMs": 1077.8616659808904,
"pairsPerSecond": 2.783288519004801,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 14009.797834034543,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 372195328,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1126.9980000215583,
"inferenceMs": 1111.429916927591,
"pairsPerSecond": 2.6992255240826384,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 14049.613959039561,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 395444224,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1285.407707968261,
"inferenceMs": 1273.9044988993555,
"pairsPerSecond": 2.354964601029338,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 14208.023666986264,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 395624448,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1054.7386250109412,
"inferenceMs": 1046.471874928102,
"pairsPerSecond": 2.866775564518746,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 13977.354584028944,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 395870208,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1073.9567499840632,
"inferenceMs": 1065.207667008508,
"pairsPerSecond": 2.816352240896928,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 13996.572709002066,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 396017664,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1100.122082978487,
"inferenceMs": 1088.489583984483,
"pairsPerSecond": 2.7561127310178897,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 14022.73804199649,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 396378112,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1151.4222499681637,
"inferenceMs": 1141.7254590196535,
"pairsPerSecond": 2.627601912789052,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 14074.038208986167,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 396525568,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1136.6623330395669,
"inferenceMs": 1119.2240430391394,
"pairsPerSecond": 2.68042847958645,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 14059.27829205757,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 419430400,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1054.836625000462,
"inferenceMs": 1044.6818339987658,
"pairsPerSecond": 2.871687725742098,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 13977.452584018465,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 419594240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1050.9531670249999,
"inferenceMs": 1042.8772069863044,
"pairsPerSecond": 2.876656983106735,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 13973.569126043003,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 419807232,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1150.1329999882728,
"inferenceMs": 1140.8666259958409,
"pairsPerSecond": 2.629579945316883,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 14072.748959006276,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 419971072,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1232.0494170417078,
"inferenceMs": 1220.9767090389505,
"pairsPerSecond": 2.4570493259952078,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 14154.665376059711,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 420151296,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1091.8622500030324,
"inferenceMs": 1083.1710839411244,
"pairsPerSecond": 2.7696455753642186,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 14014.478209021036,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 420315136,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1070.5477920128033,
"inferenceMs": 1062.8253759932704,
"pairsPerSecond": 2.8226650094765855,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 13993.163751030806,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 420806656,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1076.9130840199068,
"inferenceMs": 1069.9578759958968,
"pairsPerSecond": 2.803848700312296,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 13999.52904303791,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 420937728,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1072.050541988574,
"inferenceMs": 1064.855876029469,
"pairsPerSecond": 2.817282664754697,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 13994.666501006577,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 421117952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1093.3203750173561,
"inferenceMs": 1083.9707500417717,
"pairsPerSecond": 2.767602354477178,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 14015.93633403536,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 421314560,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1098.1946249958128,
"inferenceMs": 1089.5740410196595,
"pairsPerSecond": 2.753369561918436,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 14020.810584013816,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 421478400,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 12922.615959018003,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 14433.883958030492,
"durationMs": 1089.250290999189,
"inferenceMs": 1081.1553329695016,
"pairsPerSecond": 2.774809417773669,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 14011.866250017192,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 421642240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 2112.0374579913914,
"inferenceMs": 2073.147668095771,
"pairsPerSecond": 1.4470749219497527,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 13725.034957984462,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 634748928,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1076.201250019949,
"inferenceMs": 1068.9530009985901,
"pairsPerSecond": 2.806484473309371,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12689.19875001302,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 634945536,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1078.0555829987861,
"inferenceMs": 1068.901082035154,
"pairsPerSecond": 2.806620790661091,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12691.053082991857,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 635092992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1091.020457970444,
"inferenceMs": 1082.862499926705,
"pairsPerSecond": 2.770434843023061,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12704.017957963515,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 635256832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1088.3422500337474,
"inferenceMs": 1076.8186250934377,
"pairsPerSecond": 2.7859845010943083,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12701.339750026818,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 635420672,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.827348709106445,
"cpuScoreDifference": 0.0000023245811462402344,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1081.2375410459936,
"inferenceMs": 1072.6586660603061,
"pairsPerSecond": 2.7967890391623764,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32926531584,
"totalMs": 12694.235041039065,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 635584512,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1317.7349159959704,
"inferenceMs": 1296.0867909132503,
"pairsPerSecond": 2.3146598059888692,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12930.732415989041,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 669450240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1176.9551250035875,
"inferenceMs": 1168.7820840161294,
"pairsPerSecond": 2.5667744578112472,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12789.952624996658,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 669777920,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1145.872499968391,
"inferenceMs": 1136.0328739974648,
"pairsPerSecond": 2.640768650860974,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12758.869999961462,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 669925376,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1126.528833003249,
"inferenceMs": 1118.7664169701748,
"pairsPerSecond": 2.6815248960766556,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12739.52633299632,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 670089216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1141.0423330380581,
"inferenceMs": 1132.6621670159511,
"pairsPerSecond": 2.6486273554131623,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12754.03983303113,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 670253056,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972906112670898,
"cpuScoreDifference": 0.0000018477439880371094,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1249.990166979842,
"inferenceMs": 1241.0086250165477,
"pairsPerSecond": 2.417388517311874,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32934084608,
"totalMs": 12862.987666972913,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 670416896,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1145.2717499923892,
"inferenceMs": 1128.886165912263,
"pairsPerSecond": 2.6574867250460747,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12758.26924998546,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 693387264,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1124.13504201686,
"inferenceMs": 1114.2855419311672,
"pairsPerSecond": 2.692308108746258,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12737.132542009931,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 693567488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1140.8816250041127,
"inferenceMs": 1130.408208933659,
"pairsPerSecond": 2.6539085405527723,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12753.879124997184,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 693731328,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1379.1389589896426,
"inferenceMs": 1370.3346260008402,
"pairsPerSecond": 2.1892462928964624,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12992.136458982714,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 693878784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1114.7539170342498,
"inferenceMs": 1107.298249960877,
"pairsPerSecond": 2.709297156485162,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12727.75141702732,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 694042624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.842802047729492,
"cpuScoreDifference": 7.152557373046875e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1121.6960829915479,
"inferenceMs": 1115.0822910130955,
"pairsPerSecond": 2.6903843995893646,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32939032576,
"totalMs": 12734.693582984619,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 694206464,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1130.3136659553275,
"inferenceMs": 1113.749208918307,
"pairsPerSecond": 2.693604606833933,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12743.311165948398,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 716914688,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1109.6475000376813,
"inferenceMs": 1100.2304180292413,
"pairsPerSecond": 2.7267015625451174,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12722.645000030752,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 717078528,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1122.2120419843122,
"inferenceMs": 1111.4547919714823,
"pairsPerSecond": 2.69916511375028,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12735.209541977383,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 717242368,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1102.9020409914665,
"inferenceMs": 1095.3468330553733,
"pairsPerSecond": 2.7388585144595385,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12715.899540984537,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 717406208,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1115.1087500038557,
"inferenceMs": 1105.0603330368176,
"pairsPerSecond": 2.7147838994054707,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12728.106249996927,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 717586432,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.60305118560791,
"cpuScoreDifference": 8.195638656616211e-8,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1090.4985829838552,
"inferenceMs": 1080.2904999582097,
"pairsPerSecond": 2.777030808024372,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12703.496082976926,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 717733888,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1329.584999999497,
"inferenceMs": 1319.0197499934584,
"pairsPerSecond": 2.2744162852867658,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12942.582499992568,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 718028800,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1070.9305420168675,
"inferenceMs": 1061.0301670385525,
"pairsPerSecond": 2.82744081478222,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12683.928042009939,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 718192640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1102.823332999833,
"inferenceMs": 1093.6849599820562,
"pairsPerSecond": 2.7430202569935864,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12715.820832992904,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 718356480,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1111.1485000001267,
"inferenceMs": 1100.2633339376189,
"pairsPerSecond": 2.726619989474347,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12724.145999993198,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 718536704,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1120.074458012823,
"inferenceMs": 1109.083042072598,
"pairsPerSecond": 2.7049372194833605,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12733.071958005894,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 718684160,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 8.412825991399586e-11,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 11612.997499993071,
"placement": {
"mps:0/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 13115.285625040997,
"durationMs": 1188.157083990518,
"inferenceMs": 1176.9496260094456,
"pairsPerSecond": 2.548962108235483,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 32943177728,
"totalMs": 12801.154583983589,
"device": "mps",
"precision": "fp32",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 718864384,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 915.0098750251345,
"inferenceMs": 881.4563329797238,
"pairsPerSecond": 3.4034584445705143,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3980.5069580324925,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5074550784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 800.1867919811048,
"inferenceMs": 793.2582090143114,
"pairsPerSecond": 3.781870727474408,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3865.683874988463,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5074878464,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 809.6554580261,
"inferenceMs": 804.0023740031756,
"pairsPerSecond": 3.731332265927054,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3875.152541033458,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5075124224,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 781.1183750163764,
"inferenceMs": 772.6671250420623,
"pairsPerSecond": 3.8826551599910335,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3846.6154580237344,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5075386368,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 803.265625028871,
"inferenceMs": 793.4724589576945,
"pairsPerSecond": 3.7808495633746384,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3868.762708036229,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5075632128,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 780.4856669972651,
"inferenceMs": 772.4887910299003,
"pairsPerSecond": 3.883551495938639,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3845.982750004623,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5075894272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 913.4225830202922,
"inferenceMs": 896.3099169777706,
"pairsPerSecond": 3.347056574042573,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3978.9196660276502,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5109661696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 831.0773329576477,
"inferenceMs": 823.8565409556031,
"pairsPerSecond": 3.6414106714747407,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3896.5744159650058,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5109907456,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 843.6565410229377,
"inferenceMs": 834.7531239851378,
"pairsPerSecond": 3.5938769365461076,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3909.1536240302958,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5110136832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 912.8074169857427,
"inferenceMs": 900.618000945542,
"pairsPerSecond": 3.3310460115724494,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3978.3044999931008,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5110398976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 1049.4669590261765,
"inferenceMs": 1040.7803740235977,
"pairsPerSecond": 2.8824525085942683,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4114.9640420335345,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5110841344,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 825.4383750027046,
"inferenceMs": 817.5757909775712,
"pairsPerSecond": 3.669384579517595,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3890.9354580100626,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5111169024,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 887.2129580122419,
"inferenceMs": 874.6700010378845,
"pairsPerSecond": 3.429864973578831,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3952.7100410196,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5133139968,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 861.1934999935329,
"inferenceMs": 852.1390409441665,
"pairsPerSecond": 3.520552228983679,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3926.690583000891,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5133418496,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 887.5657499884255,
"inferenceMs": 877.7248329715803,
"pairsPerSecond": 3.417927677679294,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3953.0628329957835,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5133680640,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 816.1518329870887,
"inferenceMs": 805.7051670621149,
"pairsPerSecond": 3.723446395334732,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3881.6489159944467,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5133910016,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 835.9443750232458,
"inferenceMs": 826.7377080628648,
"pairsPerSecond": 3.6287204160910016,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3901.441458030604,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5134172160,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 820.9809160325676,
"inferenceMs": 812.8705419949256,
"pairsPerSecond": 3.690624576746844,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3886.4779990399256,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5134417920,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 857.5787500012666,
"inferenceMs": 843.0070410249755,
"pairsPerSecond": 3.5586891378183876,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3923.0758330086246,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5155700736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 803.9058750146069,
"inferenceMs": 794.5615419885144,
"pairsPerSecond": 3.775667259822356,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3869.402958021965,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5155995648,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 819.1417500493117,
"inferenceMs": 810.3562499745749,
"pairsPerSecond": 3.7020754761799215,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3884.6388330566697,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5156241408,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 804.9385420163162,
"inferenceMs": 798.7522499752231,
"pairsPerSecond": 3.755857964835853,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3870.435625023674,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5156519936,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 1044.9215830303729,
"inferenceMs": 1032.9886670224369,
"pairsPerSecond": 2.9041944948412866,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4110.418666037731,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5156782080,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 797.4227499798872,
"inferenceMs": 787.7463340410031,
"pairsPerSecond": 3.8083325435619817,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3862.919832987245,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5157027840,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 845.6950829713605,
"inferenceMs": 835.5897089932114,
"pairsPerSecond": 3.5902787788215482,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3911.1921659787185,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5157273600,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 815.5897500109859,
"inferenceMs": 805.7834160281345,
"pairsPerSecond": 3.7230848145120583,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3881.086833018344,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5157519360,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 829.3542920146137,
"inferenceMs": 820.5520420451649,
"pairsPerSecond": 3.656075235060927,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3894.8513750219718,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5157781504,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 829.6810829779133,
"inferenceMs": 822.1157070365734,
"pairsPerSecond": 3.6491213758874688,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3895.1781659852713,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5158043648,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 833.6414170335047,
"inferenceMs": 822.9327090084553,
"pairsPerSecond": 3.6454985531133826,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3899.1385000408627,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5158322176,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3065.497083007358,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4515.163624950219,
"durationMs": 798.766500025522,
"inferenceMs": 787.2421240317635,
"pairsPerSecond": 3.8107716907167895,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3864.26358303288,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2547220480,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 1002.3291659890674,
"inferenceMs": 961.1858339630999,
"pairsPerSecond": 3.1211446257281925,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3632.6540000154637,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4990730240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 870.6154170213267,
"inferenceMs": 860.5021659750491,
"pairsPerSecond": 3.4863363726698475,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3500.940251047723,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4991090688,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 868.4486249694601,
"inferenceMs": 859.4897910370491,
"pairsPerSecond": 3.490442854917729,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3498.7734589958563,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4991385600,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 798.72904100921,
"inferenceMs": 790.5949990381487,
"pairsPerSecond": 3.7946103929949606,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3429.0538750356063,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4991647744,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 835.6223750161007,
"inferenceMs": 829.001915990375,
"pairsPerSecond": 3.6188094890179134,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3465.947209042497,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4991926272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 798.6493339994922,
"inferenceMs": 791.8565000873059,
"pairsPerSecond": 3.788565225731223,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3428.9741680258885,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4992155648,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 947.6098750019446,
"inferenceMs": 932.3657070053741,
"pairsPerSecond": 3.217621559286616,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3577.934709028341,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5025890304,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 848.802666994743,
"inferenceMs": 836.7918760050088,
"pairsPerSecond": 3.5851208478774033,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3479.1275010211393,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5026152448,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 853.2471660291776,
"inferenceMs": 842.6160010276362,
"pairsPerSecond": 3.5603406490516023,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3483.572000055574,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5026447360,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 855.1313750213012,
"inferenceMs": 848.5134999500588,
"pairsPerSecond": 3.53559489645901,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3485.4562090476975,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5026725888,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 917.8445829893462,
"inferenceMs": 909.9205830134451,
"pairsPerSecond": 3.2969910297717395,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3548.1694170157425,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5027004416,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 1066.0923750256188,
"inferenceMs": 1055.4957920103334,
"pairsPerSecond": 2.842266186856224,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3696.417209052015,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5027528704,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 860.0604169769213,
"inferenceMs": 848.5742079792544,
"pairsPerSecond": 3.5353419557071226,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3490.3852510033175,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5049860096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 872.8728750138544,
"inferenceMs": 863.2805839879438,
"pairsPerSecond": 3.475115803185835,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3503.1977090402506,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5050122240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 865.9490419668145,
"inferenceMs": 858.6397090111859,
"pairsPerSecond": 3.4938985100686946,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3496.273875993211,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5050400768,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 875.4976249765605,
"inferenceMs": 866.4669989957474,
"pairsPerSecond": 3.4623361345291395,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3505.8224590029567,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5050695680,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 847.2945420071483,
"inferenceMs": 831.7273760330863,
"pairsPerSecond": 3.60695113140133,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3477.6193760335445,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5050957824,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 859.5653750235215,
"inferenceMs": 851.2365410570055,
"pairsPerSecond": 3.524284796649838,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3489.8902090499178,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5051219968,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 868.880124995485,
"inferenceMs": 856.5836239722557,
"pairsPerSecond": 3.5022850262862,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3499.204959021881,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5072470016,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 875.5058749811724,
"inferenceMs": 866.4216669858433,
"pairsPerSecond": 3.4625172872656447,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3505.8307090075687,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5072748544,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 813.6110419873148,
"inferenceMs": 804.5709999860264,
"pairsPerSecond": 3.7286951680486906,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3443.935876013711,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5073027072,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 841.7996250209399,
"inferenceMs": 833.3279169746675,
"pairsPerSecond": 3.6000233988215204,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3472.124459047336,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5073289216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 1025.4532910184935,
"inferenceMs": 1014.8871670244262,
"pairsPerSecond": 2.9559936291201487,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3655.7781250448897,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5073584128,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 861.7543330183253,
"inferenceMs": 852.2453329642303,
"pairsPerSecond": 3.5201131457834727,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3492.0791670447215,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5073862656,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 874.1355829988606,
"inferenceMs": 865.1142919552512,
"pairsPerSecond": 3.4677499006746006,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3504.460417025257,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5074124800,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 809.4709169818088,
"inferenceMs": 801.945292041637,
"pairsPerSecond": 3.7409035625889553,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3439.795751008205,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5074419712,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 833.2508750027046,
"inferenceMs": 826.1184149305336,
"pairsPerSecond": 3.6314406576353377,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3463.575709029101,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5074665472,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 836.6261250339448,
"inferenceMs": 829.7746260068379,
"pairsPerSecond": 3.6154395494557794,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3466.950959060341,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5074944000,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 859.1468749800697,
"inferenceMs": 850.8413749514148,
"pairsPerSecond": 3.525921621020496,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3489.471709006466,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5075222528,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2630.3248340263963,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3694.823959027417,
"durationMs": 824.779957998544,
"inferenceMs": 811.2267920514569,
"pairsPerSecond": 3.6981027123296824,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3455.1047920249403,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 5075517440,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 978.8412089692429,
"inferenceMs": 943.3113759732805,
"pairsPerSecond": 3.1802860395960875,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3627.7667509857565,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4875911168,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 843.2081670034677,
"inferenceMs": 834.8906669416465,
"pairsPerSecond": 3.5932848680528857,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3492.1337090199813,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4876222464,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 913.3987500099465,
"inferenceMs": 904.3040409451351,
"pairsPerSecond": 3.3174683117246104,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3562.32429202646,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4876484608,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 846.3740420411341,
"inferenceMs": 838.4285420179367,
"pairsPerSecond": 3.5781224632209865,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3495.2995840576477,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4876763136,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 851.6601250157692,
"inferenceMs": 840.2340410393663,
"pairsPerSecond": 3.5704337761524294,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3500.585667032283,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4877090816,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.83447265625,
"cpuScoreDifference": 0.0005395412445068359,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 819.4552079658024,
"inferenceMs": 806.8084989790805,
"pairsPerSecond": 3.718354484113815,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3468.380749982316,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4877369344,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 961.7790000047535,
"inferenceMs": 942.8822490153834,
"pairsPerSecond": 3.181733459435457,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3610.704542021267,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4911480832,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 886.4636669750325,
"inferenceMs": 879.056874953676,
"pairsPerSecond": 3.4127484642652868,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3535.389208991546,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4911808512,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 935.7240420067683,
"inferenceMs": 927.6971239596605,
"pairsPerSecond": 3.233814056893045,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3584.649584023282,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4912119808,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 892.985874961596,
"inferenceMs": 882.7455829596147,
"pairsPerSecond": 3.398487693296392,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3541.9114169781096,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4912398336,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 1127.276792016346,
"inferenceMs": 1120.1482089818455,
"pairsPerSecond": 2.6782170215911325,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3776.2023340328597,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4912807936,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.9921875,
"cpuScoreDifference": 0.0028174519538879395,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 945.4765419941396,
"inferenceMs": 937.4296239693649,
"pairsPerSecond": 3.2002402348851304,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 3594.402084010653,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4913102848,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 964.3044999684207,
"inferenceMs": 951.3913331320509,
"pairsPerSecond": 3.1532765703506853,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3613.2300419849344,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4935155712,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 915.579583030194,
"inferenceMs": 908.6140000727028,
"pairsPerSecond": 3.3017320883895196,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3564.5051250467077,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4935434240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 898.2921660062857,
"inferenceMs": 890.1261240243912,
"pairsPerSecond": 3.3703089023346022,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3547.2177080227993,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4935729152,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 859.0611669933423,
"inferenceMs": 849.2666670354083,
"pairsPerSecond": 3.5324593751834152,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3507.986709009856,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4936007680,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 902.5046250317246,
"inferenceMs": 895.3320420114323,
"pairsPerSecond": 3.35071220422344,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3551.430167048238,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4936286208,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.859375,
"cpuScoreDifference": 0.0016699433326721191,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 872.6154590258375,
"inferenceMs": 863.7570830178447,
"pairsPerSecond": 3.473198725639882,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 3521.541001042351,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4936499200,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 910.0559169892222,
"inferenceMs": 894.4081250228919,
"pairsPerSecond": 3.354173465187625,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3558.981459005736,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4957700096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 854.1128329816274,
"inferenceMs": 847.3336670431308,
"pairsPerSecond": 3.5405178817794978,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3503.038374998141,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4957978624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 870.1090840040706,
"inferenceMs": 862.2204580460675,
"pairsPerSecond": 3.479388562408378,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3519.0346260205843,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4958240768,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 1037.8021249780431,
"inferenceMs": 1030.5047919391654,
"pairsPerSecond": 2.911194614005348,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3686.727666994557,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4958535680,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 912.8555829520337,
"inferenceMs": 904.942040971946,
"pairsPerSecond": 3.315129438320573,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3561.7811249685474,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4958797824,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.59375,
"cpuScoreDifference": 0.00006947852671146393,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 871.9791250186972,
"inferenceMs": 865.5402919976041,
"pairsPerSecond": 3.4660431498529296,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3520.904667035211,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4959076352,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 850.3040409996174,
"inferenceMs": 844.2505830898881,
"pairsPerSecond": 3.553447353296749,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3499.229583016131,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4959371264,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 883.6686669965275,
"inferenceMs": 875.5329160485417,
"pairsPerSecond": 3.426484538741971,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3532.594209013041,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4959633408,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 840.0815419736318,
"inferenceMs": 829.4041240005754,
"pairsPerSecond": 3.617054597618469,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3489.0070839901455,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4959895552,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 881.8201249814592,
"inferenceMs": 875.1657910179347,
"pairsPerSecond": 3.427921921525977,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3530.745666997973,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 4960157696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 873.2804589672014,
"inferenceMs": 865.7666249782778,
"pairsPerSecond": 3.465137039759728,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3522.206000983715,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2549022720,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 1.938642526511103e-7,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2648.9255420165136,
"placement": {
"mps:0/torch.float16": 399
},
"parameterDtype": "torch.float16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3682.3849999927916,
"durationMs": 904.9136670073494,
"inferenceMs": 896.7960830195807,
"pairsPerSecond": 3.34524208658313,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3553.839209023863,
"device": "mps",
"precision": "fp16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 2549284864,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 1034.2657499713823,
"inferenceMs": 997.1293330308981,
"pairsPerSecond": 3.008636794267328,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4166.694999963511,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582352896,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 872.0403339830227,
"inferenceMs": 863.9057499240153,
"pairsPerSecond": 3.4726010334621162,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4004.469583975151,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582647808,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 907.7367500285618,
"inferenceMs": 898.1878339545801,
"pairsPerSecond": 3.340058600873573,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4040.1660000206903,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582909952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 874.6637080330402,
"inferenceMs": 866.3134180242196,
"pairsPerSecond": 3.462949941191063,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4007.0929580251686,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 583221248,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 896.8865839997306,
"inferenceMs": 887.4036670313217,
"pairsPerSecond": 3.380648639909342,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4029.315833991859,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 583499776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 852.7504170197062,
"inferenceMs": 844.8845000239089,
"pairsPerSecond": 3.550781201353682,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3985.1796670118347,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 583761920,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 997.5371250184253,
"inferenceMs": 980.1807499607094,
"pairsPerSecond": 3.060659985538642,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4129.966375010554,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617431040,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 928.6028330097906,
"inferenceMs": 919.681876024697,
"pairsPerSecond": 3.261997521324905,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4061.032083001919,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617725952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 1131.065291003324,
"inferenceMs": 1119.012582988944,
"pairsPerSecond": 2.680935000736842,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4263.494540995453,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618233856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 980.045541014988,
"inferenceMs": 967.6019179751165,
"pairsPerSecond": 3.1004485876568406,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4112.474791007116,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618496000,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 973.1971659930423,
"inferenceMs": 959.025209012907,
"pairsPerSecond": 3.1281763730567635,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4105.626415985171,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 618758144,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 928.9197499747388,
"inferenceMs": 919.323249021545,
"pairsPerSecond": 3.2632700230228733,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4061.3489999668673,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 619053056,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 943.2595000253059,
"inferenceMs": 927.6869990280829,
"pairsPerSecond": 3.233849351282311,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4075.6887500174344,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641417216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 900.3358749905601,
"inferenceMs": 890.78733389033,
"pairsPerSecond": 3.367807203654455,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4032.7651249826886,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641712128,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 907.5834579998627,
"inferenceMs": 900.9243749896996,
"pairsPerSecond": 3.329913234986343,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4040.012707991991,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642007040,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 1103.5065419855528,
"inferenceMs": 1093.8620010274462,
"pairsPerSecond": 2.7425763004676553,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4235.935791977681,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642301952,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 900.5098749767058,
"inferenceMs": 887.6587920240127,
"pairsPerSecond": 3.379676996337175,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4032.939124968834,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642564096,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 920.9411250194535,
"inferenceMs": 912.3399580130354,
"pairsPerSecond": 3.288247953683441,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4053.370375011582,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 642842624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 914.3849170068279,
"inferenceMs": 901.8622909206897,
"pairsPerSecond": 3.326450202211439,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4046.8141669989564,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664059904,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 1203.0536250094883,
"inferenceMs": 1188.819748989772,
"pairsPerSecond": 2.5235112409171547,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4335.482875001617,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664354816,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 816.3388329558074,
"inferenceMs": 808.5460839793086,
"pairsPerSecond": 3.7103636508080258,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3948.768082947936,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664616960,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 910.7170419883914,
"inferenceMs": 902.3585010436364,
"pairsPerSecond": 3.3246209755106255,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4043.14629198052,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664911872,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 855.013583030086,
"inferenceMs": 840.5135829234496,
"pairsPerSecond": 3.569246304819356,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3987.4428330222145,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665174016,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 902.148875000421,
"inferenceMs": 894.6610830607824,
"pairsPerSecond": 3.3532251003212385,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4034.5781249925494,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665452544,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 853.2596250297502,
"inferenceMs": 841.5799579815939,
"pairsPerSecond": 3.5647236742603283,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 3985.6888750218786,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665731072,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 899.9928330304101,
"inferenceMs": 891.399207059294,
"pairsPerSecond": 3.3654954774942336,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4032.4220830225386,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665993216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 868.6174579779617,
"inferenceMs": 859.164709050674,
"pairsPerSecond": 3.4917635331120875,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4001.04670797009,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666271744,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 896.1392079945654,
"inferenceMs": 887.7030009753071,
"pairsPerSecond": 3.3795086833140604,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4028.568457986694,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666533888,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 887.4523330014199,
"inferenceMs": 878.889707964845,
"pairsPerSecond": 3.413397577435277,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4019.8815829935484,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 666845184,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3132.4292499921285,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4189.120750001166,
"durationMs": 909.3014580430463,
"inferenceMs": 902.5970819639042,
"pairsPerSecond": 3.32374218790126,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4041.7307080351748,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 667123712,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1077.957333996892,
"inferenceMs": 1040.6537069357,
"pairsPerSecond": 2.882803357164579,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4187.837917008437,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 581369856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 991.4182919892482,
"inferenceMs": 984.623042983003,
"pairsPerSecond": 3.0468513015003524,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4101.2988750007935,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 581697536,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 951.6234999755397,
"inferenceMs": 944.9622499523684,
"pairsPerSecond": 3.1747299959879003,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4061.504082987085,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 581976064,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 883.4114169585519,
"inferenceMs": 875.0637079938315,
"pairsPerSecond": 3.428321815422778,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 3993.291999970097,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582270976,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 934.84995898325,
"inferenceMs": 927.5844590738416,
"pairsPerSecond": 3.234206837612812,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4044.730541994795,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582565888,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 990.6015409505926,
"inferenceMs": 981.1612070188858,
"pairsPerSecond": 3.0576015220934583,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4100.482123962138,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 582844416,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1198.7003330141306,
"inferenceMs": 1183.1778749474324,
"pairsPerSecond": 2.5355443703959453,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4308.580916025676,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 616366080,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 954.2600410059094,
"inferenceMs": 945.2010840177536,
"pairsPerSecond": 3.1739278030109106,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4064.1406240174547,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 616677376,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 964.0491669997573,
"inferenceMs": 956.2697919900529,
"pairsPerSecond": 3.1371899699527535,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4073.9297500113025,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 616955904,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 933.9887910173275,
"inferenceMs": 925.2918331185356,
"pairsPerSecond": 3.242220338084062,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4043.8693740288727,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617234432,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1050.1982090063393,
"inferenceMs": 1041.5097909863107,
"pairsPerSecond": 2.8804337952108905,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4160.078792017885,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617562112,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 979.3492499738932,
"inferenceMs": 972.0736250164919,
"pairsPerSecond": 3.086185987145884,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4089.2298329854384,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 617807872,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1102.7387920184992,
"inferenceMs": 1089.7267910186201,
"pairsPerSecond": 2.7529836145404443,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4212.6193750300445,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 639860736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1048.3117089606822,
"inferenceMs": 1041.3934990065172,
"pairsPerSecond": 2.880755452057249,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4158.192291972227,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 640172032,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 984.762915992178,
"inferenceMs": 977.4736670660786,
"pairsPerSecond": 3.069136388097906,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4094.6434990037233,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 640483328,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1125.2542500151321,
"inferenceMs": 1116.9992089853622,
"pairsPerSecond": 2.6857673451041038,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4235.134833026677,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 640778240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1035.4033749899827,
"inferenceMs": 1028.4084579907358,
"pairsPerSecond": 2.917128867124725,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4145.283958001528,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641040384,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1119.2601249786094,
"inferenceMs": 1109.868000028655,
"pairsPerSecond": 2.7030241433418616,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4229.140707990155,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 641335296,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 991.2144579575397,
"inferenceMs": 974.1059590596706,
"pairsPerSecond": 3.0797470974266257,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4101.095040969085,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 662831104,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 968.228249985259,
"inferenceMs": 960.1253739674576,
"pairsPerSecond": 3.124591934908786,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4078.1088329968043,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663093248,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 957.710542017594,
"inferenceMs": 948.6558329663239,
"pairsPerSecond": 3.1623692130995376,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4067.591125029139,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663371776,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1116.2502910010517,
"inferenceMs": 1108.4147919900715,
"pairsPerSecond": 2.706567993930987,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4226.130874012597,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663650304,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1035.1133749936707,
"inferenceMs": 1025.1804580329917,
"pairsPerSecond": 2.926314071335386,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4144.993958005216,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663945216,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 904.8380830208771,
"inferenceMs": 896.4628339163028,
"pairsPerSecond": 3.3464856394482623,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4014.7186660324223,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664207360,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 972.6605830364861,
"inferenceMs": 966.6229580179788,
"pairsPerSecond": 3.1035886072387298,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4082.5411660480313,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664502272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1217.0672080246732,
"inferenceMs": 1209.2456669779494,
"pairsPerSecond": 2.4808854659759594,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4326.947791036218,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664780800,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1073.0281250434928,
"inferenceMs": 1064.1508750268258,
"pairsPerSecond": 2.819149117294457,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4182.908708055038,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665059328,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 1008.2812500186265,
"inferenceMs": 998.9009169512428,
"pairsPerSecond": 3.003300877084321,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4118.161833030172,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665337856,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 974.9110420234501,
"inferenceMs": 968.6913749901578,
"pairsPerSecond": 3.096961609708232,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4084.7916250349954,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665632768,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3109.8805830115452,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4194.437333964743,
"durationMs": 930.1811250043102,
"inferenceMs": 922.4114989629015,
"pairsPerSecond": 3.2523445375225717,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4040.0617080158554,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 665894912,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 1299.7369159711525,
"inferenceMs": 1262.0924589573406,
"pairsPerSecond": 2.3770049323315083,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4447.59866601089,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 579371008,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 1058.281417004764,
"inferenceMs": 1046.4585829759017,
"pairsPerSecond": 2.866811977850714,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4206.143167044502,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 579764224,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 939.6548340446316,
"inferenceMs": 932.5593319954351,
"pairsPerSecond": 3.2169534924719247,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4087.5165840843692,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 580042752,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 969.3435000372119,
"inferenceMs": 963.6964169330895,
"pairsPerSecond": 3.11301354585019,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4117.20525007695,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 580354048,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 954.3490830110386,
"inferenceMs": 947.2528749611229,
"pairsPerSecond": 3.167052937287866,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4102.210833050776,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 580632576,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82421875,
"cpuScoreDifference": 0.0037817955017089844,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 968.664208019618,
"inferenceMs": 960.023874999024,
"pairsPerSecond": 3.1249222838370034,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16493248512,
"totalMs": 4116.525958059356,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 580894720,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 1014.4436249975115,
"inferenceMs": 1001.2541670585051,
"pairsPerSecond": 2.99624221171876,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4162.305375037249,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 614924288,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 982.4129170156084,
"inferenceMs": 973.7085830420256,
"pairsPerSecond": 3.0810039597550913,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4130.274667055346,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 615235584,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 994.1232079872862,
"inferenceMs": 984.9636669969186,
"pairsPerSecond": 3.0457976273853613,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4141.984958027024,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 615481344,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 933.2387920003384,
"inferenceMs": 926.1430830229074,
"pairsPerSecond": 3.2392403020579463,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4081.100542040076,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 615743488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 1054.3086670222692,
"inferenceMs": 1047.0006669638678,
"pairsPerSecond": 2.86532768761219,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4202.170417062007,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 616022016,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.8125,
"cpuScoreDifference": 0.016170740127563477,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 1152.0259589888155,
"inferenceMs": 1144.119291973766,
"pairsPerSecond": 2.6221041993135,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16509190144,
"totalMs": 4299.887709028553,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 616300544,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 1017.644667008426,
"inferenceMs": 1002.3866670089774,
"pairsPerSecond": 2.9928570468237603,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4165.506417048164,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 638337024,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 952.2312500048429,
"inferenceMs": 939.996459055692,
"pairsPerSecond": 3.191501383966659,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4100.093000044581,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 638648320,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 927.1499160095118,
"inferenceMs": 919.145124964416,
"pairsPerSecond": 3.26390242249954,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4075.0116660492495,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 638894080,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 923.7762920092791,
"inferenceMs": 916.6952090454288,
"pairsPerSecond": 3.27262537253135,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4071.638042049017,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 639188992,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 962.2277079615742,
"inferenceMs": 953.8825840572827,
"pairsPerSecond": 3.1450411718805884,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4110.089458001312,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 639467520,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8125,
"cpuScoreDifference": 0.013964653015136719,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 930.2317079855129,
"inferenceMs": 923.4616250032559,
"pairsPerSecond": 3.2486460928892664,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16514138112,
"totalMs": 4078.0934580252506,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 639746048,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 984.0918750269338,
"inferenceMs": 971.6539580258541,
"pairsPerSecond": 3.0875189415120716,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4131.9536250666715,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 661209088,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 905.0500839948654,
"inferenceMs": 899.2528750095516,
"pairsPerSecond": 3.336102761937942,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4052.911834034603,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 661471232,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 919.0164590254426,
"inferenceMs": 911.1563749611378,
"pairsPerSecond": 3.2925193550096763,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4066.8782090651803,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 661749760,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 1189.050999993924,
"inferenceMs": 1182.0028750807978,
"pairsPerSecond": 2.5380648924351643,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4336.912750033662,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 662044672,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 857.3215419892222,
"inferenceMs": 848.4483340289444,
"pairsPerSecond": 3.5358664513538387,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4005.18329202896,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 662306816,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.5625,
"cpuScoreDifference": 0.0004160068929195404,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 928.4050409914926,
"inferenceMs": 919.6983330184594,
"pairsPerSecond": 3.2619391514541176,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4076.2667910312302,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 662585344,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 893.1276670191437,
"inferenceMs": 884.0017080074176,
"pairsPerSecond": 3.393658601364181,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4040.9894170588814,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 662847488,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 895.7897499785759,
"inferenceMs": 888.9183320570737,
"pairsPerSecond": 3.374888211673626,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4043.6515000183135,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663158784,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 889.4952909904532,
"inferenceMs": 881.7693749442697,
"pairsPerSecond": 3.4022501634167193,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4037.357041030191,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663420928,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 919.5551659795456,
"inferenceMs": 907.7296240138821,
"pairsPerSecond": 3.304948875342775,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4067.4169160192832,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663699456,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 857.3864999925718,
"inferenceMs": 847.0197910210118,
"pairsPerSecond": 3.54182987434538,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4005.2482500323094,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 663961600,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "bf16",
"device": "mps",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0.0000019388571672607213,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 3147.8617500397377,
"placement": {
"mps:0/torch.bfloat16": 399
},
"parameterDtype": "torch.bfloat16",
"provider": "Metal/MPS",
"acceleration": "partial",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4211.147667025216,
"durationMs": 928.1275410321541,
"inferenceMs": 921.6109990375116,
"pairsPerSecond": 3.255169483798547,
"outputPlacement": "mps:0",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": 16518283264,
"totalMs": 4075.989291071892,
"device": "mps",
"precision": "bf16",
"cpuThreads": 4,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "good",
"observedResidentBytesAtCaseEnd": 664256512,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3680.2080839988776,
"inferenceMs": 3679.3314989772625,
"pairsPerSecond": 0.8153655088795084,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5855.0080839777365,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33263910912,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3396.6331250267103,
"inferenceMs": 3395.9339580032974,
"pairsPerSecond": 0.8834094058071452,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5571.433125005569,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33266597888,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3461.9797079940327,
"inferenceMs": 3461.2565829302184,
"pairsPerSecond": 0.8667372464656378,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5636.779707972892,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33266663424,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3357.05125000095,
"inferenceMs": 3356.315332930535,
"pairsPerSecond": 0.8938373491207628,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5531.851249979809,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33266663424,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3338.5745419654995,
"inferenceMs": 3337.8445010166615,
"pairsPerSecond": 0.8987836308989953,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5513.3745419443585,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33266696192,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3408.259415999055,
"inferenceMs": 3407.525126065593,
"pairsPerSecond": 0.8804043665156679,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5583.059415977914,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33266696192,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3659.508416953031,
"inferenceMs": 3658.623333030846,
"pairsPerSecond": 0.81998055741769,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5834.30841693189,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281802240,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3652.2459590341896,
"inferenceMs": 3651.4984990353696,
"pairsPerSecond": 0.8215805102460049,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5827.045959013049,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281818624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3664.850667002611,
"inferenceMs": 3664.077459019609,
"pairsPerSecond": 0.8187599835301258,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5839.65066698147,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281818624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3683.0387089867145,
"inferenceMs": 3682.287500007078,
"pairsPerSecond": 0.8147109643106991,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5857.8387089655735,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281818624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3625.8233340340666,
"inferenceMs": 3625.085249019321,
"pairsPerSecond": 0.8275667450335347,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5800.623334012926,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281818624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3652.3057090234943,
"inferenceMs": 3651.536750025116,
"pairsPerSecond": 0.8215719039331496,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5827.105709002353,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281818624,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3524.1132079972886,
"inferenceMs": 3523.3724580029957,
"pairsPerSecond": 0.851456959421305,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5698.9132079761475,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281916928,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3505.5410000495613,
"inferenceMs": 3504.775332985446,
"pairsPerSecond": 0.8559749812677816,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5680.34100002842,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281933312,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3526.2282909825444,
"inferenceMs": 3525.4375010263175,
"pairsPerSecond": 0.8509582141582844,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5701.028290961403,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281933312,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3510.213165951427,
"inferenceMs": 3509.461624082178,
"pairsPerSecond": 0.8548319717798833,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5685.013165930286,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281933312,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3530.9489999781363,
"inferenceMs": 3530.178666987922,
"pairsPerSecond": 0.8498153444906827,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5705.748999956995,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281933312,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3510.3229170199484,
"inferenceMs": 3509.612501016818,
"pairsPerSecond": 0.8547952228717071,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5685.122916998807,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281933312,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3343.4922079904936,
"inferenceMs": 3342.644707998261,
"pairsPerSecond": 0.8974929321149859,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5518.292207969353,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3357.415416976437,
"inferenceMs": 3356.670498906169,
"pairsPerSecond": 0.8937427730775491,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5532.215416955296,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3350.84679198917,
"inferenceMs": 3350.098542054184,
"pairsPerSecond": 0.8954960465612115,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5525.646791968029,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3341.2767080008052,
"inferenceMs": 3340.566498984117,
"pairsPerSecond": 0.8980512739118695,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5516.076707979664,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3345.2525409520604,
"inferenceMs": 3344.5154590299353,
"pairsPerSecond": 0.8969909204336999,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5520.052540930919,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3340.128082956653,
"inferenceMs": 3339.3902920070104,
"pairsPerSecond": 0.8983675873948136,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5514.928082935512,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3349.7094999765977,
"inferenceMs": 3348.923665995244,
"pairsPerSecond": 0.8958102062647195,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5524.509499955457,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3349.67454202706,
"inferenceMs": 3348.9334580372088,
"pairsPerSecond": 0.8958075869797315,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5524.474542005919,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3344.086957979016,
"inferenceMs": 3343.391999020241,
"pairsPerSecond": 0.897292330925937,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5518.886957957875,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3333.21758400416,
"inferenceMs": 3332.51045900397,
"pairsPerSecond": 0.9002222309293662,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5508.017583983019,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3330.26241598418,
"inferenceMs": 3329.6120000304654,
"pairsPerSecond": 0.9010058829595011,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5505.062415963039,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2174.799999978859,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3233.5313329822384,
"durationMs": 3403.5551249980927,
"inferenceMs": 3402.7349580428563,
"pairsPerSecond": 0.88164374745352,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5578.355124976952,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33281949696,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3706.2399580026977,
"inferenceMs": 3705.292207945604,
"pairsPerSecond": 0.8096527430594597,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6550.831291009672,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33258012672,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3405.188790988177,
"inferenceMs": 3404.502750025131,
"pairsPerSecond": 0.8811859529201012,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6249.780123995151,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260814336,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3452.391874976456,
"inferenceMs": 3451.718083990272,
"pairsPerSecond": 0.8691323934925547,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6296.9832079834305,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260896256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3455.6453329860233,
"inferenceMs": 3454.931915970519,
"pairsPerSecond": 0.8683239128772455,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6300.236665992998,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260896256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3383.3447500364855,
"inferenceMs": 3382.6284580281936,
"pairsPerSecond": 0.8868842786679457,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6227.93608304346,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260896256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3417.0296670054086,
"inferenceMs": 3416.3070420036092,
"pairsPerSecond": 0.8781412101180894,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6261.621000012383,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260896256,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3681.1408339999616,
"inferenceMs": 3680.330831964966,
"pairsPerSecond": 0.815144109856632,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6525.732167006936,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276018688,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3657.990207953844,
"inferenceMs": 3657.2644159314223,
"pairsPerSecond": 0.8202852347595349,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6502.5815409608185,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276018688,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3650.0374580500647,
"inferenceMs": 3649.2938329465687,
"pairsPerSecond": 0.8220768557783394,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6494.628791057039,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276051456,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3975.86120798951,
"inferenceMs": 3975.066000013612,
"pairsPerSecond": 0.7547044501876766,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6820.4525409964845,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282850816,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3671.1792920250446,
"inferenceMs": 3670.4513329896145,
"pairsPerSecond": 0.8173381766531894,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6515.770625032019,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282850816,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3687.654583947733,
"inferenceMs": 3686.8769170250744,
"pairsPerSecond": 0.8136968137305456,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6532.245916954707,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282850816,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3550.590957980603,
"inferenceMs": 3549.764374969527,
"pairsPerSecond": 0.8451265163270881,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6395.182290987577,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282932736,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3524.1835840279236,
"inferenceMs": 3523.434541071765,
"pairsPerSecond": 0.8514419567129107,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6368.774917034898,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282949120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3549.5432080351748,
"inferenceMs": 3548.8087920239195,
"pairsPerSecond": 0.8453540824015687,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6394.134541042149,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282949120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3537.7712080371566,
"inferenceMs": 3537.0179589954205,
"pairsPerSecond": 0.8481721141308698,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6382.362541044131,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282949120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3529.3454159982502,
"inferenceMs": 3528.616041992791,
"pairsPerSecond": 0.8501916797685207,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6373.936749005225,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282949120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3542.1564169810154,
"inferenceMs": 3541.3898350670934,
"pairsPerSecond": 0.8471250383941884,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6386.74774998799,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282949120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3390.966417035088,
"inferenceMs": 3390.213416016195,
"pairsPerSecond": 0.8849000437044077,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6235.557750042062,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282949120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3371.7524579842575,
"inferenceMs": 3371.0669159772806,
"pairsPerSecond": 0.8899259714428697,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6216.343790991232,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282949120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3401.6504579922184,
"inferenceMs": 3400.9488340234384,
"pairsPerSecond": 0.882106772671113,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6246.241790999193,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282949120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3385.885709023569,
"inferenceMs": 3385.1191669818945,
"pairsPerSecond": 0.8862317253884864,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6230.477042030543,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282949120,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3462.5015420024283,
"inferenceMs": 3461.7816240061074,
"pairsPerSecond": 0.8666057902659625,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6307.092875009403,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33283014656,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3430.278041982092,
"inferenceMs": 3429.536125040613,
"pairsPerSecond": 0.8747538706752866,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6274.869374989066,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282998272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3401.9886670284905,
"inferenceMs": 3401.1301670689136,
"pairsPerSecond": 0.8820597426841188,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6246.580000035465,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282998272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3370.6352089648135,
"inferenceMs": 3369.9367090011947,
"pairsPerSecond": 0.8902244341820772,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6215.226541971788,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282998272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3420.8792909630574,
"inferenceMs": 3420.1292919460684,
"pairsPerSecond": 0.8771598217250398,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6265.470623970032,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282998272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3380.1469579921104,
"inferenceMs": 3379.457040980924,
"pairsPerSecond": 0.8877165661881642,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6224.738290999085,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282998272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3350.711416976992,
"inferenceMs": 3349.9515840085223,
"pairsPerSecond": 0.8955353308152074,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6195.302749983966,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282998272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2844.5913330069743,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 4493.174707982689,
"durationMs": 3364.2110420041718,
"inferenceMs": 3363.470416981727,
"pairsPerSecond": 0.8919358959880808,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 6208.802375011146,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33282998272,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3622.209125023801,
"inferenceMs": 3621.278832957614,
"pairsPerSecond": 0.8284366209795019,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5778.019874996971,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33258242048,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3490.229291957803,
"inferenceMs": 3489.505250006914,
"pairsPerSecond": 0.8597207297493121,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5646.0400419309735,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260978176,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3431.324541044887,
"inferenceMs": 3430.6113340426236,
"pairsPerSecond": 0.8744797086835272,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5587.135291018058,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260978176,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3455.0920410547405,
"inferenceMs": 3454.352665983606,
"pairsPerSecond": 0.8684695194970106,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5610.902791027911,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260978176,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3467.818457982503,
"inferenceMs": 3467.0458339969628,
"pairsPerSecond": 0.8652899741280513,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5623.629207955673,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260978176,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 0,
"topIndex": 0,
"positiveLogitMargin": 9.82731819152832,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3451.772083994001,
"inferenceMs": 3450.960375077557,
"pairsPerSecond": 0.8693232242437956,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
93
],
[
1,
95
],
[
1,
90
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5607.582833967172,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33260978176,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3702.706540992949,
"inferenceMs": 3701.870709017385,
"pairsPerSecond": 0.8104010744330702,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5858.51729096612,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276018688,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3720.173832960427,
"inferenceMs": 3719.3814170314,
"pairsPerSecond": 0.8065857366127377,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5875.984582933597,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276952576,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3691.558958031237,
"inferenceMs": 3690.741500002332,
"pairsPerSecond": 0.8128447901317674,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5847.369708004408,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276936192,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3690.786749997642,
"inferenceMs": 3689.932583074551,
"pairsPerSecond": 0.8130229841490273,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5846.597499970812,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276936192,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3701.232000021264,
"inferenceMs": 3700.4351659561507,
"pairsPerSecond": 0.8107154606030866,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5857.042749994434,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276936192,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 4.972908020019531,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3666.971334023401,
"inferenceMs": 3666.1794160027057,
"pairsPerSecond": 0.8182905579866433,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
101
],
[
1,
100
],
[
1,
92
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5822.782083996572,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33276936192,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3593.970666988753,
"inferenceMs": 3593.2045410154387,
"pairsPerSecond": 0.8349093311432253,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5749.781416961923,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277034496,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3548.151124967262,
"inferenceMs": 3547.3177510430105,
"pairsPerSecond": 0.8457094093467991,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5703.961874940433,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3514.76166700013,
"inferenceMs": 3513.971332984511,
"pairsPerSecond": 0.8537349100830651,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5670.5724169733,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3524.8622079961933,
"inferenceMs": 3524.0773750119843,
"pairsPerSecond": 0.851286643497661,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5680.672957969364,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3537.0591249666177,
"inferenceMs": 3536.259958986193,
"pairsPerSecond": 0.8483539204680154,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5692.869874939788,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 1,
"topIndex": 1,
"positiveLogitMargin": 2.8428077697753906,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3514.640041976236,
"inferenceMs": 3513.8751239865087,
"pairsPerSecond": 0.8537582851255354,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
96
],
[
1,
98
],
[
1,
95
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5670.450791949406,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3377.392750000581,
"inferenceMs": 3376.5946659841575,
"pairsPerSecond": 0.8884690929065383,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5533.203499973752,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3367.5983749562874,
"inferenceMs": 3366.8743330053985,
"pairsPerSecond": 0.8910341471884065,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5523.409124929458,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3351.3247090158984,
"inferenceMs": 3350.6412080605514,
"pairsPerSecond": 0.8953510130487792,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5507.135458989069,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3347.2333750105463,
"inferenceMs": 3346.5368329780176,
"pairsPerSecond": 0.8964491203075625,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5503.044124983717,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3362.653332995251,
"inferenceMs": 3361.850665998645,
"pairsPerSecond": 0.8923656337093259,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5518.464082968421,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": 1,
"expectedIndex": 2,
"topIndex": 2,
"positiveLogitMargin": 11.603020668029785,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3354.6388749964535,
"inferenceMs": 3353.8832080666907,
"pairsPerSecond": 0.894485530320335,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
90
],
[
1,
88
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5510.449624969624,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3356.780665984843,
"inferenceMs": 3355.9901239932515,
"pairsPerSecond": 0.8939239655539679,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5512.591415958013,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3385.5857910239138,
"inferenceMs": 3384.849041060079,
"pairsPerSecond": 0.8863024505992295,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5541.396540997084,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3353.530083957594,
"inferenceMs": 3352.845708024688,
"pairsPerSecond": 0.8947623187132684,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5509.340833930764,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3350.097500020638,
"inferenceMs": 3349.394916964229,
"pairsPerSecond": 0.8956841681479268,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5505.908249993809,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3362.4859169940464,
"inferenceMs": 3361.8116679717787,
"pairsPerSecond": 0.8923759854191761,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5518.296666967217,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
},
{
"precision": "fp32",
"device": "cpu",
"status": "pass",
"quality": {
"finite": true,
"completeDocuments": true,
"top1Accuracy": null,
"expectedIndex": null,
"topIndex": 1,
"positiveLogitMargin": null,
"cpuScoreDifference": 0,
"passed": true
},
"runtime": {
"engine": "transformers",
"version": "5.22.1",
"torchVersion": "2.10.0",
"loadMs": 2155.8107499731705,
"placement": {
"cpu/torch.float32": 399
},
"parameterDtype": "torch.float32",
"provider": "PyTorch CPU",
"acceleration": "cpu",
"accelerationDetails": "Qwen3 forward/logits on selected device; tokenization and ranking/serialization on CPU",
"yesTokenId": 9693,
"noTokenId": 2152,
"processInitMs": 3220.3270419850014,
"durationMs": 3353.40233304305,
"inferenceMs": 3352.683540957514,
"pairsPerSecond": 0.8948055977699616,
"outputPlacement": "cpu",
"pairCount": 3,
"tokenShapes": [
[
1,
89
],
[
1,
88
],
[
1,
89
]
],
"truncatedPairs": [
false,
false,
false
],
"maxLength": 512,
"scoreFunction": "sigmoid(yesLogit-noLogit)",
"driverAllocatedMemoryBytes": null,
"totalMs": 5509.2130830162205,
"device": "cpu",
"precision": "fp32",
"cpuThreads": 1,
"artifactRevision": "77d193c791ed757ca307ee72715aa132723da912",
"fallback": false,
"grade": "acceptable",
"observedResidentBytesAtCaseEnd": 33277050880,
"memoryMeasurementScope": "Sampled process RSS after request, not peak; MPS driver allocation separately recorded"
}
}
]