{ "complete": true, "model": "HuggingFaceTB/SmolVLM-500M-Instruct", "revision": "a7da5b986cb59b408707209984f360a5f4ad7e47", "source_sha256": "b5c3fd5dd20a1c9c7b649c5a08350d358fe61a39dd8d898f5e70812912eab747", "cases_sha256": "498a940b2266aaa9ebea89ea9cb2545fcef3ad3aeab8c2548e8c3ec22a87183f", "weights_sha256": "d05b567eeaf534e83d375551f068ed57b5f52d37c657197f644af5ef9db091a2", "weights_bytes": 1015025832, "environment": { "python": "3.12.12", "torch": "2.14.1", "transformers": "4.57.3", "pillow": "12.0.0", "platform": "macOS-26.6.2-arm64-arm-64bit", "device": "cpu", "dtype": "float32", "threads": 4 }, "config": { "max_new_tokens": 32, "do_sample": false, "attention": "eager", "image_size": 512, "image_splitting": false, "image_seq_len": 64, "image_token_id": 49190, "vision_patch_size": 16, "vision_hidden_size": 768, "scale_factor": 4, "text_hidden_size": 960, "eos_token_id": 49279 }, "processor": { "_processor_class": "Idefics3Processor", "image_processor_type": "Idefics3ImageProcessor", "do_convert_rgb": true, "do_resize": true, "size": { "longest_edge": 512 }, "resample": 1, "do_image_splitting": false, "max_image_size": { "longest_edge": 512 }, "do_rescale": true, "rescale_factor": 0.00392156862745098, "do_normalize": true, "image_mean": [ 0.5, 0.5, 0.5 ], "image_std": [ 0.5, 0.5, 0.5 ], "do_pad": true }, "rows": [ { "id": "note-a", "image": "note-a.png", "question": "What is the pickup time on this note? Answer with the time only.", "kind": "time", "expected": "17:30", "image_sha256": "3a2b41f989d3217de593971b1ecb2520e208451e2d505d9cde4fce6c5d019836", "image_size": [ 512, 512 ], "prompt": "<|im_start|>User:What is the pickup time on this note? Answer with the time only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 1780, 314, 260, 40619, 655, 335, 451, 4109, 47, 19842, 351, 260, 655, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 90, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 216, 33, 39, 42, 35, 32, 30, 49279 ], "answer": " 17:30.", "seconds": 1.194359625224024, "stop_reason": "eos", "score": { "exact": true } }, { "id": "note-b", "image": "note-b.png", "question": "What is the pickup time on this note? Answer with the time only.", "kind": "time", "expected": "18:45", "image_sha256": "83d4434f824067ea075bf82ee92e31a28c0fb19422593e01b1f6f80ea3c555c4", "image_size": [ 512, 512 ], "prompt": "<|im_start|>User:What is the pickup time on this note? Answer with the time only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 1780, 314, 260, 40619, 655, 335, 451, 4109, 47, 19842, 351, 260, 655, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 90, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 216, 33, 40, 42, 36, 37, 30, 49279 ], "answer": " 18:45.", "seconds": 0.6700815409421921, "stop_reason": "eos", "score": { "exact": true } }, { "id": "note-small", "image": "note-small.png", "question": "What is the pickup time on this note? Answer with the time only.", "kind": "time", "expected": "17:30", "image_sha256": "fa1427e9458c527f19bcda5f5a8be916e2ac742e67d0bce2e5202424cede16b2", "image_size": [ 64, 64 ], "prompt": "<|im_start|>User:What is the pickup time on this note? Answer with the time only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 1780, 314, 260, 40619, 655, 335, 451, 4109, 47, 19842, 351, 260, 655, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 90, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 216, 33, 39, 42, 35, 32, 30, 49279 ], "answer": " 17:30.", "seconds": 0.6657232088036835, "stop_reason": "eos", "score": { "exact": true } }, { "id": "note-crop", "image": "note-crop.png", "question": "What is the pickup time on this note? Answer with the time only.", "kind": "time", "expected": "17:30", "image_sha256": "e01bb878cca2fe50ee773de42a6497efcfe59b29f9a018d0029e6ca40dc3b9fa", "image_size": [ 456, 90 ], "prompt": "<|im_start|>User:What is the pickup time on this note? Answer with the time only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 1780, 314, 260, 40619, 655, 335, 451, 4109, 47, 19842, 351, 260, 655, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 90, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 216, 33, 39, 42, 35, 32, 30, 49279 ], "answer": " 17:30.", "seconds": 0.6367049580439925, "stop_reason": "eos", "score": { "exact": true } }, { "id": "note-blank", "image": "blank.png", "question": "What is the pickup time on this note? Answer with the time only.", "kind": "blank", "expected": null, "image_sha256": "b4d63041dafe3244fa279df657034d40d8e679ba7051476e6a30b8cac1aaf382", "image_size": [ 512, 512 ], "prompt": "<|im_start|>User:What is the pickup time on this note? Answer with the time only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 1780, 314, 260, 40619, 655, 335, 451, 4109, 47, 19842, 351, 260, 655, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 90, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 216, 33, 42, 32, 32, 30, 49279 ], "answer": " 1:00.", "seconds": 0.9693480827845633, "stop_reason": "eos", "score": { "invented_time": true } }, { "id": "chart-a", "image": "chart-a.png", "question": "Which bar is the highest? Answer with A, B, or C only.", "kind": "choice", "expected": "B", "image_sha256": "9b8ce09e349c3268c1441f823ba19e7108d0c6eb39428ec690229054cb083f4a", "image_size": [ 512, 512 ], "prompt": "<|im_start|>User:Which bar is the highest? Answer with A, B, or C only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 14413, 2753, 314, 260, 4919, 47, 19842, 351, 330, 28, 389, 28, 355, 340, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 91, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 389, 30, 49279 ], "answer": " B.", "seconds": 0.8421713751740754, "stop_reason": "eos", "score": { "exact": true } }, { "id": "chart-b", "image": "chart-b.png", "question": "Which bar is the highest? Answer with A, B, or C only.", "kind": "choice", "expected": "A", "image_sha256": "626b3897f9308a93dd1c8b7279be763dd8e982102596364f41fd54b109513baa", "image_size": [ 512, 512 ], "prompt": "<|im_start|>User:Which bar is the highest? Answer with A, B, or C only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 14413, 2753, 314, 260, 4919, 47, 19842, 351, 330, 28, 389, 28, 355, 340, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 91, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 330, 30, 49279 ], "answer": " A.", "seconds": 0.5409914581105113, "stop_reason": "eos", "score": { "exact": true } }, { "id": "chart-value", "image": "chart-a.png", "question": "What is the value of bar B? Answer with the number only.", "kind": "number", "expected": "7", "image_sha256": "9b8ce09e349c3268c1441f823ba19e7108d0c6eb39428ec690229054cb083f4a", "image_size": [ 512, 512 ], "prompt": "<|im_start|>User:What is the value of bar B? Answer with the number only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 1780, 314, 260, 1685, 282, 2753, 389, 47, 19842, 351, 260, 1230, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 89, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 216, 39, 30, 49279 ], "answer": " 7.", "seconds": 0.5431343750096858, "stop_reason": "eos", "score": { "exact": true } }, { "id": "position-a", "image": "position-a.png", "question": "Is the red square to the left or to the right of the blue circle? Answer left or right only.", "kind": "choice", "expected": "left", "image_sha256": "2c79c8bb9398899bf37f6c5627f696450eba814439298c4a07b00b67e0b508c2", "image_size": [ 512, 512 ], "prompt": "<|im_start|>User:Is the red square to the left or to the right of the blue circle? Answer left or right only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 4182, 260, 2382, 5222, 288, 260, 2049, 355, 288, 260, 1048, 282, 260, 4461, 7842, 47, 19842, 2049, 355, 1048, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 97, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 19937, 30, 49279 ], "answer": " Left.", "seconds": 0.6107187089510262, "stop_reason": "eos", "score": { "exact": true } }, { "id": "position-b", "image": "position-b.png", "question": "Is the red square to the left or to the right of the blue circle? Answer left or right only.", "kind": "choice", "expected": "right", "image_sha256": "f78d797ce3f97cc80aff81d7714abb875d9dcdb2591ee54a55e294b2854b4c66", "image_size": [ 512, 512 ], "prompt": "<|im_start|>User:Is the red square to the left or to the right of the blue circle? Answer left or right only.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 4182, 260, 2382, 5222, 288, 260, 2049, 355, 288, 260, 1048, 282, 260, 4461, 7842, 47, 19842, 2049, 355, 1048, 805, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 97, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 19937, 30, 49279 ], "answer": " Left.", "seconds": 0.5982103748247027, "stop_reason": "eos", "score": { "exact": false } }, { "id": "box-a", "image": "position-a.png", "question": "Locate the red square in this 512 by 512 image. Return only [x_min, y_min, x_max, y_max] in pixels, with the origin at the top left.", "kind": "box", "expected": [ 64, 176, 184, 296 ], "image_sha256": "2c79c8bb9398899bf37f6c5627f696450eba814439298c4a07b00b67e0b508c2", "image_size": [ 512, 512 ], "prompt": "<|im_start|>User:Locate the red square in this 512 by 512 image. Return only [x_min, y_min, x_max, y_max] in pixels, with the origin at the top left.\nAssistant:", "input_ids": [ 1, 11126, 42, 49189, 49152, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49190, 49189, 60, 11003, 260, 2382, 5222, 281, 451, 216, 37, 33, 34, 411, 216, 37, 33, 34, 2443, 30, 13632, 805, 933, 104, 79, 1659, 28, 329, 79, 1659, 28, 1792, 79, 3831, 28, 329, 79, 3831, 77, 281, 17218, 28, 351, 260, 5062, 418, 260, 1466, 2049, 30, 49279, 198, 9519, 9531, 42 ], "input_tokens": 123, "visual_tokens": 64, "pixel_values_shape": [ 1, 1, 3, 512, 512 ], "valid_pixels": 262144, "connector_shapes": [ [ 1, 64, 960 ] ], "output_ids": [ 933, 33, 28, 216, 33, 2880, 49279 ], "answer": " [1, 1].", "seconds": 0.6990775000303984, "stop_reason": "eos", "score": { "format_valid": false, "iou": null } } ] }