Skip to content

Commit 1049093

Browse files
fix(workflows): preserve image dimensions on empty VLM detections
The numpy serialiser returned {"width": null, "height": null} for empty VLM-as-detector results because sv.Detections.data is per-row — zero rows means the serialiser per-row loop never executes and IMAGE_DIMENSIONS_KEY is never read. The tensor-native path stores image dimensions in image_metadata (a dict on the Detections object), so it correctly emits real width/height for empty detections. Fix: store IMAGE_DIMENSIONS_KEY in sv.Detections.metadata (a free-form dict that survives zero rows) and teach the numpy serialiser to read it as a fallback when the per-row loop yields no rows. Before: serialise_sv_detections(parse_...(empty_input)) returns {image: {width: null, height: null}, predictions: []} After: serialise_sv_detections(parse_...(empty_input)) returns {image: {width: 640, height: 480}, predictions: []} All 7 VLM-as-detector parsers updated (anthropic, gemini, muse, openai, qwen, spacexai, v2/llm). 1905 tests pass.
1 parent 11dd0a3 commit 1049093

16 files changed

Lines changed: 173 additions & 9 deletions

inference/core/workflows/core_steps/common/serializers.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -250,10 +250,16 @@ def serialise_sv_detections(detections: sv.Detections) -> dict:
250250
} # TODO: this breaks the contract of
251251
# standard inference, but to fix that problem, we would need sv.Detections to provide
252252
# detection-level metadata.
253+
if image_dimensions is None:
254+
# Zero-row detections carry image dimensions in ``metadata`` (not per-row
255+
# ``data``, which the loop above never iterates). Producers that return
256+
# empty detections with ``metadata={IMAGE_DIMENSIONS_KEY: [h, w]}`` get
257+
# the same ``image.width`` / ``image.height`` as the tensor-native path.
258+
image_dimensions = detections.metadata.get(IMAGE_DIMENSIONS_KEY)
253259
if image_dimensions is not None:
254260
image_metadata = {
255-
"width": image_dimensions[1].item(),
256-
"height": image_dimensions[0].item(),
261+
"width": int(image_dimensions[1]),
262+
"height": int(image_dimensions[0]),
257263
}
258264
return {"image": image_metadata, "predictions": serialized_detections}
259265

inference/core/workflows/core_steps/common/utils.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,26 @@ def attach_parents_coordinates_to_batch_of_sv_detections(
228228
return result
229229

230230

231+
def empty_detections_with_image_metadata(
232+
image_height: int,
233+
image_width: int,
234+
) -> sv.Detections:
235+
"""Create a zero-row ``sv.Detections`` that carries image dimensions in
236+
``metadata`` so the numpy serialiser can emit real ``image.width`` /
237+
``image.height`` for empty VLM results — matching the tensor-native path.
238+
239+
``sv.Detections.data`` is per-row, so zero rows means the serialiser's
240+
per-row loop never executes and any keys stored there are invisible.
241+
``metadata`` is a free-form dict on ``sv.Detections`` that survives zero
242+
rows and is read by the serialiser as a fallback when no per-row
243+
``IMAGE_DIMENSIONS_KEY`` is found.
244+
"""
245+
return sv.Detections(
246+
xyxy=np.empty((0, 4), dtype=np.float32),
247+
metadata={IMAGE_DIMENSIONS_KEY: [image_height, image_width]},
248+
)
249+
250+
231251
def attach_parents_coordinates_to_sv_detections(
232252
detections: sv.Detections,
233253
image: WorkflowImageData,

inference/core/workflows/core_steps/formatters/vlm_as_detector/anthropic_detection_parsing.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from inference.core.workflows.core_steps.common.utils import (
99
attach_parents_coordinates_to_sv_detections,
1010
compute_anthropic_upload_dimensions,
11+
empty_detections_with_image_metadata,
1112
)
1213
from inference.core.workflows.core_steps.formatters.vlm_as_detector.gemini_detection_parsing import (
1314
create_classes_index,
@@ -88,7 +89,11 @@ def parse_anthropic_object_detection_response(
8889
if not isinstance(parsed_data, list):
8990
raise ValueError("Unexpected Anthropic Claude object detection response format")
9091
if len(parsed_data) == 0:
91-
return sv.Detections.empty()
92+
image_height, image_width = image.numpy_image.shape[:2]
93+
return empty_detections_with_image_metadata(
94+
image_height=image_height,
95+
image_width=image_width,
96+
)
9297

9398
class_name2id = create_classes_index(classes=classes)
9499
image_height, image_width = image.numpy_image.shape[:2]

inference/core/workflows/core_steps/formatters/vlm_as_detector/gemini_detection_parsing.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
from inference.core.workflows.core_steps.common.utils import (
99
attach_parents_coordinates_to_sv_detections,
10+
empty_detections_with_image_metadata,
1011
)
1112
from inference.core.workflows.execution_engine.constants import (
1213
DETECTION_ID_KEY,
@@ -77,7 +78,10 @@ def parse_gemini_object_detection_response(
7778
image_height, image_width = image.numpy_image.shape[:2]
7879
detections = extract_gemini_detection_entries(parsed_data=parsed_data)
7980
if len(detections) == 0:
80-
return sv.Detections.empty()
81+
return empty_detections_with_image_metadata(
82+
image_height=image_height,
83+
image_width=image_width,
84+
)
8185

8286
xyxy, class_id, class_name, confidence = [], [], [], []
8387
for detection in detections:

inference/core/workflows/core_steps/formatters/vlm_as_detector/muse_detection_parsing.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from inference.core.logger import logger
1111
from inference.core.workflows.core_steps.common.utils import (
1212
attach_parents_coordinates_to_sv_detections,
13+
empty_detections_with_image_metadata,
1314
)
1415
from inference.core.workflows.core_steps.formatters.vlm_as_detector.gemini_detection_parsing import (
1516
create_classes_index,
@@ -195,7 +196,10 @@ def parse_muse_object_detection_response(
195196
confidence.append(1.0)
196197

197198
if not xyxy:
198-
return sv.Detections.empty()
199+
return empty_detections_with_image_metadata(
200+
image_height=image_height,
201+
image_width=image_width,
202+
)
199203

200204
xyxy = np.array(xyxy).round(0)
201205
detection_ids = np.array([str(uuid4()) for _ in range(len(xyxy))])

inference/core/workflows/core_steps/formatters/vlm_as_detector/openai_detection_parsing.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from inference.core.workflows.core_steps.common.utils import (
99
DETECTION_MAX_EDGE_PIXELS,
1010
attach_parents_coordinates_to_sv_detections,
11+
empty_detections_with_image_metadata,
1112
scale_dimensions_to_max_edge,
1213
)
1314
from inference.core.workflows.core_steps.formatters.vlm_as_detector.gemini_detection_parsing import (
@@ -89,7 +90,11 @@ def parse_openai_object_detection_response(
8990
if not isinstance(parsed_data, list):
9091
raise ValueError("Unexpected OpenAI object detection response format")
9192
if len(parsed_data) == 0:
92-
return sv.Detections.empty()
93+
image_height, image_width = image.numpy_image.shape[:2]
94+
return empty_detections_with_image_metadata(
95+
image_height=image_height,
96+
image_width=image_width,
97+
)
9398

9499
class_name2id = create_classes_index(classes=classes)
95100
image_height, image_width = image.numpy_image.shape[:2]

inference/core/workflows/core_steps/formatters/vlm_as_detector/qwen_detection_parsing.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from inference.core.logger import logger
99
from inference.core.workflows.core_steps.common.utils import (
1010
attach_parents_coordinates_to_sv_detections,
11+
empty_detections_with_image_metadata,
1112
)
1213
from inference.core.workflows.core_steps.formatters.vlm_as_detector.gemini_detection_parsing import (
1314
create_classes_index,
@@ -183,7 +184,10 @@ def parse_qwen_object_detection_response(
183184
confidence.append(1.0)
184185

185186
if not xyxy:
186-
return sv.Detections.empty()
187+
return empty_detections_with_image_metadata(
188+
image_height=image_height,
189+
image_width=image_width,
190+
)
187191

188192
xyxy = np.array(xyxy).round(0)
189193
confidence = np.array(confidence)

inference/core/workflows/core_steps/formatters/vlm_as_detector/spacexai_detection_parsing.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
from inference.core.workflows.core_steps.common.utils import (
99
attach_parents_coordinates_to_sv_detections,
10+
empty_detections_with_image_metadata,
1011
)
1112
from inference.core.workflows.core_steps.formatters.vlm_as_detector.gemini_detection_parsing import (
1213
create_classes_index,
@@ -99,7 +100,11 @@ def parse_spacexai_object_detection_response(
99100
"""
100101
detections = extract_spacexai_detection_entries(parsed_data=parsed_data)
101102
if len(detections) == 0:
102-
return sv.Detections.empty()
103+
image_height, image_width = image.numpy_image.shape[:2]
104+
return empty_detections_with_image_metadata(
105+
image_height=image_height,
106+
image_width=image_width,
107+
)
103108

104109
class_name2id = create_classes_index(classes=classes)
105110
image_height, image_width = image.numpy_image.shape[:2]

inference/core/workflows/core_steps/formatters/vlm_as_detector/v2.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
from inference.core.workflows.core_steps.common.utils import (
1515
attach_parents_coordinates_to_sv_detections,
16+
empty_detections_with_image_metadata,
1617
)
1718
from inference.core.workflows.core_steps.common.vlms import VLM_TASKS_METADATA
1819
from inference.core.workflows.core_steps.formatters.vlm_as_detector.anthropic_detection_parsing import (
@@ -398,7 +399,10 @@ def parse_llm_object_detection_response(
398399
class_name2id = create_classes_index(classes=classes)
399400
image_height, image_width = image.numpy_image.shape[:2]
400401
if len(parsed_data["detections"]) == 0:
401-
return sv.Detections.empty()
402+
return empty_detections_with_image_metadata(
403+
image_height=image_height,
404+
image_width=image_width,
405+
)
402406
xyxy, class_id, class_name, confidence = [], [], [], []
403407
for detection in parsed_data["detections"]:
404408
xyxy.append(

tests/workflows/unit_tests/core_steps/common/test_serializers.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,34 @@ def test_serialise_sv_detections() -> None:
195195
}
196196

197197

198+
def test_serialise_sv_detections_empty_without_metadata_returns_none_dims() -> None:
199+
# given — plain sv.Detections.empty() has no metadata, so the serialiser
200+
# cannot recover image dimensions (the historical behaviour).
201+
detections = sv.Detections.empty()
202+
203+
# when
204+
result = serialise_sv_detections(detections=detections)
205+
206+
# then
207+
assert result == {"image": {"width": None, "height": None}, "predictions": []}
208+
209+
210+
def test_serialise_sv_detections_empty_with_metadata_returns_real_dims() -> None:
211+
# given — zero-row detections carrying IMAGE_DIMENSIONS_KEY in metadata
212+
# (the contract producers use to pass image-level info past zero rows).
213+
detections = sv.Detections(
214+
xyxy=np.empty((0, 4), dtype=np.float32),
215+
metadata={"image_dimensions": [480, 640]},
216+
)
217+
218+
# when
219+
result = serialise_sv_detections(detections=detections)
220+
221+
# then — the serialiser reads image dimensions from metadata as a fallback
222+
# when the per-row loop never executes, matching the tensor-native path.
223+
assert result == {"image": {"width": 640, "height": 480}, "predictions": []}
224+
225+
198226
def test_serialise_sv_detections_skips_padded_keypoint_slots() -> None:
199227
# given the padded, rectangular (n, max_kps, 2) keypoint layout that
200228
# add_inference_keypoints_to_sv_detections produces when detections carry

0 commit comments

Comments
 (0)