Skip to content

Commit a6caf5e

Browse files
SkalskiPcursoragentPawelPeczek-Roboflow
authored
Add SpaceXAI Grok workflow block (Grok 4.6 / 4.5) (#2799)
* Add SpaceXAI Grok workflow block with percent-format detection support. Introduces roboflow_core/spacexai@v1 for Grok 4.6/4.5 via xAI Responses API (managed rf_key proxy or direct key) and registers a spacexai parser in vlm_as_detector@v2 for the vlm-exam percent box_2d contract. Co-authored-by: Cursor <cursoragent@cursor.com> * Apply review fixes to SpaceXAI block. Fixes black formatting, drops the future-annotations import for parity with other blocks, corrects the reasoning_effort retry description (direct-key path only) and reuses the shared image-content helper in the detection prompt. Co-authored-by: Cursor <cursoragent@cursor.com> * Gate SpaceXAI managed key behind env flag and match vlm-exam image handling. Requires a user-provided xAI API key by default so the block works without the platform-side apiproxy/xai proxy; the rf_key option is enabled only via WORKFLOWS_SPACEXAI_MANAGED_KEY_ENABLED. Detection images are now sent at original resolution as PNG, matching the vlm-exam benchmark setup. Co-authored-by: Cursor <cursoragent@cursor.com> * Prune low-value SpaceXAI tests. Removes tests that pinned framework defaults, the unvalidated reasoning-error heuristic, and the flag-disabled proxy internals (payload shape and response extraction), plus a malformed-JSON case already covered by the block's generic handling. Keeps validator, key-routing, encoding, and parser contract tests. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Paweł Pęczek <146137186+PawelPeczek-Roboflow@users.noreply.github.com>
1 parent 3256020 commit a6caf5e

10 files changed

Lines changed: 1614 additions & 24 deletions

File tree

‎inference/core/env.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -944,6 +944,12 @@ def _reset_offline_mode_lock_after_fork() -> None:
944944
WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS = int(
945945
os.getenv("WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS", "8")
946946
)
947+
# Enables the Roboflow-managed (rf_key) API key option in the SpaceXAI block.
948+
# Off by default until the platform-side xAI proxy (apiproxy/xai) is deployed;
949+
# with the flag off users must provide their own xAI API key.
950+
WORKFLOWS_SPACEXAI_MANAGED_KEY_ENABLED = str2bool(
951+
os.getenv("WORKFLOWS_SPACEXAI_MANAGED_KEY_ENABLED", False)
952+
)
947953
ALLOW_CUSTOM_PYTHON_EXECUTION_IN_WORKFLOWS = str2bool(
948954
os.getenv("ALLOW_CUSTOM_PYTHON_EXECUTION_IN_WORKFLOWS", True)
949955
)

‎inference/core/workflows/core_steps/formatters/vlm_as_detector/spacexai_detection_parsing.py‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
from typing import List, Union
2+
from uuid import uuid4
3+
4+
import numpy as np
5+
import supervision as sv
6+
from supervision.config import CLASS_NAME_DATA_FIELD
7+
8+
from inference.core.workflows.core_steps.common.utils import (
9+
attach_parents_coordinates_to_sv_detections,
10+
)
11+
from inference.core.workflows.core_steps.formatters.vlm_as_detector.gemini_detection_parsing import (
12+
create_classes_index,
13+
get_gemini_detection_class_name,
14+
scale_confidence,
15+
)
16+
from inference.core.workflows.execution_engine.constants import (
17+
DETECTION_ID_KEY,
18+
IMAGE_DIMENSIONS_KEY,
19+
INFERENCE_ID_KEY,
20+
PREDICTION_TYPE_KEY,
21+
)
22+
from inference.core.workflows.execution_engine.entities.base import WorkflowImageData
23+
24+
SPACEXAI_BOX_COORDINATE_SCALE = 100.0
25+
26+
27+
def extract_spacexai_detection_entries(
28+
parsed_data: Union[dict, list],
29+
) -> List[dict]:
30+
"""Extract detection entries from SpaceXAI object-detection JSON.
31+
32+
Args:
33+
parsed_data: JSON payload extracted from the VLM output.
34+
35+
Returns:
36+
List of detection dictionaries.
37+
38+
Raises:
39+
ValueError: If the response is not a JSON list or a
40+
``{"detections": [...]}`` wrapper.
41+
"""
42+
if isinstance(parsed_data, list):
43+
return parsed_data
44+
if isinstance(parsed_data, dict) and "detections" in parsed_data:
45+
return parsed_data["detections"]
46+
raise ValueError("Unexpected SpaceXAI object detection response format")
47+
48+
49+
def convert_spacexai_detection_to_pixel_xyxy(
50+
detection: dict,
51+
image_height: int,
52+
image_width: int,
53+
) -> List[float]:
54+
"""Convert a percent ``box_2d`` entry into original-image pixel coordinates.
55+
56+
SpaceXAI Grok detection prompts ask for ``[x_min, y_min, x_max, y_max]`` as
57+
percentages of image width and height (floats 0-100). Coordinates are
58+
clamped to ``[0, 100]`` before scaling.
59+
60+
Args:
61+
detection: Detection entry with a ``box_2d`` field.
62+
image_height: Original image height in pixels.
63+
image_width: Original image width in pixels.
64+
65+
Returns:
66+
``[x_min, y_min, x_max, y_max]`` in pixel coordinates of the original
67+
image.
68+
"""
69+
x_min, y_min, x_max, y_max = detection["box_2d"]
70+
scale = SPACEXAI_BOX_COORDINATE_SCALE
71+
x_min = min(max(float(x_min), 0.0), scale)
72+
x_max = min(max(float(x_max), 0.0), scale)
73+
y_min = min(max(float(y_min), 0.0), scale)
74+
y_max = min(max(float(y_max), 0.0), scale)
75+
return [
76+
x_min / scale * image_width,
77+
y_min / scale * image_height,
78+
x_max / scale * image_width,
79+
y_max / scale * image_height,
80+
]
81+
82+
83+
def parse_spacexai_object_detection_response(
84+
image: WorkflowImageData,
85+
parsed_data: Union[dict, list],
86+
classes: List[str],
87+
inference_id: str,
88+
) -> sv.Detections:
89+
"""Parse SpaceXAI Grok object-detection output into detections.
90+
91+
Args:
92+
image: Workflow image the detections refer to.
93+
parsed_data: JSON list of detection entries produced by the model.
94+
classes: Class names used to map labels onto class ids.
95+
inference_id: Identifier attached to every parsed detection.
96+
97+
Returns:
98+
Parsed detections in the original image's coordinate space.
99+
"""
100+
detections = extract_spacexai_detection_entries(parsed_data=parsed_data)
101+
if len(detections) == 0:
102+
return sv.Detections.empty()
103+
104+
class_name2id = create_classes_index(classes=classes)
105+
image_height, image_width = image.numpy_image.shape[:2]
106+
107+
xyxy, class_id, class_name, confidence = [], [], [], []
108+
for detection in detections:
109+
xyxy.append(
110+
convert_spacexai_detection_to_pixel_xyxy(
111+
detection=detection,
112+
image_height=image_height,
113+
image_width=image_width,
114+
)
115+
)
116+
label = get_gemini_detection_class_name(detection=detection)
117+
class_id.append(class_name2id.get(label, -1))
118+
class_name.append(label)
119+
confidence.append(scale_confidence(detection.get("confidence", 1.0)))
120+
121+
xyxy = np.array(xyxy).round(0) if len(xyxy) > 0 else np.empty((0, 4))
122+
confidence = np.array(confidence) if len(confidence) > 0 else np.empty(0)
123+
class_id = np.array(class_id).astype(int) if len(class_id) > 0 else np.empty(0)
124+
class_name = np.array(class_name) if len(class_name) > 0 else np.empty(0)
125+
detection_ids = np.array([str(uuid4()) for _ in range(len(xyxy))])
126+
dimensions = np.array([[image_height, image_width]] * len(xyxy))
127+
inference_ids = np.array([inference_id] * len(xyxy))
128+
prediction_type = np.array(["object-detection"] * len(xyxy))
129+
data = {
130+
CLASS_NAME_DATA_FIELD: class_name,
131+
IMAGE_DIMENSIONS_KEY: dimensions,
132+
INFERENCE_ID_KEY: inference_ids,
133+
DETECTION_ID_KEY: detection_ids,
134+
PREDICTION_TYPE_KEY: prediction_type,
135+
}
136+
detections_result = sv.Detections(
137+
xyxy=xyxy,
138+
confidence=confidence,
139+
class_id=class_id,
140+
mask=None,
141+
tracker_id=None,
142+
data=data,
143+
)
144+
return attach_parents_coordinates_to_sv_detections(
145+
detections=detections_result,
146+
image=image,
147+
)

‎inference/core/workflows/core_steps/formatters/vlm_as_detector/v2.py‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
from inference.core.workflows.core_steps.formatters.vlm_as_detector.openai_detection_parsing import (
2222
parse_openai_object_detection_response,
2323
)
24+
from inference.core.workflows.core_steps.formatters.vlm_as_detector.spacexai_detection_parsing import (
25+
parse_spacexai_object_detection_response,
26+
)
2427
from inference.core.workflows.execution_engine.constants import (
2528
DETECTION_ID_KEY,
2629
IMAGE_DIMENSIONS_KEY,
@@ -145,7 +148,7 @@
145148
146149
## Requirements
147150
148-
This block requires an image input (for metadata and dimensions) and a VLM output string containing JSON detection data. The JSON can be raw JSON or wrapped in Markdown code blocks (```json ... ```). The block supports four model types: "openai", "google-gemini", "anthropic-claude", and "florence-2". It supports multiple task types: "object-detection", "open-vocabulary-object-detection", "object-detection-and-caption", "phrase-grounded-object-detection", "region-proposal", and "ocr-with-text-detection". The `classes` parameter is required for OpenAI, Gemini, and Claude models (to map class names to IDs) but optional for Florence-2 (some tasks don't require it). Classes are mapped to IDs by index (first class = 0, second = 1, etc.). Classes not in the list get class_id = -1. The block outputs object detection predictions in standard format (compatible with detection blocks), error_status (boolean), and inference_id (INFERENCE_ID_KIND) for tracking.
151+
This block requires an image input (for metadata and dimensions) and a VLM output string containing JSON detection data. The JSON can be raw JSON or wrapped in Markdown code blocks (```json ... ```). The block supports five model types: "openai", "google-gemini", "anthropic-claude", "spacexai", and "florence-2". It supports multiple task types: "object-detection", "open-vocabulary-object-detection", "object-detection-and-caption", "phrase-grounded-object-detection", "region-proposal", and "ocr-with-text-detection". The `classes` parameter is required for OpenAI, Gemini, Claude, and SpaceXAI models (to map class names to IDs) but optional for Florence-2 (some tasks don't require it). Classes are mapped to IDs by index (first class = 0, second = 1, etc.). Classes not in the list get class_id = -1. The block outputs object detection predictions in standard format (compatible with detection blocks), error_status (boolean), and inference_id (INFERENCE_ID_KIND) for tracking.
149152
"""
150153

151154
SHORT_DESCRIPTION = "Parses raw string into object-detection prediction."
@@ -213,22 +216,28 @@ class BlockManifest(WorkflowBlockManifest):
213216
json_schema_extra={
214217
"relevant_for": {
215218
"model_type": {
216-
"values": ["openai", "google-gemini", "anthropic-claude"],
219+
"values": [
220+
"openai",
221+
"google-gemini",
222+
"anthropic-claude",
223+
"spacexai",
224+
],
217225
"required": True,
218226
},
219227
}
220228
},
221229
)
222-
model_type: Literal["openai", "google-gemini", "anthropic-claude", "florence-2"] = (
223-
Field(
224-
description="Type of the VLM/LLM model that generated the prediction. Determines which parser is used to extract detection data from the JSON output. Supported models: 'openai' (GPT-4V), 'google-gemini' (Gemini Vision), 'anthropic-claude' (Claude Vision), 'florence-2' (Microsoft Florence-2). Each model type has different JSON output formats, so the correct model type must be specified for proper parsing.",
225-
examples=[
226-
["openai"],
227-
["google-gemini"],
228-
["anthropic-claude"],
229-
["florence-2"],
230-
],
231-
)
230+
model_type: Literal[
231+
"openai", "google-gemini", "anthropic-claude", "spacexai", "florence-2"
232+
] = Field(
233+
description="Type of the VLM/LLM model that generated the prediction. Determines which parser is used to extract detection data from the JSON output. Supported models: 'openai' (GPT-4V), 'google-gemini' (Gemini Vision), 'anthropic-claude' (Claude Vision), 'spacexai' (Grok), 'florence-2' (Microsoft Florence-2). Each model type has different JSON output formats, so the correct model type must be specified for proper parsing.",
234+
examples=[
235+
["openai"],
236+
["google-gemini"],
237+
["anthropic-claude"],
238+
["spacexai"],
239+
["florence-2"],
240+
],
232241
)
233242
task_type: Literal[tuple(SUPPORTED_TASKS)] = Field(
234243
description="Task type performed by the VLM/LLM model. Determines which parser and format handler is used. Supported task types: 'object-detection' (standard object detection), 'open-vocabulary-object-detection' (detect objects with custom classes), 'object-detection-and-caption' (detection with captions), 'phrase-grounded-object-detection' (ground phrases to detections), 'region-proposal' (propose regions of interest), 'ocr-with-text-detection' (OCR with text region detection). The task type must match what the VLM/LLM was asked to perform.",
@@ -517,6 +526,7 @@ def parse_openai_detection_response(
517526
("openai", "object-detection"): parse_openai_detection_response,
518527
("google-gemini", "object-detection"): parse_gemini_object_detection_response,
519528
("anthropic-claude", "object-detection"): parse_llm_object_detection_response,
529+
("spacexai", "object-detection"): parse_spacexai_object_detection_response,
520530
# Florence 2
521531
("florence-2", "object-detection"): partial(
522532
parse_florence2_object_detection_response, florence_task_type="<OD>"

‎inference/core/workflows/core_steps/formatters/vlm_as_detector/v2_tensor.py‎

Lines changed: 76 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@
2323
from inference.core.workflows.core_steps.formatters.vlm_as_detector.openai_detection_parsing import (
2424
convert_openai_detection_to_pixel_xyxy,
2525
)
26+
from inference.core.workflows.core_steps.formatters.vlm_as_detector.spacexai_detection_parsing import (
27+
convert_spacexai_detection_to_pixel_xyxy,
28+
extract_spacexai_detection_entries,
29+
)
2630
from inference.core.workflows.execution_engine.constants import (
2731
CLASS_NAME_KEY,
2832
CLASS_NAMES_KEY,
@@ -158,7 +162,7 @@
158162
159163
## Requirements
160164
161-
This block requires an image input (for metadata and dimensions) and a VLM output string containing JSON detection data. The JSON can be raw JSON or wrapped in Markdown code blocks (```json ... ```). The block supports four model types: "openai", "google-gemini", "anthropic-claude", and "florence-2". It supports multiple task types: "object-detection", "open-vocabulary-object-detection", "object-detection-and-caption", "phrase-grounded-object-detection", "region-proposal", and "ocr-with-text-detection". The `classes` parameter is required for OpenAI, Gemini, and Claude models (to map class names to IDs) but optional for Florence-2 (some tasks don't require it). Classes are mapped to IDs by index (first class = 0, second = 1, etc.). Classes not in the list get class_id = -1. The block outputs object detection predictions in standard format (compatible with detection blocks), error_status (boolean), and inference_id (INFERENCE_ID_KIND) for tracking.
165+
This block requires an image input (for metadata and dimensions) and a VLM output string containing JSON detection data. The JSON can be raw JSON or wrapped in Markdown code blocks (```json ... ```). The block supports five model types: "openai", "google-gemini", "anthropic-claude", "spacexai", and "florence-2". It supports multiple task types: "object-detection", "open-vocabulary-object-detection", "object-detection-and-caption", "phrase-grounded-object-detection", "region-proposal", and "ocr-with-text-detection". The `classes` parameter is required for OpenAI, Gemini, Claude, and SpaceXAI models (to map class names to IDs) but optional for Florence-2 (some tasks don't require it). Classes are mapped to IDs by index (first class = 0, second = 1, etc.). Classes not in the list get class_id = -1. The block outputs object detection predictions in standard format (compatible with detection blocks), error_status (boolean), and inference_id (INFERENCE_ID_KIND) for tracking.
162166
"""
163167

164168
SHORT_DESCRIPTION = "Parses raw string into object-detection prediction."
@@ -226,22 +230,28 @@ class BlockManifest(WorkflowBlockManifest):
226230
json_schema_extra={
227231
"relevant_for": {
228232
"model_type": {
229-
"values": ["openai", "google-gemini", "anthropic-claude"],
233+
"values": [
234+
"openai",
235+
"google-gemini",
236+
"anthropic-claude",
237+
"spacexai",
238+
],
230239
"required": True,
231240
},
232241
}
233242
},
234243
)
235-
model_type: Literal["openai", "google-gemini", "anthropic-claude", "florence-2"] = (
236-
Field(
237-
description="Type of the VLM/LLM model that generated the prediction. Determines which parser is used to extract detection data from the JSON output. Supported models: 'openai' (GPT-4V), 'google-gemini' (Gemini Vision), 'anthropic-claude' (Claude Vision), 'florence-2' (Microsoft Florence-2). Each model type has different JSON output formats, so the correct model type must be specified for proper parsing.",
238-
examples=[
239-
["openai"],
240-
["google-gemini"],
241-
["anthropic-claude"],
242-
["florence-2"],
243-
],
244-
)
244+
model_type: Literal[
245+
"openai", "google-gemini", "anthropic-claude", "spacexai", "florence-2"
246+
] = Field(
247+
description="Type of the VLM/LLM model that generated the prediction. Determines which parser is used to extract detection data from the JSON output. Supported models: 'openai' (GPT-4V), 'google-gemini' (Gemini Vision), 'anthropic-claude' (Claude Vision), 'spacexai' (Grok), 'florence-2' (Microsoft Florence-2). Each model type has different JSON output formats, so the correct model type must be specified for proper parsing.",
248+
examples=[
249+
["openai"],
250+
["google-gemini"],
251+
["anthropic-claude"],
252+
["spacexai"],
253+
["florence-2"],
254+
],
245255
)
246256
task_type: Literal[tuple(SUPPORTED_TASKS)] = Field(
247257
description="Task type performed by the VLM/LLM model. Determines which parser and format handler is used. Supported task types: 'object-detection' (standard object detection), 'open-vocabulary-object-detection' (detect objects with custom classes), 'object-detection-and-caption' (detection with captions), 'phrase-grounded-object-detection' (ground phrases to detections), 'region-proposal' (propose regions of interest), 'ocr-with-text-detection' (OCR with text region detection). The task type must match what the VLM/LLM was asked to perform.",
@@ -699,6 +709,59 @@ def parse_openai_object_detection_response(
699709
)
700710

701711

712+
def parse_spacexai_object_detection_response(
713+
image: WorkflowImageData,
714+
parsed_data: Union[dict, list],
715+
classes: List[str],
716+
inference_id: str,
717+
) -> Detections:
718+
"""Parse SpaceXAI Grok object-detection output into native detections.
719+
720+
Coordinates are percentages of image width/height (floats 0-100) in
721+
``[x_min, y_min, x_max, y_max]`` order. Tensor-native sibling of the
722+
numpy ``spacexai_detection_parsing.parse_spacexai_object_detection_response``.
723+
"""
724+
class_name2id = create_classes_index(classes=classes)
725+
image_height, image_width = image._read_shape_without_materialization()
726+
detections = extract_spacexai_detection_entries(parsed_data=parsed_data)
727+
if len(detections) == 0:
728+
return empty_native_detections(
729+
image=image,
730+
image_height=image_height,
731+
image_width=image_width,
732+
inference_id=inference_id,
733+
class_names={idx: class_name for class_name, idx in class_name2id.items()},
734+
)
735+
736+
xyxy, class_id, class_name, confidence = [], [], [], []
737+
for detection in detections:
738+
xyxy.append(
739+
convert_spacexai_detection_to_pixel_xyxy(
740+
detection=detection,
741+
image_height=image_height,
742+
image_width=image_width,
743+
)
744+
)
745+
label = get_gemini_detection_class_name(detection=detection)
746+
class_id.append(class_name2id.get(label, -1))
747+
class_name.append(label)
748+
confidence.append(scale_confidence(detection.get("confidence", 1.0)))
749+
750+
xyxy = np.array(xyxy).round(0) if len(xyxy) > 0 else np.empty((0, 4))
751+
confidence = np.array(confidence) if len(confidence) > 0 else np.empty(0)
752+
class_id = np.array(class_id).astype(int) if len(class_id) > 0 else np.empty(0)
753+
return native_detections_from_parsed(
754+
image=image,
755+
image_height=image_height,
756+
image_width=image_width,
757+
inference_id=inference_id,
758+
xyxy=xyxy,
759+
class_id=class_id,
760+
class_name=class_name,
761+
confidence=confidence,
762+
)
763+
764+
702765
def parse_openai_detection_response(
703766
image: WorkflowImageData,
704767
parsed_data: Union[dict, list],
@@ -748,6 +811,7 @@ def parse_openai_detection_response(
748811
("openai", "object-detection"): parse_openai_detection_response,
749812
("google-gemini", "object-detection"): parse_gemini_object_detection_response,
750813
("anthropic-claude", "object-detection"): parse_llm_object_detection_response,
814+
("spacexai", "object-detection"): parse_spacexai_object_detection_response,
751815
# Florence 2
752816
("florence-2", "object-detection"): partial(
753817
parse_florence2_object_detection_response, florence_task_type="<OD>"

‎inference/core/workflows/core_steps/loader.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -627,6 +627,9 @@
627627
from inference.core.workflows.core_steps.models.foundation.openrouter.v1 import (
628628
OpenRouterBlockV1,
629629
)
630+
from inference.core.workflows.core_steps.models.foundation.spacexai.v1 import (
631+
SpaceXAIBlockV1,
632+
)
630633

631634
if not ENABLE_TENSOR_DATA_REPRESENTATION:
632635
from inference.core.workflows.core_steps.models.foundation.perception_encoder.v1 import (
@@ -1723,6 +1726,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]:
17231726
AnthropicClaudeBlockV1,
17241727
AnthropicClaudeBlockV2,
17251728
AnthropicClaudeBlockV3,
1729+
SpaceXAIBlockV1,
17261730
CosineSimilarityBlockV1,
17271731
BackgroundColorVisualizationBlockV1,
17281732
BarcodeDetectorBlockV1,

‎inference/core/workflows/core_steps/models/foundation/spacexai/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)