diff --git a/development/benchmark_scripts/benchmark_remote_stream_pipeline.py b/development/benchmark_scripts/benchmark_remote_stream_pipeline.py new file mode 100644 index 0000000000..fc2bfadb99 --- /dev/null +++ b/development/benchmark_scripts/benchmark_remote_stream_pipeline.py @@ -0,0 +1,572 @@ +"""Benchmark remote workflow stream pipelining against sequential execution. + +Runs a video workflow through `InferencePipeline.init_with_workflow` once per +requested pipeline depth. Several workflow shapes are available via +--workflow: tracking (remote model -> ByteTrack -> visualization), sam3 +(text-prompt segmentation), two-models (two models -> consensus), and +preprocessed-tracking (stateless crop before the model plus a stateless +side branch). Depth 1 is the sequential baseline; depths +above 1 enable `WORKFLOWS_STREAM_LOOKAHEAD_DEPTH`, which keeps that +many remote model requests in flight while ByteTrack consumes results in +strict frame order. + +Each scenario runs in a subprocess because the pipeline depth env variable is +read at import time. The parent aggregates per-scenario results, verifies that +every pipelined run emits frames in order with tracker ids identical to the +sequential baseline, and writes a combined summary. + +Example: + ROBOFLOW_API_KEY=... python development/benchmark_scripts/benchmark_remote_stream_pipeline.py \ + --api-url https://serverless.roboflow.com \ + --model-id rfdetr-nano \ + --frames 300 --warmup 10 --resize-width 640 \ + --depths 1,8,16,32 \ + --output-dir /tmp/remote_stream_pipeline_benchmark +""" + +import argparse +import json +import os +import platform +import statistics +import subprocess +import sys +import time +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +DEFAULT_API_URL = "https://serverless.roboflow.com" +DEFAULT_MODEL_ID = "rfdetr-nano" +DEFAULT_VIDEO_URL = ( + "https://media.roboflow.com/supervision/video-examples/vehicles-2.mp4" +) +SCENARIO_RESULT_MARKER = "SCENARIO_RESULT_JSON " + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Compare sequential remote workflow execution against remote " + "stream pipelining across pipeline depths." + ) + ) + parser.add_argument("--api-url", default=DEFAULT_API_URL) + parser.add_argument( + "--core-model-url", + default="https://infer.roboflow.com", + help="Hosted core-model API used by the sam3 workflow.", + ) + parser.add_argument("--model-id", default=DEFAULT_MODEL_ID) + parser.add_argument("--video-url", default=DEFAULT_VIDEO_URL) + parser.add_argument( + "--frames", type=int, default=300, help="Measured frames per scenario." + ) + parser.add_argument( + "--warmup", + type=int, + default=10, + help="Frames run before measurement to warm the model and sessions.", + ) + parser.add_argument( + "--resize-width", + type=int, + default=640, + help="Width the benchmark clip is scaled to; 0 keeps the original.", + ) + parser.add_argument( + "--depths", + default="1,8,16,32", + help="Comma-separated pipeline depths; depth 1 is the baseline.", + ) + parser.add_argument( + "--workflow", + choices=["tracking", "sam3", "two-models", "preprocessed-tracking"], + default="tracking", + help=( + "Workflow shape: tracking (model -> ByteTrack -> viz), sam3 " + "(SAM3 text-prompt segmentation -> ByteTrack -> viz), two-models " + "(two detection models -> consensus -> ByteTrack -> viz), " + "preprocessed-tracking (static crop -> model -> ByteTrack -> viz " + "plus a stateless blur side branch)." + ), + ) + parser.add_argument( + "--second-model-id", + default="yolov8n-640", + help="Second model for the two-models workflow.", + ) + parser.add_argument( + "--sam3-class-names", + default="car,truck", + help="Comma-separated text prompts for the sam3 workflow.", + ) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument( + "--run-single-depth", + type=int, + default=None, + help="Internal: run one scenario in this process and print its result.", + ) + return parser.parse_args() + + +def prepare_clip( + video_url: str, + frames: int, + resize_width: int, + output_dir: Path, +) -> Dict[str, Any]: + import cv2 + + source_path = output_dir / "source-video.mp4" + if not source_path.exists(): + urllib.request.urlretrieve(video_url, source_path) + clip_path = output_dir / f"clip-{frames}f-w{resize_width}.mp4" + capture = cv2.VideoCapture(str(source_path)) + fps = capture.get(cv2.CAP_PROP_FPS) + source_width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH)) + source_height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT)) + if resize_width and resize_width < source_width: + clip_width = resize_width + clip_height = round(source_height * resize_width / source_width) + else: + clip_width, clip_height = source_width, source_height + if not clip_path.exists(): + writer = cv2.VideoWriter( + str(clip_path), + cv2.VideoWriter_fourcc(*"mp4v"), + fps, + (clip_width, clip_height), + ) + written = 0 + while written < frames: + success, frame = capture.read() + if not success: + break + if (clip_width, clip_height) != (source_width, source_height): + frame = cv2.resize( + frame, (clip_width, clip_height), interpolation=cv2.INTER_AREA + ) + writer.write(frame) + written += 1 + writer.release() + capture.release() + return { + "clip_path": str(clip_path), + "fps": fps, + "width": clip_width, + "height": clip_height, + } + + +def build_workflow_specification(args: argparse.Namespace) -> Dict[str, Any]: + if args.workflow == "tracking": + detection_steps = [ + { + "type": "roboflow_core/roboflow_object_detection_model@v3", + "name": "model", + "images": "$inputs.image", + "model_id": args.model_id, + "confidence_mode": "custom", + "custom_confidence": 0.35, + }, + ] + tracker_detections_selector = "$steps.model.predictions" + elif args.workflow == "sam3": + detection_steps = [ + { + "type": "roboflow_core/sam3@v3", + "name": "sam3", + "images": "$inputs.image", + "class_names": args.sam3_class_names.split(","), + "output_format": "rle", + }, + ] + tracker_detections_selector = "$steps.sam3.predictions" + elif args.workflow == "preprocessed-tracking": + # Stateless preprocessing UPSTREAM of the model plus a stateless side + # branch — shapes only executable ahead of stream order with the + # frontier scheduler. + detection_steps = [ + { + "type": "roboflow_core/absolute_static_crop@v1", + "name": "crop", + "images": "$inputs.image", + "x_center": 320, + "y_center": 180, + "width": 600, + "height": 340, + }, + { + "type": "roboflow_core/image_blur@v1", + "name": "blur", + "image": "$inputs.image", + }, + { + "type": "roboflow_core/roboflow_object_detection_model@v3", + "name": "model", + "images": "$steps.crop.crops", + "model_id": args.model_id, + "confidence_mode": "custom", + "custom_confidence": 0.35, + }, + ] + tracker_detections_selector = "$steps.model.predictions" + else: + detection_steps = [ + { + "type": "roboflow_core/roboflow_object_detection_model@v3", + "name": "model_a", + "images": "$inputs.image", + "model_id": args.model_id, + "confidence_mode": "custom", + "custom_confidence": 0.35, + }, + { + "type": "roboflow_core/roboflow_object_detection_model@v3", + "name": "model_b", + "images": "$inputs.image", + "model_id": args.second_model_id, + "confidence_mode": "custom", + "custom_confidence": 0.35, + }, + { + "type": "roboflow_core/detections_consensus@v1", + "name": "consensus", + "predictions_batches": [ + "$steps.model_a.predictions", + "$steps.model_b.predictions", + ], + "required_votes": 1, + }, + ] + tracker_detections_selector = "$steps.consensus.predictions" + return { + "version": "1.0.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": detection_steps + + [ + { + "type": "roboflow_core/byte_tracker@v3", + "name": "byte_tracker", + "image": "$inputs.image", + "detections": tracker_detections_selector, + }, + { + "type": "roboflow_core/bounding_box_visualization@v1", + "name": "bounding_box_visualization", + "image": "$inputs.image", + "predictions": "$steps.byte_tracker.tracked_detections", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "tracked_detections", + "coordinates_system": "own", + "selector": "$steps.byte_tracker.tracked_detections", + }, + { + "type": "JsonField", + "name": "visualization", + "coordinates_system": "own", + "selector": "$steps.bounding_box_visualization.image", + }, + ] + + ( + [ + { + "type": "JsonField", + "name": "blurred", + "coordinates_system": "own", + "selector": "$steps.blur.image", + } + ] + if args.workflow == "preprocessed-tracking" + else [] + ), + } + + +def run_scenario(args: argparse.Namespace, depth: int) -> Dict[str, Any]: + from inference.core.interfaces.stream.inference_pipeline import InferencePipeline + + workflow_specification = build_workflow_specification(args=args) + clip = json.loads((args.output_dir / "clip-metadata.json").read_text()) + + def execute_pipeline(video_path: str) -> Dict[str, Any]: + emissions: List[Dict[str, Any]] = [] + + def on_prediction(predictions, video_frame) -> None: + tracked = predictions.get("tracked_detections") if predictions else None + tracker_ids = ( + [int(tracker_id) for tracker_id in tracked.tracker_id] + if tracked is not None and tracked.tracker_id is not None + else [] + ) + emissions.append( + { + "frame_id": video_frame.frame_id, + "emitted_at": time.perf_counter(), + "tracker_ids": tracker_ids, + "has_visualization": ( + predictions.get("visualization") is not None + if predictions + else False + ), + } + ) + + pipeline = InferencePipeline.init_with_workflow( + video_reference=video_path, + workflow_specification=workflow_specification, + api_key=os.environ["ROBOFLOW_API_KEY"], + on_prediction=on_prediction, + ) + runner_type = type(pipeline._on_video_frame).__name__ + started_at = time.perf_counter() + pipeline.start() + pipeline.join() + finished_at = time.perf_counter() + return { + "runner_type": runner_type, + "started_at": started_at, + "finished_at": finished_at, + "emissions": emissions, + } + + warmup_clip = args.output_dir / f"warmup-{args.warmup}f.mp4" + if args.warmup > 0 and warmup_clip.exists(): + execute_pipeline(video_path=str(warmup_clip)) + measured = execute_pipeline(video_path=clip["clip_path"]) + + emissions = measured["emissions"] + if not emissions: + raise RuntimeError( + "Scenario emitted no frames — the pipeline likely hit an error on " + "the inference thread (check credentials/URLs and rerun with logs)." + ) + frame_ids = [emission["frame_id"] for emission in emissions] + emission_times = [emission["emitted_at"] for emission in emissions] + wall_seconds = measured["finished_at"] - measured["started_at"] + steady_seconds = ( + emission_times[-1] - emission_times[0] if len(emission_times) > 1 else None + ) + inter_emission_ms = [ + (later - earlier) * 1000 + for earlier, later in zip(emission_times, emission_times[1:]) + ] + return { + "depth": depth, + "runner_type": measured["runner_type"], + "frames_emitted": len(emissions), + "wall_seconds": round(wall_seconds, 3), + "wall_fps": round(len(emissions) / wall_seconds, 2) if wall_seconds else None, + "steady_fps": ( + round((len(emissions) - 1) / steady_seconds, 2) if steady_seconds else None + ), + "first_emission_latency_seconds": ( + round(emission_times[0] - measured["started_at"], 3) + if emission_times + else None + ), + "inter_emission_ms": summarize_values(values=inter_emission_ms), + "emitted_in_order": frame_ids == sorted(frame_ids), + "unique_frames": len(set(frame_ids)) == len(frame_ids), + "all_have_visualization": all( + emission["has_visualization"] for emission in emissions + ), + "total_tracked_detections": sum( + len(emission["tracker_ids"]) for emission in emissions + ), + "tracker_ids_per_frame": { + str(emission["frame_id"]): emission["tracker_ids"] for emission in emissions + }, + } + + +def summarize_values(values: List[float]) -> Optional[Dict[str, float]]: + if not values: + return None + ordered = sorted(values) + return { + "count": len(values), + "min": round(ordered[0], 2), + "max": round(ordered[-1], 2), + "avg": round(statistics.fmean(values), 2), + "median": round(statistics.median(values), 2), + "p95": round(percentile(ordered, 0.95), 2), + } + + +def percentile(ordered_values: List[float], fraction: float) -> float: + if len(ordered_values) == 1: + return ordered_values[0] + position = fraction * (len(ordered_values) - 1) + lower_index = int(position) + upper_index = min(lower_index + 1, len(ordered_values) - 1) + weight = position - lower_index + return ( + ordered_values[lower_index] * (1 - weight) + + ordered_values[upper_index] * weight + ) + + +def check_tracker_parity( + baseline: Dict[str, Any], candidate: Dict[str, Any] +) -> Dict[str, Any]: + baseline_ids = baseline["tracker_ids_per_frame"] + candidate_ids = candidate["tracker_ids_per_frame"] + mismatched_frames = [ + frame_id + for frame_id, tracker_ids in baseline_ids.items() + if candidate_ids.get(frame_id) != tracker_ids + ] + return { + "frames_compared": len(baseline_ids), + "mismatched_frames": len(mismatched_frames), + "matches_baseline": not mismatched_frames + and len(candidate_ids) == len(baseline_ids), + "first_mismatches": mismatched_frames[:5], + } + + +def run_child_scenarios(args: argparse.Namespace, depths: List[int]) -> List[dict]: + scenario_results = [] + for depth in depths: + child_env = { + **os.environ, + "WORKFLOWS_STEP_EXECUTION_MODE": "remote", + "WORKFLOWS_REMOTE_API_TARGET": "hosted", + "HOSTED_DETECT_URL": args.api_url, + "HOSTED_CORE_MODEL_URL": args.core_model_url, + "WORKFLOWS_STREAM_LOOKAHEAD_DEPTH": str(depth), + } + command = [ + sys.executable, + os.path.abspath(__file__), + "--api-url", + args.api_url, + "--model-id", + args.model_id, + "--video-url", + args.video_url, + "--frames", + str(args.frames), + "--warmup", + str(args.warmup), + "--resize-width", + str(args.resize_width), + "--workflow", + args.workflow, + "--second-model-id", + args.second_model_id, + "--sam3-class-names", + args.sam3_class_names, + "--output-dir", + str(args.output_dir), + "--run-single-depth", + str(depth), + ] + print(f"Running scenario depth={depth}...", flush=True) + completed = subprocess.run( + command, env=child_env, capture_output=True, text=True + ) + result_lines = [ + line + for line in completed.stdout.splitlines() + if line.startswith(SCENARIO_RESULT_MARKER) + ] + if completed.returncode != 0 or not result_lines: + raise RuntimeError( + f"Scenario depth={depth} failed (exit {completed.returncode}).\n" + f"stdout tail: {completed.stdout[-2000:]}\n" + f"stderr tail: {completed.stderr[-2000:]}" + ) + scenario_results.append( + json.loads(result_lines[-1][len(SCENARIO_RESULT_MARKER) :]) + ) + return scenario_results + + +def main() -> None: + args = parse_args() + if "ROBOFLOW_API_KEY" not in os.environ: + raise RuntimeError("ROBOFLOW_API_KEY must be set") + args.output_dir.mkdir(parents=True, exist_ok=True) + + if args.run_single_depth is not None: + result = run_scenario(args=args, depth=args.run_single_depth) + print(SCENARIO_RESULT_MARKER + json.dumps(result), flush=True) + return + + clip = prepare_clip( + video_url=args.video_url, + frames=args.frames, + resize_width=args.resize_width, + output_dir=args.output_dir, + ) + (args.output_dir / "clip-metadata.json").write_text(json.dumps(clip)) + if args.warmup > 0: + warmup_clip = prepare_clip( + video_url=args.video_url, + frames=args.warmup, + resize_width=args.resize_width, + output_dir=args.output_dir, + ) + os.replace( + warmup_clip["clip_path"], args.output_dir / f"warmup-{args.warmup}f.mp4" + ) + + depths = [int(depth.strip()) for depth in args.depths.split(",")] + scenario_results = run_child_scenarios(args=args, depths=depths) + + baseline = next( + (result for result in scenario_results if result["depth"] == 1), None + ) + for result in scenario_results: + if baseline is not None and result["depth"] != 1: + result["tracker_parity_vs_sequential"] = check_tracker_parity( + baseline=baseline, candidate=result + ) + if baseline["wall_fps"]: + result["speedup_vs_sequential"] = round( + result["wall_fps"] / baseline["wall_fps"], 2 + ) + + summary = { + "created_at": datetime.now(timezone.utc).isoformat(), + "api_url": args.api_url, + "workflow": args.workflow, + "model_id": args.model_id, + "video_url": args.video_url, + "frames": args.frames, + "warmup": args.warmup, + "clip": clip, + "platform": platform.platform(), + "python": platform.python_version(), + "scenarios": [ + { + key: value + for key, value in result.items() + if key != "tracker_ids_per_frame" + } + for result in scenario_results + ], + } + summary_path = args.output_dir / "summary.json" + summary_path.write_text(json.dumps(summary, indent=2)) + for result in scenario_results: + scenario_path = args.output_dir / f"scenario-depth-{result['depth']}.json" + scenario_path.write_text(json.dumps(result, indent=2)) + + print("SUMMARY_JSON_START") + print(json.dumps(summary, indent=2)) + print("SUMMARY_JSON_END") + print(f"Results written to {summary_path}") + + +if __name__ == "__main__": + main() diff --git a/development/benchmark_scripts/benchmark_remote_stream_pipeline_overhead.py b/development/benchmark_scripts/benchmark_remote_stream_pipeline_overhead.py new file mode 100644 index 0000000000..2d5f89085d --- /dev/null +++ b/development/benchmark_scripts/benchmark_remote_stream_pipeline_overhead.py @@ -0,0 +1,255 @@ +"""Measure the scheduler-side overhead of stream-lookahead execution. + +All remote calls are mocked to return instantly, so the numbers isolate the +execution engine's own costs — workflow compilation, frontier computation, +and the per-frame difference between the classic single-pass run and the +lookahead two-pass (deferred run + resume) — with zero network time. + +Example: + python development/benchmark_scripts/benchmark_lookahead_overhead.py \ + --frames 200 --compiles 30 --output /tmp/lookahead_overhead.json +""" + +import argparse +import json +import os +import statistics +import time +from datetime import datetime +from typing import Any, Callable, Dict, List +from unittest.mock import MagicMock, patch + +import numpy as np + +WORKFLOW_SPECIFICATION = { + "version": "1.0.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": [ + { + "type": "roboflow_core/roboflow_object_detection_model@v3", + "name": "model", + "images": "$inputs.image", + "model_id": "some-project/1", + "confidence_mode": "custom", + "custom_confidence": 0.35, + }, + { + "type": "roboflow_core/byte_tracker@v3", + "name": "byte_tracker", + "image": "$inputs.image", + "detections": "$steps.model.predictions", + }, + { + "type": "roboflow_core/bounding_box_visualization@v1", + "name": "bounding_box_visualization", + "image": "$inputs.image", + "predictions": "$steps.byte_tracker.tracked_detections", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "tracked_detections", + "coordinates_system": "own", + "selector": "$steps.byte_tracker.tracked_detections", + }, + { + "type": "JsonField", + "name": "visualization", + "coordinates_system": "own", + "selector": "$steps.bounding_box_visualization.image", + }, + ], +} + +CANNED_PREDICTION = { + "predictions": [ + { + "x": 100.0, + "y": 100.0, + "width": 50.0, + "height": 50.0, + "confidence": 0.9, + "class": "car", + "class_id": 0, + "detection_id": "00000000-0000-0000-0000-000000000000", + } + ], + "image": {"width": 640, "height": 360}, + "time": 0.0, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Measure compilation, frontier and per-frame scheduler overhead." + ) + parser.add_argument("--frames", type=int, default=200) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--compiles", type=int, default=30) + parser.add_argument("--depth", type=int, default=4) + parser.add_argument("--output", type=str, default=None) + return parser.parse_args() + + +def time_repeated(task: Callable[[], Any], repeats: int) -> Dict[str, float]: + durations = [] + for _ in range(repeats): + started_at = time.perf_counter() + task() + durations.append((time.perf_counter() - started_at) * 1000) + return { + "repeats": repeats, + "avg_ms": round(statistics.fmean(durations), 4), + "median_ms": round(statistics.median(durations), 4), + "min_ms": round(min(durations), 4), + "max_ms": round(max(durations), 4), + } + + +def build_engine(): + from inference.core.workflows.execution_engine.core import ExecutionEngine + + return ExecutionEngine.init( + workflow_definition=WORKFLOW_SPECIFICATION, + init_parameters={ + "workflows_core.api_key": "overhead-benchmark", + "workflows_core.model_manager": MagicMock(), + "workflows_core.step_execution_mode": _step_execution_mode(), + }, + ) + + +def _step_execution_mode(): + from inference.core.workflows.core_steps.common.entities import StepExecutionMode + + return StepExecutionMode.REMOTE + + +def make_video_frame(frame_id: int, image: np.ndarray): + from inference.core.interfaces.camera.entities import VideoFrame + + return VideoFrame( + image=image, + frame_id=frame_id, + frame_timestamp=datetime.fromtimestamp(1_700_000_000 + frame_id / 30), + fps=30, + measured_fps=30, + source_id=0, + comes_from_video_file=True, + ) + + +def run_per_frame_measurement( + runner: Callable[[List[Any]], Any], + image: np.ndarray, + frames: int, + warmup: int, + drain: Callable[[], Any], +) -> Dict[str, float]: + for frame_id in range(warmup): + runner([make_video_frame(frame_id, image)]) + started_at = time.perf_counter() + for frame_id in range(warmup, warmup + frames): + runner([make_video_frame(frame_id, image)]) + drain() + total_ms = (time.perf_counter() - started_at) * 1000 + return {"frames": frames, "avg_ms_per_frame": round(total_ms / frames, 4)} + + +def main() -> None: + args = parse_args() + os.environ.setdefault("WORKFLOWS_STREAM_LOOKAHEAD_DEPTH", str(args.depth)) + # DEBUG logging renders more lines on the sequential path than the + # lookahead path (~0.3 ms per line) and would fabricate an overhead gap. + os.environ["LOG_LEVEL"] = os.environ.get("OVERHEAD_BENCHMARK_LOG_LEVEL", "WARNING") + from inference.core.interfaces.stream.model_handlers import ( + workflows as workflows_module, + ) + from inference.core.interfaces.stream.model_handlers.workflows import ( + LookaheadPipelinedWorkflowRunner, + WorkflowRunner, + wrap_workflow_runner_for_stream_pipeline, + ) + from inference.core.workflows.core_steps.models.roboflow.object_detection import ( + v3 as object_detection_v3, + ) + from inference.core.workflows.execution_engine.v1.executor.core import ( + compute_stream_lookahead_frontier, + ) + + mock_client = MagicMock() + mock_client.infer.return_value = CANNED_PREDICTION + image = np.zeros((360, 640, 3), dtype=np.uint8) + results: Dict[str, Any] = {} + + with patch.object( + object_detection_v3, "InferenceHTTPClient", MagicMock(return_value=mock_client) + ), patch.object(object_detection_v3, "InferenceConfiguration", MagicMock()): + results["compilation"] = time_repeated(build_engine, repeats=args.compiles) + + engine = build_engine() + compiled_workflow = engine._engine._compiled_workflow + results["frontier_computation"] = time_repeated( + lambda: compute_stream_lookahead_frontier(workflow=compiled_workflow), + repeats=1000, + ) + + def build_workflow_runner(): + return WorkflowRunner( + workflows_parameters=None, + execution_engine=engine, + image_input_name="image", + video_metadata_input_name="video_metadata", + ) + + workflows_module.WORKFLOWS_STREAM_LOOKAHEAD_DEPTH = 1 + sequential_runner = build_workflow_runner() + results["sequential_per_frame"] = run_per_frame_measurement( + runner=sequential_runner, + image=image, + frames=args.frames, + warmup=args.warmup, + drain=lambda: None, + ) + + workflows_module.WORKFLOWS_STREAM_LOOKAHEAD_DEPTH = args.depth + lookahead_engine = build_engine() + lookahead_runner = wrap_workflow_runner_for_stream_pipeline( + workflow_runner=WorkflowRunner( + workflows_parameters=None, + execution_engine=lookahead_engine, + image_input_name="image", + video_metadata_input_name="video_metadata", + ), + execution_engine=lookahead_engine, + ) + if not isinstance(lookahead_runner, LookaheadPipelinedWorkflowRunner): + raise RuntimeError( + f"Expected the lookahead runner to activate, got " + f"{type(lookahead_runner).__name__} — results would be invalid." + ) + try: + results["lookahead_per_frame"] = run_per_frame_measurement( + runner=lookahead_runner, + image=image, + frames=args.frames, + warmup=args.warmup, + drain=lookahead_runner.flush, + ) + finally: + lookahead_runner.close() + + results["scheduler_overhead_ms_per_frame"] = round( + results["lookahead_per_frame"]["avg_ms_per_frame"] + - results["sequential_per_frame"]["avg_ms_per_frame"], + 4, + ) + print("OVERHEAD_RESULT " + json.dumps(results, indent=2)) + if args.output: + with open(args.output, "w") as f: + json.dump(results, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/docs/workflows/execution_engine_changelog.md b/docs/workflows/execution_engine_changelog.md index f23ba875c0..7b73230648 100644 --- a/docs/workflows/execution_engine_changelog.md +++ b/docs/workflows/execution_engine_changelog.md @@ -2,6 +2,32 @@ Below you can find the changelog for Execution Engine. +## Execution Engine `v1.13.0` | inference `v1.3.5` + +**What changed** + +* **Stream-lookahead execution for video pipelines (opt-in, default off)** — With + `WORKFLOWS_STREAM_LOOKAHEAD_DEPTH=N` (N > 1), `InferencePipeline` video workflows may + keep up to N frames' long-latency step executions (remote model requests, external + API calls) in flight concurrently, while stateful steps (trackers, counters, sinks) + still observe frames strictly in stream order. Emission order and outputs are + identical to sequential execution. +* **Per-block statefulness declaration** — `WorkflowBlockManifest` gained the + `is_stateful_for_video_processing()` classmethod (default `True`, conservative). + Blocks that are pure functions of their per-frame inputs may declare `False`; + blocks additionally declaring `is_async_stream_step()` on the block class certify + their `run()` as long-latency and re-entrant, letting the engine offload it to a + lookahead worker pool with per-output `Future` placeholders built from + `describe_outputs()`. +* **New engine entry points** — `run_stream_lookahead(...)` (deferred pass: executes + the stateless-ancestor frontier of the DAG and returns the frame's live + `ExecutionDataManager`) and `resume_stream_lookahead(...)` (emission pass: runs the + remaining steps on that state). Both exist on `ExecutionEngine` and + `ExecutionEngineV1` only — like `flush_stream_pipeline`, they are intentionally not + part of the `BaseExecutionEngine` ABC. Default `run(...)` behavior is unchanged; + with the env variable unset (default `1`) nothing about compilation or execution + differs. + ## Execution Engine `v1.12.0` | inference `v1.3.2` **What changed** diff --git a/inference/core/env.py b/inference/core/env.py index 1636dbffbc..5671e0c071 100644 --- a/inference/core/env.py +++ b/inference/core/env.py @@ -741,6 +741,15 @@ WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS = int( os.getenv("WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS", "8") ) +# Number of video frames whose async step executions (remote models, API +# calls) may be in flight concurrently in stream-lookahead scheduling; +# 1 disables lookahead. The old name is honored as a fallback. +WORKFLOWS_STREAM_LOOKAHEAD_DEPTH = int( + os.getenv( + "WORKFLOWS_STREAM_LOOKAHEAD_DEPTH", + os.getenv("WORKFLOWS_REMOTE_EXECUTION_PIPELINE_DEPTH", "1"), + ) +) ALLOW_CUSTOM_PYTHON_EXECUTION_IN_WORKFLOWS = str2bool( os.getenv("ALLOW_CUSTOM_PYTHON_EXECUTION_IN_WORKFLOWS", True) ) diff --git a/inference/core/interfaces/stream/inference_pipeline.py b/inference/core/interfaces/stream/inference_pipeline.py index fc043bd818..fd33091e10 100644 --- a/inference/core/interfaces/stream/inference_pipeline.py +++ b/inference/core/interfaces/stream/inference_pipeline.py @@ -670,6 +670,9 @@ def init_with_workflow( on_video_frame = wrap_workflow_runner_for_stream_pipeline( workflow_runner=workflow_runner, execution_engine=execution_engine, + allow_lookahead=( + not isinstance(video_reference, list) or len(video_reference) <= 1 + ), ) except ImportError as error: raise CannotInitialiseModelError( @@ -821,12 +824,13 @@ def init_with_custom_logic( except ValueError: predictions_queue_size = 512 if ( - _rfdetr_stream_pipeline_enabled() + _uses_buffered_stream_dispatch(on_video_frame) and "INFERENCE_PIPELINE_PREDICTIONS_QUEUE_SIZE" not in os.environ ): - # Stream-pipelined RF-DETR returns async response futures. Letting - # the producer queue hundreds of full-resolution VideoFrame objects - # can exhaust host memory on 4K videos before dispatch catches up. + # Stream-pipelined runs buffer frames already; letting the + # producer queue hundreds of full-resolution VideoFrame objects + # on top can exhaust host memory on 4K videos before dispatch + # catches up. predictions_queue_size = min(predictions_queue_size, 4) predictions_queue = Queue(maxsize=predictions_queue_size) return cls( @@ -858,6 +862,7 @@ def __init__( sink_mode: SinkMode = SinkMode.ADAPTIVE, ): self._on_video_frame = on_video_frame + self._buffered_stream_dispatch = _uses_buffered_stream_dispatch(on_video_frame) self._video_sources = video_sources self._on_prediction = on_prediction self._max_fps = max_fps @@ -931,7 +936,7 @@ def _execute_inference(self) -> None: frames=video_frames, ) predictions = self._on_video_frame(video_frames) - if _rfdetr_stream_pipeline_enabled(): + if self._buffered_stream_dispatch: self._queue_inference_result( inference_result=predictions, fallback_video_frames=video_frames, @@ -951,10 +956,20 @@ def _execute_inference(self) -> None: }, status_update_handlers=self._status_update_handlers, ) - if _rfdetr_stream_pipeline_enabled(): + if self._buffered_stream_dispatch: self._drain_inference_handler() except Exception as error: + if self._buffered_stream_dispatch: + # Deliver buffered frames whose requests already completed + # before the stream shuts down. + try: + self._drain_inference_handler() + except Exception as drain_error: + logger.exception( + "Failed to drain buffered frames after inference " "error: %s", + drain_error, + ) payload = { "error_type": error.__class__.__name__, "error_message": str(error), @@ -968,7 +983,7 @@ def _execute_inference(self) -> None: ) logger.exception(f"Encountered inference error: {error}") finally: - if _rfdetr_stream_pipeline_enabled(): + if self._buffered_stream_dispatch: self._close_inference_handler() self._predictions_queue.put(None) send_inference_pipeline_status_update( @@ -987,7 +1002,7 @@ def _dispatch_inference_results(self) -> None: self._predictions_queue.task_done() break predictions, video_frames = inference_results - if _rfdetr_stream_pipeline_enabled(): + if self._buffered_stream_dispatch: predictions = _resolve_prediction_futures(predictions) if self._on_prediction is not None: self._handle_predictions_dispatching( @@ -1044,10 +1059,24 @@ def _normalise_inference_result( return inference_result, fallback_video_frames def _drain_inference_handler(self) -> None: + from inference.core.interfaces.stream.model_handlers.workflows import ( + StreamLookaheadDrainError, + ) + flush_fn = getattr(self._on_video_frame, "flush", None) if not callable(flush_fn): return None - flush_result = flush_fn() + try: + flush_result = flush_fn() + except StreamLookaheadDrainError as error: + # Dispatch the frames that did drain, then let the emission + # failure surface through the normal inference error path. + for result in error.drained_results: + self._queue_inference_result( + inference_result=result, + fallback_video_frames=[], + ) + raise if flush_result is None: return None if isinstance(flush_result, list) and all( @@ -1181,8 +1210,16 @@ def _resolve_prediction_futures(value: Any) -> Any: ) -def _rfdetr_stream_pipeline_enabled() -> bool: - try: - return int(os.getenv("RFDETR_PIPELINE_DEPTH", "1").strip()) > 1 - except ValueError: - return False +def _uses_buffered_stream_dispatch(on_video_frame: InferenceHandler) -> bool: + # Buffered dispatch (bounded predictions queue + drain/close on stream end + # + late future resolution) is correct only when the handler actually + # buffers frames across calls. The stream-pipeline runners + # (RF-DETR PipelinedWorkflowRunner, LookaheadPipelinedWorkflowRunner) + # expose flush(); a plain WorkflowRunner does not. Keying on the actual + # handler rather than the env var means a depth flag set on a workflow + # that does not qualify for pipelining (e.g. no async steps, multi-source, + # stateful-fed model — wrap falls back to a plain runner) keeps the normal + # dispatch path instead of needlessly capping the queue and draining + # nothing. This mirrors how _drain_inference_handler / _close_inference_handler + # already discover the handler's capabilities. + return callable(getattr(on_video_frame, "flush", None)) diff --git a/inference/core/interfaces/stream/model_handlers/workflows.py b/inference/core/interfaces/stream/model_handlers/workflows.py index 7539653318..dd3fb824db 100644 --- a/inference/core/interfaces/stream/model_handlers/workflows.py +++ b/inference/core/interfaces/stream/model_handlers/workflows.py @@ -1,15 +1,46 @@ +import weakref +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import thread as _futures_thread from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from threading import Barrier, Thread +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple +from inference.core import logger +from inference.core.env import WORKFLOWS_STREAM_LOOKAHEAD_DEPTH from inference.core.interfaces.camera.entities import VideoFrame from inference.core.interfaces.stream.entities import InferenceHandlerResult from inference.core.workflows.execution_engine.core import ExecutionEngine from inference.core.workflows.execution_engine.entities.base import VideoMetadata +from inference.core.workflows.execution_engine.v1.executor.core import ( + compute_async_stream_step_selectors, + compute_stream_lookahead_frontier, +) + +if TYPE_CHECKING: + from inference.core.workflows.execution_engine.v1.executor.execution_data_manager.manager import ( + ExecutionDataManager, + ) @dataclass(frozen=True) class _StreamPipelineStep: step: Any + name: Optional[str] = None + + +class StreamLookaheadDrainError(RuntimeError): + """A frame emission failed while draining the stream-lookahead buffer. + + Raised only after every buffered frame was drained best-effort; the + frames that did emit successfully are carried in ``drained_results`` so + the pipeline can dispatch them before propagating the error. + """ + + def __init__( + self, message: str, drained_results: List[InferenceHandlerResult] + ) -> None: + super().__init__(message) + self.drained_results = drained_results class WorkflowRunner: @@ -61,18 +92,44 @@ def _flush_stream_pipeline(self, video_frames: List[VideoFrame]) -> List[dict]: _is_preview=self._is_preview, ) + def _run_stream_lookahead( + self, + video_frames: List[VideoFrame], + frontier_step_selectors: Set[str], + async_step_selectors: Set[str], + lookahead_executor: "ThreadPoolExecutor", + ) -> "ExecutionDataManager": + workflows_parameters, _ = self._build_workflows_parameters( + video_frames=video_frames + ) + return self._execution_engine.run_stream_lookahead( + runtime_parameters=workflows_parameters, + frontier_step_selectors=frontier_step_selectors, + async_step_selectors=async_step_selectors, + lookahead_executor=lookahead_executor, + ) + + def _resume_stream_lookahead( + self, + execution_data_manager: "ExecutionDataManager", + frontier_step_selectors: Set[str], + video_frames: List[VideoFrame], + ) -> List[dict]: + return self._execution_engine.resume_stream_lookahead( + execution_data_manager=execution_data_manager, + frontier_step_selectors=frontier_step_selectors, + serialize_results=self._serialize_results, + fps=_resolve_stream_fps(video_frames=video_frames), + _is_preview=self._is_preview, + ) + def _build_workflows_parameters( self, video_frames: List[VideoFrame], ) -> tuple[Dict[str, Any], float]: workflows_parameters: Dict[str, Any] = dict(self._workflows_parameters or {}) # TODO: pass fps reflecting each stream to workflows_parameters - fps = video_frames[0].fps - if video_frames[0].measured_fps: - fps = video_frames[0].measured_fps - if fps is None: - # for FPS reporting we expect 0 when FPS cannot be determined - fps = 0 + fps = _resolve_stream_fps(video_frames=video_frames) video_metadata_for_images = [ VideoMetadata( video_identifier=( @@ -104,6 +161,16 @@ def _build_workflows_parameters( return workflows_parameters, fps +def _resolve_stream_fps(video_frames: List[VideoFrame]) -> float: + fps = video_frames[0].fps + if video_frames[0].measured_fps: + fps = video_frames[0].measured_fps + if fps is None: + # for FPS reporting we expect 0 when FPS cannot be determined + fps = 0 + return fps + + class PipelinedWorkflowRunner: def __init__( self, @@ -165,31 +232,214 @@ def flush(self) -> Optional[List[InferenceHandlerResult]]: return results def close(self) -> None: - for stream_step in self._stream_steps: - close_fn = getattr(stream_step.step, "close_stream_pipeline", None) - if callable(close_fn): - close_fn() + _close_stream_steps(stream_steps=self._stream_steps) def _stream_buffer_depth(self) -> int: - return max( - (_stream_step_depth(stream_step) for stream_step in self._stream_steps), - default=0, + return _max_stream_buffer_depth(stream_steps=self._stream_steps) + + +class LookaheadPipelinedWorkflowRunner: + """Pipelined runner for steps that defer downstream execution. + + Each frame gets a deferred pass that executes only the workflow's + stream-lookahead frontier (stateless steps; pipelined model steps launch + their remote requests and register future-bearing outputs), and the + frame's live execution state is buffered. Once the buffer exceeds the + pipeline depth, the oldest frame is emitted by resuming its execution + state: the remaining steps run with that frame's own inputs, in frame + order, with in-flight futures resolved at input assembly. + """ + + def __init__( + self, + workflow_runner: WorkflowRunner, + frontier_step_selectors: Set[str], + async_step_selectors: Set[str], + buffer_depth: int, + ) -> None: + self._workflow_runner = workflow_runner + self._frontier_step_selectors = frontier_step_selectors + self._async_step_selectors = async_step_selectors + self._buffer_depth = buffer_depth + max_workers = max(1, (buffer_depth + 1) * len(async_step_selectors)) + self._lookahead_executor = ThreadPoolExecutor( + max_workers=max_workers, + thread_name_prefix="workflows_stream_lookahead", + ) + _prespawn_daemon_workers( + executor=self._lookahead_executor, max_workers=max_workers ) + # GC fallback: if the owner is dropped without close(), still reap + # the pool so it cannot outlive the runner. + weakref.finalize(self, _shutdown_lookahead_executor, self._lookahead_executor) + self._pending_frames: List[Tuple[List[VideoFrame], "ExecutionDataManager"]] = [] + + def __call__( + self, video_frames: List[VideoFrame] + ) -> Optional[InferenceHandlerResult]: + execution_data_manager = self._workflow_runner._run_stream_lookahead( + video_frames=video_frames, + frontier_step_selectors=self._frontier_step_selectors, + async_step_selectors=self._async_step_selectors, + lookahead_executor=self._lookahead_executor, + ) + self._pending_frames.append((video_frames, execution_data_manager)) + if len(self._pending_frames) <= self._buffer_depth: + return None + return self._emit_oldest_frame() + + def flush(self) -> Optional[List[InferenceHandlerResult]]: + if not self._pending_frames: + return None + results = [] + first_error: Optional[Exception] = None + failed_frames = 0 + while self._pending_frames: + try: + results.append(self._emit_oldest_frame()) + except Exception as error: + # One failed frame must not drop the rest of the buffered + # frames whose requests already completed — keep draining + # and re-raise the first failure afterwards. + failed_frames += 1 + if first_error is None: + first_error = error + logger.exception( + "Failed to emit a buffered frame during stream-lookahead " + "drain: %s", + error, + ) + if first_error is not None: + raise StreamLookaheadDrainError( + f"Failed to emit {failed_frames} buffered frame(s) during " + "stream-lookahead drain.", + drained_results=results, + ) from first_error + return results + + def close(self) -> None: + _shutdown_lookahead_executor(executor=self._lookahead_executor) + + def _emit_oldest_frame(self) -> InferenceHandlerResult: + emit_video_frames, execution_data_manager = self._pending_frames.pop(0) + predictions = self._workflow_runner._resume_stream_lookahead( + execution_data_manager=execution_data_manager, + frontier_step_selectors=self._frontier_step_selectors, + video_frames=emit_video_frames, + ) + return InferenceHandlerResult( + predictions=predictions, + video_frames=emit_video_frames, + ) + + +def _shutdown_lookahead_executor(executor: ThreadPoolExecutor) -> None: + executor.shutdown(wait=False, cancel_futures=True) + # shutdown(wait=False) leaves the workers registered in + # concurrent.futures' interpreter-shutdown hook, which joins every pool + # thread regardless of its daemon flag — a request hung past close() + # would still block process exit. Detach them so that join skips them; + # their daemon flag (set at spawn) keeps threading's own shutdown from + # waiting on them as well. + # + # `executor._threads` and `concurrent.futures.thread._threads_queues` are + # CPython internals (validated on 3.9-3.12; the detach degrades safely via + # pop(..., None) if a future CPython reworks the shutdown hook — worst case + # is the pre-existing "hung request blocks exit" behavior, never a crash). + # `test_lookahead_runner_pool_threads_are_daemon` guards the detach. + for worker in executor._threads: + _futures_thread._threads_queues.pop(worker, None) + + +def _prespawn_daemon_workers(executor: ThreadPoolExecutor, max_workers: int) -> None: + """Force all lookahead pool threads into existence as daemon threads. + + ThreadPoolExecutor spawns worker threads lazily on submit(), and a new + thread inherits the daemon flag of the thread that created it. The + daemon flag alone is NOT sufficient to let the interpreter exit while a + request hangs: concurrent.futures joins every registered pool thread at + interpreter shutdown regardless of daemonness. It is one half of the + lifecycle guarantee — close() detaches the workers from that shutdown + hook, and the daemon flag set here keeps threading's own shutdown from + waiting on them afterwards. + """ + + def _spawn_all_workers() -> None: + all_workers_started = Barrier(parties=max_workers + 1) + for _ in range(max_workers): + executor.submit(all_workers_started.wait) + all_workers_started.wait() + + spawner = Thread(target=_spawn_all_workers, daemon=True) + spawner.start() + spawner.join() def wrap_workflow_runner_for_stream_pipeline( workflow_runner: WorkflowRunner, execution_engine: ExecutionEngine, + allow_lookahead: bool = True, ): stream_steps = _stream_pipeline_steps(execution_engine=execution_engine) - if not stream_steps: + compiled_workflow = _compiled_workflow_with_graph(execution_engine=execution_engine) + async_step_selectors = ( + compute_async_stream_step_selectors(workflow=compiled_workflow) + if compiled_workflow is not None and WORKFLOWS_STREAM_LOOKAHEAD_DEPTH > 1 + else set() + ) + if not async_step_selectors: + if stream_steps: + return PipelinedWorkflowRunner( + workflow_runner=workflow_runner, + stream_steps=stream_steps, + ) + return workflow_runner + if not allow_lookahead: + # Multi-source pipelines feed frame batches larger than one image; + # buffering them would add latency without any overlap. + logger.warning( + "Stream lookahead is disabled: it currently supports " + "single-source pipelines only. Falling back to sequential " + "execution." + ) + return workflow_runner + if stream_steps: + # A workflow mixing async lookahead steps with a local stream + # pipeline (RF-DETR) would leave the latter's flush queue undrained + # under the lookahead runner. + logger.warning( + "Stream lookahead is disabled for this workflow: it mixes async " + "steps with a locally stream-pipelined model. Falling back to " + "sequential execution." + ) + return workflow_runner + frontier_step_selectors = compute_stream_lookahead_frontier( + workflow=compiled_workflow + ) + if not async_step_selectors <= frontier_step_selectors: + logger.warning( + "Stream lookahead is disabled for this workflow. Every async " + "step must sit in the stream-lookahead frontier: fed only by " + "steps that are stateless for video processing and not by " + "another async step. Falling back to sequential execution." + ) return workflow_runner - return PipelinedWorkflowRunner( + return LookaheadPipelinedWorkflowRunner( workflow_runner=workflow_runner, - stream_steps=stream_steps, + frontier_step_selectors=frontier_step_selectors, + async_step_selectors=async_step_selectors, + buffer_depth=WORKFLOWS_STREAM_LOOKAHEAD_DEPTH - 1, ) +def _compiled_workflow_with_graph(execution_engine: ExecutionEngine) -> Optional[Any]: + engine = getattr(execution_engine, "_engine", None) + compiled_workflow = getattr(engine, "_compiled_workflow", None) + if getattr(compiled_workflow, "execution_graph", None) is None: + return None + return compiled_workflow + + def _stream_pipeline_steps( execution_engine: ExecutionEngine, ) -> List[_StreamPipelineStep]: @@ -197,10 +447,10 @@ def _stream_pipeline_steps( compiled_workflow = getattr(engine, "_compiled_workflow", None) steps = getattr(compiled_workflow, "steps", {}) stream_steps = [] - for initialised_step in steps.values(): + for step_name, initialised_step in steps.items(): step_instance = getattr(initialised_step, "step", None) if _is_stream_pipeline_step(step_instance=step_instance): - stream_steps.append(_StreamPipelineStep(step=step_instance)) + stream_steps.append(_StreamPipelineStep(step=step_instance, name=step_name)) return stream_steps @@ -212,6 +462,24 @@ def _is_stream_pipeline_step(step_instance: Any) -> bool: return callable(can_activate_pipeline) and can_activate_pipeline() +def _close_stream_steps(stream_steps: List[_StreamPipelineStep]) -> None: + for stream_step in stream_steps: + close_fn = getattr(stream_step.step, "close_stream_pipeline", None) + if not callable(close_fn): + continue + try: + close_fn() + except Exception as error: + logger.exception("Failed to close stream pipeline step: %s", error) + + +def _max_stream_buffer_depth(stream_steps: List[_StreamPipelineStep]) -> int: + return max( + (_stream_step_depth(stream_step) for stream_step in stream_steps), + default=0, + ) + + def _stream_step_depth(stream_step: _StreamPipelineStep) -> int: get_depth = getattr(stream_step.step, "stream_pipeline_depth", None) if not callable(get_depth): diff --git a/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py b/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py index b151d82098..86a00bf141 100644 --- a/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py @@ -138,6 +138,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/fusion/detections_consensus/v1.py b/inference/core/workflows/core_steps/fusion/detections_consensus/v1.py index 94fe58688e..510bce18dd 100644 --- a/inference/core/workflows/core_steps/fusion/detections_consensus/v1.py +++ b/inference/core/workflows/core_steps/fusion/detections_consensus/v1.py @@ -224,6 +224,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/integrations/roboflow/visual_search_classifier/v1.py b/inference/core/workflows/core_steps/integrations/roboflow/visual_search_classifier/v1.py index 2039ceb479..6283b39b27 100644 --- a/inference/core/workflows/core_steps/integrations/roboflow/visual_search_classifier/v1.py +++ b/inference/core/workflows/core_steps/integrations/roboflow/visual_search_classifier/v1.py @@ -184,6 +184,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="message", kind=[STRING_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -201,6 +205,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + def run( self, image: Union[WorkflowImageData, Batch[WorkflowImageData]], diff --git a/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v1.py b/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v1.py index 3759405d72..65f921c239 100644 --- a/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v1.py @@ -245,6 +245,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -268,6 +272,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v2.py b/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v2.py index 748239e7ef..1ab4f563b4 100644 --- a/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v2.py @@ -334,6 +334,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -357,6 +361,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v3.py b/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v3.py index a056620b8c..141a25c020 100644 --- a/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v3.py +++ b/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v3.py @@ -351,6 +351,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -374,6 +378,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/clip_comparison/v1.py b/inference/core/workflows/core_steps/models/foundation/clip_comparison/v1.py index d028125751..442c486492 100644 --- a/inference/core/workflows/core_steps/models/foundation/clip_comparison/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/clip_comparison/v1.py @@ -95,6 +95,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="prediction_type", kind=[PREDICTION_TYPE_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -129,6 +133,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/clip_comparison/v2.py b/inference/core/workflows/core_steps/models/foundation/clip_comparison/v2.py index d8c68f1799..6338b464a0 100644 --- a/inference/core/workflows/core_steps/models/foundation/clip_comparison/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/clip_comparison/v2.py @@ -119,6 +119,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="root_parent_id", kind=[PARENT_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -153,6 +157,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/depth_estimation/v1.py b/inference/core/workflows/core_steps/models/foundation/depth_estimation/v1.py index 91dea9846a..e5c7b999a8 100644 --- a/inference/core/workflows/core_steps/models/foundation/depth_estimation/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/depth_estimation/v1.py @@ -114,6 +114,10 @@ def get_parameters_accepting_batches(cls) -> List[str]: # Only images can be passed in as a list/batch return ["images"] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -172,6 +176,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode == StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/easy_ocr/v1.py b/inference/core/workflows/core_steps/models/foundation/easy_ocr/v1.py index 2f8c4747e4..b63b8fd85c 100644 --- a/inference/core/workflows/core_steps/models/foundation/easy_ocr/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/easy_ocr/v1.py @@ -140,6 +140,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="prediction_type", kind=[PREDICTION_TYPE_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -177,6 +181,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/florence2/v1.py b/inference/core/workflows/core_steps/models/foundation/florence2/v1.py index 0a538e9367..18ae084e24 100644 --- a/inference/core/workflows/core_steps/models/foundation/florence2/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/florence2/v1.py @@ -318,6 +318,10 @@ def get_restrictions(cls) -> List[RuntimeRestriction]: class BlockManifest(BaseManifest): + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_supported_model_variants(cls) -> Optional[List[str]]: """Return list of model_id variants that can satisfy this block.""" @@ -376,6 +380,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/florence2/v2.py b/inference/core/workflows/core_steps/models/foundation/florence2/v2.py index 811e9ce984..dcf2ce10a9 100644 --- a/inference/core/workflows/core_steps/models/foundation/florence2/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/florence2/v2.py @@ -3,6 +3,7 @@ import supervision as sv from pydantic import ConfigDict, Field +from inference.core.workflows.core_steps.common.entities import StepExecutionMode from inference.core.workflows.core_steps.models.foundation.florence2.v1 import ( LONG_DESCRIPTION, BaseManifest, @@ -22,6 +23,10 @@ class V2BlockManifest(BaseManifest): + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_supported_model_variants(cls) -> Optional[List[str]]: """Return list of model_id variants that can satisfy this block.""" @@ -65,6 +70,12 @@ class Florence2BlockV2(Florence2BlockV1): def get_manifest(cls) -> Type[WorkflowBlockManifest]: return V2BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/glm_ocr/v1.py b/inference/core/workflows/core_steps/models/foundation/glm_ocr/v1.py index 72f9bc5669..26fd9e3a72 100644 --- a/inference/core/workflows/core_steps/models/foundation/glm_ocr/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/glm_ocr/v1.py @@ -232,6 +232,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: def get_parameters_accepting_batches(cls) -> List[str]: return ["images"] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -309,6 +313,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode == StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/google_gemini/v1.py b/inference/core/workflows/core_steps/models/foundation/google_gemini/v1.py index 0b726f5d7f..92bfc18d6e 100644 --- a/inference/core/workflows/core_steps/models/foundation/google_gemini/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/google_gemini/v1.py @@ -252,6 +252,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -275,6 +279,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/google_gemini/v2.py b/inference/core/workflows/core_steps/models/foundation/google_gemini/v2.py index 19e8d9bf0b..8011d90ab5 100644 --- a/inference/core/workflows/core_steps/models/foundation/google_gemini/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/google_gemini/v2.py @@ -330,6 +330,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -353,6 +357,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/google_gemini/v3.py b/inference/core/workflows/core_steps/models/foundation/google_gemini/v3.py index a1ee71fb80..a4ff276c6e 100644 --- a/inference/core/workflows/core_steps/models/foundation/google_gemini/v3.py +++ b/inference/core/workflows/core_steps/models/foundation/google_gemini/v3.py @@ -370,6 +370,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -393,6 +397,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/google_gemma/v1.py b/inference/core/workflows/core_steps/models/foundation/google_gemma/v1.py index f36e0eecf2..3e0adfd6c5 100644 --- a/inference/core/workflows/core_steps/models/foundation/google_gemma/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/google_gemma/v1.py @@ -265,6 +265,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -286,6 +290,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/google_gemma/v2.py b/inference/core/workflows/core_steps/models/foundation/google_gemma/v2.py index 10f9d45fbb..74aa561c03 100644 --- a/inference/core/workflows/core_steps/models/foundation/google_gemma/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/google_gemma/v2.py @@ -190,6 +190,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -201,6 +205,10 @@ class GoogleGemmaBlockV2(OpenRouterWorkflowBlockBase): def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/kimi_openrouter/v1.py b/inference/core/workflows/core_steps/models/foundation/kimi_openrouter/v1.py index db5f5e4920..a18f0c6ac1 100644 --- a/inference/core/workflows/core_steps/models/foundation/kimi_openrouter/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/kimi_openrouter/v1.py @@ -265,6 +265,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -286,6 +290,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/kimi_openrouter/v2.py b/inference/core/workflows/core_steps/models/foundation/kimi_openrouter/v2.py index 51b70e7c3c..b601e9797e 100644 --- a/inference/core/workflows/core_steps/models/foundation/kimi_openrouter/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/kimi_openrouter/v2.py @@ -190,6 +190,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -201,6 +205,10 @@ class KimiOpenrouterBlockV2(OpenRouterWorkflowBlockBase): def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/llama_vision/v1.py b/inference/core/workflows/core_steps/models/foundation/llama_vision/v1.py index 683ccaae1b..df5fd4afb0 100644 --- a/inference/core/workflows/core_steps/models/foundation/llama_vision/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/llama_vision/v1.py @@ -1,644 +1,652 @@ -import base64 -import json -from functools import partial -from typing import Any, Dict, List, Literal, Optional, Type, Union - -from openai import OpenAI -from pydantic import ConfigDict, Field, field_validator, model_validator - -from inference.core.env import WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS -from inference.core.managers.base import ModelManager -from inference.core.utils.image_utils import encode_image_to_jpeg_bytes, load_image -from inference.core.workflows.core_steps.common.utils import run_in_parallel -from inference.core.workflows.core_steps.common.vlms import VLM_TASKS_METADATA -from inference.core.workflows.execution_engine.entities.base import ( - Batch, - OutputDefinition, - WorkflowImageData, -) -from inference.core.workflows.execution_engine.entities.types import ( - FLOAT_KIND, - IMAGE_KIND, - LANGUAGE_MODEL_OUTPUT_KIND, - LIST_OF_VALUES_KIND, - STRING_KIND, - ImageInputField, - Selector, -) -from inference.core.workflows.prototypes.block import ( - AirGappedAvailability, - BlockResult, - WorkflowBlock, - WorkflowBlockManifest, -) - -MODEL_VERSION_MAPPING = { - "11B (Free) - OpenRouter": "meta-llama/llama-3.2-11b-vision-instruct:free", - "11B (Regular) - OpenRouter": "meta-llama/llama-3.2-11b-vision-instruct", - "90B (Free) - OpenRouter": "meta-llama/llama-3.2-90b-vision-instruct:free", - "90B (Regular) - OpenRouter": "meta-llama/llama-3.2-90b-vision-instruct", -} - -ModelVersion = Literal[ - "11B (Free) - OpenRouter", - "11B (Regular) - OpenRouter", - "90B (Free) - OpenRouter", - "90B (Regular) - OpenRouter", -] - -SUPPORTED_TASK_TYPES_LIST = [ - "unconstrained", - "ocr", - "structured-answering", - "classification", - "multi-label-classification", - "visual-question-answering", - "caption", - "detailed-caption", -] -SUPPORTED_TASK_TYPES = set(SUPPORTED_TASK_TYPES_LIST) - -RELEVANT_TASKS_METADATA = { - k: v for k, v in VLM_TASKS_METADATA.items() if k in SUPPORTED_TASK_TYPES -} -RELEVANT_TASKS_DOCS_DESCRIPTION = "\n\n".join( - f"* **{v['name']}** (`{k}`) - {v['description']}" - for k, v in RELEVANT_TASKS_METADATA.items() -) - - -LONG_DESCRIPTION = f""" -Ask a question to Llama 3.2 Vision model with vision capabilities. - -You can specify arbitrary text prompts or predefined ones, the block supports the following types of prompt: - -{RELEVANT_TASKS_DOCS_DESCRIPTION} - -!!! warning "Issues with structured prompting" - - Model tends to be quite unpredictable when structured output (in our case JSON document) is expected. - That problems may impact tasks like `structured-answering`, `classification` or `multi-label-classification`. - - The cause seems to be quite sensitive "filters" of inappropriate content embedded in model. - - -#### 🛠️ API providers and model variants - -Llama Vision 3.2 model is exposed via [OpenRouter API](https://openrouter.ai/) and we require -passing [OpenRouter API Key](https://openrouter.ai/docs/api-keys) to run. - -There are different versions of the model supported: - -* smaller version (`11B`) is faster and cheaper, yet you can expect better quality of results using `90B` version - -* `Regular` version is paid (and usually faster) API, whereas `Free` is free for use for OpenRouter clients -(state at 01.01.2025) - - -As for now, OpenRouter is the only provider for Llama 3.2 Vision model, but we will keep you posted if -the state of the matter changes. - - -!!! warning "API Usage Charges" - - OpenRouter is external third party providing access to the model and incurring charges on the usage. - Please check out pricing before use: - - * [Llama 3.2 Vision 11B (Regular)](https://openrouter.ai/meta-llama/llama-3.2-11b-vision-instruct/providers) - - * [Llama 3.2 Vision 90B (Regular)](https://openrouter.ai/meta-llama/llama-3.2-90b-vision-instruct/providers) - -#### 💡 Further reading and Acceptable Use Policy - -!!! warning "Model license" - - Check out [model license](https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/LICENSE) before - use. - -[Click here](https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/MODEL_CARD_VISION.md) for the original model card. - -Usage of this model is subject to Meta's [Acceptable Use Policy](https://www.llama.com/llama3/use-policy/). -""" - - -TaskType = Literal[tuple(SUPPORTED_TASK_TYPES_LIST)] - -TASKS_REQUIRING_PROMPT = { - "unconstrained", - "visual-question-answering", -} - -TASKS_REQUIRING_CLASSES = { - "classification", - "multi-label-classification", -} - -TASKS_REQUIRING_OUTPUT_STRUCTURE = { - "structured-answering", -} - - -class BlockManifest(WorkflowBlockManifest): - model_config = ConfigDict( - json_schema_extra={ - "name": "Llama 3.2 Vision", - "version": "v1", - "deprecated": True, - "deprecation_message": "Use the Llama 3.2 Vision v2 block, which adds a Roboflow-managed API key option and user-selectable privacy controls.", - "short_description": "Run Llama model with Vision capabilities", - "long_description": LONG_DESCRIPTION, - "license": "Llama 3.2 Community", - "block_type": "model", - "search_keywords": ["LMM", "VLM", "Llama", "Vision", "Meta"], - "is_vlm_block": True, - "task_type_property": "task_type", - "ui_manifest": { - "section": "model", - "icon": "far fa-brands fa-meta", - }, - }, - protected_namespaces=(), - ) - type: Literal["roboflow_core/llama_3_2_vision@v1"] - images: Selector(kind=[IMAGE_KIND]) = ImageInputField - task_type: TaskType = Field( - default="unconstrained", - description="Task type to be performed by model. Value determines required parameters and output response.", - json_schema_extra={ - "values_metadata": RELEVANT_TASKS_METADATA, - "recommended_parsers": { - "structured-answering": "roboflow_core/json_parser@v1", - "classification": "roboflow_core/vlm_as_classifier@v1", - "multi-label-classification": "roboflow_core/vlm_as_classifier@v1", - }, - "always_visible": True, - }, - ) - prompt: Optional[Union[Selector(kind=[STRING_KIND]), str]] = Field( - default=None, - description="Text prompt to the Llama model", - examples=["my prompt", "$inputs.prompt"], - json_schema_extra={ - "relevant_for": { - "task_type": {"values": TASKS_REQUIRING_PROMPT, "required": True}, - }, - "multiline": True, - }, - ) - output_structure: Optional[Dict[str, str]] = Field( - default=None, - description="Dictionary with structure of expected JSON response", - examples=[{"my_key": "description"}, "$inputs.output_structure"], - json_schema_extra={ - "relevant_for": { - "task_type": { - "values": TASKS_REQUIRING_OUTPUT_STRUCTURE, - "required": True, - }, - }, - }, - ) - classes: Optional[Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]]] = Field( - default=None, - description="List of classes to be used", - examples=[["class-a", "class-b"], "$inputs.classes"], - json_schema_extra={ - "relevant_for": { - "task_type": { - "values": TASKS_REQUIRING_CLASSES, - "required": True, - }, - }, - }, - ) - api_key: Union[Selector(kind=[STRING_KIND]), str] = Field( - description="Your Llama Vision API key (dependent on provider, ex: OpenRouter API key)", - examples=["xxx-xxx", "$inputs.llama_api_key"], - private=True, - ) - model_version: Union[ - Selector(kind=[STRING_KIND]), - Literal[ - "11B (Free) - OpenRouter", - "11B (Regular) - OpenRouter", - "90B (Free) - OpenRouter", - "90B (Regular) - OpenRouter", - ], - ] = Field( - default="11B (Free) - OpenRouter", - description="Model to be used", - examples=["11B (Free) - OpenRouter", "$inputs.llama_model"], - ) - max_tokens: int = Field( - default=500, - description="Maximum number of tokens the model can generate in it's response.", - gt=1, - ) - temperature: Union[float, Selector(kind=[FLOAT_KIND])] = Field( - default=0.1, - description="Temperature to sample from the model - value in range 0.0-2.0, the higher - the more " - 'random / "creative" the generations are.', - ) - max_concurrent_requests: Optional[int] = Field( - default=None, - description="Number of concurrent requests that can be executed by block when batch of input images provided. " - "If not given - block defaults to value configured globally in Workflows Execution Engine. " - "Please restrict if you hit limits.", - ) - - @model_validator(mode="after") - def validate(self) -> "BlockManifest": - if self.task_type in TASKS_REQUIRING_PROMPT and self.prompt is None: - raise ValueError( - f"`prompt` parameter required to be set for task `{self.task_type}`" - ) - if self.task_type in TASKS_REQUIRING_CLASSES and self.classes is None: - raise ValueError( - f"`classes` parameter required to be set for task `{self.task_type}`" - ) - if ( - self.task_type in TASKS_REQUIRING_OUTPUT_STRUCTURE - and self.output_structure is None - ): - raise ValueError( - f"`output_structure` parameter required to be set for task `{self.task_type}`" - ) - return self - - @classmethod - def get_air_gapped_availability(cls) -> AirGappedAvailability: - return AirGappedAvailability(available=False, reason="requires_internet") - - @field_validator("temperature") - @classmethod - def validate_temperature(cls, value: Union[str, float]) -> Union[str, float]: - if isinstance(value, str): - return value - if value < 0.0 or value > 2.0: - raise ValueError( - "'temperature' parameter required to be in range [0.0, 2.0]" - ) - return value - - @classmethod - def get_parameters_accepting_batches(cls) -> List[str]: - return ["images"] - - @classmethod - def describe_outputs(cls) -> List[OutputDefinition]: - return [ - OutputDefinition( - name="output", kind=[STRING_KIND, LANGUAGE_MODEL_OUTPUT_KIND] - ), - OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), - ] - - @classmethod - def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.3.0,<2.0.0" - - -class LlamaVisionBlockV1(WorkflowBlock): - - def __init__( - self, - model_manager: ModelManager, - ): - self._model_manager = model_manager - - @classmethod - def get_init_parameters(cls) -> List[str]: - return ["model_manager"] - - @classmethod - def get_manifest(cls) -> Type[WorkflowBlockManifest]: - return BlockManifest - - @classmethod - def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.3.0,<2.0.0" - - def run( - self, - images: Batch[WorkflowImageData], - task_type: TaskType, - prompt: Optional[str], - output_structure: Optional[Dict[str, str]], - classes: Optional[List[str]], - api_key: str, - model_version: ModelVersion, - max_tokens: int, - temperature: float, - max_concurrent_requests: Optional[int], - ) -> BlockResult: - inference_images = [i.to_inference_format() for i in images] - raw_outputs = run_llama_vision_32_llm_prompting( - images=inference_images, - task_type=task_type, - prompt=prompt, - output_structure=output_structure, - classes=classes, - llama_api_key=api_key, - llama_model_version=model_version, - max_tokens=max_tokens, - temperature=temperature, - max_concurrent_requests=max_concurrent_requests, - ) - return [ - {"output": raw_output, "classes": classes} for raw_output in raw_outputs - ] - - -def run_llama_vision_32_llm_prompting( - images: List[Dict[str, Any]], - task_type: TaskType, - prompt: Optional[str], - output_structure: Optional[Dict[str, str]], - classes: Optional[List[str]], - llama_api_key: Optional[str], - llama_model_version: ModelVersion, - max_tokens: int, - temperature: float, - max_concurrent_requests: Optional[int], -) -> List[str]: - if task_type not in PROMPT_BUILDERS: - raise ValueError(f"Task type: {task_type} not supported.") - model_version_id = MODEL_VERSION_MAPPING.get(llama_model_version) - if not llama_model_version: - raise ValueError( - f"Invalid model name: '{llama_model_version}'. Please use one of {list(MODEL_VERSION_MAPPING.keys())}." - ) - llama_prompts = [] - for image in images: - loaded_image, _ = load_image(image) - base64_image = base64.b64encode( - encode_image_to_jpeg_bytes(loaded_image) - ).decode("ascii") - generated_prompt = PROMPT_BUILDERS[task_type]( - base64_image=base64_image, - prompt=prompt, - output_structure=output_structure, - classes=classes, - ) - llama_prompts.append(generated_prompt) - return execute_llama_vision_32_requests( - llama_api_key=llama_api_key, - llama_prompts=llama_prompts, - model_version_id=model_version_id, - max_tokens=max_tokens, - temperature=temperature, - max_concurrent_requests=max_concurrent_requests, - ) - - -def execute_llama_vision_32_requests( - llama_api_key: str, - llama_prompts: List[List[dict]], - model_version_id: str, - max_tokens: int, - temperature: float, - max_concurrent_requests: Optional[int], -) -> List[str]: - client = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=llama_api_key) - tasks = [ - partial( - execute_llama_vision_32_request, - client=client, - prompt=prompt, - llama_model_version=model_version_id, - max_tokens=max_tokens, - temperature=temperature, - ) - for prompt in llama_prompts - ] - max_workers = ( - max_concurrent_requests - or WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS - ) - return run_in_parallel( - tasks=tasks, - max_workers=max_workers, - ) - - -def execute_llama_vision_32_request( - client: OpenAI, - prompt: List[dict], - llama_model_version: str, - max_tokens: int, - temperature: float, -) -> str: - response = client.chat.completions.create( - model=llama_model_version, - messages=prompt, - max_tokens=max_tokens, - temperature=temperature, - ) - if response.choices is None: - error_detail = getattr(response, "error", {}).get("message", "N/A") - raise RuntimeError( - "OpenRouter provider failed in delivering response. This issue happens from time " - "to time, especially when using Free offering - raise issue to OpenRouter if that's " - f"problematic for you. Details: {error_detail}" - ) - return response.choices[0].message.content - - -def prepare_unconstrained_prompt( - base64_image: str, - prompt: str, - **kwargs, -) -> List[dict]: - return [ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}", - }, - }, - ], - } - ] - - -def prepare_classification_prompt( - base64_image: str, classes: List[str], **kwargs -) -> List[dict]: - serialised_classes = ", ".join(classes) - return [ - { - "role": "system", - "content": "You act as single-class classification model. You must provide reasonable predictions. " - "You are only allowed to produce JSON document in Markdown ```json [...]``` markers. " - 'Expected structure of json: {"class_name": "class-name", "confidence": 0.4}. ' - "`class-name` must be one of the class names defined by user. You are only allowed to return " - "single JSON document, even if there are potentially multiple classes. You are not allowed to return list. " - "You cannot discuss the result, you are only allowed to return JSON document.", - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": f"List of all classes to be recognised by model: {serialised_classes}", - }, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}", - }, - }, - ], - }, - ] - - -def prepare_multi_label_classification_prompt( - base64_image: str, classes: List[str], **kwargs -) -> List[dict]: - serialised_classes = ", ".join(classes) - return [ - { - "role": "system", - "content": "You act as multi-label classification model. You must provide reasonable predictions. " - "You are only allowed to produce JSON document in Markdown ```json``` markers. " - 'Expected structure of json: {"predicted_classes": [{"class": "class-name-1", "confidence": 0.9}, ' - '{"class": "class-name-2", "confidence": 0.7}]}. ' - "`class-name-X` must be one of the class names defined by user and `confidence` is a float value in range " - "0.0-1.0 that represent how sure you are that the class is present in the image. Only return class names " - "that are visible. You cannot discuss the result, you are only allowed to return JSON document.", - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": f"List of all classes to be recognised by model: {serialised_classes}", - }, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}", - }, - }, - ], - }, - ] - - -def prepare_vqa_prompt(base64_image: str, prompt: str, **kwargs) -> List[dict]: - return [ - { - "role": "system", - "content": "You act as Visual Question Answering model. Your task is to provide answer to question" - "submitted by user. If this is open-question - answer with few sentences, for ABCD question, " - "return only the indicator of the answer.", - }, - { - "role": "user", - "content": [ - {"type": "text", "text": f"Question: {prompt}"}, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}", - }, - }, - ], - }, - ] - - -def prepare_ocr_prompt(base64_image: str, **kwargs) -> List[dict]: - return [ - { - "role": "system", - "content": "You act as OCR model. Your task is to read text from the image and return it in " - "paragraphs representing the structure of texts in the image. You should only return " - "recognised text, nothing else.", - }, - { - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}", - }, - }, - ], - }, - ] - - -def prepare_caption_prompt( - base64_image: str, short_description: bool, **kwargs -) -> List[dict]: - caption_detail_level = "Caption should be short." - if not short_description: - caption_detail_level = "Caption should be extensive." - return [ - { - "role": "system", - "content": f"You act as image caption model. Your task is to provide description of the image. " - f"{caption_detail_level}", - }, - { - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}", - }, - }, - ], - }, - ] - - -def prepare_structured_answering_prompt( - base64_image: str, output_structure: Dict[str, str], **kwargs -) -> List[dict]: - output_structure_serialised = json.dumps(output_structure, indent=4) - return [ - { - "role": "system", - "content": "You are supposed to produce responses in JSON wrapped in Markdown markers: " - "```json\nyour-response\n```. User is to provide you dictionary with keys and values. " - "Each key must be present in your response. Values in user dictionary represent " - "descriptions for JSON fields to be generated. Provide only JSON Markdown in response.", - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": f"Specification of requirements regarding output fields: \n" - f"{output_structure_serialised}", - }, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}", - }, - }, - ], - }, - ] - - -PROMPT_BUILDERS = { - "unconstrained": prepare_unconstrained_prompt, - "ocr": prepare_ocr_prompt, - "visual-question-answering": prepare_vqa_prompt, - "caption": partial(prepare_caption_prompt, short_description=True), - "detailed-caption": partial(prepare_caption_prompt, short_description=False), - "classification": prepare_classification_prompt, - "multi-label-classification": prepare_multi_label_classification_prompt, - "structured-answering": prepare_structured_answering_prompt, -} +import base64 +import json +from functools import partial +from typing import Any, Dict, List, Literal, Optional, Type, Union + +from openai import OpenAI +from pydantic import ConfigDict, Field, field_validator, model_validator + +from inference.core.env import WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS +from inference.core.managers.base import ModelManager +from inference.core.utils.image_utils import encode_image_to_jpeg_bytes, load_image +from inference.core.workflows.core_steps.common.utils import run_in_parallel +from inference.core.workflows.core_steps.common.vlms import VLM_TASKS_METADATA +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + FLOAT_KIND, + IMAGE_KIND, + LANGUAGE_MODEL_OUTPUT_KIND, + LIST_OF_VALUES_KIND, + STRING_KIND, + ImageInputField, + Selector, +) +from inference.core.workflows.prototypes.block import ( + AirGappedAvailability, + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) + +MODEL_VERSION_MAPPING = { + "11B (Free) - OpenRouter": "meta-llama/llama-3.2-11b-vision-instruct:free", + "11B (Regular) - OpenRouter": "meta-llama/llama-3.2-11b-vision-instruct", + "90B (Free) - OpenRouter": "meta-llama/llama-3.2-90b-vision-instruct:free", + "90B (Regular) - OpenRouter": "meta-llama/llama-3.2-90b-vision-instruct", +} + +ModelVersion = Literal[ + "11B (Free) - OpenRouter", + "11B (Regular) - OpenRouter", + "90B (Free) - OpenRouter", + "90B (Regular) - OpenRouter", +] + +SUPPORTED_TASK_TYPES_LIST = [ + "unconstrained", + "ocr", + "structured-answering", + "classification", + "multi-label-classification", + "visual-question-answering", + "caption", + "detailed-caption", +] +SUPPORTED_TASK_TYPES = set(SUPPORTED_TASK_TYPES_LIST) + +RELEVANT_TASKS_METADATA = { + k: v for k, v in VLM_TASKS_METADATA.items() if k in SUPPORTED_TASK_TYPES +} +RELEVANT_TASKS_DOCS_DESCRIPTION = "\n\n".join( + f"* **{v['name']}** (`{k}`) - {v['description']}" + for k, v in RELEVANT_TASKS_METADATA.items() +) + + +LONG_DESCRIPTION = f""" +Ask a question to Llama 3.2 Vision model with vision capabilities. + +You can specify arbitrary text prompts or predefined ones, the block supports the following types of prompt: + +{RELEVANT_TASKS_DOCS_DESCRIPTION} + +!!! warning "Issues with structured prompting" + + Model tends to be quite unpredictable when structured output (in our case JSON document) is expected. + That problems may impact tasks like `structured-answering`, `classification` or `multi-label-classification`. + + The cause seems to be quite sensitive "filters" of inappropriate content embedded in model. + + +#### 🛠️ API providers and model variants + +Llama Vision 3.2 model is exposed via [OpenRouter API](https://openrouter.ai/) and we require +passing [OpenRouter API Key](https://openrouter.ai/docs/api-keys) to run. + +There are different versions of the model supported: + +* smaller version (`11B`) is faster and cheaper, yet you can expect better quality of results using `90B` version + +* `Regular` version is paid (and usually faster) API, whereas `Free` is free for use for OpenRouter clients +(state at 01.01.2025) + + +As for now, OpenRouter is the only provider for Llama 3.2 Vision model, but we will keep you posted if +the state of the matter changes. + + +!!! warning "API Usage Charges" + + OpenRouter is external third party providing access to the model and incurring charges on the usage. + Please check out pricing before use: + + * [Llama 3.2 Vision 11B (Regular)](https://openrouter.ai/meta-llama/llama-3.2-11b-vision-instruct/providers) + + * [Llama 3.2 Vision 90B (Regular)](https://openrouter.ai/meta-llama/llama-3.2-90b-vision-instruct/providers) + +#### 💡 Further reading and Acceptable Use Policy + +!!! warning "Model license" + + Check out [model license](https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/LICENSE) before + use. + +[Click here](https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/MODEL_CARD_VISION.md) for the original model card. + +Usage of this model is subject to Meta's [Acceptable Use Policy](https://www.llama.com/llama3/use-policy/). +""" + + +TaskType = Literal[tuple(SUPPORTED_TASK_TYPES_LIST)] + +TASKS_REQUIRING_PROMPT = { + "unconstrained", + "visual-question-answering", +} + +TASKS_REQUIRING_CLASSES = { + "classification", + "multi-label-classification", +} + +TASKS_REQUIRING_OUTPUT_STRUCTURE = { + "structured-answering", +} + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Llama 3.2 Vision", + "version": "v1", + "deprecated": True, + "deprecation_message": "Use the Llama 3.2 Vision v2 block, which adds a Roboflow-managed API key option and user-selectable privacy controls.", + "short_description": "Run Llama model with Vision capabilities", + "long_description": LONG_DESCRIPTION, + "license": "Llama 3.2 Community", + "block_type": "model", + "search_keywords": ["LMM", "VLM", "Llama", "Vision", "Meta"], + "is_vlm_block": True, + "task_type_property": "task_type", + "ui_manifest": { + "section": "model", + "icon": "far fa-brands fa-meta", + }, + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/llama_3_2_vision@v1"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + task_type: TaskType = Field( + default="unconstrained", + description="Task type to be performed by model. Value determines required parameters and output response.", + json_schema_extra={ + "values_metadata": RELEVANT_TASKS_METADATA, + "recommended_parsers": { + "structured-answering": "roboflow_core/json_parser@v1", + "classification": "roboflow_core/vlm_as_classifier@v1", + "multi-label-classification": "roboflow_core/vlm_as_classifier@v1", + }, + "always_visible": True, + }, + ) + prompt: Optional[Union[Selector(kind=[STRING_KIND]), str]] = Field( + default=None, + description="Text prompt to the Llama model", + examples=["my prompt", "$inputs.prompt"], + json_schema_extra={ + "relevant_for": { + "task_type": {"values": TASKS_REQUIRING_PROMPT, "required": True}, + }, + "multiline": True, + }, + ) + output_structure: Optional[Dict[str, str]] = Field( + default=None, + description="Dictionary with structure of expected JSON response", + examples=[{"my_key": "description"}, "$inputs.output_structure"], + json_schema_extra={ + "relevant_for": { + "task_type": { + "values": TASKS_REQUIRING_OUTPUT_STRUCTURE, + "required": True, + }, + }, + }, + ) + classes: Optional[Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]]] = Field( + default=None, + description="List of classes to be used", + examples=[["class-a", "class-b"], "$inputs.classes"], + json_schema_extra={ + "relevant_for": { + "task_type": { + "values": TASKS_REQUIRING_CLASSES, + "required": True, + }, + }, + }, + ) + api_key: Union[Selector(kind=[STRING_KIND]), str] = Field( + description="Your Llama Vision API key (dependent on provider, ex: OpenRouter API key)", + examples=["xxx-xxx", "$inputs.llama_api_key"], + private=True, + ) + model_version: Union[ + Selector(kind=[STRING_KIND]), + Literal[ + "11B (Free) - OpenRouter", + "11B (Regular) - OpenRouter", + "90B (Free) - OpenRouter", + "90B (Regular) - OpenRouter", + ], + ] = Field( + default="11B (Free) - OpenRouter", + description="Model to be used", + examples=["11B (Free) - OpenRouter", "$inputs.llama_model"], + ) + max_tokens: int = Field( + default=500, + description="Maximum number of tokens the model can generate in it's response.", + gt=1, + ) + temperature: Union[float, Selector(kind=[FLOAT_KIND])] = Field( + default=0.1, + description="Temperature to sample from the model - value in range 0.0-2.0, the higher - the more " + 'random / "creative" the generations are.', + ) + max_concurrent_requests: Optional[int] = Field( + default=None, + description="Number of concurrent requests that can be executed by block when batch of input images provided. " + "If not given - block defaults to value configured globally in Workflows Execution Engine. " + "Please restrict if you hit limits.", + ) + + @model_validator(mode="after") + def validate(self) -> "BlockManifest": + if self.task_type in TASKS_REQUIRING_PROMPT and self.prompt is None: + raise ValueError( + f"`prompt` parameter required to be set for task `{self.task_type}`" + ) + if self.task_type in TASKS_REQUIRING_CLASSES and self.classes is None: + raise ValueError( + f"`classes` parameter required to be set for task `{self.task_type}`" + ) + if ( + self.task_type in TASKS_REQUIRING_OUTPUT_STRUCTURE + and self.output_structure is None + ): + raise ValueError( + f"`output_structure` parameter required to be set for task `{self.task_type}`" + ) + return self + + @classmethod + def get_air_gapped_availability(cls) -> AirGappedAvailability: + return AirGappedAvailability(available=False, reason="requires_internet") + + @field_validator("temperature") + @classmethod + def validate_temperature(cls, value: Union[str, float]) -> Union[str, float]: + if isinstance(value, str): + return value + if value < 0.0 or value > 2.0: + raise ValueError( + "'temperature' parameter required to be in range [0.0, 2.0]" + ) + return value + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="output", kind=[STRING_KIND, LANGUAGE_MODEL_OUTPUT_KIND] + ), + OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), + ] + + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class LlamaVisionBlockV1(WorkflowBlock): + + def __init__( + self, + model_manager: ModelManager, + ): + self._model_manager = model_manager + + @classmethod + def get_init_parameters(cls) -> List[str]: + return ["model_manager"] + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + def run( + self, + images: Batch[WorkflowImageData], + task_type: TaskType, + prompt: Optional[str], + output_structure: Optional[Dict[str, str]], + classes: Optional[List[str]], + api_key: str, + model_version: ModelVersion, + max_tokens: int, + temperature: float, + max_concurrent_requests: Optional[int], + ) -> BlockResult: + inference_images = [i.to_inference_format() for i in images] + raw_outputs = run_llama_vision_32_llm_prompting( + images=inference_images, + task_type=task_type, + prompt=prompt, + output_structure=output_structure, + classes=classes, + llama_api_key=api_key, + llama_model_version=model_version, + max_tokens=max_tokens, + temperature=temperature, + max_concurrent_requests=max_concurrent_requests, + ) + return [ + {"output": raw_output, "classes": classes} for raw_output in raw_outputs + ] + + +def run_llama_vision_32_llm_prompting( + images: List[Dict[str, Any]], + task_type: TaskType, + prompt: Optional[str], + output_structure: Optional[Dict[str, str]], + classes: Optional[List[str]], + llama_api_key: Optional[str], + llama_model_version: ModelVersion, + max_tokens: int, + temperature: float, + max_concurrent_requests: Optional[int], +) -> List[str]: + if task_type not in PROMPT_BUILDERS: + raise ValueError(f"Task type: {task_type} not supported.") + model_version_id = MODEL_VERSION_MAPPING.get(llama_model_version) + if not llama_model_version: + raise ValueError( + f"Invalid model name: '{llama_model_version}'. Please use one of {list(MODEL_VERSION_MAPPING.keys())}." + ) + llama_prompts = [] + for image in images: + loaded_image, _ = load_image(image) + base64_image = base64.b64encode( + encode_image_to_jpeg_bytes(loaded_image) + ).decode("ascii") + generated_prompt = PROMPT_BUILDERS[task_type]( + base64_image=base64_image, + prompt=prompt, + output_structure=output_structure, + classes=classes, + ) + llama_prompts.append(generated_prompt) + return execute_llama_vision_32_requests( + llama_api_key=llama_api_key, + llama_prompts=llama_prompts, + model_version_id=model_version_id, + max_tokens=max_tokens, + temperature=temperature, + max_concurrent_requests=max_concurrent_requests, + ) + + +def execute_llama_vision_32_requests( + llama_api_key: str, + llama_prompts: List[List[dict]], + model_version_id: str, + max_tokens: int, + temperature: float, + max_concurrent_requests: Optional[int], +) -> List[str]: + client = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=llama_api_key) + tasks = [ + partial( + execute_llama_vision_32_request, + client=client, + prompt=prompt, + llama_model_version=model_version_id, + max_tokens=max_tokens, + temperature=temperature, + ) + for prompt in llama_prompts + ] + max_workers = ( + max_concurrent_requests + or WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS + ) + return run_in_parallel( + tasks=tasks, + max_workers=max_workers, + ) + + +def execute_llama_vision_32_request( + client: OpenAI, + prompt: List[dict], + llama_model_version: str, + max_tokens: int, + temperature: float, +) -> str: + response = client.chat.completions.create( + model=llama_model_version, + messages=prompt, + max_tokens=max_tokens, + temperature=temperature, + ) + if response.choices is None: + error_detail = getattr(response, "error", {}).get("message", "N/A") + raise RuntimeError( + "OpenRouter provider failed in delivering response. This issue happens from time " + "to time, especially when using Free offering - raise issue to OpenRouter if that's " + f"problematic for you. Details: {error_detail}" + ) + return response.choices[0].message.content + + +def prepare_unconstrained_prompt( + base64_image: str, + prompt: str, + **kwargs, +) -> List[dict]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}", + }, + }, + ], + } + ] + + +def prepare_classification_prompt( + base64_image: str, classes: List[str], **kwargs +) -> List[dict]: + serialised_classes = ", ".join(classes) + return [ + { + "role": "system", + "content": "You act as single-class classification model. You must provide reasonable predictions. " + "You are only allowed to produce JSON document in Markdown ```json [...]``` markers. " + 'Expected structure of json: {"class_name": "class-name", "confidence": 0.4}. ' + "`class-name` must be one of the class names defined by user. You are only allowed to return " + "single JSON document, even if there are potentially multiple classes. You are not allowed to return list. " + "You cannot discuss the result, you are only allowed to return JSON document.", + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": f"List of all classes to be recognised by model: {serialised_classes}", + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}", + }, + }, + ], + }, + ] + + +def prepare_multi_label_classification_prompt( + base64_image: str, classes: List[str], **kwargs +) -> List[dict]: + serialised_classes = ", ".join(classes) + return [ + { + "role": "system", + "content": "You act as multi-label classification model. You must provide reasonable predictions. " + "You are only allowed to produce JSON document in Markdown ```json``` markers. " + 'Expected structure of json: {"predicted_classes": [{"class": "class-name-1", "confidence": 0.9}, ' + '{"class": "class-name-2", "confidence": 0.7}]}. ' + "`class-name-X` must be one of the class names defined by user and `confidence` is a float value in range " + "0.0-1.0 that represent how sure you are that the class is present in the image. Only return class names " + "that are visible. You cannot discuss the result, you are only allowed to return JSON document.", + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": f"List of all classes to be recognised by model: {serialised_classes}", + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}", + }, + }, + ], + }, + ] + + +def prepare_vqa_prompt(base64_image: str, prompt: str, **kwargs) -> List[dict]: + return [ + { + "role": "system", + "content": "You act as Visual Question Answering model. Your task is to provide answer to question" + "submitted by user. If this is open-question - answer with few sentences, for ABCD question, " + "return only the indicator of the answer.", + }, + { + "role": "user", + "content": [ + {"type": "text", "text": f"Question: {prompt}"}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}", + }, + }, + ], + }, + ] + + +def prepare_ocr_prompt(base64_image: str, **kwargs) -> List[dict]: + return [ + { + "role": "system", + "content": "You act as OCR model. Your task is to read text from the image and return it in " + "paragraphs representing the structure of texts in the image. You should only return " + "recognised text, nothing else.", + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}", + }, + }, + ], + }, + ] + + +def prepare_caption_prompt( + base64_image: str, short_description: bool, **kwargs +) -> List[dict]: + caption_detail_level = "Caption should be short." + if not short_description: + caption_detail_level = "Caption should be extensive." + return [ + { + "role": "system", + "content": f"You act as image caption model. Your task is to provide description of the image. " + f"{caption_detail_level}", + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}", + }, + }, + ], + }, + ] + + +def prepare_structured_answering_prompt( + base64_image: str, output_structure: Dict[str, str], **kwargs +) -> List[dict]: + output_structure_serialised = json.dumps(output_structure, indent=4) + return [ + { + "role": "system", + "content": "You are supposed to produce responses in JSON wrapped in Markdown markers: " + "```json\nyour-response\n```. User is to provide you dictionary with keys and values. " + "Each key must be present in your response. Values in user dictionary represent " + "descriptions for JSON fields to be generated. Provide only JSON Markdown in response.", + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": f"Specification of requirements regarding output fields: \n" + f"{output_structure_serialised}", + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}", + }, + }, + ], + }, + ] + + +PROMPT_BUILDERS = { + "unconstrained": prepare_unconstrained_prompt, + "ocr": prepare_ocr_prompt, + "visual-question-answering": prepare_vqa_prompt, + "caption": partial(prepare_caption_prompt, short_description=True), + "detailed-caption": partial(prepare_caption_prompt, short_description=False), + "classification": prepare_classification_prompt, + "multi-label-classification": prepare_multi_label_classification_prompt, + "structured-answering": prepare_structured_answering_prompt, +} diff --git a/inference/core/workflows/core_steps/models/foundation/llama_vision/v2.py b/inference/core/workflows/core_steps/models/foundation/llama_vision/v2.py index d23d41d1e4..1ca5a4181b 100644 --- a/inference/core/workflows/core_steps/models/foundation/llama_vision/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/llama_vision/v2.py @@ -190,6 +190,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -201,6 +205,10 @@ class LlamaVisionBlockV2(OpenRouterWorkflowBlockBase): def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/lmm_classifier/v1.py b/inference/core/workflows/core_steps/models/foundation/lmm_classifier/v1.py index 321bcad270..2f1d6faf88 100644 --- a/inference/core/workflows/core_steps/models/foundation/lmm_classifier/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/lmm_classifier/v1.py @@ -120,6 +120,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="prediction_type", kind=[PREDICTION_TYPE_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -163,6 +167,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/moondream2/v1.py b/inference/core/workflows/core_steps/models/foundation/moondream2/v1.py index 04df2eb6bf..0dc6581e7f 100644 --- a/inference/core/workflows/core_steps/models/foundation/moondream2/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/moondream2/v1.py @@ -106,6 +106,10 @@ def get_parameters_accepting_batches(cls) -> List[str]: # Only images can be passed in as a list/batch return ["images"] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -160,6 +164,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode == StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/ocr/v1.py b/inference/core/workflows/core_steps/models/foundation/ocr/v1.py index ce066c500e..8030c14e29 100644 --- a/inference/core/workflows/core_steps/models/foundation/ocr/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/ocr/v1.py @@ -102,6 +102,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="prediction_type", kind=[PREDICTION_TYPE_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -133,6 +137,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/openai/v2.py b/inference/core/workflows/core_steps/models/foundation/openai/v2.py index 1d80b5c8be..e614187f97 100644 --- a/inference/core/workflows/core_steps/models/foundation/openai/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/openai/v2.py @@ -241,6 +241,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -264,6 +268,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/openai/v3.py b/inference/core/workflows/core_steps/models/foundation/openai/v3.py index 329d1bec04..097d2bae7f 100644 --- a/inference/core/workflows/core_steps/models/foundation/openai/v3.py +++ b/inference/core/workflows/core_steps/models/foundation/openai/v3.py @@ -251,6 +251,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -274,6 +278,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/openai/v4.py b/inference/core/workflows/core_steps/models/foundation/openai/v4.py index 849a2ad189..7d1347ad23 100644 --- a/inference/core/workflows/core_steps/models/foundation/openai/v4.py +++ b/inference/core/workflows/core_steps/models/foundation/openai/v4.py @@ -353,6 +353,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.4.0,<2.0.0" @@ -376,6 +380,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/openrouter/v1.py b/inference/core/workflows/core_steps/models/foundation/openrouter/v1.py index ce9f16bc08..f32b1dacbb 100644 --- a/inference/core/workflows/core_steps/models/foundation/openrouter/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/openrouter/v1.py @@ -227,6 +227,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -238,6 +242,10 @@ class OpenRouterBlockV1(OpenRouterWorkflowBlockBase): def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/qwen/v1.py b/inference/core/workflows/core_steps/models/foundation/qwen/v1.py index 136d345f46..18f55f3653 100644 --- a/inference/core/workflows/core_steps/models/foundation/qwen/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/qwen/v1.py @@ -109,6 +109,10 @@ def get_parameters_accepting_batches(cls) -> List[str]: # Only images can be passed in as a list/batch return ["images"] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -166,6 +170,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode == StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/qwen3_5_openrouter/v1.py b/inference/core/workflows/core_steps/models/foundation/qwen3_5_openrouter/v1.py index 0acec0fb97..3613a76ff2 100644 --- a/inference/core/workflows/core_steps/models/foundation/qwen3_5_openrouter/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/qwen3_5_openrouter/v1.py @@ -277,6 +277,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -298,6 +302,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/qwen3_5vl/v1.py b/inference/core/workflows/core_steps/models/foundation/qwen3_5vl/v1.py index 70f70f396d..fe6da77940 100644 --- a/inference/core/workflows/core_steps/models/foundation/qwen3_5vl/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/qwen3_5vl/v1.py @@ -133,6 +133,10 @@ def get_parameters_accepting_batches(cls) -> List[str]: # Only images can be passed in as a list/batch return ["images"] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -190,6 +194,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode == StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/qwen3_5vl/v2.py b/inference/core/workflows/core_steps/models/foundation/qwen3_5vl/v2.py index 6fc41c5ab5..6d4994a57d 100644 --- a/inference/core/workflows/core_steps/models/foundation/qwen3_5vl/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/qwen3_5vl/v2.py @@ -108,6 +108,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: def get_parameters_accepting_batches(cls) -> List[str]: return ["images"] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -164,6 +168,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode == StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/qwen3_6_openrouter/v1.py b/inference/core/workflows/core_steps/models/foundation/qwen3_6_openrouter/v1.py index 015908e7ed..66a873a5db 100644 --- a/inference/core/workflows/core_steps/models/foundation/qwen3_6_openrouter/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/qwen3_6_openrouter/v1.py @@ -274,6 +274,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -295,6 +299,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/qwen3vl/v1.py b/inference/core/workflows/core_steps/models/foundation/qwen3vl/v1.py index 8ae4f04d5b..8b994e6af8 100644 --- a/inference/core/workflows/core_steps/models/foundation/qwen3vl/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/qwen3vl/v1.py @@ -106,6 +106,10 @@ def get_parameters_accepting_batches(cls) -> List[str]: # Only images can be passed in as a list/batch return ["images"] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -163,6 +167,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode == StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/qwen_vlm/v1.py b/inference/core/workflows/core_steps/models/foundation/qwen_vlm/v1.py index 58c1a1cb31..18051d1edf 100644 --- a/inference/core/workflows/core_steps/models/foundation/qwen_vlm/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/qwen_vlm/v1.py @@ -692,6 +692,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -731,6 +735,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode == StepExecutionMode.REMOTE + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/models/foundation/seg_preview/v1.py b/inference/core/workflows/core_steps/models/foundation/seg_preview/v1.py index 9e1c12c734..5446f221be 100644 --- a/inference/core/workflows/core_steps/models/foundation/seg_preview/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/seg_preview/v1.py @@ -106,6 +106,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -156,6 +160,10 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # External API call — remote regardless of execution mode; request path is re-entrant. + return True + def run( self, images: Batch[WorkflowImageData], @@ -183,8 +191,7 @@ def run_via_request( threshold: float, ) -> BlockResult: predictions = [] - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) diff --git a/inference/core/workflows/core_steps/models/foundation/segment_anything2/v1.py b/inference/core/workflows/core_steps/models/foundation/segment_anything2/v1.py index 84d5195d74..013c2721a8 100644 --- a/inference/core/workflows/core_steps/models/foundation/segment_anything2/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/segment_anything2/v1.py @@ -144,6 +144,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -204,6 +208,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/segment_anything3/v1.py b/inference/core/workflows/core_steps/models/foundation/segment_anything3/v1.py index baf78ec5b6..3700abe0ca 100644 --- a/inference/core/workflows/core_steps/models/foundation/segment_anything3/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/segment_anything3/v1.py @@ -138,6 +138,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -193,6 +197,15 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return ( + SAM3_EXEC_MODE != "remote" + and self._step_execution_mode is StepExecutionMode.REMOTE + ) + def run( self, images: Batch[WorkflowImageData], @@ -244,8 +257,7 @@ def run_locally( threshold: float, ) -> BlockResult: predictions = [] - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) @@ -319,8 +331,7 @@ def run_remotely( threshold: float, ) -> BlockResult: predictions = [] - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) @@ -393,8 +404,7 @@ def run_via_request( threshold: float, ) -> BlockResult: predictions = [] - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) diff --git a/inference/core/workflows/core_steps/models/foundation/segment_anything3/v2.py b/inference/core/workflows/core_steps/models/foundation/segment_anything3/v2.py index 152bc24b2e..0bfa8c817f 100644 --- a/inference/core/workflows/core_steps/models/foundation/segment_anything3/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/segment_anything3/v2.py @@ -189,6 +189,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -244,6 +248,15 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return ( + SAM3_EXEC_MODE != "remote" + and self._step_execution_mode is StepExecutionMode.REMOTE + ) + def run( self, images: Batch[WorkflowImageData], @@ -310,8 +323,7 @@ def run_locally( nms_iou_threshold: float = 0.9, ) -> BlockResult: predictions = [] - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) @@ -397,8 +409,7 @@ def run_remotely( nms_iou_threshold: float = 0.9, ) -> BlockResult: predictions = [] - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) @@ -478,8 +489,7 @@ def run_via_request( nms_iou_threshold: float = 0.9, ) -> BlockResult: predictions = [] - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) diff --git a/inference/core/workflows/core_steps/models/foundation/segment_anything3/v3.py b/inference/core/workflows/core_steps/models/foundation/segment_anything3/v3.py index c12ed80d68..fcfcf4fa31 100644 --- a/inference/core/workflows/core_steps/models/foundation/segment_anything3/v3.py +++ b/inference/core/workflows/core_steps/models/foundation/segment_anything3/v3.py @@ -216,6 +216,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -271,6 +275,15 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return ( + SAM3_EXEC_MODE != "remote" + and self._step_execution_mode is StepExecutionMode.REMOTE + ) + def run( self, images: Batch[WorkflowImageData], @@ -346,8 +359,7 @@ def run_locally( nms_iou_threshold: float = 0.9, output_format: Literal["rle", "polygons"] = "rle", ) -> BlockResult: - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) @@ -437,8 +449,7 @@ def run_remotely( nms_iou_threshold: float = 0.9, output_format: Literal["rle", "polygons"] = "rle", ) -> BlockResult: - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) @@ -522,8 +533,7 @@ def run_via_request( nms_iou_threshold: float = 0.9, output_format: Literal["rle", "polygons"] = "rle", ) -> BlockResult: - if class_names is None: - class_names = [] + class_names = [] if class_names is None else list(class_names) if len(class_names) == 0: class_names.append(None) diff --git a/inference/core/workflows/core_steps/models/foundation/segment_anything3_3d/v1.py b/inference/core/workflows/core_steps/models/foundation/segment_anything3_3d/v1.py index 237c263f86..ad183ca926 100644 --- a/inference/core/workflows/core_steps/models/foundation/segment_anything3_3d/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/segment_anything3_3d/v1.py @@ -110,6 +110,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -160,6 +164,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/segment_anything3_interactive/v1.py b/inference/core/workflows/core_steps/models/foundation/segment_anything3_interactive/v1.py index e4823d1ecb..40b7c59fa0 100644 --- a/inference/core/workflows/core_steps/models/foundation/segment_anything3_interactive/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/segment_anything3_interactive/v1.py @@ -207,6 +207,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -262,6 +266,15 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return ( + SAM3_EXEC_MODE != "remote" + and self._step_execution_mode is StepExecutionMode.REMOTE + ) + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/smolvlm/v1.py b/inference/core/workflows/core_steps/models/foundation/smolvlm/v1.py index 8c0d7242c5..34fa3890ae 100644 --- a/inference/core/workflows/core_steps/models/foundation/smolvlm/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/smolvlm/v1.py @@ -93,6 +93,10 @@ def get_parameters_accepting_batches(cls) -> List[str]: # Only images can be passed in as a list/batch return ["images"] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -147,6 +151,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode == StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/foundation/yolo_world/v1.py b/inference/core/workflows/core_steps/models/foundation/yolo_world/v1.py index a5a0310f39..0b783fec16 100644 --- a/inference/core/workflows/core_steps/models/foundation/yolo_world/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/yolo_world/v1.py @@ -115,6 +115,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -154,6 +158,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v1.py b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v1.py index a7c2e2f5ba..a414014e95 100644 --- a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v1.py @@ -181,6 +181,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -206,6 +210,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v2.py b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v2.py index 39be31b3c7..20314957d7 100644 --- a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v2.py +++ b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v2.py @@ -179,6 +179,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -204,6 +208,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v3.py b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v3.py index c5fbdf4e75..0c618facb4 100644 --- a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v3.py +++ b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v3.py @@ -238,6 +238,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -270,6 +274,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v4.py b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v4.py index a50cc8cdd3..e5521029dc 100644 --- a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v4.py +++ b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v4.py @@ -215,6 +215,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -240,6 +244,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v1.py b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v1.py index cabdd0795d..ed7b255b6d 100644 --- a/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v1.py @@ -163,6 +163,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -188,6 +192,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v2.py b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v2.py index 2f658fe05b..6392b1c264 100644 --- a/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v2.py +++ b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v2.py @@ -160,6 +160,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -185,6 +189,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v3.py b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v3.py index 919427c9d9..859bb5000b 100644 --- a/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v3.py +++ b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v3.py @@ -195,6 +195,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -220,6 +224,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v1.py b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v1.py index 2adb1ee17c..e88e8c97b7 100644 --- a/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v1.py @@ -110,6 +110,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name=INFERENCE_ID_KEY, kind=[STRING_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -135,6 +139,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v2.py b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v2.py index 8e6d9a031e..d888105486 100644 --- a/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v2.py +++ b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v2.py @@ -114,6 +114,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -139,6 +143,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v3.py b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v3.py index 64df2418f8..798000b3f7 100644 --- a/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v3.py +++ b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v3.py @@ -154,6 +154,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -179,6 +183,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v1.py b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v1.py index 3778013480..236121bdfe 100644 --- a/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v1.py @@ -116,6 +116,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name=INFERENCE_ID_KEY, kind=[STRING_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -141,6 +145,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v2.py b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v2.py index 7dd3a120c9..5b50c86615 100644 --- a/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v2.py +++ b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v2.py @@ -113,6 +113,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -138,6 +142,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v3.py b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v3.py index 8cdd804f74..4be693d020 100644 --- a/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v3.py +++ b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v3.py @@ -154,6 +154,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -179,6 +183,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/object_detection/v1.py b/inference/core/workflows/core_steps/models/roboflow/object_detection/v1.py index 16e796b816..5c3ad27703 100644 --- a/inference/core/workflows/core_steps/models/roboflow/object_detection/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/object_detection/v1.py @@ -153,6 +153,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -178,6 +182,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/object_detection/v2.py b/inference/core/workflows/core_steps/models/roboflow/object_detection/v2.py index 6b8ffef162..5e1c95f988 100644 --- a/inference/core/workflows/core_steps/models/roboflow/object_detection/v2.py +++ b/inference/core/workflows/core_steps/models/roboflow/object_detection/v2.py @@ -150,6 +150,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -175,6 +179,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/object_detection/v3.py b/inference/core/workflows/core_steps/models/roboflow/object_detection/v3.py index 8976dd99f9..bea91adb0e 100644 --- a/inference/core/workflows/core_steps/models/roboflow/object_detection/v3.py +++ b/inference/core/workflows/core_steps/models/roboflow/object_detection/v3.py @@ -188,6 +188,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -213,6 +217,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/semantic_segmentation/v1.py b/inference/core/workflows/core_steps/models/roboflow/semantic_segmentation/v1.py index 0eedf0a21f..b2c7559cc8 100644 --- a/inference/core/workflows/core_steps/models/roboflow/semantic_segmentation/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/semantic_segmentation/v1.py @@ -103,6 +103,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -128,6 +132,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/models/roboflow/semantic_segmentation/v2.py b/inference/core/workflows/core_steps/models/roboflow/semantic_segmentation/v2.py index e1276a599e..26d832acea 100644 --- a/inference/core/workflows/core_steps/models/roboflow/semantic_segmentation/v2.py +++ b/inference/core/workflows/core_steps/models/roboflow/semantic_segmentation/v2.py @@ -151,6 +151,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" @@ -176,6 +180,12 @@ def get_init_parameters(cls) -> List[str]: def get_manifest(cls) -> Type[WorkflowBlockManifest]: return BlockManifest + def is_async_stream_step(self) -> bool: + # The remote request path is re-entrant (fresh client per call, + # thread-local connection pooling in the SDK executors), so the + # stream scheduler may execute run() ahead of stream order. + return self._step_execution_mode is StepExecutionMode.REMOTE + def run( self, images: Batch[WorkflowImageData], diff --git a/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py b/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py index e2c30a0906..6b9f89af17 100644 --- a/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py +++ b/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py @@ -107,6 +107,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="crops", kind=[IMAGE_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/transformations/relative_static_crop/v1.py b/inference/core/workflows/core_steps/transformations/relative_static_crop/v1.py index 7dad71fe5f..adb5f8a142 100644 --- a/inference/core/workflows/core_steps/transformations/relative_static_crop/v1.py +++ b/inference/core/workflows/core_steps/transformations/relative_static_crop/v1.py @@ -113,6 +113,10 @@ def describe_outputs(cls) -> List[OutputDefinition]: OutputDefinition(name="crops", kind=[IMAGE_KIND]), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/visualizations/common/base.py b/inference/core/workflows/core_steps/visualizations/common/base.py index 051103b9e2..c061a9b0d9 100644 --- a/inference/core/workflows/core_steps/visualizations/common/base.py +++ b/inference/core/workflows/core_steps/visualizations/common/base.py @@ -56,6 +56,11 @@ def describe_outputs(cls) -> List[OutputDefinition]: ), ] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + # Visualizations render purely from per-frame inputs. + return False + class VisualizationBlock(WorkflowBlock, ABC): def __init__(self, *args, **kwargs): diff --git a/inference/core/workflows/core_steps/visualizations/heatmap/v1.py b/inference/core/workflows/core_steps/visualizations/heatmap/v1.py index f058d10a2f..44baba6215 100644 --- a/inference/core/workflows/core_steps/visualizations/heatmap/v1.py +++ b/inference/core/workflows/core_steps/visualizations/heatmap/v1.py @@ -148,6 +148,12 @@ class HeatmapManifest(PredictionsVisualizationManifest): examples=[25, "$inputs.motion_threshold"], ) + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + # Accumulates cross-frame state (trajectory/track history) and must + # observe frames strictly in stream order. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/core_steps/visualizations/trace/v1.py b/inference/core/workflows/core_steps/visualizations/trace/v1.py index 0a276c9313..347a104322 100644 --- a/inference/core/workflows/core_steps/visualizations/trace/v1.py +++ b/inference/core/workflows/core_steps/visualizations/trace/v1.py @@ -132,6 +132,12 @@ def ensure_max_entries_per_file_is_correct(cls, value: Any) -> Any: raise ValueError("`trace_length` and `thickness` cannot be lower than 1.") return value + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + # Accumulates cross-frame state (trajectory/track history) and must + # observe frames strictly in stream order. + return True + @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: return ">=1.3.0,<2.0.0" diff --git a/inference/core/workflows/execution_engine/core.py b/inference/core/workflows/execution_engine/core.py index d01983fcac..97bf6b5ded 100644 --- a/inference/core/workflows/execution_engine/core.py +++ b/inference/core/workflows/execution_engine/core.py @@ -1,9 +1,14 @@ from concurrent.futures import ThreadPoolExecutor -from typing import Any, Callable, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Type, Union from packaging.specifiers import SpecifierSet from packaging.version import Version +if TYPE_CHECKING: + from inference.core.workflows.execution_engine.v1.executor.execution_data_manager.manager import ( + ExecutionDataManager, + ) + from inference.core.workflows.errors import ( NotSupportedExecutionEngineError, WorkflowDefinitionError, @@ -86,6 +91,36 @@ def run( resolve_output_futures=resolve_output_futures, ) + def run_stream_lookahead( + self, + runtime_parameters: Dict[str, Any], + frontier_step_selectors: Set[str], + async_step_selectors: Set[str], + lookahead_executor: ThreadPoolExecutor, + ) -> "ExecutionDataManager": + return self._engine.run_stream_lookahead( + runtime_parameters=runtime_parameters, + frontier_step_selectors=frontier_step_selectors, + async_step_selectors=async_step_selectors, + lookahead_executor=lookahead_executor, + ) + + def resume_stream_lookahead( + self, + execution_data_manager: "ExecutionDataManager", + frontier_step_selectors: Set[str], + serialize_results: bool = False, + fps: float = 0, + _is_preview: bool = False, + ) -> List[Dict[str, Any]]: + return self._engine.resume_stream_lookahead( + execution_data_manager=execution_data_manager, + frontier_step_selectors=frontier_step_selectors, + serialize_results=serialize_results, + fps=fps, + _is_preview=_is_preview, + ) + def flush_stream_pipeline( self, runtime_parameters: Dict[str, Any], diff --git a/inference/core/workflows/execution_engine/v1/core.py b/inference/core/workflows/execution_engine/v1/core.py index d063ba6d66..2bbe8553fc 100644 --- a/inference/core/workflows/execution_engine/v1/core.py +++ b/inference/core/workflows/execution_engine/v1/core.py @@ -1,6 +1,6 @@ import os from concurrent.futures import ThreadPoolExecutor -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Set, Union from packaging.version import Version @@ -19,8 +19,13 @@ ) from inference.core.workflows.execution_engine.v1.executor.core import ( flush_stream_pipeline_workflow, + resume_stream_lookahead_workflow, + run_stream_lookahead_workflow, run_workflow, ) +from inference.core.workflows.execution_engine.v1.executor.execution_data_manager.manager import ( + ExecutionDataManager, +) from inference.core.workflows.execution_engine.v1.executor.runtime_input_assembler import ( assemble_runtime_parameters, ) @@ -32,7 +37,7 @@ legacy_step_error_handler, ) -EXECUTION_ENGINE_V1_VERSION = Version("1.12.0") +EXECUTION_ENGINE_V1_VERSION = Version("1.13.0") DEFAULT_WORKFLOWS_STEP_ERROR_HANDLER = os.getenv( "DEFAULT_WORKFLOWS_STEP_ERROR_HANDLER", "extended_roboflow_errors" @@ -115,16 +120,9 @@ def __init__( self._executor = executor self._step_error_handler = step_error_handler - def run( - self, - runtime_parameters: Dict[str, Any], - fps: float = 0, - _is_preview: bool = False, - serialize_results: bool = False, - defer_stream_pipeline_flush: bool = False, - resolve_output_futures: bool = True, - ) -> List[Dict[str, Any]]: - self._profiler.start_workflow_run() + def _prepare_runtime_parameters( + self, runtime_parameters: Dict[str, Any] + ) -> Dict[str, Any]: runtime_parameters = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=self._compiled_workflow.workflow_definition.inputs, @@ -137,19 +135,37 @@ def run( input_substitutions=self._compiled_workflow.input_substitutions, profiler=self._profiler, ) - usage_workflow_id = self._internal_id - if self._workflow_id and not usage_workflow_id: + return runtime_parameters + + def _resolve_usage_workflow_id(self) -> Optional[str]: + if self._internal_id: + return self._internal_id + if self._workflow_id: logger.debug( "Workflow ID is set to '%s' however internal Workflow ID is missing", self._workflow_id, ) - usage_workflow_id = self._workflow_id + return self._workflow_id + + def run( + self, + runtime_parameters: Dict[str, Any], + fps: float = 0, + _is_preview: bool = False, + serialize_results: bool = False, + defer_stream_pipeline_flush: bool = False, + resolve_output_futures: bool = True, + ) -> List[Dict[str, Any]]: + self._profiler.start_workflow_run() + runtime_parameters = self._prepare_runtime_parameters( + runtime_parameters=runtime_parameters + ) result = run_workflow( workflow=self._compiled_workflow, runtime_parameters=runtime_parameters, max_concurrent_steps=self._max_concurrent_steps, usage_fps=fps, - usage_workflow_id=usage_workflow_id, + usage_workflow_id=self._resolve_usage_workflow_id(), usage_workflow_preview=_is_preview, kinds_serializers=self._compiled_workflow.kinds_serializers, serialize_results=serialize_results, @@ -162,25 +178,67 @@ def run( self._profiler.end_workflow_run() return result - def flush_stream_pipeline( + def run_stream_lookahead( self, runtime_parameters: Dict[str, Any], + frontier_step_selectors: Set[str], + async_step_selectors: Set[str], + lookahead_executor: ThreadPoolExecutor, + ) -> ExecutionDataManager: + self._profiler.start_workflow_run() + runtime_parameters = self._prepare_runtime_parameters( + runtime_parameters=runtime_parameters + ) + execution_data_manager = run_stream_lookahead_workflow( + workflow=self._compiled_workflow, + runtime_parameters=runtime_parameters, + max_concurrent_steps=self._max_concurrent_steps, + frontier_step_selectors=frontier_step_selectors, + async_step_selectors=async_step_selectors, + lookahead_executor=lookahead_executor, + profiler=self._profiler, + executor=self._executor, + step_error_handler=self._step_error_handler, + ) + self._profiler.end_workflow_run() + return execution_data_manager + + def resume_stream_lookahead( + self, + execution_data_manager: ExecutionDataManager, + frontier_step_selectors: Set[str], + serialize_results: bool = False, fps: float = 0, _is_preview: bool = False, - serialize_results: bool = False, ) -> List[Dict[str, Any]]: self._profiler.start_workflow_run() - runtime_parameters = assemble_runtime_parameters( - runtime_parameters=runtime_parameters, - defined_inputs=self._compiled_workflow.workflow_definition.inputs, - kinds_deserializers=self._compiled_workflow.kinds_deserializers, - prevent_local_images_loading=self._prevent_local_images_loading, + result = resume_stream_lookahead_workflow( + workflow=self._compiled_workflow, + execution_data_manager=execution_data_manager, + max_concurrent_steps=self._max_concurrent_steps, + kinds_serializers=self._compiled_workflow.kinds_serializers, + frontier_step_selectors=frontier_step_selectors, + serialize_results=serialize_results, + usage_fps=fps, + usage_workflow_id=self._resolve_usage_workflow_id(), + usage_workflow_preview=_is_preview, profiler=self._profiler, + executor=self._executor, + step_error_handler=self._step_error_handler, ) - validate_runtime_input( - runtime_parameters=runtime_parameters, - input_substitutions=self._compiled_workflow.input_substitutions, - profiler=self._profiler, + self._profiler.end_workflow_run() + return result + + def flush_stream_pipeline( + self, + runtime_parameters: Dict[str, Any], + fps: float = 0, + _is_preview: bool = False, + serialize_results: bool = False, + ) -> List[Dict[str, Any]]: + self._profiler.start_workflow_run() + runtime_parameters = self._prepare_runtime_parameters( + runtime_parameters=runtime_parameters ) result = flush_stream_pipeline_workflow( workflow=self._compiled_workflow, diff --git a/inference/core/workflows/execution_engine/v1/executor/core.py b/inference/core/workflows/execution_engine/v1/executor/core.py index 71b148629c..b1e266158b 100644 --- a/inference/core/workflows/execution_engine/v1/executor/core.py +++ b/inference/core/workflows/execution_engine/v1/executor/core.py @@ -3,7 +3,7 @@ from concurrent.futures import ThreadPoolExecutor from datetime import datetime from functools import partial -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Set from uuid import uuid4 import cv2 @@ -57,11 +57,13 @@ ) from inference.core.workflows.execution_engine.v1.executor.flow_coordinator import ( ParallelStepExecutionCoordinator, + establish_execution_order, ) from inference.core.workflows.execution_engine.v1.executor.output_constructor import ( construct_workflow_output, ) from inference.core.workflows.execution_engine.v1.executor.utils import ( + chain_output_future, run_steps_in_parallel, ) from inference.core.workflows.prototypes.block import WorkflowBlock @@ -106,7 +108,7 @@ def run_workflow( serialize_results: bool = False, profiler: Optional[WorkflowsProfiler] = None, executor: Optional[ThreadPoolExecutor] = None, - step_error_handler: Optional[Callable[[Exception], None]] = None, + step_error_handler: Optional[Callable[[str, Exception], None]] = None, defer_stream_pipeline_flush: bool = False, resolve_output_futures: bool = True, ) -> List[Dict[str, Any]]: @@ -133,7 +135,7 @@ def _run_workflow( serialize_results: bool = False, profiler: Optional[WorkflowsProfiler] = None, executor: Optional[ThreadPoolExecutor] = None, - step_error_handler: Optional[Callable[[Exception], None]] = None, + step_error_handler: Optional[Callable[[str, Exception], None]] = None, defer_stream_pipeline_flush: bool = False, resolve_output_futures: bool = True, ) -> List[Dict[str, Any]]: @@ -141,26 +143,15 @@ def _run_workflow( execution_graph=workflow.execution_graph, runtime_parameters=runtime_parameters, ) - execution_coordinator = ParallelStepExecutionCoordinator.init( - execution_graph=workflow.execution_graph, - ) - workflow_execution_id = get_or_create_workflow_execution_id() try: - next_steps = execution_coordinator.get_steps_to_execute_next(profiler=profiler) - while next_steps is not None: - execute_steps( - next_steps=next_steps, - workflow=workflow, - execution_data_manager=execution_data_manager, - max_concurrent_steps=max_concurrent_steps, - workflow_execution_id=workflow_execution_id, - profiler=profiler, - executor=executor, - step_error_handler=step_error_handler, - ) - next_steps = execution_coordinator.get_steps_to_execute_next( - profiler=profiler - ) + _execute_workflow_steps( + workflow=workflow, + execution_data_manager=execution_data_manager, + max_concurrent_steps=max_concurrent_steps, + profiler=profiler, + executor=executor, + step_error_handler=step_error_handler, + ) if not defer_stream_pipeline_flush: with profiler.profile_execution_phase( name="stream_pipeline_flush", @@ -195,15 +186,12 @@ def flush_stream_pipeline_workflow( serialize_results: bool = False, profiler: Optional[WorkflowsProfiler] = None, executor: Optional[ThreadPoolExecutor] = None, - step_error_handler: Optional[Callable[[Exception], None]] = None, + step_error_handler: Optional[Callable[[str, Exception], None]] = None, ) -> List[Dict[str, Any]]: execution_data_manager = ExecutionDataManager.init( execution_graph=workflow.execution_graph, runtime_parameters=runtime_parameters, ) - execution_coordinator = ParallelStepExecutionCoordinator.init( - execution_graph=workflow.execution_graph, - ) flushed_step_selectors = flush_stream_pipeline_outputs( workflow=workflow, execution_data_manager=execution_data_manager, @@ -212,29 +200,20 @@ def flush_stream_pipeline_workflow( workflow=workflow, step_selectors=flushed_step_selectors, ) - workflow_execution_id = get_or_create_workflow_execution_id() - next_steps = execution_coordinator.get_steps_to_execute_next(profiler=profiler) - while next_steps is not None: - runnable_steps = [ - step_selector - for step_selector in next_steps - if step_selector in downstream_step_selectors + _execute_workflow_steps( + workflow=workflow, + execution_data_manager=execution_data_manager, + max_concurrent_steps=max_concurrent_steps, + profiler=profiler, + executor=executor, + step_error_handler=step_error_handler, + step_execution_filter=lambda step_selector: ( + step_selector in downstream_step_selectors and execution_data_manager.all_inputs_impacting_step_are_registered( step_selector=step_selector ) - ] - if runnable_steps: - execute_steps( - next_steps=runnable_steps, - workflow=workflow, - execution_data_manager=execution_data_manager, - max_concurrent_steps=max_concurrent_steps, - profiler=profiler, - executor=executor, - step_error_handler=step_error_handler, - workflow_execution_id=workflow_execution_id, - ) - next_steps = execution_coordinator.get_steps_to_execute_next(profiler=profiler) + ), + ) return construct_workflow_output( workflow_outputs=workflow.workflow_definition.outputs, execution_graph=workflow.execution_graph, @@ -291,6 +270,372 @@ def flush_stream_pipeline_outputs( return flushed_step_selectors +def _execute_workflow_steps( + workflow: CompiledWorkflow, + execution_data_manager: ExecutionDataManager, + max_concurrent_steps: int, + profiler: Optional[WorkflowsProfiler] = None, + executor: Optional[ThreadPoolExecutor] = None, + step_error_handler: Optional[Callable[[str, Exception], None]] = None, + step_execution_filter: Optional[Callable[[str], bool]] = None, + async_step_selectors: Optional[Set[str]] = None, + lookahead_executor: Optional[ThreadPoolExecutor] = None, +) -> None: + execution_coordinator = ParallelStepExecutionCoordinator.init( + execution_graph=workflow.execution_graph, + ) + workflow_execution_id = get_or_create_workflow_execution_id() + next_steps = execution_coordinator.get_steps_to_execute_next(profiler=profiler) + while next_steps is not None: + runnable_steps = ( + next_steps + if step_execution_filter is None + else [ + step_selector + for step_selector in next_steps + if step_execution_filter(step_selector) + ] + ) + if async_step_selectors and lookahead_executor is not None: + for step_selector in runnable_steps: + if step_selector in async_step_selectors: + launch_async_simd_step( + step_selector=step_selector, + workflow=workflow, + execution_data_manager=execution_data_manager, + lookahead_executor=lookahead_executor, + workflow_execution_id=workflow_execution_id, + step_error_handler=step_error_handler, + ) + runnable_steps = [ + step_selector + for step_selector in runnable_steps + if step_selector not in async_step_selectors + ] + if runnable_steps: + execute_steps( + next_steps=runnable_steps, + workflow=workflow, + execution_data_manager=execution_data_manager, + max_concurrent_steps=max_concurrent_steps, + workflow_execution_id=workflow_execution_id, + profiler=profiler, + executor=executor, + step_error_handler=step_error_handler, + ) + next_steps = execution_coordinator.get_steps_to_execute_next(profiler=profiler) + + +def launch_async_simd_step( + step_selector: str, + workflow: CompiledWorkflow, + execution_data_manager: ExecutionDataManager, + lookahead_executor: ThreadPoolExecutor, + workflow_execution_id: Optional[str] = None, + step_error_handler: Optional[Callable[[str, Exception], None]] = None, +) -> None: + """Execute a declared async step on the lookahead pool. + + Inputs are assembled on the calling thread; the block's whole ``run()`` + executes on a worker thread, and per-output future placeholders (built + from the block's declared outputs) are registered so downstream steps + resolve them at input assembly during the resume pass. + """ + step_name = get_last_chunk_of_selector(selector=step_selector) + step = workflow.steps[step_name] + step_input = execution_data_manager.get_simd_step_input( + step_selector=step_selector, + ) + if not step_input.indices: + execution_data_manager.register_simd_step_output( + step_selector=step_selector, + indices=[], + outputs=[], + ) + return + if remote_processing_times is not None: + processing_time_collector = remote_processing_times.get() + else: + processing_time_collector = None + if apply_duration_minimum is not None: + duration_minimum_value = apply_duration_minimum.get() + else: + duration_minimum_value = None + result_future = lookahead_executor.submit( + partial( + _run_async_simd_step, + step_name=step_name, + workflow=workflow, + parameters=step_input.parameters, + workflow_execution_id=workflow_execution_id, + processing_time_collector=processing_time_collector, + duration_minimum_value=duration_minimum_value, + debug_collector=current_debug_collector.get(), + debug_trace=current_debug_trace.get(), + otel_ctx=capture_context(), + step_error_handler=step_error_handler, + expected_output_elements=len(step_input.indices), + ) + ) + output_names = [ + output.name + for output in step.block_specification.manifest_class.describe_outputs() + ] + outputs = [ + { + output_name: chain_output_future( + result_future=result_future, + element_index=element_index, + output_name=output_name, + ) + for output_name in output_names + } + for element_index in range(len(step_input.indices)) + ] + execution_data_manager.register_simd_step_output( + step_selector=step_selector, + indices=step_input.indices, + outputs=outputs, + ) + + +def _run_async_simd_step( + step_name: str, + workflow: CompiledWorkflow, + parameters: Dict[str, Any], + workflow_execution_id: Optional[str], + processing_time_collector, + duration_minimum_value, + debug_collector, + debug_trace, + otel_ctx, + step_error_handler: Optional[Callable[[str, Exception], None]] = None, + expected_output_elements: Optional[int] = None, +) -> Any: + # Lookahead workers need the same ContextVar / OTel rebinding as + # safe_execute_step: values set on the stream thread do not propagate + # into pool threads, and pool threads are reused across frames. + if execution_id is not None and workflow_execution_id: + execution_id.set(workflow_execution_id) + if remote_processing_times is not None and processing_time_collector is not None: + remote_processing_times.set(processing_time_collector) + if apply_duration_minimum is not None and duration_minimum_value is not None: + apply_duration_minimum.set(duration_minimum_value) + current_debug_collector.set(debug_collector) + current_debug_trace.set(debug_trace) + current_debug_step_name.set(step_name) + _otel_token = attach_context(otel_ctx) + try: + with start_span("workflow.step", {"workflow.step": step_name}): + try: + result = workflow.steps[step_name].step.run(**parameters) + if ( + expected_output_elements is not None + and isinstance(result, list) + and len(result) != expected_output_elements + ): + # Sequential execution fails fast at output registration + # when a block breaks its batch contract; without this + # check the mismatch would surface at resume time as an + # index error blamed on the consuming step. + raise ValueError( + f"Async step {step_name} returned {len(result)} output " + f"elements for {expected_output_elements} input indices." + ) + return result + except WorkflowError: + raise + except Exception as error: + if step_error_handler: + step_error_handler(step_name, error) + logger.exception( + f"Execution of async step {step_name} encountered error." + ) + error_traceback = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + block_traceback = BlockTraceback( + traceback=error_traceback, + error_line=getattr(error, "error_line", None), + code_snippet=getattr(error, "code_snippet", None), + stdout=getattr(error, "stdout", None), + stderr=getattr(error, "stderr", None), + ) + raise StepExecutionError( + block_id=step_name, + block_type=workflow.steps[step_name].manifest.type, + block_traceback=block_traceback, + public_message=str(error), + context="workflow_execution | step_execution", + inner_error=error, + ) from error + finally: + detach_context(_otel_token) + + +def compute_async_stream_step_selectors(workflow: CompiledWorkflow) -> Set[str]: + """Steps declared and eligible for async launch in the deferred pass. + + A step qualifies when its block declares ``is_async_stream_step()`` and + the engine can offload it generically: SIMD with batch input, no output + dimensionality offset, and concrete declared outputs to build future + placeholders from. + """ + async_step_selectors = set() + for step_name, step in workflow.steps.items(): + declares_fn = getattr(step.step, "is_async_stream_step", None) + if not callable(declares_fn) or not declares_fn(): + continue + manifest_class = step.block_specification.manifest_class + output_names = [output.name for output in manifest_class.describe_outputs()] + if ( + not manifest_class.accepts_batch_input() + or manifest_class.get_output_dimensionality_offset() != 0 + or not output_names + or any("*" in output_name for output_name in output_names) + ): + continue + async_step_selectors.add(construct_step_selector(step_name=step_name)) + return async_step_selectors + + +@execution_phase( + name="workflow_lookahead_execution", + categories=["execution_engine_operation"], +) +def run_stream_lookahead_workflow( + workflow: CompiledWorkflow, + runtime_parameters: Dict[str, Any], + max_concurrent_steps: int, + frontier_step_selectors: Set[str], + async_step_selectors: Set[str], + lookahead_executor: ThreadPoolExecutor, + profiler: Optional[WorkflowsProfiler] = None, + executor: Optional[ThreadPoolExecutor] = None, + step_error_handler: Optional[Callable[[str, Exception], None]] = None, +) -> ExecutionDataManager: + """Deferred pass of stream-lookahead execution for one video frame. + + Runs only the frontier steps — declared async steps have their whole + ``run()`` offloaded to the lookahead pool and register future-bearing + outputs — and returns the live ExecutionDataManager so the caller can + buffer it and resume the remaining steps at ordered emission time. + """ + with start_span("workflow.run_stream_lookahead"): + execution_data_manager = ExecutionDataManager.init( + execution_graph=workflow.execution_graph, + runtime_parameters=runtime_parameters, + ) + _execute_workflow_steps( + workflow=workflow, + execution_data_manager=execution_data_manager, + max_concurrent_steps=max_concurrent_steps, + profiler=profiler, + executor=executor, + step_error_handler=step_error_handler, + step_execution_filter=lambda step_selector: step_selector + in frontier_step_selectors, + async_step_selectors=async_step_selectors, + lookahead_executor=lookahead_executor, + ) + return execution_data_manager + + +@usage_collector("workflows") +def resume_stream_lookahead_workflow( + workflow: CompiledWorkflow, + execution_data_manager: ExecutionDataManager, + max_concurrent_steps: int, + kinds_serializers: Optional[Dict[str, Callable[[Any], Any]]], + frontier_step_selectors: Set[str], + serialize_results: bool = False, + profiler: Optional[WorkflowsProfiler] = None, + executor: Optional[ThreadPoolExecutor] = None, + step_error_handler: Optional[Callable[[str, Exception], None]] = None, +) -> List[Dict[str, Any]]: + """Emission pass of stream-lookahead execution for one buffered frame. + + Usage is collected here rather than on the deferred pass, which returns + as soon as async work is launched. Counting is exactly once per frame, + but the recorded duration is an approximation: in steady state the + frame's futures have already resolved by emission time (that wait is + hidden behind newer frames' work — the point of the lookahead), so the + duration reflects the resume-pass compute, not the remote inference + wall time. + + Resumes the frame's ExecutionDataManager from the deferred pass and runs + every step outside the frontier. Frontier steps must not be re-executed: + their outputs (including in-flight futures resolved at input assembly) + are already registered. + """ + with start_span("workflow.resume_stream_lookahead"): + _execute_workflow_steps( + workflow=workflow, + execution_data_manager=execution_data_manager, + max_concurrent_steps=max_concurrent_steps, + profiler=profiler, + executor=executor, + step_error_handler=step_error_handler, + step_execution_filter=lambda step_selector: step_selector + not in frontier_step_selectors, + ) + return construct_workflow_output( + workflow_outputs=workflow.workflow_definition.outputs, + execution_graph=workflow.execution_graph, + execution_data_manager=execution_data_manager, + serialize_results=serialize_results, + kinds_serializers=kinds_serializers, + ) + + +def compute_stream_lookahead_frontier(workflow: CompiledWorkflow) -> Set[str]: + """Steps that may execute ahead of stream order in the deferred pass. + + Stream-lookahead terminology: per video frame, the *deferred pass* + (``run_stream_lookahead_workflow``) executes only the *frontier*; the + *resume pass* (``resume_stream_lookahead_workflow``) later executes the + remaining steps in frame order at emission time. + + A step belongs to the frontier when its manifest declares it stateless + for video processing and every step feeding it is itself in the frontier + and is not an async-launching (stream-pipelined) step — async steps are + frontier sinks: their future-bearing outputs must only be consumed at + resume time, otherwise input assembly would block the deferred pass. + """ + step_selectors = { + construct_step_selector(step_name=step_name) for step_name in workflow.steps + } + async_step_selectors = compute_async_stream_step_selectors(workflow=workflow) + frontier: Set[str] = set() + for execution_layer in establish_execution_order( + execution_graph=workflow.execution_graph + ): + for step_selector in execution_layer: + step_name = get_last_chunk_of_selector(selector=step_selector) + manifest_class = workflow.steps[ + step_name + ].block_specification.manifest_class + is_stateful_fn = getattr( + manifest_class, "is_stateful_for_video_processing", None + ) + if not callable(is_stateful_fn) or is_stateful_fn(): + # Manifests without the declaration (e.g. dynamic Custom + # Python blocks, built without the WorkflowBlockManifest + # base) are conservatively treated as stateful. + continue + feeding_step_selectors = [ + predecessor + for predecessor in workflow.execution_graph.predecessors(step_selector) + if predecessor in step_selectors + ] + if all( + predecessor in frontier and predecessor not in async_step_selectors + for predecessor in feeding_step_selectors + ): + frontier.add(step_selector) + return frontier + + def _downstream_step_selectors( workflow: CompiledWorkflow, step_selectors: List[str], diff --git a/inference/core/workflows/execution_engine/v1/executor/utils.py b/inference/core/workflows/execution_engine/v1/executor/utils.py index c22edb3c7b..2b4eb5cedf 100644 --- a/inference/core/workflows/execution_engine/v1/executor/utils.py +++ b/inference/core/workflows/execution_engine/v1/executor/utils.py @@ -116,3 +116,29 @@ def maybe_resolve_futures( if not contains_future(value): return value return resolve_futures(value=value, timeout=timeout, context=context) + + +def chain_output_future( + result_future: Future, + element_index: int, + output_name: str, +) -> Future: + # Chained via callback rather than executor.submit() — selector tasks + # waiting on the producing task in the same pool could exhaust its workers. + output_future: Future = Future() + + def _propagate_result(done_future: Future) -> None: + error = done_future.exception() + if error is not None: + output_future.set_exception(error) + return + try: + output_future.set_result(done_future.result()[element_index][output_name]) + except Exception as selection_error: + # concurrent.futures swallows callback exceptions — without this + # the chained future would never resolve and consumers would hang + # until the future-resolution timeout. + output_future.set_exception(selection_error) + + result_future.add_done_callback(_propagate_result) + return output_future diff --git a/inference/core/workflows/prototypes/block.py b/inference/core/workflows/prototypes/block.py index a3bd853c92..c330eae257 100644 --- a/inference/core/workflows/prototypes/block.py +++ b/inference/core/workflows/prototypes/block.py @@ -245,6 +245,26 @@ def get_restrictions(cls) -> List[RuntimeRestriction]: """ return [] + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + """Whether the block must observe video frames strictly in stream order. + + Stateful blocks keep cross-frame state (trackers, counters, + aggregators) or have order-sensitive side effects (notifications, + sinks), so the stream scheduler must run them frame by frame, in + order. Stateless blocks — pure functions of their per-frame inputs — + may be executed ahead of stream order, which lets the stream + scheduler overlap several frames' expensive steps (e.g. remote model + requests). + + This is a scheduling contract about video-frame ordering, distinct + from the "stateless HTTP runtime" caveats in ``RuntimeRestriction``. + + The default is conservative: blocks are assumed stateful unless they + override this to return ``False``. + """ + return True + @classmethod def get_supported_model_variants(cls) -> Optional[List[str]]: """Return model IDs whose cached weights enable this block to run offline. diff --git a/tests/inference/hosted_platform_tests/test_workflows.py b/tests/inference/hosted_platform_tests/test_workflows.py index c63183da37..3e6a60465b 100644 --- a/tests/inference/hosted_platform_tests/test_workflows.py +++ b/tests/inference/hosted_platform_tests/test_workflows.py @@ -129,7 +129,7 @@ def test_get_versions_of_execution_engine(object_detection_service_url: str) -> # then response.raise_for_status() response_data = response.json() - assert response_data["versions"] == ["1.12.0"] + assert response_data["versions"] == ["1.13.0"] FUNCTION = """ diff --git a/tests/inference/integration_tests/test_workflow_endpoints.py b/tests/inference/integration_tests/test_workflow_endpoints.py index 580f1bde37..a075d6a631 100644 --- a/tests/inference/integration_tests/test_workflow_endpoints.py +++ b/tests/inference/integration_tests/test_workflow_endpoints.py @@ -691,7 +691,7 @@ def test_get_versions_of_execution_engine(server_url: str) -> None: # then response.raise_for_status() response_data = response.json() - assert response_data["versions"] == ["1.12.0"] + assert response_data["versions"] == ["1.13.0"] def test_getting_block_schema_using_get_endpoint(server_url) -> None: diff --git a/tests/inference/unit_tests/core/interfaces/stream/test_interface_pipeline.py b/tests/inference/unit_tests/core/interfaces/stream/test_interface_pipeline.py index 18516ba080..3a19698679 100644 --- a/tests/inference/unit_tests/core/interfaces/stream/test_interface_pipeline.py +++ b/tests/inference/unit_tests/core/interfaces/stream/test_interface_pipeline.py @@ -29,6 +29,7 @@ VideoSource, lock_state_transition, ) +from inference.core.interfaces.stream import inference_pipeline from inference.core.interfaces.stream.entities import ( InferenceHandlerResult, ModelConfig, @@ -40,6 +41,9 @@ from inference.core.interfaces.stream.model_handlers.roboflow_models import ( default_process_frame, ) +from inference.core.interfaces.stream.model_handlers.workflows import ( + StreamLookaheadDrainError, +) from inference.core.interfaces.stream.sinks import active_learning_sink, multi_sink from inference.core.interfaces.stream.watchdog import BasePipelineWatchDog @@ -203,6 +207,33 @@ def test_inference_pipeline_drain_enqueues_flush_results_with_bound_frames() -> assert watchdog.ready_frames == [[frame_1], [frame_2]] +def test_inference_pipeline_drain_dispatches_partial_results_and_reraises() -> None: + # given - a failed tail frame must not disappear as a normal empty drain: + # the frames that drained are dispatched, the failure propagates + frame_1 = VideoFrame( + image=np.zeros((8, 8, 3), dtype=np.uint8), + frame_id=1, + frame_timestamp=datetime.now(), + source_id=0, + ) + drained = [InferenceHandlerResult(predictions=["p1"], video_frames=[frame_1])] + handler = _FlushableInferenceHandler(results=None) + handler.flush = lambda: (_ for _ in ()).throw( + StreamLookaheadDrainError("final frame failed", drained_results=drained) + ) + pipeline = object.__new__(InferencePipeline) + pipeline._on_video_frame = handler + pipeline._watchdog = _PredictionReadyWatchdog() + pipeline._predictions_queue = Queue() + pipeline._status_update_handlers = [] + + # when / then + with pytest.raises(StreamLookaheadDrainError): + pipeline._drain_inference_handler() + assert pipeline._predictions_queue.get_nowait() == (["p1"], [frame_1]) + assert pipeline._predictions_queue.empty() + + def test_resolve_prediction_futures_recursively_resolves_nested_values() -> None: inner = Future() inner.set_result("resolved") @@ -743,3 +774,63 @@ def on_prediction( assert frames_by_sources[1] == list( range(1, 431 * 2 + 1) ), "Order of prediction frames violated for source 1" + + +def test_buffered_stream_dispatch_keys_on_actual_handler_not_env() -> None: + # given - buffered dispatch must follow whether the handler actually + # buffers (exposes flush()), NOT whether a depth env var is set: a depth + # flag on a workflow that does not qualify for pipelining falls back to a + # plain WorkflowRunner, which must still use the normal dispatch path + class _BufferingRunner: + def __call__(self, video_frames): + return None + + def flush(self): + return None + + class _PlainRunner: + def __call__(self, video_frames): + return [] + + # when / then + assert inference_pipeline._uses_buffered_stream_dispatch(_BufferingRunner()) + assert not inference_pipeline._uses_buffered_stream_dispatch(_PlainRunner()) + assert not inference_pipeline._uses_buffered_stream_dispatch(lambda frames: []) + + +def test_init_with_custom_logic_caps_queue_only_for_buffering_handler() -> None: + # given - the predictions queue is capped to 4 only when the handler + # buffers; a non-buffering handler keeps the full queue even if a depth + # env var happens to be set (the bug: cap + buffered path with nothing + # buffered starves throughput) + class _BufferingRunner: + def __call__(self, video_frames): + return None + + def flush(self): + return None + + class _PlainRunner: + def __call__(self, video_frames): + return [] + + buffering_pipeline = object.__new__(InferencePipeline) + buffering_pipeline.__init__( + on_video_frame=_BufferingRunner(), + video_sources=[], + predictions_queue=Queue(), + watchdog=BasePipelineWatchDog(), + status_update_handlers=[], + ) + plain_pipeline = object.__new__(InferencePipeline) + plain_pipeline.__init__( + on_video_frame=_PlainRunner(), + video_sources=[], + predictions_queue=Queue(), + watchdog=BasePipelineWatchDog(), + status_update_handlers=[], + ) + + # then + assert buffering_pipeline._buffered_stream_dispatch is True + assert plain_pipeline._buffered_stream_dispatch is False diff --git a/tests/inference/unit_tests/core/interfaces/stream/test_workflows.py b/tests/inference/unit_tests/core/interfaces/stream/test_workflows.py index fced34648b..9dc317706f 100644 --- a/tests/inference/unit_tests/core/interfaces/stream/test_workflows.py +++ b/tests/inference/unit_tests/core/interfaces/stream/test_workflows.py @@ -1,14 +1,20 @@ from concurrent.futures import Future from datetime import datetime from types import SimpleNamespace -from typing import Optional +from typing import List, Literal, Optional +import networkx as nx import numpy as np import pytest from inference.core.interfaces.camera.entities import VideoFrame +from inference.core.interfaces.stream.model_handlers import ( + workflows as workflows_module, +) from inference.core.interfaces.stream.model_handlers.workflows import ( + LookaheadPipelinedWorkflowRunner, PipelinedWorkflowRunner, + StreamLookaheadDrainError, WorkflowRunner, wrap_workflow_runner_for_stream_pipeline, ) @@ -16,7 +22,16 @@ from inference.core.workflows.core_steps.models.roboflow.instance_segmentation.v3 import ( RoboflowInstanceSegmentationModelBlockV3, ) -from inference.core.workflows.execution_engine.entities.base import Batch, VideoMetadata +from inference.core.workflows.execution_engine.constants import ( + NODE_COMPILATION_OUTPUT_PROPERTY, +) +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + VideoMetadata, +) +from inference.core.workflows.execution_engine.v1.compiler.entities import NodeCategory +from inference.core.workflows.prototypes.block import WorkflowBlockManifest from inference_models.models.base.async_handoff import attach_async_response_future @@ -831,3 +846,345 @@ def test_close_stream_pipeline_detaches_response_executor_finalizer() -> None: assert block._stream_response_executor is None assert block._stream_response_executor_finalizer is None assert executor._shutdown + + +# --- Stream lookahead (engine-offloaded async steps) --- + + +class _AsyncModelManifest(WorkflowBlockManifest): + type: Literal["test/async_model@v1"] + name: str + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [OutputDefinition(name="predictions")] + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + + +class _StatefulStepManifest(WorkflowBlockManifest): + type: Literal["test/stateful_step@v1"] + name: str + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [OutputDefinition(name="consumed")] + + +class _FakeAsyncStep: + def is_async_stream_step(self) -> bool: + return True + + +class _FakeLookaheadExecutionEngine: + def __init__(self) -> None: + self._engine = SimpleNamespace(_compiled_workflow=SimpleNamespace(steps={})) + self.lookahead_calls = [] + self.edms = [] + self.resume_calls = [] + self.fail_resume_for_frames = set() + + def run_stream_lookahead( + self, + runtime_parameters, + frontier_step_selectors, + async_step_selectors, + lookahead_executor, + ): + frame_number = runtime_parameters["image"][0]["video_metadata"].frame_number + self.lookahead_calls.append( + { + "frame_number": frame_number, + "frontier_step_selectors": set(frontier_step_selectors), + "async_step_selectors": set(async_step_selectors), + "lookahead_executor": lookahead_executor, + } + ) + execution_data_manager = SimpleNamespace(frame_number=frame_number) + self.edms.append(execution_data_manager) + return execution_data_manager + + def resume_stream_lookahead( + self, + execution_data_manager, + frontier_step_selectors, + serialize_results, + fps, + _is_preview, + ): + self.resume_calls.append(execution_data_manager) + if execution_data_manager.frame_number in self.fail_resume_for_frames: + raise RuntimeError( + f"resume failed for frame {execution_data_manager.frame_number}" + ) + return [{"result": f"resumed-frame-{execution_data_manager.frame_number}"}] + + +def _initialised_step(step, manifest_class) -> SimpleNamespace: + return SimpleNamespace( + step=step, + block_specification=SimpleNamespace(manifest_class=manifest_class), + ) + + +def _compiled_workflow_namespace(steps: dict, edges: dict) -> SimpleNamespace: + graph = nx.DiGraph() + for step_name in steps: + graph.add_node( + f"$steps.{step_name}", + **{ + NODE_COMPILATION_OUTPUT_PROPERTY: SimpleNamespace( + node_category=NodeCategory.STEP_NODE + ) + }, + ) + for edge_start, edge_ends in edges.items(): + for edge_end in edge_ends: + if edge_end not in graph.nodes: + graph.add_node( + edge_end, + **{ + NODE_COMPILATION_OUTPUT_PROPERTY: SimpleNamespace( + node_category=NodeCategory.OUTPUT_NODE + ) + }, + ) + graph.add_edge(edge_start, edge_end) + return SimpleNamespace(steps=steps, execution_graph=graph) + + +def _make_wrap_case_engine(case: str) -> SimpleNamespace: + if case == "async_linear": + compiled_workflow = _compiled_workflow_namespace( + steps={ + "model": _initialised_step(_FakeAsyncStep(), _AsyncModelManifest), + "tracker": _initialised_step(SimpleNamespace(), _StatefulStepManifest), + }, + edges={ + "$steps.model": ["$steps.tracker"], + "$steps.tracker": ["$outputs.tracked"], + }, + ) + elif case == "rfdetr_only": + compiled_workflow = _compiled_workflow_namespace( + steps={ + "segmentation": _initialised_step( + _FakePipelinedStep(stream_buffer_depth=1), _StatefulStepManifest + ), + }, + edges={"$steps.segmentation": ["$outputs.predictions"]}, + ) + elif case == "mixed": + compiled_workflow = _compiled_workflow_namespace( + steps={ + "model": _initialised_step(_FakeAsyncStep(), _AsyncModelManifest), + "segmentation": _initialised_step( + _FakePipelinedStep(stream_buffer_depth=1), _StatefulStepManifest + ), + }, + edges={ + "$steps.model": ["$outputs.model"], + "$steps.segmentation": ["$outputs.predictions"], + }, + ) + elif case == "no_graph": + compiled_workflow = SimpleNamespace( + steps={"model": _initialised_step(_FakeAsyncStep(), _AsyncModelManifest)} + ) + elif case == "stateful_fed_async": + compiled_workflow = _compiled_workflow_namespace( + steps={ + "stateful_pre": _initialised_step( + SimpleNamespace(), _StatefulStepManifest + ), + "model": _initialised_step(_FakeAsyncStep(), _AsyncModelManifest), + }, + edges={ + "$steps.stateful_pre": ["$steps.model"], + "$steps.model": ["$outputs.predictions"], + }, + ) + else: + raise ValueError(f"Unknown wrap case: {case}") + return SimpleNamespace( + _engine=SimpleNamespace(_compiled_workflow=compiled_workflow) + ) + + +@pytest.mark.parametrize( + "case, lookahead_depth, allow_lookahead, expected", + [ + ("async_linear", 4, True, "lookahead"), + ("rfdetr_only", 4, True, "pipelined"), + ("mixed", 4, True, "fallback"), + ("async_linear", 1, True, "fallback"), + ("async_linear", 4, False, "fallback"), + ("no_graph", 4, True, "fallback"), + ("stateful_fed_async", 4, True, "fallback"), + ], + ids=[ + "async_in_frontier_activates", + "rfdetr_only_keeps_pipelined_runner", + "mixed_async_and_pipelined_falls_back", + "lookahead_depth_disabled_falls_back", + "lookahead_disallowed_falls_back", + "missing_execution_graph_falls_back", + "async_fed_by_stateful_step_falls_back", + ], +) +def test_wrap_selects_runner_for_workflow_shape_and_configuration( + monkeypatch, + case: str, + lookahead_depth: int, + allow_lookahead: bool, + expected: str, +) -> None: + # given + monkeypatch.setattr( + workflows_module, "WORKFLOWS_STREAM_LOOKAHEAD_DEPTH", lookahead_depth + ) + engine = _make_wrap_case_engine(case) + workflow_runner = WorkflowRunner( + workflows_parameters=None, + execution_engine=engine, + image_input_name="image", + video_metadata_input_name="video_metadata", + ) + + # when + runner = wrap_workflow_runner_for_stream_pipeline( + workflow_runner=workflow_runner, + execution_engine=engine, + allow_lookahead=allow_lookahead, + ) + + # then + if expected == "lookahead": + assert isinstance(runner, LookaheadPipelinedWorkflowRunner) + runner.close() + elif expected == "pipelined": + assert type(runner) is PipelinedWorkflowRunner + else: + assert runner is workflow_runner + + +def test_lookahead_runner_emits_frames_in_order_from_their_own_execution_state() -> ( + None +): + # given + engine = _FakeLookaheadExecutionEngine() + workflow_runner = WorkflowRunner( + workflows_parameters=None, + execution_engine=engine, + image_input_name="image", + video_metadata_input_name="video_metadata", + ) + runner = LookaheadPipelinedWorkflowRunner( + workflow_runner=workflow_runner, + frontier_step_selectors={"$steps.model"}, + async_step_selectors={"$steps.model"}, + buffer_depth=2, + ) + frames = [_make_frame(frame_id) for frame_id in range(1, 5)] + + # when + results = [runner([frame]) for frame in frames] + flushed_results = runner.flush() + + # then - frames 1..4 at buffer depth 2 emit [1, 2], flush yields [3, 4], + # each emission resuming ITS OWN buffered execution state, in frame order + assert results[0] is None + assert results[1] is None + assert results[2].predictions == [{"result": "resumed-frame-1"}] + assert results[2].video_frames == [frames[0]] + assert results[3].predictions == [{"result": "resumed-frame-2"}] + assert results[3].video_frames == [frames[1]] + assert [result.video_frames for result in flushed_results] == [ + [frames[2]], + [frames[3]], + ] + assert flushed_results[0].predictions == [{"result": "resumed-frame-3"}] + assert flushed_results[1].predictions == [{"result": "resumed-frame-4"}] + assert [edm.frame_number for edm in engine.resume_calls] == [1, 2, 3, 4] + assert all(resumed is edm for resumed, edm in zip(engine.resume_calls, engine.edms)) + assert engine.lookahead_calls[0]["frontier_step_selectors"] == {"$steps.model"} + assert engine.lookahead_calls[0]["async_step_selectors"] == {"$steps.model"} + assert engine.lookahead_calls[0]["lookahead_executor"] is runner._lookahead_executor + + # then - close shuts down the runner-owned lookahead pool + runner.close() + assert runner._lookahead_executor._shutdown + + +def test_lookahead_runner_flush_skips_failing_frame_and_drains_the_rest() -> None: + # given + engine = _FakeLookaheadExecutionEngine() + workflow_runner = WorkflowRunner( + workflows_parameters=None, + execution_engine=engine, + image_input_name="image", + video_metadata_input_name="video_metadata", + ) + runner = LookaheadPipelinedWorkflowRunner( + workflow_runner=workflow_runner, + frontier_step_selectors={"$steps.model"}, + async_step_selectors={"$steps.model"}, + buffer_depth=2, + ) + frame_1 = _make_frame(1) + frame_2 = _make_frame(2) + assert runner([frame_1]) is None + assert runner([frame_2]) is None + engine.fail_resume_for_frames.add(1) + + # when - the drain still emits the remaining frames, then re-raises so + # a failed tail frame cannot disappear as a normal empty drain + with pytest.raises(StreamLookaheadDrainError) as error: + runner.flush() + + # then + drained_results = error.value.drained_results + assert len(drained_results) == 1 + assert drained_results[0].predictions == [{"result": "resumed-frame-2"}] + assert drained_results[0].video_frames == [frame_2] + assert isinstance(error.value.__cause__, RuntimeError) + assert [edm.frame_number for edm in engine.resume_calls] == [1, 2] + runner.close() + + +def test_lookahead_runner_pool_threads_are_daemon() -> None: + # given - a hung remote request must not keep the interpreter alive + # after close() (close() uses shutdown(wait=False)) + engine = _FakeLookaheadExecutionEngine() + workflow_runner = WorkflowRunner( + workflows_parameters=None, + execution_engine=engine, + image_input_name="image", + video_metadata_input_name="video_metadata", + ) + runner = LookaheadPipelinedWorkflowRunner( + workflow_runner=workflow_runner, + frontier_step_selectors={"$steps.model"}, + async_step_selectors={"$steps.model"}, + buffer_depth=2, + ) + + # when / then - every pool worker exists already and is a daemon thread + pool_threads = runner._lookahead_executor._threads + assert len(pool_threads) == 3 # (buffer_depth + 1) * len(async_steps) + assert all(thread.daemon for thread in pool_threads) + + # after close(), the workers are detached from concurrent.futures' + # interpreter-shutdown hook (which joins pool threads regardless of the + # daemon flag) - both halves are needed for a hung request not to block + # process exit + runner.close() + from concurrent.futures.thread import _threads_queues + + assert all(thread not in _threads_queues for thread in pool_threads) diff --git a/tests/workflows/unit_tests/core_steps/models/foundation/test_sam3_class_names_copy.py b/tests/workflows/unit_tests/core_steps/models/foundation/test_sam3_class_names_copy.py new file mode 100644 index 0000000000..9460bdef4f --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/models/foundation/test_sam3_class_names_copy.py @@ -0,0 +1,97 @@ +"""Regression tests: async-stream blocks must not mutate the caller's +``class_names`` list. + +When these blocks run as async stream steps, a scalar (non-batch) +``class_names`` list is shared across frames. The blocks default an empty +list to ``[None]`` internally; they must do so on a fresh copy so the +caller's list is never mutated (which would corrupt subsequent frames). +""" + +from unittest.mock import MagicMock + +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.models.foundation.seg_preview.v1 import ( + SegPreviewBlockV1, +) +from inference.core.workflows.core_steps.models.foundation.segment_anything3.v1 import ( + SegmentAnything3BlockV1, +) +from inference.core.workflows.core_steps.models.foundation.segment_anything3.v2 import ( + SegmentAnything3BlockV2, +) + + +def test_sam3_v1_run_locally_does_not_mutate_caller_class_names(): + block = SegmentAnything3BlockV1( + model_manager=MagicMock(), + api_key="test_key", + step_execution_mode=StepExecutionMode.LOCAL, + ) + shared_class_names = [] + + # With no images the empty-list -> [None] defaulting still runs before the + # (skipped) per-image loop, which is exactly where the mutation would occur. + block.run_locally( + images=[], + model_id="sam3/sam3_final", + class_names=shared_class_names, + threshold=0.5, + ) + assert shared_class_names == [] + + # A second call with the same list must behave identically. + block.run_locally( + images=[], + model_id="sam3/sam3_final", + class_names=shared_class_names, + threshold=0.5, + ) + assert shared_class_names == [] + + +def test_sam3_v2_run_locally_does_not_mutate_caller_class_names(): + block = SegmentAnything3BlockV2( + model_manager=MagicMock(), + api_key="test_key", + step_execution_mode=StepExecutionMode.LOCAL, + ) + shared_class_names = [] + + block.run_locally( + images=[], + model_id="sam3/sam3_final", + class_names=shared_class_names, + confidence=0.5, + ) + assert shared_class_names == [] + + block.run_locally( + images=[], + model_id="sam3/sam3_final", + class_names=shared_class_names, + confidence=0.5, + ) + assert shared_class_names == [] + + +def test_seg_preview_v1_run_via_request_does_not_mutate_caller_class_names(): + block = SegPreviewBlockV1( + model_manager=MagicMock(), + api_key="test_key", + step_execution_mode=StepExecutionMode.LOCAL, + ) + shared_class_names = [] + + block.run_via_request( + images=[], + class_names=shared_class_names, + threshold=0.5, + ) + assert shared_class_names == [] + + block.run_via_request( + images=[], + class_names=shared_class_names, + threshold=0.5, + ) + assert shared_class_names == [] diff --git a/tests/workflows/unit_tests/execution_engine/executor/test_stream_lookahead.py b/tests/workflows/unit_tests/execution_engine/executor/test_stream_lookahead.py new file mode 100644 index 0000000000..788cdcccef --- /dev/null +++ b/tests/workflows/unit_tests/execution_engine/executor/test_stream_lookahead.py @@ -0,0 +1,709 @@ +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, get_args +from unittest.mock import MagicMock + +import networkx as nx +import pytest +from pydantic import BaseModel + +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.models.foundation.segment_anything3 import ( + v3 as segment_anything3_v3, +) +from inference.core.workflows.core_steps.models.foundation.segment_anything3.v3 import ( + SegmentAnything3BlockV3, +) +from inference.core.workflows.core_steps.models.roboflow.object_detection.v3 import ( + BlockManifest as ObjectDetectionModelV3Manifest, +) +from inference.core.workflows.core_steps.models.roboflow.object_detection.v3 import ( + RoboflowObjectDetectionModelBlockV3, +) +from inference.core.workflows.core_steps.visualizations.common.base import ( + VisualizationManifest, +) +from inference.core.workflows.core_steps.visualizations.heatmap.v1 import ( + HeatmapManifest, +) +from inference.core.workflows.core_steps.visualizations.trace.v1 import TraceManifest +from inference.core.workflows.errors import StepExecutionError +from inference.core.workflows.execution_engine.constants import ( + NODE_COMPILATION_OUTPUT_PROPERTY, +) +from inference.core.workflows.execution_engine.entities.base import ( + JsonField, + OutputDefinition, +) +from inference.core.workflows.execution_engine.entities.types import WILDCARD_KIND +from inference.core.workflows.execution_engine.profiling.core import ( + NullWorkflowsProfiler, +) +from inference.core.workflows.execution_engine.v1.compiler.entities import ( + BlockSpecification, + CompiledWorkflow, + DynamicStepInputDefinition, + InitialisedStep, + NodeCategory, + NodeInputCategory, + OutputNode, + ParameterSpecification, + ParsedWorkflowDefinition, + StepNode, +) +from inference.core.workflows.execution_engine.v1.executor.core import ( + compute_async_stream_step_selectors, + compute_stream_lookahead_frontier, + launch_async_simd_step, + resume_stream_lookahead_workflow, + run_stream_lookahead_workflow, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) + + +class StatelessModelManifest(WorkflowBlockManifest): + type: Literal["test/async_model@v1"] + name: str + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="predictions"), + OutputDefinition(name="inference_id"), + ] + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def is_stateful_for_video_processing(cls) -> bool: + return False + + +class ScalarModelManifest(StatelessModelManifest): + type: Literal["test/scalar_model@v1"] + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return [] + + +class WildcardOutputModelManifest(StatelessModelManifest): + type: Literal["test/wildcard_model@v1"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [OutputDefinition(name="*")] + + +class DimensionalityOffsetModelManifest(StatelessModelManifest): + type: Literal["test/dim_offset_model@v1"] + + @classmethod + def get_output_dimensionality_offset(cls) -> int: + return 1 + + +class StatefulConsumerManifest(WorkflowBlockManifest): + type: Literal["test/consumer@v1"] + name: str + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [OutputDefinition(name="consumed")] + + +class BareManifest(WorkflowBlockManifest): + type: Literal["test/bare@v1"] + name: str + + +class DynamicBlockManifest(BaseModel): + # Dynamic Custom Python block manifests are plain pydantic models built + # via create_model, not WorkflowBlockManifest subclasses - they have no + # is_stateful_for_video_processing method at all. + type: Literal["test/dynamic_block@v1"] + name: str + + +class AsyncModelBlock(WorkflowBlock): + def __init__(self, run_fn: Optional[Callable[..., BlockResult]] = None) -> None: + self.run_calls: List[dict] = [] + self._run_fn = run_fn + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return StatelessModelManifest + + def is_async_stream_step(self) -> bool: + return True + + def run(self, **kwargs) -> BlockResult: + self.run_calls.append(kwargs) + if self._run_fn is not None: + return self._run_fn(**kwargs) + return [ + {"predictions": f"prediction-{image}", "inference_id": f"id-{image}"} + for image in kwargs["images"] + ] + + +class PlainStepBlock(WorkflowBlock): + # Stateless, non-async step (e.g. a static crop, or an undeclared model). + def __init__(self) -> None: + self.run_calls = 0 + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return StatelessModelManifest + + def run(self, **kwargs) -> BlockResult: + self.run_calls += 1 + return {"predictions": f"plain-output-{self.run_calls}", "inference_id": None} + + +class ConsumerBlock(WorkflowBlock): + def __init__(self) -> None: + self.seen_predictions: List[Any] = [] + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return StatefulConsumerManifest + + def run(self, predictions: Any) -> BlockResult: + self.seen_predictions.append(predictions) + return {"consumed": predictions} + + +@pytest.mark.parametrize( + "manifest_class, block_factory, expected_selectors", + [ + (StatelessModelManifest, AsyncModelBlock, {"$steps.model"}), + (ScalarModelManifest, AsyncModelBlock, set()), + (WildcardOutputModelManifest, AsyncModelBlock, set()), + (DimensionalityOffsetModelManifest, AsyncModelBlock, set()), + (StatelessModelManifest, PlainStepBlock, set()), + ], + ids=[ + "declared_and_eligible", + "scalar", + "wildcard_output", + "dim_offset", + "undeclared", + ], +) +def test_compute_async_stream_step_selectors_eligibility( + manifest_class: Type[WorkflowBlockManifest], + block_factory: Type[WorkflowBlock], + expected_selectors: set, +) -> None: + # given + workflow = _lookahead_compiled_workflow( + steps={"model": _step_spec(block_factory(), manifest_class)}, + edges=[], + ) + + # when / then + assert compute_async_stream_step_selectors(workflow=workflow) == expected_selectors + + +_FRONTIER_CASES = { + "linear_model_into_tracker": lambda: ( + { + "model": _step_spec(AsyncModelBlock(), StatelessModelManifest), + "tracker": _step_spec(ConsumerBlock(), StatefulConsumerManifest), + }, + [("$steps.model", "$steps.tracker")], + {"$steps.model"}, + ), + "model_fed_by_stateful_step": lambda: ( + { + "stateful_pre": _step_spec(ConsumerBlock(), StatefulConsumerManifest), + "model": _step_spec(AsyncModelBlock(), StatelessModelManifest), + }, + [("$steps.stateful_pre", "$steps.model")], + set(), + ), + "model_chained_after_model": lambda: ( + { + "model_a": _step_spec(AsyncModelBlock(), StatelessModelManifest), + "model_b": _step_spec(AsyncModelBlock(), StatelessModelManifest), + }, + [("$steps.model_a", "$steps.model_b")], + {"$steps.model_a"}, + ), + "stateless_side_branch": lambda: ( + { + "crop": _step_spec(PlainStepBlock(), StatelessModelManifest), + "model": _step_spec(AsyncModelBlock(), StatelessModelManifest), + "side": _step_spec(PlainStepBlock(), StatelessModelManifest), + "tracker": _step_spec(ConsumerBlock(), StatefulConsumerManifest), + }, + [ + ("$steps.crop", "$steps.model"), + ("$steps.crop", "$steps.side"), + ("$steps.model", "$steps.tracker"), + ], + {"$steps.crop", "$steps.model", "$steps.side"}, + ), + # Dynamic Custom Python manifests lack the statefulness declaration + # entirely and must be treated as stateful, not crash. + "dynamic_manifest_without_statefulness_method": lambda: ( + {"custom": _step_spec(PlainStepBlock(), DynamicBlockManifest)}, + [], + set(), + ), +} + + +@pytest.mark.parametrize("case_name", list(_FRONTIER_CASES)) +def test_compute_stream_lookahead_frontier_shapes(case_name: str) -> None: + # given + steps, edges, expected_frontier = _FRONTIER_CASES[case_name]() + workflow = _lookahead_compiled_workflow(steps=steps, edges=edges) + + # when / then + assert compute_stream_lookahead_frontier(workflow=workflow) == expected_frontier + + +def test_launch_async_simd_step_registers_per_output_chained_futures() -> None: + # given + block = AsyncModelBlock() + workflow = _lookahead_compiled_workflow( + steps={"model": _step_spec(block, StatelessModelManifest)}, + edges=[], + ) + execution_data_manager = MagicMock() + execution_data_manager.get_simd_step_input.return_value = SimpleNamespace( + indices=[(0,), (1,)], + parameters={"images": ["a", "b"]}, + ) + + # when + with ThreadPoolExecutor(max_workers=1) as lookahead_executor: + launch_async_simd_step( + step_selector="$steps.model", + workflow=workflow, + execution_data_manager=execution_data_manager, + lookahead_executor=lookahead_executor, + ) + + # then - one future per declared output per batch element, resolving + # to that element's slice of the offloaded run() result + registered = execution_data_manager.register_simd_step_output.call_args + assert registered.kwargs["indices"] == [(0,), (1,)] + outputs = registered.kwargs["outputs"] + assert len(outputs) == 2 + assert outputs[0]["predictions"].result(timeout=5) == "prediction-a" + assert outputs[0]["inference_id"].result(timeout=5) == "id-a" + assert outputs[1]["predictions"].result(timeout=5) == "prediction-b" + assert outputs[1]["inference_id"].result(timeout=5) == "id-b" + assert block.run_calls == [{"images": ["a", "b"]}] + + +def test_launch_async_simd_step_with_empty_indices_registers_empty_output() -> None: + # given + block = AsyncModelBlock() + workflow = _lookahead_compiled_workflow( + steps={"model": _step_spec(block, StatelessModelManifest)}, + edges=[], + ) + execution_data_manager = MagicMock() + execution_data_manager.get_simd_step_input.return_value = SimpleNamespace( + indices=[], + parameters={"images": []}, + ) + lookahead_executor = MagicMock() + + # when + launch_async_simd_step( + step_selector="$steps.model", + workflow=workflow, + execution_data_manager=execution_data_manager, + lookahead_executor=lookahead_executor, + ) + + # then + execution_data_manager.register_simd_step_output.assert_called_once_with( + step_selector="$steps.model", + indices=[], + outputs=[], + ) + lookahead_executor.submit.assert_not_called() + + +@pytest.mark.parametrize( + "run_fn, expected_error", + [ + # a raising run() is wrapped the same way as on the sync path + ( + lambda **kwargs: (_ for _ in ()).throw(RuntimeError("remote failed")), + StepExecutionError, + ), + (lambda **kwargs: [{}], KeyError), + ], + ids=["run_raises", "malformed_payload"], +) +def test_launch_async_simd_step_surfaces_errors_through_output_futures( + run_fn: Callable[..., BlockResult], + expected_error: Type[Exception], +) -> None: + # given - the offloaded run() either raises or returns a payload the + # chained futures cannot select from; either way consumers must get the + # error instead of hanging + block = AsyncModelBlock(run_fn=run_fn) + workflow = _lookahead_compiled_workflow( + steps={"model": _step_spec(block, StatelessModelManifest)}, + edges=[], + ) + execution_data_manager = MagicMock() + execution_data_manager.get_simd_step_input.return_value = SimpleNamespace( + indices=[(0,)], + parameters={"images": ["a"]}, + ) + + # when + with ThreadPoolExecutor(max_workers=1) as lookahead_executor: + launch_async_simd_step( + step_selector="$steps.model", + workflow=workflow, + execution_data_manager=execution_data_manager, + lookahead_executor=lookahead_executor, + ) + + # then + outputs = execution_data_manager.register_simd_step_output.call_args.kwargs[ + "outputs" + ] + with pytest.raises(expected_error): + outputs[0]["predictions"].result(timeout=5) + + +def test_launch_async_simd_step_flags_batch_contract_violation_on_model_step() -> None: + # given - a block returning fewer output elements than input indices + # must be blamed at the model step (with a traceback attached), not + # surface later as an index error on the consuming step + block = AsyncModelBlock( + run_fn=lambda **kwargs: [{"predictions": "only-one", "inference_id": "i"}] + ) + workflow = _lookahead_compiled_workflow( + steps={"model": _step_spec(block, StatelessModelManifest)}, + edges=[], + ) + execution_data_manager = MagicMock() + execution_data_manager.get_simd_step_input.return_value = SimpleNamespace( + indices=[(0,), (1,)], + parameters={"images": ["a", "b"]}, + ) + + # when + with ThreadPoolExecutor(max_workers=1) as lookahead_executor: + launch_async_simd_step( + step_selector="$steps.model", + workflow=workflow, + execution_data_manager=execution_data_manager, + lookahead_executor=lookahead_executor, + ) + outputs = execution_data_manager.register_simd_step_output.call_args.kwargs[ + "outputs" + ] + + # then + with pytest.raises(StepExecutionError) as error: + outputs[0]["predictions"].result(timeout=5) + assert error.value.block_id == "model" + assert "1 output elements for 2 input indices" in str(error.value) + assert error.value.block_traceback is not None + + +def test_launch_async_simd_step_rebinds_execution_context_in_worker() -> None: + # given - the offloaded run() bypasses safe_execute_step, so the launch + # must rebind the request thread's execution context on the worker + from inference_sdk.config import execution_id + + seen_context = {} + + def run_fn(**kwargs) -> BlockResult: + seen_context["execution_id"] = execution_id.get() + return [{"predictions": "p", "inference_id": "i"}] + + block = AsyncModelBlock(run_fn=run_fn) + workflow = _lookahead_compiled_workflow( + steps={"model": _step_spec(block, StatelessModelManifest)}, + edges=[], + ) + execution_data_manager = MagicMock() + execution_data_manager.get_simd_step_input.return_value = SimpleNamespace( + indices=[(0,)], + parameters={"images": ["a"]}, + ) + + # when + with ThreadPoolExecutor(max_workers=1) as lookahead_executor: + launch_async_simd_step( + step_selector="$steps.model", + workflow=workflow, + execution_data_manager=execution_data_manager, + lookahead_executor=lookahead_executor, + workflow_execution_id="test-execution-id", + ) + outputs = execution_data_manager.register_simd_step_output.call_args.kwargs[ + "outputs" + ] + assert outputs[0]["predictions"].result(timeout=5) == "p" + + # then + assert seen_context["execution_id"] == "test-execution-id" + + +def test_stream_lookahead_run_and_resume_split_execution_between_passes() -> None: + # given + model = PlainStepBlock() + consumer = ConsumerBlock() + workflow = _lookahead_compiled_workflow( + steps={ + "model": _step_spec(model, StatelessModelManifest), + "consumer": _step_spec( + consumer, + StatefulConsumerManifest, + inputs={"predictions": "$steps.model.predictions"}, + ), + }, + edges=[("$steps.model", "$steps.consumer")], + output_selector="$steps.consumer.consumed", + ) + frontier = compute_stream_lookahead_frontier(workflow=workflow) + assert frontier == {"$steps.model"} + + # when - the deferred pass runs only the frontier + with ThreadPoolExecutor(max_workers=1) as lookahead_executor: + execution_data_manager = run_stream_lookahead_workflow( + workflow=workflow, + runtime_parameters={}, + max_concurrent_steps=1, + frontier_step_selectors=frontier, + async_step_selectors=set(), + lookahead_executor=lookahead_executor, + profiler=NullWorkflowsProfiler.init(), + ) + + # then + assert model.run_calls == 1 + assert consumer.seen_predictions == [] + + # when - the resume pass runs only the remainder on the same state; + # usage kwargs are accepted here (and only here): the deferred pass + # returns before async work completes, so it must not record usage + result = resume_stream_lookahead_workflow( + workflow=workflow, + execution_data_manager=execution_data_manager, + max_concurrent_steps=1, + kinds_serializers={}, + frontier_step_selectors=frontier, + serialize_results=False, + usage_fps=30, + usage_workflow_id="test-workflow", + profiler=NullWorkflowsProfiler.init(), + ) + + # then - no step executed twice, outputs correct + assert model.run_calls == 1 + assert consumer.seen_predictions == ["plain-output-1"] + assert result == [{"result": "plain-output-1"}] + + +@pytest.mark.parametrize( + "manifest_class, expected_stateful", + [ + (BareManifest, True), + (VisualizationManifest, False), + (TraceManifest, True), + (HeatmapManifest, True), + (ObjectDetectionModelV3Manifest, False), + ], + ids=["default", "visualization_base", "trace", "heatmap", "object_detection_v3"], +) +def test_is_stateful_for_video_processing_metadata( + manifest_class: Type[WorkflowBlockManifest], + expected_stateful: bool, +) -> None: + assert manifest_class.is_stateful_for_video_processing() is expected_stateful + + +@pytest.mark.parametrize( + "block_class, sam3_exec_mode, execution_mode, expected", + [ + (RoboflowObjectDetectionModelBlockV3, None, StepExecutionMode.REMOTE, True), + (RoboflowObjectDetectionModelBlockV3, None, StepExecutionMode.LOCAL, False), + (SegmentAnything3BlockV3, "local", StepExecutionMode.REMOTE, True), + (SegmentAnything3BlockV3, "remote", StepExecutionMode.REMOTE, False), + (SegmentAnything3BlockV3, "local", StepExecutionMode.LOCAL, False), + ], + ids=["od_remote", "od_local", "sam3_remote", "sam3_proxy_mode", "sam3_local"], +) +def test_model_blocks_declare_async_stream_step_only_for_reentrant_remote_execution( + monkeypatch, + block_class: Type[WorkflowBlock], + sam3_exec_mode: Optional[str], + execution_mode: StepExecutionMode, + expected: bool, +) -> None: + # given + if sam3_exec_mode is not None: + monkeypatch.setattr(segment_anything3_v3, "SAM3_EXEC_MODE", sam3_exec_mode) + block = block_class( + model_manager=MagicMock(), + api_key="key", + step_execution_mode=execution_mode, + ) + + # when / then + assert block.is_async_stream_step() is expected + + +def test_all_registry_manifests_declare_boolean_statefulness() -> None: + # Every loaded block manifest must answer is_stateful_for_video_processing() + # with a bool (a broken override on a future adoption fails here, not at + # stream time). + from inference.core.workflows.execution_engine.introspection.blocks_loader import ( + load_workflow_blocks, + ) + + for block in load_workflow_blocks(): + value = block.manifest_class.is_stateful_for_video_processing() + assert isinstance(value, bool), block.manifest_class.__name__ + + +def test_cross_frame_state_blocks_are_never_declared_stateless_for_lookahead() -> None: + # Registry-wide consistency guard that scales to every future adoption: a + # block keeping per-video HTTP state (declares + # STATEFUL_VIDEO_HTTP_SOFT_RESTRICTION — trackers, counters, aggregators) + # must NOT also declare itself stateless for video processing. That + # contradiction would place it in the stream-lookahead frontier and let the + # scheduler run it ahead of stream order, silently corrupting its state. + # This is the automated declaration-level check for the class of + # mis-adoption the class_names shared-mutation bug demonstrated. + from inference.core.workflows.execution_engine.introspection.blocks_loader import ( + load_workflow_blocks, + ) + from inference.core.workflows.prototypes.block import ( + STATEFUL_VIDEO_HTTP_SOFT_RESTRICTION, + ) + + offenders = [] + for block in load_workflow_blocks(): + manifest = block.manifest_class + keeps_cross_frame_state = ( + STATEFUL_VIDEO_HTTP_SOFT_RESTRICTION in manifest.get_restrictions() + ) + if keeps_cross_frame_state and not manifest.is_stateful_for_video_processing(): + offenders.append(manifest.__name__) + assert offenders == [], ( + "Blocks declaring per-video HTTP state must be stateful for video " + f"processing (else the lookahead frontier corrupts them): {offenders}" + ) + + +def _step_spec( + block: WorkflowBlock, + manifest_class: Type[WorkflowBlockManifest], + inputs: Optional[Dict[str, str]] = None, +) -> dict: + return {"block": block, "manifest_class": manifest_class, "inputs": inputs or {}} + + +def _lookahead_compiled_workflow( + steps: Dict[str, dict], + edges: List[Tuple[str, str]], + output_selector: Optional[str] = None, +) -> CompiledWorkflow: + manifests = { + step_name: spec["manifest_class"]( + type=get_args(spec["manifest_class"].model_fields["type"].annotation)[0], + name=step_name, + ) + for step_name, spec in steps.items() + } + graph = nx.DiGraph() + for step_name, spec in steps.items(): + graph.add_node( + f"$steps.{step_name}", + **{ + NODE_COMPILATION_OUTPUT_PROPERTY: StepNode( + node_category=NodeCategory.STEP_NODE, + name=step_name, + selector=f"$steps.{step_name}", + data_lineage=[], + step_manifest=manifests[step_name], + input_data={ + parameter_name: DynamicStepInputDefinition( + parameter_specification=ParameterSpecification( + parameter_name=parameter_name + ), + category=NodeInputCategory.NON_BATCH_STEP_OUTPUT, + data_lineage=[], + selector=selector, + ) + for parameter_name, selector in spec["inputs"].items() + }, + ) + }, + ) + for edge_start, edge_end in edges: + graph.add_edge(edge_start, edge_end) + workflow_outputs = [] + if output_selector is not None: + workflow_output = JsonField( + type="JsonField", + name="result", + selector=output_selector, + ) + workflow_outputs.append(workflow_output) + graph.add_node( + "$outputs.result", + **{ + NODE_COMPILATION_OUTPUT_PROPERTY: OutputNode( + node_category=NodeCategory.OUTPUT_NODE, + name="result", + selector="$outputs.result", + data_lineage=[], + output_manifest=workflow_output, + kind=[WILDCARD_KIND], + ) + }, + ) + output_step_selector = ".".join(output_selector.split(".")[:2]) + graph.add_edge(output_step_selector, "$outputs.result") + + return CompiledWorkflow( + workflow_definition=ParsedWorkflowDefinition( + version="1.0", + inputs=[], + steps=list(manifests.values()), + outputs=workflow_outputs, + ), + execution_graph=graph, + steps={ + step_name: InitialisedStep( + block_specification=BlockSpecification( + block_source="test", + identifier=manifests[step_name].type, + block_class=type(spec["block"]), + manifest_class=spec["manifest_class"], + ), + manifest=manifests[step_name], + step=spec["block"], + ) + for step_name, spec in steps.items() + }, + input_substitutions=[], + workflow_json={}, + init_parameters={}, + kinds_serializers={}, + kinds_deserializers={}, + ) diff --git a/tests/workflows/unit_tests/execution_engine/test_remote_stream_pipeline_ordering.py b/tests/workflows/unit_tests/execution_engine/test_remote_stream_pipeline_ordering.py new file mode 100644 index 0000000000..da351bc152 --- /dev/null +++ b/tests/workflows/unit_tests/execution_engine/test_remote_stream_pipeline_ordering.py @@ -0,0 +1,246 @@ +"""Ordering-correctness tests for remote stream pipelining on a real +ExecutionEngine. + +The workflow is a remote object-detection model feeding a real ByteTrack +step. The mocked HTTP client answers with a distinct canned prediction per +frame and sleeps longer for earlier frames, so remote responses complete in +reverse submission order. Emissions must still come out strictly in frame +order, each with its own frame's detections, and the tracker must behave +exactly as in a sequential (depth 1) run. + +Lives under unit_tests: everything remote is mocked (MagicMock model manager, +patched `InferenceHTTPClient`), unlike tests/workflows/integration_tests +which load real models. +""" + +import base64 +import time +from datetime import datetime +from typing import List, Optional +from unittest.mock import MagicMock + +import cv2 +import numpy as np + +from inference.core.env import WORKFLOWS_MAX_CONCURRENT_STEPS +from inference.core.interfaces.camera.entities import VideoFrame +from inference.core.interfaces.stream.model_handlers import ( + workflows as workflows_module, +) +from inference.core.interfaces.stream.model_handlers.workflows import ( + LookaheadPipelinedWorkflowRunner, + WorkflowRunner, + wrap_workflow_runner_for_stream_pipeline, +) +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.models.roboflow.object_detection import ( + v3 as object_detection_v3, +) +from inference.core.workflows.execution_engine.core import ExecutionEngine + +FRAMES_NUMBER = 8 +IMAGE_SIZE = 240 +PIXEL_VALUE_STEP = 25 +STATIC_OBJECT_FIRST_FRAME = 4 + +WORKFLOW_DEFINITION = { + "version": "1.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": [ + { + "type": "roboflow_core/roboflow_object_detection_model@v3", + "name": "model", + "images": "$inputs.image", + "model_id": "workspace/model/1", + }, + { + "type": "roboflow_core/byte_tracker@v3", + "name": "tracker", + "image": "$inputs.image", + "detections": "$steps.model.predictions", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "tracked_detections", + "selector": "$steps.tracker.tracked_detections", + } + ], +} + + +def _make_frame(frame_id: int) -> VideoFrame: + # Each frame carries a distinct uniform pixel value so the mocked HTTP + # client can recover the frame index from the base64 payload it receives. + return VideoFrame( + image=np.full( + (IMAGE_SIZE, IMAGE_SIZE, 3), frame_id * PIXEL_VALUE_STEP, dtype=np.uint8 + ), + frame_id=frame_id, + frame_timestamp=datetime.fromtimestamp(frame_id + 1), + fps=30.0, + measured_fps=None, + source_id=0, + comes_from_video_file=True, + ) + + +def _frame_index_of(base64_image: str) -> int: + image = cv2.imdecode( + np.frombuffer(base64.b64decode(base64_image), dtype=np.uint8), + cv2.IMREAD_COLOR, + ) + return int(round(float(image.mean()) / PIXEL_VALUE_STEP)) + + +def _canned_box(cx: int, cy: int, detection_id: str) -> dict: + return { + "width": 20, + "height": 20, + "x": cx, + "y": cy, + "confidence": 0.9, + "class_id": 0, + "class": "car", + "detection_id": detection_id, + "parent_id": "image", + } + + +def _moving_box_xyxy(frame_index: int) -> List[float]: + cx = 40 + 2 * frame_index + return [cx - 10, 90, cx + 10, 110] + + +def _canned_prediction(frame_index: int) -> dict: + # One object drifting 2px per frame (uniquely identifying the frame by + # its coordinates) plus a second, static object appearing mid-stream. + boxes = [_canned_box(40 + 2 * frame_index, 100, f"moving-{frame_index}")] + if frame_index >= STATIC_OBJECT_FIRST_FRAME: + boxes.append(_canned_box(180, 180, f"static-{frame_index}")) + return { + "predictions": boxes, + "image": {"width": IMAGE_SIZE, "height": IMAGE_SIZE}, + "time": 0.1, + } + + +def _patch_remote_http_client( + monkeypatch, + delays: Optional[List[float]], + completion_order: Optional[List[int]] = None, +) -> None: + def _infer(inference_input, model_id): + frame_index = _frame_index_of(inference_input[0]) + if delays is not None: + time.sleep(delays[frame_index]) + if completion_order is not None: + completion_order.append(frame_index) + return _canned_prediction(frame_index) + + mock_client = MagicMock() + mock_client.infer.side_effect = _infer + monkeypatch.setattr( + object_detection_v3, + "InferenceHTTPClient", + MagicMock(return_value=mock_client), + ) + monkeypatch.setattr(object_detection_v3, "InferenceConfiguration", MagicMock()) + + +def _init_wrapped_runner(monkeypatch, pipeline_depth: int): + monkeypatch.setattr( + workflows_module, + "WORKFLOWS_STREAM_LOOKAHEAD_DEPTH", + pipeline_depth, + ) + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_DEFINITION, + init_parameters={ + "workflows_core.model_manager": MagicMock(), + "workflows_core.api_key": "key", + "workflows_core.step_execution_mode": StepExecutionMode.REMOTE, + }, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + workflow_runner = WorkflowRunner( + workflows_parameters=None, + execution_engine=execution_engine, + image_input_name="image", + video_metadata_input_name="video_metadata", + ) + runner = wrap_workflow_runner_for_stream_pipeline( + workflow_runner=workflow_runner, + execution_engine=execution_engine, + ) + return execution_engine, runner + + +def test_remote_stream_pipelining_emits_in_frame_order_with_tracker_parity( + monkeypatch, +) -> None: + # given - remote responses for early frames land LAST: frame 0 is the + # slowest, so with 4 requests in flight completion order is reversed + completion_order: List[int] = [] + delays = [0.08, 0.06, 0.04, 0.02, 0.015, 0.01, 0.005, 0.002] + _patch_remote_http_client( + monkeypatch, delays=delays, completion_order=completion_order + ) + _, runner = _init_wrapped_runner(monkeypatch, pipeline_depth=4) + assert isinstance(runner, LookaheadPipelinedWorkflowRunner) + + try: + # when + emissions = [] + for frame_id in range(FRAMES_NUMBER): + result = runner([_make_frame(frame_id)]) + if result is not None: + emissions.append(result) + flushed_results = runner.flush() + emissions.extend(flushed_results or []) + + # then - the delays actually reversed completion of the in-flight batch + assert completion_order.index(3) < completion_order.index(0) + + # then - every frame emitted exactly once, in strictly ascending order + emitted_frame_ids = [ + emission.video_frames[0].frame_id for emission in emissions + ] + assert emitted_frame_ids == list(range(FRAMES_NUMBER)) + + # then - each emission carries its own frame's detections + for frame_index, emission in enumerate(emissions): + tracked_detections = emission.predictions[0]["tracked_detections"] + canned_xyxy = [_moving_box_xyxy(frame_index)] + ( + [[170, 170, 190, 190]] + if frame_index >= STATIC_OBJECT_FIRST_FRAME + else [] + ) + assert tracked_detections.xyxy[0].tolist() == _moving_box_xyxy(frame_index) + for tracked_xyxy in tracked_detections.xyxy.tolist(): + assert tracked_xyxy in canned_xyxy + finally: + runner.close() + assert runner._lookahead_executor._shutdown + + # given - a sequential reference run: same canned responses, no delays, + # pipelining disabled, fresh engine (fresh tracker state) + _patch_remote_http_client(monkeypatch, delays=None) + _, sequential_runner = _init_wrapped_runner(monkeypatch, pipeline_depth=1) + assert isinstance(sequential_runner, WorkflowRunner) + + # when + sequential_results = [ + sequential_runner([_make_frame(frame_id)]) for frame_id in range(FRAMES_NUMBER) + ] + + # then - tracked boxes and tracker id sequences match the sequential run + for emission, sequential_result in zip(emissions, sequential_results): + pipelined_detections = emission.predictions[0]["tracked_detections"] + sequential_detections = sequential_result[0]["tracked_detections"] + assert pipelined_detections.xyxy.tolist() == sequential_detections.xyxy.tolist() + assert ( + pipelined_detections.tracker_id.tolist() + == sequential_detections.tracker_id.tolist() + )