From ad9cd307dd08e3c63c9f85f191aac572bc0d2b58 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 12 Aug 2026 12:38:56 -0300 Subject: [PATCH] fix(tune): accept MOT-layout ground truth in Tuner validation `Tuner` passes `gt_dir` straight to `evaluate_mot_sequences`, which auto-detects both the flat `{seq}.txt` layout and the `{seq}/gt/gt.txt` layout that MOTChallenge, DanceTrack and SportsMOT ship in. Eager validation only looked for the flat layout, so a downloaded dataset was rejected at construction with "Missing ground-truth files" even though evaluation would have read it fine. The workaround was to flatten the tree by hand. Validation now detects the layout with the same helper evaluation uses, so the two cannot drift apart, and missing files are reported at the path actually searched. `_get_paths` is split so the ground-truth rule lives in one place rather than being restated in the tuner. Co-authored-by: Cursor --- src/trackers/eval/evaluate.py | 31 +++++++++++++++++-------- src/trackers/tune/tuner.py | 18 +++++++++------ tests/tune/test_tuner.py | 43 +++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 16 deletions(-) diff --git a/src/trackers/eval/evaluate.py b/src/trackers/eval/evaluate.py index 0c6fdb29b..c45f64876 100644 --- a/src/trackers/eval/evaluate.py +++ b/src/trackers/eval/evaluate.py @@ -338,6 +338,27 @@ def _discover_sequences( return sorted(p.parent.parent.name for p in gt_dir.glob("*/gt/gt.txt") if not p.name.startswith(".")) +def _ground_truth_path( + gt_dir: Path, + seq_name: str, + data_format: Literal["flat", "mot"], +) -> Path: + """Get the ground truth file path for a sequence. + + Args: + gt_dir: Ground truth directory (direct parent of sequences). + seq_name: Sequence name. + data_format: Directory format. + + Returns: + Path to the sequence's ground truth file, which is not guaranteed to exist. + """ + if data_format == "flat": + return gt_dir / f"{seq_name}.txt" + # MOT format: gt_dir/{seq}/gt/gt.txt + return gt_dir / seq_name / "gt" / "gt.txt" + + def _get_paths( gt_dir: Path, tracker_dir: Path, @@ -355,16 +376,8 @@ def _get_paths( Returns: Tuple of (gt_path, tracker_path). """ - if data_format == "flat": - gt_path = gt_dir / f"{seq_name}.txt" - else: - # MOT format: gt_dir/{seq}/gt/gt.txt - gt_path = gt_dir / seq_name / "gt" / "gt.txt" - # Tracker files are always flat: tracker_dir/{seq}.txt - tracker_path = tracker_dir / f"{seq_name}.txt" - - return gt_path, tracker_path + return _ground_truth_path(gt_dir, seq_name, data_format), tracker_dir / f"{seq_name}.txt" def _parse_seqmap(seqmap_path: str | Path) -> list[str]: diff --git a/src/trackers/tune/tuner.py b/src/trackers/tune/tuner.py index c5c705ecd..bf8165d75 100644 --- a/src/trackers/tune/tuner.py +++ b/src/trackers/tune/tuner.py @@ -17,7 +17,7 @@ import supervision as sv from trackers.core.base import BaseTracker -from trackers.eval.evaluate import evaluate_mot_sequences +from trackers.eval.evaluate import _detect_format, _ground_truth_path, evaluate_mot_sequences from trackers.eval.results import BenchmarkResult from trackers.io.frames import load_mot_frame_image from trackers.io.mot import _mot_frame_to_detections, _MOTOutput, load_mot_file @@ -201,7 +201,8 @@ def _validate_sequence_files(self) -> None: """Validate that every selected sequence has required MOT files. This performs eager filesystem validation so configuration errors are reported during tuner initialization - rather than later during trial execution. + rather than later during trial execution. Ground truth is accepted in either the flat `{seq}.txt` layout or the + MOT `{seq}/gt/gt.txt` layout, matching what `evaluate_mot_sequences` detects. """ missing_detection_files = [ str(self._detections_dir / f"{seq_name}.txt") @@ -213,11 +214,14 @@ def _validate_sequence_files(self) -> None: "Missing detection files for selected sequences: " + ", ".join(missing_detection_files) ) - missing_gt_files = [ - str(self._gt_dir / f"{seq_name}.txt") - for seq_name in self._sequences - if not (self._gt_dir / f"{seq_name}.txt").is_file() - ] + # Ground truth may use either layout that `evaluate_mot_sequences` accepts, so validation + # resolves paths the same way rather than assuming the flat one. + gt_format = _detect_format(self._gt_dir) + missing_gt_files = [] + for seq_name in self._sequences: + gt_path = _ground_truth_path(self._gt_dir, seq_name, gt_format) + if not gt_path.is_file(): + missing_gt_files.append(str(gt_path)) if missing_gt_files: raise FileNotFoundError("Missing ground-truth files for selected sequences: " + ", ".join(missing_gt_files)) diff --git a/tests/tune/test_tuner.py b/tests/tune/test_tuner.py index 4deaeb3ab..5508735ec 100644 --- a/tests/tune/test_tuner.py +++ b/tests/tune/test_tuner.py @@ -257,6 +257,49 @@ def test_objective_normalized_to_uppercase(self, tmp_path: Path) -> None: assert tuner._objective_metric == "MOTA" +class TestTunerGroundTruthLayout: + """Ground-truth validation must accept every layout evaluation accepts. + + `Tuner` hands `gt_dir` straight to `evaluate_mot_sequences`, which auto-detects the flat `{seq}.txt` layout and the + MOT `{seq}/gt/gt.txt` layout that downloaded datasets ship in. Eager validation has to agree, or it rejects trees + that would evaluate fine. + """ + + def test_accepts_mot_layout_ground_truth(self, tmp_path: Path) -> None: + """A MOT-layout ground-truth tree initializes instead of being reported as missing.""" + det_dir = tmp_path / "det" + det_dir.mkdir() + (det_dir / "seq1.txt").write_text(_MOT_LINE) + gt_dir = tmp_path / "gt" + (gt_dir / "seq1" / "gt").mkdir(parents=True) + (gt_dir / "seq1" / "gt" / "gt.txt").write_text(_MOT_LINE) + + tuner = Tuner("bytetrack", gt_dir, det_dir) + + assert tuner._sequences == ["seq1"] + + def test_reports_missing_ground_truth_at_mot_layout_path(self, tmp_path: Path) -> None: + """A sequence missing from a MOT-layout tree is reported at its MOT path.""" + det_dir = tmp_path / "det" + det_dir.mkdir() + (det_dir / "seq1.txt").write_text(_MOT_LINE) + (det_dir / "seq2.txt").write_text(_MOT_LINE) + gt_dir = tmp_path / "gt" + (gt_dir / "seq1" / "gt").mkdir(parents=True) + (gt_dir / "seq1" / "gt" / "gt.txt").write_text(_MOT_LINE) + + with pytest.raises(FileNotFoundError, match=r"seq2.*gt.*gt\.txt"): + Tuner("bytetrack", gt_dir, det_dir) + + def test_still_reports_missing_flat_ground_truth(self, tmp_path: Path) -> None: + """The flat layout keeps reporting missing files at the flat path.""" + gt_dir, det_dir = _setup_dirs(tmp_path) + (det_dir / "seq2.txt").write_text(_MOT_LINE) + + with pytest.raises(FileNotFoundError, match=r"seq2\.txt"): + Tuner("bytetrack", gt_dir, det_dir) + + class TestTunerSeed: def test_create_optuna_study_uses_seeded_sampler(self) -> None: with patch.object(optuna, "create_study", wraps=optuna.create_study) as mock_create: