fix(datasets): name WebDataset shards as file:// URLs so Windows can open them

webdataset routes every shard string through gopen, which dispatches on the scheme urlparse reads off it. A POSIX
path parses with an empty scheme and reaches the local-file branch; a Windows path does not, because
C:\shards\train-000000.tar parses as scheme "c" and no handler is registered for it. Every streaming test failed on
both windows-latest jobs with "no gopen handler defined" while the same tests passed on ubuntu and macos.

Naming the shards as file:// URLs picks the branch that resolves back to a local path on both platforms, and
Path.as_uri() percent-encodes a directory containing spaces or other reserved characters, which url2pathname
reverses on the way back in.

Author: JESUSROYETH

src/rfdetr/datasets/webdataset_io.py

@@ -730,6 +730,22 @@ def _decode(self, sample: dict[str, Any]) -> tuple[Any, dict[str, Any]]:
             image, target = self._transforms(image, target)
         return image, target
 
+    def _shard_urls(self) -> list[str]:
+        """Return this split's shards as ``file://`` URLs.
+
+        ``webdataset`` hands every shard string to ``gopen``, which dispatches on the scheme
+        ``urllib.parse.urlparse`` reads off it. A plain filesystem path only reaches the local-file branch when that
+        scheme comes back empty, which is true of a POSIX path and false of a Windows one: ``C:\\shards\\train.tar``
+        parses as scheme ``c``, no handler is registered for it, and the read fails with ``no gopen handler
+        defined``. Naming the shards as ``file://`` URLs instead picks the one branch that resolves back to a local
+        path on both platforms, and it is the spelling :meth:`pathlib.Path.as_uri` already percent-encodes for a
+        directory containing spaces or other reserved characters.
+
+        Returns:
+            One ``file://`` URL per shard, in index order.
+        """
+        return [(self._shard_dir / shard).resolve().as_uri() for shard in self.index.shards]
+
     def __iter__(self) -> Iterator[tuple[Any, Any]]:
         """Iterate this worker's share of the split.
 
@@ -738,7 +754,7 @@ def __iter__(self) -> Iterator[tuple[Any, Any]]:
         """
         wds = _require_webdataset()
         shard_seed, buffer_seed = self._epoch_seeds()
-        urls = [str(self._shard_dir / shard) for shard in self.index.shards]
+        urls = self._shard_urls()
         pipeline = wds.WebDataset(
             urls,
             # A split with fewer shards than workers legitimately leaves some workers with nothing to read;

tests/datasets/test_webdataset_io.py

@@ -20,6 +20,8 @@
 from dataclasses import replace
 from pathlib import Path
 from typing import Any
+from urllib.parse import urlparse
+from urllib.request import url2pathname
 
 import numpy as np
 import pytest
@@ -353,6 +355,16 @@ def _require_webdataset(self) -> None:
         """
         pytest.importorskip("webdataset")
 
+    def test_shard_urls_carry_the_file_scheme(self, tmp_path: Path) -> None:
+        dataset = WebDatasetDetection(_pack(tmp_path, count=4), "train", transforms=None)
+        assert {urlparse(url).scheme for url in dataset._shard_urls()} == {"file"}
+
+    def test_shard_urls_resolve_back_to_the_shards_they_name(self, tmp_path: Path) -> None:
+        shard_dir = _pack(tmp_path / "shard dir", count=4)
+        dataset = WebDatasetDetection(shard_dir, "train", transforms=None)
+        opened = [Path(url2pathname(urlparse(url).path)) for url in dataset._shard_urls()]
+        assert opened == [(shard_dir / name).resolve() for name in dataset.index.shards]
+
     def test_every_sample_is_streamed_once(self, tmp_path: Path) -> None:
         dataset = WebDatasetDetection(_pack(tmp_path, count=12), "train", transforms=None)
         image_ids = [int(target["image_id"]) for _, target in dataset]