cleanup / fix listings handling

Author: shcheklein

src/datachain/lib/listing.py

@@ -129,6 +129,24 @@ def _sanitize_ds_name(raw: str) -> str:
     return _DS_ENCODE_RE.sub(lambda m: f"_x{ord(m.group()):02x}", raw)
 
 
+_DS_DECODE_RE = re.compile(r"_x([0-9a-f]{2}|_x)")
+
+
+def _desanitize_ds_name(encoded: str) -> str:
+    """Reverse :func:`_sanitize_ds_name`.
+
+    ``_xHH`` → the original character, ``_x_x`` → literal ``_x``.
+    """
+
+    def _repl(m: re.Match) -> str:
+        tok = m.group(1)
+        if tok == "_x":  # escaped literal "_x"
+            return "_x"
+        return chr(int(tok, 16))  # _xHH → original char
+
+    return _DS_DECODE_RE.sub(_repl, encoded)
+
+
 def parse_listing_uri(uri: str) -> tuple[str, str, str]:
     """
     Parsing uri and returns listing dataset name, listing uri and listing path
@@ -156,10 +174,14 @@ def is_listing_dataset(name: str) -> bool:
 
 
 def listing_uri_from_name(dataset_name: str) -> str:
-    """Returns clean storage URI from listing dataset name"""
+    """Returns clean storage URI from listing dataset name.
+
+    Strips the ``lst__`` prefix and reverses the ``_xHH`` encoding
+    applied by :func:`_sanitize_ds_name`.
+    """
     if not is_listing_dataset(dataset_name):
         raise ValueError(f"Dataset {dataset_name} is not a listing")
-    return dataset_name.removeprefix(LISTING_PREFIX)
+    return _desanitize_ds_name(dataset_name.removeprefix(LISTING_PREFIX))
 
 
 @contextmanager

src/datachain/lib/listing_info.py

@@ -1,19 +1,13 @@
 from datetime import datetime, timedelta, timezone
 
-from datachain.client import Client
 from datachain.lib.dataset_info import DatasetInfo
-from datachain.lib.listing import LISTING_PREFIX, LISTING_TTL
+from datachain.lib.listing import LISTING_PREFIX, LISTING_TTL, _desanitize_ds_name
 
 
 class ListingInfo(DatasetInfo):
     @property
     def uri(self) -> str:
-        return self.name.removeprefix(LISTING_PREFIX)
-
-    @property
-    def storage_uri(self) -> str:
-        uri, _ = Client.parse_url(self.uri)
-        return uri
+        return _desanitize_ds_name(self.name.removeprefix(LISTING_PREFIX))
 
     @property
     def expires(self) -> datetime | None:

tests/unit/test_listing.py

@@ -9,6 +9,8 @@
 from datachain.lib.file import File
 from datachain.lib.listing import (
     LISTING_PREFIX,
+    _desanitize_ds_name,
+    _sanitize_ds_name,
     get_listing,
     is_listing_dataset,
     listing_uri_from_name,
@@ -212,6 +214,13 @@ def test_is_listing_dataset(name, is_listing):
 
 def test_listing_uri_from_name():
     assert listing_uri_from_name("lst__s3://my-bucket") == "s3://my-bucket"
+    # Encoded name round-trips back to the original URI
+    assert listing_uri_from_name("lst__s3://bucket/v1_x2e0/") == "s3://bucket/v1.0/"
+    # Escaped _x round-trips
+    assert (
+        listing_uri_from_name("lst__s3://bucket/export_x_xml/")
+        == "s3://bucket/export_xml/"
+    )
     with pytest.raises(ValueError):
         listing_uri_from_name("s3://my-bucket")
 
@@ -250,3 +259,27 @@ def test_parse_listing_uri_no_collision_percent_vs_literal():
 
 def test_parse_listing_uri_no_collision_dot_vs_underscore():
     assert _ds_name("s3://b/v1.0/") != _ds_name("s3://b/v1_0/")
+
+
+@pytest.mark.parametrize(
+    "raw",
+    [
+        "s3://my-bucket/dogs/",
+        "s3://bucket/v1.0/",
+        "s3://bucket/dir%25/",
+        "s3://bucket/user@host/",
+        "s3://bucket/export_xml/",
+        "s3://my.company.data/path/to/files/",
+        "gs://bucket-with_underscores_x_and.dots/",
+        "file:///home/user/path with spaces/",
+        "s3://b/_x_x_x/edge/",
+    ],
+)
+def test_sanitize_desanitize_roundtrip(raw):
+    """_desanitize_ds_name is the exact inverse of _sanitize_ds_name."""
+    assert _desanitize_ds_name(_sanitize_ds_name(raw)) == raw
+
+
+def test_desanitize_plain_passthrough():
+    """Strings with no _xHH sequences pass through unchanged."""
+    assert _desanitize_ds_name("s3://my-bucket/dogs/") == "s3://my-bucket/dogs/"