From 1dff876e710992c6c589a8950aea53d8b4826283 Mon Sep 17 00:00:00 2001 From: Ayush Raj Date: Fri, 18 Sep 2026 12:03:03 +0530 Subject: [PATCH] fix(download): extract clean filenames from URLs with query parameters and fragments --- databusclient/api/deploy.py | 11 ++++--- databusclient/api/download.py | 3 +- databusclient/api/utils.py | 37 ++++++++++++++++++------ tests/test_deploy.py | 18 ++++++++++++ tests/test_download.py | 54 +++++++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 14 deletions(-) diff --git a/databusclient/api/deploy.py b/databusclient/api/deploy.py index 2c8cd08..146adda 100644 --- a/databusclient/api/deploy.py +++ b/databusclient/api/deploy.py @@ -12,6 +12,8 @@ import requests +from databusclient.api.utils import get_filename_from_url + _debug = False @@ -131,11 +133,12 @@ def _get_extensions(distribution_str: str) -> Tuple[str, str, str]: format_extension = "file" compression = "none" - # get the last segment of the URL - last_segment = str(distribution_str).split("|")[0].split("/")[-1] + # get the last segment of the URL (clean filename without query params or fragments) + url = str(distribution_str).split("|")[0] + last_segment = get_filename_from_url(url) - # cut of fragments and split by dots - dot_splits = last_segment.split("#")[0].rsplit(".", 2) + # split by dots + dot_splits = last_segment.rsplit(".", 2) if len(dot_splits) > 1: # if only format is given (no compression) diff --git a/databusclient/api/download.py b/databusclient/api/download.py index 356a2dd..0647119 100644 --- a/databusclient/api/download.py +++ b/databusclient/api/download.py @@ -18,6 +18,7 @@ fetch_databus_jsonld, get_databus_id_parts_from_file_url, compute_sha256_and_length, + get_filename_from_url, ) from databusclient.filehandling.format import ( convert_file, @@ -446,7 +447,7 @@ def _download_file( if not local_dir_was_given: print(f"Local directory not given, using {localDir}") - file = url.split("/")[-1] + file = get_filename_from_url(url) filename = os.path.join(localDir, file) print(f"Download file: {url}") dirpath = os.path.dirname(filename) diff --git a/databusclient/api/utils.py b/databusclient/api/utils.py index df4f49d..5efec16 100644 --- a/databusclient/api/utils.py +++ b/databusclient/api/utils.py @@ -4,11 +4,32 @@ `download`, `deploy` and `delete` modules. """ -from typing import Optional, Tuple import hashlib +import posixpath +from typing import Optional, Tuple +from urllib.parse import unquote, urlparse + import requests +def get_filename_from_url(url: str) -> str: + """Extract a clean, filesystem-safe filename from a URL. + + Strips query parameters and URL fragments, unquotes percent-encoded + characters, and falls back to 'downloaded_file' if the path is empty. + + Args: + url: The download URL. + + Returns: + The clean filename string. + """ + parsed = urlparse(url) + clean_path = parsed.path + basename = posixpath.basename(unquote(clean_path)) + return basename or "downloaded_file" + + def get_databus_id_parts_from_file_url( uri: str, ) -> Tuple[ @@ -21,6 +42,9 @@ def get_databus_id_parts_from_file_url( ]: """Extract databus ID parts from a given databus URI. + The returned tuple is (host, accountId, groupId, artifactId, versionId, fileId). + Missing parts are returned as ``None``. Query strings and fragments are stripped. + Args: uri: The full databus URI of the form "http(s)://host/accountId/groupId/artifactId/versionId/fileId". @@ -28,14 +52,9 @@ def get_databus_id_parts_from_file_url( A tuple containing (host, accountId, groupId, artifactId, versionId, fileId). Each element is a string or None if not present. """ - """Split a Databus URI into its six parts. - - The returned tuple is (host, accountId, groupId, artifactId, versionId, fileId). - Missing parts are returned as ``None``. - """ - - uri = uri.removeprefix("https://").removeprefix("http://") - parts = uri.strip("/").split("/") + clean_uri = uri.split("?")[0].split("#")[0] + clean_uri = clean_uri.removeprefix("https://").removeprefix("http://") + parts = clean_uri.strip("/").split("/") parts += [None] * (6 - len(parts)) # pad with None if less than 6 parts return tuple(parts[:6]) # return only the first 6 parts diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 679f11f..9bb586f 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -128,3 +128,21 @@ def test_empty_cvs(): } assert dataset == correct_dataset + + +def test_get_extensions_with_query_parameters(): + from databusclient.api.deploy import _get_extensions + + # Distribution URL with query parameters + ext_part, fmt, comp = _get_extensions( + "https://example.org/dataset.ttl.gz?token=123&expires=456" + ) + assert fmt == "ttl" + assert comp == "gz" + assert ext_part == ".ttl.gz" + + # Distribution URL with fragment and single extension + ext_part, fmt, comp = _get_extensions("https://example.org/dataset.nt#distribution") + assert fmt == "nt" + assert comp == "none" + assert ext_part == ".nt" diff --git a/tests/test_download.py b/tests/test_download.py index b66521a..ac5e7b4 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -213,3 +213,57 @@ def test_converted_output_gets_graph_sidecar_at_final_path(monkeypatch, tmp_path assert not (tmp_path / "data.ttl.bz2").exists() assert not (tmp_path / "data.ttl.bz2.graph").exists() assert (tmp_path / "data.ttl.graph").read_text(encoding="utf-8") == url + + +def test_get_filename_from_url(): + from databusclient.api.utils import get_filename_from_url + + # Query parameters and fragments + assert ( + get_filename_from_url("https://example.org/path/file.ttl.bz2?token=xyz123&expires=1700") + == "file.ttl.bz2" + ) + assert get_filename_from_url("https://example.org/path/file.ttl#fragment") == "file.ttl" + + # Query parameter containing slashes + assert ( + get_filename_from_url("https://example.org/file.ttl?redirect=http://other.org/abc") + == "file.ttl" + ) + + # Percent-encoded characters + assert get_filename_from_url("https://example.org/path/my%20data.ttl") == "my data.ttl" + + # Root or empty path fallback + assert get_filename_from_url("https://example.org/") == "downloaded_file" + assert get_filename_from_url("https://example.org") == "downloaded_file" + + +def test_download_url_with_query_params_and_fragments(monkeypatch, tmp_path): + import databusclient.api.download as dl + + url = "https://example.org/data.ttl?token=xyz123&expires=9999#header" + content = b"

.\n" + _mock_download(monkeypatch, {url: content}) + + dl._download_file(url, localDir=str(tmp_path)) + + # File should be cleanly named 'data.ttl' without query parameters + target = tmp_path / "data.ttl" + assert target.exists() + assert target.read_bytes() == content + + +def test_download_url_with_query_params_and_decompression(monkeypatch, tmp_path): + import databusclient.api.download as dl + + url = "https://example.org/data.ttl.bz2?token=secret123&foo=bar" + compressed_content = bz2.compress(b"

.\n") + _mock_download(monkeypatch, {url: compressed_content}) + + dl._download_file(url, localDir=str(tmp_path), compression="none") + + # Clean decompression to data.ttl + assert (tmp_path / "data.ttl").exists() + assert not (tmp_path / "data.ttl.bz2").exists() + assert (tmp_path / "data.ttl").read_text(encoding="utf-8") == "

.\n"