diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py index aff1e8793bb4..82f85870533f 100755 --- a/paimon-python/pypaimon/api/rest_api.py +++ b/paimon-python/pypaimon/api/rest_api.py @@ -16,6 +16,7 @@ # under the License. import logging +import platform from typing import Callable, Dict, List, Optional, Union import re @@ -50,6 +51,7 @@ from pypaimon.api.resource_paths import ResourcePaths from pypaimon.api.rest_util import RESTUtil from pypaimon.api.typedef import T +from pypaimon import build_info from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions from pypaimon.common.identifier import Identifier @@ -60,6 +62,7 @@ class RESTApi: HEADER_PREFIX = "header." + USER_AGENT_HEADER = "User-Agent" READ_VIA_HEADER = "X-Paimon-Read-Via" MAX_RESULTS = "maxResults" PAGE_TOKEN = "pageToken" @@ -87,7 +90,9 @@ def __init__(self, options: Union[Options, Dict[str, str]], config_required: boo self.logger = logging.getLogger(self.__class__.__name__) self.client = HttpClient(uri) auth_provider = AuthProviderFactory.create_auth_provider(options) + client_user_agent = self._configured_user_agent(options.to_map()) base_headers = RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX) + self._set_user_agent(base_headers, client_user_agent) if config_required: warehouse = options.get(CatalogOptions.WAREHOUSE) @@ -108,11 +113,41 @@ def __init__(self, options: Union[Options, Dict[str, str]], config_required: boo base_headers.update( RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX) ) + override_user_agent = self._configured_user_agent( + config_response.overrides or {}) + default_user_agent = self._configured_user_agent( + config_response.defaults or {}) + if override_user_agent is not None: + user_agent = override_user_agent + elif client_user_agent is not None: + user_agent = client_user_agent + else: + user_agent = default_user_agent + self._set_user_agent(base_headers, user_agent) self.rest_auth_function = RESTAuthFunction(base_headers, auth_provider) self.options = options self.resource_paths = ResourcePaths.for_catalog_properties(options) + @classmethod + def _configured_user_agent(cls, options: Dict[str, str]) -> Optional[str]: + user_agent = None + for key, value in options.items(): + if key.lower() == (cls.HEADER_PREFIX + cls.USER_AGENT_HEADER).lower() and value is not None: + user_agent = str(value) + return user_agent + + @classmethod + def _set_user_agent(cls, headers: Dict[str, str], user_agent: Optional[str]) -> None: + for key in list(headers): + if key.lower() == cls.USER_AGENT_HEADER.lower(): + del headers[key] + headers[cls.USER_AGENT_HEADER] = ( + user_agent if user_agent is not None + else "PyPaimon/{} Python/{}".format( + build_info.sdk_version(), platform.python_version()) + ) + def __build_paged_query_params( self, max_results: Optional[int], diff --git a/paimon-python/pypaimon/build_info.py b/paimon-python/pypaimon/build_info.py index e034f356fd4e..a448f2de7948 100644 --- a/paimon-python/pypaimon/build_info.py +++ b/paimon-python/pypaimon/build_info.py @@ -84,3 +84,15 @@ def _load_full_version(): def full_version(): """Return ``-`` for snapshot provenance.""" return _FULL_VERSION + + +def sdk_version(): + """Return the SDK version embedded in the build metadata.""" + try: + full = full_version() + if not full.startswith("python-"): + return "unknown" + version, separator, _ = full[len("python-"):].rpartition("-") + return version if separator and version else "unknown" + except Exception: + return "unknown" diff --git a/paimon-python/pypaimon/common/options/config.py b/paimon-python/pypaimon/common/options/config.py index 1792dbd1df43..ab642d73f06e 100644 --- a/paimon-python/pypaimon/common/options/config.py +++ b/paimon-python/pypaimon/common/options/config.py @@ -131,7 +131,7 @@ class CatalogOptions: "If not set, will be automatically selected based on endpoint host.") PREFIX = ConfigOptions.key("prefix").string_type().no_default_value().with_description("Prefix") HTTP_USER_AGENT_HEADER = ConfigOptions.key( - "header.HTTP_USER_AGENT").string_type().no_default_value().with_description("HTTP User Agent header") + "header.User-Agent").string_type().no_default_value().with_description("HTTP User Agent header") SYNC_ALL_PROPERTIES = ConfigOptions.key("sync-all-properties").boolean_type().default_value(True).with_description( "Sync all table properties to the catalog metastore") RESOLVING_FILE_IO_ENABLED = ( diff --git a/paimon-python/pypaimon/filesystem/pvfs.py b/paimon-python/pypaimon/filesystem/pvfs.py index a56128ca4025..77713774a321 100644 --- a/paimon-python/pypaimon/filesystem/pvfs.py +++ b/paimon-python/pypaimon/filesystem/pvfs.py @@ -19,6 +19,7 @@ import importlib import logging import posixpath +import platform import time from abc import ABC from dataclasses import dataclass @@ -31,6 +32,7 @@ from fsspec.implementations.local import LocalFileSystem from readerwriterlock import rwlock +from pypaimon import build_info from pypaimon.api.api_response import GetTableResponse, GetTableTokenResponse from pypaimon.api.client import AlreadyExistsException, NoSuchResourceException from pypaimon.api.rest_api import RESTApi @@ -149,7 +151,13 @@ class PaimonVirtualFileSystem(fsspec.AbstractFileSystem): def __init__(self, options: Union[Options, Dict[str, str]] = None, **kwargs): if isinstance(options, dict): options = Options(options) - options.set(CatalogOptions.HTTP_USER_AGENT_HEADER, 'PythonPVFS') + if not any(key.lower() == CatalogOptions.HTTP_USER_AGENT_HEADER.key().lower() + for key in options.to_map()): + options.set( + CatalogOptions.HTTP_USER_AGENT_HEADER, + "PythonPVFS PyPaimon/{} Python/{}".format( + build_info.sdk_version(), platform.python_version()), + ) self.options = options self.warehouse = options.get(CatalogOptions.WAREHOUSE) cache_expired_time = ( diff --git a/paimon-python/pypaimon/tests/user_agent_test.py b/paimon-python/pypaimon/tests/user_agent_test.py new file mode 100644 index 000000000000..7d52065148c8 --- /dev/null +++ b/paimon-python/pypaimon/tests/user_agent_test.py @@ -0,0 +1,243 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import platform +import unittest +from unittest.mock import patch + +import requests + +from pypaimon.api.api_response import ConfigResponse +from pypaimon.api.rest_api import RESTApi +from pypaimon.api.typedef import RESTAuthParameter +from pypaimon.common.options.config import CatalogOptions, OssOptions +from pypaimon.filesystem.pvfs import PaimonVirtualFileSystem + + +class UserAgentTest(unittest.TestCase): + + @staticmethod + def prepared_user_agent(rest_api): + headers = rest_api.rest_auth_function.apply( + RESTAuthParameter("GET", "/v1/config", "")) + request = requests.Request("GET", "http://catalog/v1/config", headers=headers) + return requests.Session().prepare_request(request).headers["User-Agent"] + + def test_rest_api_uses_sdk_and_python_versions_in_default_user_agent(self): + with patch( + "pypaimon.api.rest_api.build_info.full_version", + return_value="python-2.3.0-deadbeef"), patch( + "pypaimon.api.rest_api.platform.python_version", + return_value="3.11.9"): + rest_api = RESTApi( + { + CatalogOptions.URI.key(): "http://catalog", + CatalogOptions.TOKEN_PROVIDER.key(): "bear", + CatalogOptions.TOKEN.key(): "token", + }, + config_required=False, + ) + + self.assertEqual( + "PyPaimon/2.3.0 Python/3.11.9", + rest_api.rest_auth_function.init_header["User-Agent"], + ) + + def test_rest_api_uses_unknown_version_when_version_lookup_fails(self): + with patch( + "pypaimon.api.rest_api.build_info.full_version", + side_effect=RuntimeError("unavailable")), patch( + "pypaimon.api.rest_api.platform.python_version", + return_value="3.11.9"): + rest_api = RESTApi( + { + CatalogOptions.URI.key(): "http://catalog", + CatalogOptions.TOKEN_PROVIDER.key(): "bear", + CatalogOptions.TOKEN.key(): "token", + }, + config_required=False, + ) + + self.assertEqual( + "PyPaimon/unknown Python/3.11.9", + rest_api.rest_auth_function.init_header["User-Agent"], + ) + + def test_rest_api_uses_unknown_version_when_version_info_is_invalid(self): + with patch( + "pypaimon.api.rest_api.build_info.full_version", + return_value="UNKNOWN"), patch( + "pypaimon.api.rest_api.platform.python_version", + return_value="3.11.9"): + rest_api = RESTApi( + { + CatalogOptions.URI.key(): "http://catalog", + CatalogOptions.TOKEN_PROVIDER.key(): "bear", + CatalogOptions.TOKEN.key(): "token", + }, + config_required=False, + ) + + self.assertEqual( + "PyPaimon/unknown Python/3.11.9", + rest_api.rest_auth_function.init_header["User-Agent"], + ) + + def test_rest_api_preserves_configured_user_agent(self): + rest_api = RESTApi( + { + CatalogOptions.URI.key(): "http://catalog", + CatalogOptions.TOKEN_PROVIDER.key(): "bear", + CatalogOptions.TOKEN.key(): "token", + "header.User-Agent": "custom-client/1.0", + }, + config_required=False, + ) + + self.assertEqual( + "custom-client/1.0", + rest_api.rest_auth_function.init_header["User-Agent"], + ) + + def test_rest_api_preserves_lowercase_configured_user_agent_on_wire(self): + rest_api = RESTApi( + { + CatalogOptions.URI.key(): "http://catalog", + CatalogOptions.TOKEN_PROVIDER.key(): "bear", + CatalogOptions.TOKEN.key(): "token", + "header.user-agent": "custom-client/1.0", + }, + config_required=False, + ) + + self.assertEqual("custom-client/1.0", self.prepared_user_agent(rest_api)) + + def test_rest_config_merge_preserves_lowercase_client_user_agent(self): + with patch("pypaimon.api.rest_api.HttpClient") as http_client_class: + http_client_class.return_value.get_with_params.return_value = ConfigResponse( + defaults={"header.User-Agent": "server-default/1.0"}, + overrides=None, + ) + rest_api = RESTApi( + { + CatalogOptions.URI.key(): "http://catalog", + CatalogOptions.WAREHOUSE.key(): "warehouse", + CatalogOptions.TOKEN_PROVIDER.key(): "bear", + CatalogOptions.TOKEN.key(): "token", + "header.user-agent": "custom-client/1.0", + }, + ) + + self.assertEqual("custom-client/1.0", self.prepared_user_agent(rest_api)) + + def test_rest_config_override_wins_over_lowercase_client_user_agent(self): + with patch("pypaimon.api.rest_api.HttpClient") as http_client_class: + http_client_class.return_value.get_with_params.return_value = ConfigResponse( + defaults={}, + overrides={"header.USER-AGENT": "required-client/2.0"}, + ) + rest_api = RESTApi( + { + CatalogOptions.URI.key(): "http://catalog", + CatalogOptions.WAREHOUSE.key(): "warehouse", + CatalogOptions.TOKEN_PROVIDER.key(): "bear", + CatalogOptions.TOKEN.key(): "token", + "header.user-agent": "custom-client/1.0", + }, + ) + + self.assertEqual("required-client/2.0", self.prepared_user_agent(rest_api)) + + def test_rest_config_request_uses_default_user_agent(self): + with patch("pypaimon.api.rest_api.HttpClient") as http_client_class, patch( + "pypaimon.api.rest_api.build_info.full_version", + return_value="python-2.3.0-deadbeef"), patch( + "pypaimon.api.rest_api.platform.python_version", + return_value="3.11.9"): + http_client = http_client_class.return_value + http_client.get_with_params.return_value = ConfigResponse( + defaults={}, overrides=None) + RESTApi( + { + CatalogOptions.URI.key(): "http://catalog", + CatalogOptions.WAREHOUSE.key(): "warehouse", + CatalogOptions.TOKEN_PROVIDER.key(): "bear", + CatalogOptions.TOKEN.key(): "token", + }, + ) + + config_auth_function = http_client.get_with_params.call_args[0][3] + config_headers = config_auth_function.apply( + RESTAuthParameter("GET", "/v1/config", "")) + self.assertEqual( + "PyPaimon/2.3.0 Python/3.11.9", + config_headers["User-Agent"], + ) + + def test_pvfs_adds_default_user_agent(self): + pvfs = PaimonVirtualFileSystem( + {OssOptions.OSS_ACCESS_KEY_ID.key(): "ak"}, + skip_instance_cache=True, + ) + + self.assertIn("header.User-Agent", pvfs.options.to_map()) + + def test_pvfs_includes_sdk_and_python_versions_in_default_user_agent(self): + with patch( + "pypaimon.filesystem.pvfs.build_info.full_version", + return_value="python-2.3.0-deadbeef"): + pvfs = PaimonVirtualFileSystem( + {OssOptions.OSS_ACCESS_KEY_ID.key(): "ak"}, + skip_instance_cache=True, + ) + + self.assertEqual( + "PythonPVFS PyPaimon/2.3.0 Python/{}".format( + platform.python_version()), + pvfs.options.get(CatalogOptions.HTTP_USER_AGENT_HEADER), + ) + + def test_pvfs_preserves_configured_user_agent(self): + pvfs = PaimonVirtualFileSystem( + { + OssOptions.OSS_ACCESS_KEY_ID.key(): "ak", + "header.User-Agent": "custom-client/1.0", + }, + skip_instance_cache=True, + ) + + self.assertEqual( + "custom-client/1.0", + pvfs.options.get(CatalogOptions.HTTP_USER_AGENT_HEADER), + ) + + def test_pvfs_preserves_lowercase_configured_user_agent(self): + pvfs = PaimonVirtualFileSystem( + { + OssOptions.OSS_ACCESS_KEY_ID.key(): "ak", + "header.user-agent": "custom-client/1.0", + }, + skip_instance_cache=True, + ) + + self.assertNotIn("header.User-Agent", pvfs.options.to_map()) + self.assertEqual( + "custom-client/1.0", pvfs.options.to_map()["header.user-agent"]) + + +if __name__ == "__main__": + unittest.main()