diff --git a/HISTORY.rst b/HISTORY.rst index 08671f662..09441b632 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,16 @@ Release History =============== +0.2.14b1 +++++++++ +* Migrate azdev and ``azure-cli-diff-tool`` to PEP 621 ``pyproject.toml``. Both ``setup.py`` files are gone; wheel contents are unchanged. +* Build with ``python -m build`` in CI, and check metadata with ``twine check`` instead of ``setup.py check -r -s``. +* Drop the ``setuptools<80`` cap. It guarded ``setup.py develop``, which azdev stopped calling in 0.2.12. The ``>=78.1.1`` floor for CVE-2025-47273 stays. +* Fix ``azdev cli check-versions`` running ``python -m setup.py bdist_wheel``, which could never work. +* Fix ``azure-cli-diff-tool``'s sdist missing ``HISTORY.rst``, which broke building a wheel from it. +* Fix ``scripts/ci/extract_version.sh`` producing an empty version. +* Set ``Description-Content-Type`` and stop reading the long description through ``codecs.open``, which wrote ``\r\r\n`` line endings on Windows. + 0.2.13 ++++++ * `azdev extension add` / `azdev setup`: regenerate `*.egg-info` after the editable install so development extensions stay discoverable. `--no-build-isolation` (0.2.12) means pip no longer leaves an `*.egg-info` in the source tree, which made `azdev extension add` a silent no-op for discovery. (#554) diff --git a/azdev.pyproj b/azdev.pyproj index 6d40c214d..80ddd1e8d 100644 --- a/azdev.pyproj +++ b/azdev.pyproj @@ -71,7 +71,6 @@ - diff --git a/azdev/__init__.py b/azdev/__init__.py index fe6044c9a..9cea2c521 100644 --- a/azdev/__init__.py +++ b/azdev/__init__.py @@ -4,4 +4,4 @@ # license information. # ----------------------------------------------------------------------------- -__VERSION__ = '0.2.13' +__VERSION__ = '0.2.14b1' diff --git a/azdev/operations/code_gen.py b/azdev/operations/code_gen.py index 7e054e340..1d5a7cbf2 100644 --- a/azdev/operations/code_gen.py +++ b/azdev/operations/code_gen.py @@ -30,7 +30,6 @@ def _ensure_dir(path): def _generate_files(env, generation_kwargs, file_list, dest_path): - # allow sending a single item if not isinstance(file_list, list): file_list = [file_list] diff --git a/azdev/operations/extensions/__init__.py b/azdev/operations/extensions/__init__.py index e464087b1..5adf6272c 100644 --- a/azdev/operations/extensions/__init__.py +++ b/azdev/operations/extensions/__init__.py @@ -301,9 +301,8 @@ def update_extension_index(extensions): except IndexError: raise CLIError('unable to parse extension name') - # TODO: Update this! - extensions_dir = tempfile.mkdtemp() - ext_dir = tempfile.mkdtemp(dir=extensions_dir) + # Create temporary directories for extraction and caching + ext_dir = tempfile.mkdtemp() whl_cache_dir = tempfile.mkdtemp() whl_cache = {} ext_file = get_whl_from_url(ext_path, ext_path.split("/")[-1], whl_cache_dir, whl_cache) diff --git a/azdev/operations/latest_index.py b/azdev/operations/latest_index.py index de47a4b4e..859ca362a 100644 --- a/azdev/operations/latest_index.py +++ b/azdev/operations/latest_index.py @@ -17,10 +17,7 @@ def _resolve_cli_repo_path(cli_path): - if cli_path: - resolved = os.path.abspath(os.path.expanduser(cli_path)) - else: - resolved = get_cli_repo_path() + resolved = os.path.abspath(os.path.expanduser(cli_path)) if cli_path else get_cli_repo_path() if not resolved or resolved == '_NONE_': raise CLIError('Azure CLI repo path is not configured. Specify `--cli` or run `azdev setup`.') diff --git a/azdev/operations/pypi.py b/azdev/operations/pypi.py index 36f63d9f6..8f34fd42a 100644 --- a/azdev/operations/pypi.py +++ b/azdev/operations/pypi.py @@ -7,6 +7,7 @@ import os import re import sys +import tempfile from docutils import core, io @@ -14,16 +15,14 @@ from knack.util import CLIError from azdev.utilities import ( - display, heading, subheading, cmd, py_cmd, get_path_table, - pip_cmd, COMMAND_MODULE_PREFIX, require_azure_cli, find_files) + build_package_wheel, display, heading, subheading, cmd, get_package_config, + get_path_table, pip_cmd, COMMAND_MODULE_PREFIX, require_azure_cli) logger = get_logger(__name__) HISTORY_NAME = 'HISTORY.rst' RELEASE_HISTORY_TITLE = 'Release History' -SETUP_PY_NAME = 'setup.py' - # modules which should not be included in setup.py # because they aren't on PyPI EXCLUDED_MODULES = ['azure-cli-testsdk'] @@ -59,7 +58,20 @@ def check_history(): display('OK') -def _check_history_headings(mod_path): +def _get_built_package_metadata(mod_path): + from pkginfo import Wheel + + with tempfile.TemporaryDirectory(prefix='azdev-package-metadata-') as output_dir: + wheel_path = build_package_wheel(mod_path, output_dir) + metadata = Wheel(wheel_path) + return { + 'description': metadata.description, + 'description_content_type': metadata.description_content_type, + 'version': metadata.version, + } + + +def _check_history_headings(mod_path, actual_version=None): history_path = os.path.join(mod_path, HISTORY_NAME) source_path = None @@ -93,41 +105,57 @@ def _check_history_headings(mod_path): "line after the 'Release History' heading.".format(history_path)) first_version_history = all_versions[0] - actual_version = cmd(sys.executable + ' setup.py --version', cwd=mod_path) - # command can output warnings as well, so we just want the last line, which should have the version - actual_version = actual_version.result.splitlines()[-1].strip() + if actual_version is None: + actual_version_result = cmd(sys.executable + ' setup.py --version', cwd=mod_path) + # command can output warnings as well, so we just want the last line, which should have the version + actual_version = actual_version_result.result.splitlines()[-1].strip() if first_version_history != actual_version: - errors.append("The topmost version in {} does not match version {} defined in setup.py.".format( + errors.append("The topmost version in {} does not match package version {}.".format( history_path, actual_version)) return errors def _check_readme_render(mod_path): errors = [] - result = cmd(sys.executable + ' setup.py check -r -s', cwd=mod_path) - if result.exit_code: - # this outputs some warnings we don't care about - error_lines = [] - target_line = 'The following syntax errors were detected' - suppress = True - logger.debug(result.error.output) - - # TODO: Checks for syntax errors but potentially not other things - for line in result.error.output.splitlines(): - line = str(line).strip() - if not suppress and line: - error_lines.append(line) - if target_line in line: - suppress = False - errors.append(os.linesep.join(error_lines)) - errors += _check_history_headings(mod_path) + package_config = get_package_config(mod_path) + if package_config and os.path.basename(package_config) == 'setup.py': + result = cmd(sys.executable + ' setup.py check -r -s', cwd=mod_path) + if result.exit_code: + # this outputs some warnings we don't care about + error_lines = [] + target_line = 'The following syntax errors were detected' + suppress = True + logger.debug(result.error.output) + + # TODO: Checks for syntax errors but potentially not other things + for line in result.error.output.splitlines(): + line = str(line).strip() + if not suppress and line: + error_lines.append(line) + if target_line in line: + suppress = False + errors.append(os.linesep.join(error_lines)) + errors += _check_history_headings(mod_path) + return errors + + metadata = _get_built_package_metadata(mod_path) + content_type = (metadata.get('description_content_type') or 'text/x-rst').split(';', 1)[0].strip().lower() + if content_type == 'text/x-rst' and metadata.get('description'): + _, publication = core.publish_programmatically( + source_class=io.StringInput, source=metadata['description'], + source_path=None, destination_class=io.NullOutput, destination=None, + destination_path=None, reader=None, reader_name='standalone', parser=None, + parser_name='restructuredtext', writer=None, writer_name='null', settings=None, + settings_spec=None, settings_overrides={}, config_section=None, enable_exit_status=None) + if publication.writer.document.reporter.max_level >= 2: + errors.append('The package long description contains reStructuredText warnings or errors.') + errors += _check_history_headings(mod_path, actual_version=metadata['version']) return errors # endregion # region verify PyPI versions def verify_versions(): - import tempfile import shutil require_azure_cli() @@ -174,25 +202,6 @@ def verify_versions(): display('OK!') -def _get_module_versions(results, modules): - - version_pattern = re.compile(r'.*(?P\d+.\d+.\d+).*') - - for mod, mod_path in modules: - if not mod.startswith(COMMAND_MODULE_PREFIX) and mod != 'azure-cli': - mod = '{}{}'.format(COMMAND_MODULE_PREFIX, mod) - - setup_path = find_files(mod_path, 'setup.py') - with open(setup_path[0], 'r') as f: - local_version = 'Unknown' - for line in f.readlines(): - if line.strip().startswith('VERSION'): - local_version = version_pattern.match(line).group('ver') - break - results[mod]['local_version'] = local_version - return results - - # pylint: disable=too-many-statements def _compare_module_against_pypi(results, root_dir, mod, mod_path): import zipfile @@ -230,10 +239,7 @@ def _compare_module_against_pypi(results, root_dir, mod, mod_path): # build from source and extract the version setup_path = os.path.normpath(mod_path.strip()) os.chdir(setup_path) - py_cmd('setup.py bdist_wheel -d {}'.format(build_dir)) - if len(os.listdir(build_dir)) != 1: - raise CLIError('Unexpectedly found multiple build files found in {}.'.format(build_dir)) - build_path = os.path.join(build_dir, os.listdir(build_dir)[0]) + build_path = build_package_wheel(setup_path, build_dir) build_version = version_pattern.match(build_path).group(1) results[mod].update({ diff --git a/azdev/operations/tests/test_pypi_packaging.py b/azdev/operations/tests/test_pypi_packaging.py new file mode 100644 index 000000000..d3ffa9c88 --- /dev/null +++ b/azdev/operations/tests/test_pypi_packaging.py @@ -0,0 +1,51 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import os +import unittest +from unittest.mock import patch + +from knack.util import CommandResultItem + +from azdev.operations.pypi import _check_readme_render + + +class TestPackageHistoryValidation(unittest.TestCase): + + @patch('azdev.operations.pypi._check_history_headings', return_value=[]) + @patch('azdev.operations.pypi.cmd') + @patch('azdev.operations.pypi.get_package_config') + def test_legacy_package_keeps_setup_py_check( + self, get_package_config_mock, cmd_mock, check_history_mock): + package_dir = os.path.join('repo', 'src', 'legacy') + get_package_config_mock.return_value = os.path.join(package_dir, 'setup.py') + cmd_mock.return_value = CommandResultItem('', exit_code=0, error=None) + + self.assertEqual([], _check_readme_render(package_dir)) + + self.assertTrue(cmd_mock.call_args.args[0].endswith(' setup.py check -r -s')) + check_history_mock.assert_called_once_with(package_dir) + + @patch('azdev.operations.pypi._check_history_headings', return_value=[]) + @patch('azdev.operations.pypi._get_built_package_metadata') + @patch('azdev.operations.pypi.get_package_config') + def test_pyproject_package_uses_built_metadata( + self, get_package_config_mock, get_metadata_mock, check_history_mock): + package_dir = os.path.join('repo', 'src', 'modern') + get_package_config_mock.return_value = os.path.join(package_dir, 'pyproject.toml') + get_metadata_mock.return_value = { + 'description': None, + 'description_content_type': None, + 'version': '2.3.4', + } + + self.assertEqual([], _check_readme_render(package_dir)) + + check_history_mock.assert_called_once_with(package_dir, actual_version='2.3.4') + + +if __name__ == '__main__': + unittest.main() diff --git a/azdev/utilities/__init__.py b/azdev/utilities/__init__.py index 74e82ca17..7d92d69a0 100644 --- a/azdev/utilities/__init__.py +++ b/azdev/utilities/__init__.py @@ -52,6 +52,11 @@ get_name_index, calc_selected_mod_names ) +from .packaging import ( + build_package_wheel, + find_package_configs, + get_package_config, +) from .testing import test_cmd from .tools import ( require_virtual_env, @@ -99,4 +104,7 @@ 'diff_branches_detail', 'diff_branch_file_patch', 'calc_selected_mod_names', + 'build_package_wheel', + 'find_package_configs', + 'get_package_config', ] diff --git a/azdev/utilities/packaging.py b/azdev/utilities/packaging.py new file mode 100644 index 000000000..fffd77cc2 --- /dev/null +++ b/azdev/utilities/packaging.py @@ -0,0 +1,120 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import os + +from knack.log import get_logger +from knack.util import CLIError + +from .command import py_cmd, quote_arg + +try: + import tomllib +except ImportError: # pragma: no cover - Python 3.10 compatibility + import tomli as tomllib + +logger = get_logger(__name__) + +SETUP_PY = 'setup.py' +PYPROJECT_TOML = 'pyproject.toml' + + +def get_package_config(package_dir): + """Return the packaging configuration file for a source package. + + ``setup.py`` takes precedence while repositories transition incrementally. + A ``pyproject.toml`` qualifies only when it declares packaging configuration + through ``[build-system]`` or ``[project]``; tool-only files such as a + ``[tool.ruff]`` configuration are deliberately ignored. + """ + setup_path = os.path.join(package_dir, SETUP_PY) + if os.path.isfile(setup_path): + return setup_path + + pyproject_path = os.path.join(package_dir, PYPROJECT_TOML) + if not os.path.isfile(pyproject_path): + return None + + try: + with open(pyproject_path, 'rb') as stream: + pyproject = tomllib.load(stream) + except (OSError, tomllib.TOMLDecodeError) as ex: + raise CLIError("Unable to read packaging configuration '{}': {}".format(pyproject_path, ex)) + + if 'build-system' in pyproject or 'project' in pyproject: + return pyproject_path + return None + + +def find_package_configs(root_paths): + """Find packaging configuration files in immediate child directories. + + Azure CLI distributions live directly under ``src``. Restricting discovery + to one level prevents nested tools and vendored projects from being treated + as CLI distributions. + + A package whose configuration cannot be read is skipped rather than aborting + discovery. Nearly every command resolves paths through this function, so an + in-progress migration in one package must not break unrelated packages. + """ + if isinstance(root_paths, str): + root_paths = [root_paths] + + configs = [] + for root_path in root_paths: + if not os.path.isdir(root_path): + continue + with os.scandir(root_path) as entries: + package_dirs = sorted( + (entry.path for entry in entries if entry.is_dir()), + key=os.path.normcase, + ) + for package_dir in package_dirs: + try: + package_config = get_package_config(package_dir) + except CLIError as ex: + logger.warning("Skipping '%s' during package discovery: %s", package_dir, ex) + continue + if package_config: + configs.append(package_config) + return configs + + +def build_package_wheel(package_dir, output_dir): + """Build one wheel and return its path, preserving the legacy build path. + + Existing ``setup.py`` packages continue to use their current setuptools + command. Pyproject-only packages use the public PEP 517 interface exposed by + ``build``. Callers consume the resulting wheel identically in both cases. + """ + package_config = get_package_config(package_dir) + if not package_config: + raise CLIError("No setup.py or packaging-enabled pyproject.toml found in '{}'".format(package_dir)) + + os.makedirs(output_dir, exist_ok=True) + existing_wheels = set(os.listdir(output_dir)) + if os.path.basename(package_config) == SETUP_PY: + result = py_cmd( + 'setup.py bdist_wheel -d {}'.format(quote_arg(output_dir)), + is_module=False, + cwd=package_dir, + ) + else: + result = py_cmd( + 'build --wheel --outdir {} {}'.format(quote_arg(output_dir), quote_arg(package_dir)), + cwd=package_dir, + ) + if result.error: + raise result.error # pylint: disable=raising-bad-type + + built_wheels = [ + os.path.join(output_dir, filename) + for filename in os.listdir(output_dir) + if filename.endswith('.whl') and filename not in existing_wheels + ] + if len(built_wheels) != 1: + raise CLIError("Expected one wheel for '{}', found {}".format(package_dir, len(built_wheels))) + return built_wheels[0] diff --git a/azdev/utilities/path.py b/azdev/utilities/path.py index 923c4c6bf..7cea5aa9c 100644 --- a/azdev/utilities/path.py +++ b/azdev/utilities/path.py @@ -10,6 +10,7 @@ from knack.util import CLIError from .const import COMMAND_MODULE_PREFIX, EXTENSION_PREFIX, ENV_VAR_VIRTUAL_ENV +from .packaging import find_package_configs logger = logging.getLogger(__name__) @@ -134,7 +135,7 @@ def get_name_index(invert=False, include_whl_extensions=False): ) ) modules_paths = glob(paths) - core_paths = glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py'))) + core_paths = find_package_configs(os.path.join(cli_repo_path, 'src')) ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x] whl_ext_paths = [] if include_whl_extensions: @@ -209,7 +210,7 @@ def get_path_table(include_only=None, include_whl_extensions=False): ) ) modules_paths = glob(paths) - core_paths = glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py'))) + core_paths = find_package_configs(os.path.join(cli_repo_path, 'src')) ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x] whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x] diff --git a/azdev/utilities/tests/test_packaging.py b/azdev/utilities/tests/test_packaging.py new file mode 100644 index 000000000..2e8bf49fa --- /dev/null +++ b/azdev/utilities/tests/test_packaging.py @@ -0,0 +1,145 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from knack.util import CLIError + +from knack.util import CommandResultItem + +from azdev.utilities.packaging import build_package_wheel, find_package_configs, get_package_config + + +class TestPackageConfig(unittest.TestCase): + + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + + def _make_package(self, name, setup_py=None, pyproject=None): + package_dir = os.path.join(self.root, name) + os.makedirs(package_dir) + if setup_py is not None: + with open(os.path.join(package_dir, 'setup.py'), 'w', encoding='utf-8') as stream: + stream.write(setup_py) + if pyproject is not None: + with open(os.path.join(package_dir, 'pyproject.toml'), 'w', encoding='utf-8') as stream: + stream.write(pyproject) + return package_dir + + def test_setup_py_package(self): + package_dir = self._make_package('legacy', setup_py='from setuptools import setup\n') + + self.assertEqual(os.path.join(package_dir, 'setup.py'), get_package_config(package_dir)) + + def test_pyproject_build_system_package(self): + package_dir = self._make_package( + 'modern', + pyproject='[build-system]\nrequires = ["setuptools"]\n', + ) + + self.assertEqual(os.path.join(package_dir, 'pyproject.toml'), get_package_config(package_dir)) + + def test_pyproject_project_package(self): + package_dir = self._make_package('pep621', pyproject='[project]\nname = "pep621"\n') + + self.assertEqual(os.path.join(package_dir, 'pyproject.toml'), get_package_config(package_dir)) + + def test_tool_only_pyproject_is_ignored(self): + package_dir = self._make_package('tool-only', pyproject='[tool.ruff]\ntarget-version = "py310"\n') + + self.assertIsNone(get_package_config(package_dir)) + + def test_setup_py_takes_precedence_over_pyproject(self): + package_dir = self._make_package( + 'transitioning', + setup_py='from setuptools import setup\n', + pyproject='this is not valid TOML', + ) + + self.assertEqual(os.path.join(package_dir, 'setup.py'), get_package_config(package_dir)) + + def test_malformed_packaging_pyproject_reports_path(self): + package_dir = self._make_package('malformed', pyproject='[build-system\n') + + with self.assertRaisesRegex(CLIError, r'malformed.*pyproject\.toml'): + get_package_config(package_dir) + + def test_find_package_configs_is_immediate_and_deduplicated(self): + legacy_dir = self._make_package( + 'legacy', + setup_py='from setuptools import setup\n', + pyproject='[tool.ruff]\n', + ) + modern_dir = self._make_package( + 'modern', + pyproject='[build-system]\nrequires = ["setuptools"]\n', + ) + self._make_package('tool-only', pyproject='[tool.mypy]\nstrict = true\n') + nested_dir = os.path.join(self.root, 'tools', 'nested-package') + os.makedirs(nested_dir) + with open(os.path.join(nested_dir, 'pyproject.toml'), 'w', encoding='utf-8') as stream: + stream.write('[project]\nname = "nested"\n') + + self.assertEqual( + [os.path.join(legacy_dir, 'setup.py'), os.path.join(modern_dir, 'pyproject.toml')], + find_package_configs(self.root), + ) + + def test_find_package_configs_skips_unreadable_pyproject(self): + legacy_dir = self._make_package('legacy', setup_py='from setuptools import setup\n') + self._make_package('zz-broken', pyproject='[build-system\n') + + with self.assertLogs(level='WARNING') as logs: + configs = find_package_configs(self.root) + + self.assertEqual([os.path.join(legacy_dir, 'setup.py')], configs) + self.assertTrue(any('zz-broken' in message for message in logs.output)) + + @patch('azdev.utilities.packaging.py_cmd') + def test_build_legacy_package_preserves_setup_py_command(self, py_cmd_mock): + package_dir = self._make_package('legacy', setup_py='from setuptools import setup\n') + output_dir = os.path.join(self.root, 'dist') + + def _build(*_, **__): + with open(os.path.join(output_dir, 'legacy-1.0.0-py3-none-any.whl'), 'wb'): + pass + return CommandResultItem('', exit_code=0, error=None) + + py_cmd_mock.side_effect = _build + + wheel_path = build_package_wheel(package_dir, output_dir) + + self.assertEqual(os.path.join(output_dir, 'legacy-1.0.0-py3-none-any.whl'), wheel_path) + command = py_cmd_mock.call_args.args[0] + self.assertTrue(command.startswith('setup.py bdist_wheel -d ')) + self.assertFalse(py_cmd_mock.call_args.kwargs['is_module']) + + @patch('azdev.utilities.packaging.py_cmd') + def test_build_pyproject_package_uses_pep517_frontend(self, py_cmd_mock): + package_dir = self._make_package('modern', pyproject='[project]\nname = "modern"\n') + output_dir = os.path.join(self.root, 'dist') + + def _build(*_, **__): + with open(os.path.join(output_dir, 'modern-1.0.0-py3-none-any.whl'), 'wb'): + pass + return CommandResultItem('', exit_code=0, error=None) + + py_cmd_mock.side_effect = _build + + build_package_wheel(package_dir, output_dir) + + command = py_cmd_mock.call_args.args[0] + self.assertTrue(command.startswith('build --wheel --outdir ')) + self.assertNotIn('setup.py', command) + + +if __name__ == '__main__': + unittest.main() diff --git a/azdev/utilities/tests/test_path.py b/azdev/utilities/tests/test_path.py index 30a7f45c9..77b85ae5c 100644 --- a/azdev/utilities/tests/test_path.py +++ b/azdev/utilities/tests/test_path.py @@ -5,12 +5,38 @@ # ----------------------------------------------------------------------------- +import importlib.util +import sys +import types import unittest import os +import shutil +import tempfile +from unittest.mock import patch -from azdev.utilities import get_path_table +from azdev.utilities import get_name_index, get_path_table +def _azure_cli_installed(): + try: + return importlib.util.find_spec('azure.cli.core.extension') is not None + except (ImportError, ValueError): + return False + + +def _stub_azure_cli_extension(extensions_dir): + """Provide a minimal `azure.cli.core.extension` so path discovery can be unit tested. + + `get_path_table` and `get_name_index` import `EXTENSIONS_DIR` to locate wheel + installed extensions, which otherwise requires a full azure-cli installation. + """ + modules = {name: types.ModuleType(name) + for name in ('azure', 'azure.cli', 'azure.cli.core', 'azure.cli.core.extension')} + modules['azure.cli.core.extension'].EXTENSIONS_DIR = extensions_dir + return patch.dict(sys.modules, modules) + + +@unittest.skipUnless(_azure_cli_installed(), 'azure-cli is not installed. Run `azdev setup` first.') class TestGetPathTable(unittest.TestCase): def setUp(self): self.path_table = get_path_table() @@ -48,5 +74,52 @@ def test_extension_modules_directory_exist(self): self.assertTrue(os.path.isdir(mod_path)) +class TestCorePackageDiscovery(unittest.TestCase): + + def setUp(self): + self.cli_repo = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.cli_repo) + self.src_dir = os.path.join(self.cli_repo, 'src') + os.makedirs(self.src_dir) + self.extensions_dir = os.path.join(self.cli_repo, 'extensions') + os.makedirs(self.extensions_dir) + azure_cli_stub = _stub_azure_cli_extension(self.extensions_dir) + azure_cli_stub.start() + self.addCleanup(azure_cli_stub.stop) + + def _write(self, relative_path, content=''): + path = os.path.join(self.cli_repo, relative_path) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w', encoding='utf-8') as stream: + stream.write(content) + + @patch('azdev.utilities.path.get_ext_repo_paths', return_value=[]) + @patch('azdev.utilities.path.get_cli_repo_path') + def test_core_tables_support_both_package_formats(self, get_cli_repo_path_mock, _): + get_cli_repo_path_mock.return_value = self.cli_repo + self._write('src/legacy/setup.py', 'from setuptools import setup\n') + self._write( + 'src/modern/pyproject.toml', + '[build-system]\nrequires = ["setuptools"]\n', + ) + self._write('src/tool-only/pyproject.toml', '[tool.ruff]\ntarget-version = "py310"\n') + self._write('src/tools/nested/pyproject.toml', '[project]\nname = "nested"\n') + + path_table = get_path_table() + name_index = get_name_index() + + self.assertEqual( + { + 'legacy': os.path.join(self.src_dir, 'legacy'), + 'modern': os.path.join(self.src_dir, 'modern'), + }, + path_table['core'], + ) + self.assertEqual('azure-cli-legacy', name_index['legacy']) + self.assertEqual('azure-cli-modern', name_index['modern']) + self.assertNotIn('tool-only', path_table['core']) + self.assertNotIn('tools', path_table['core']) + + if __name__ == '__main__': unittest.main() diff --git a/azure-cli-diff-tool/pyproject.toml b/azure-cli-diff-tool/pyproject.toml index 7ba69198b..bd63b16b2 100644 --- a/azure-cli-diff-tool/pyproject.toml +++ b/azure-cli-diff-tool/pyproject.toml @@ -5,5 +5,30 @@ # ----------------------------------------------------------------------------- [build-system] -# Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel"] +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-cli-diff-tool" +description = "A tool for cli metadata management" +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +license = "MIT" +dynamic = ["version", "readme"] +dependencies = [ + "deepdiff~=8.6.1", + "requests~=2.32.3", +] + +[tool.setuptools.dynamic] +version = { attr = "azure_cli_diff_tool.__VERSION__" } +readme = { file = ["README.rst", "HISTORY.rst"], content-type = "text/x-rst" } + +[tool.setuptools] +include-package-data = true + +# Equivalent to the previous `find_packages()` call, which also picked up the +# top-level `tests` package. +[tool.setuptools.packages.find] + +[tool.setuptools.package-data] +azure_cli_diff_tool = ["data/*"] diff --git a/azure-cli-diff-tool/setup.py b/azure-cli-diff-tool/setup.py deleted file mode 100644 index 6907e2429..000000000 --- a/azure-cli-diff-tool/setup.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -# ----------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ----------------------------------------------------------------------------- - -"""Azure Command Diff Tools package that can be installed using setuptools""" -import os -import re -from setuptools import setup, find_packages - -diff_tool_path = os.path.dirname(os.path.realpath(__file__)) -with open(os.path.join(diff_tool_path, 'azure_cli_diff_tool', '__init__.py'), 'r') as version_file: - __VERSION__ = re.search(r'^__VERSION__\s*=\s*[\'"]([^\'"]*)[\'"]', - version_file.read(), re.MULTILINE).group(1) - -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() -with open('HISTORY.rst', 'r', encoding='utf-8') as f: - HISTORY = f.read() - -setup(name="azure-cli-diff-tool", - version=__VERSION__, - description="A tool for cli metadata management", - long_description=README + '\n\n' + HISTORY, - license='MIT', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - packages=find_packages(), - include_package_data=True, - install_requires=["deepdiff~=8.6.1", "requests~=2.32.3"], - package_data={ - "azure_cli_diff_tool": ["data/*"] - } - ) diff --git a/pyproject.toml b/pyproject.toml index 7ba69198b..c498390bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,5 +5,107 @@ # ----------------------------------------------------------------------------- [build-system] -# Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel"] +# setuptools >= 78.1.1 addresses CVE-2025-47273 and provides PEP 639 license +# expression support used below. +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "azdev" +description = "Microsoft Azure CLI Developer Tools" +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +license = "MIT" +license-files = ["LICENSE"] +keywords = ["azure"] +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + "Environment :: Console", + "Natural Language :: English", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +# `version` is the single source of truth in azdev/__init__.py; `readme` is the +# concatenation of README.rst and HISTORY.rst, neither of which PEP 621 can +# express statically. +dynamic = ["version", "readme"] +dependencies = [ + "azure-multiapi-storage", + "build", + "docutils", + "flake8", + "gitpython", + "jinja2", + "knack", + "pylint>=4,<5", + "pytest-xdist", # depends on pytest-forked + "pytest-forked", + "pytest>=5.0.0", + "pyyaml", + "requests", + "sphinx==1.6.7", + "tox", + "jsbeautifier~=1.14.7", + "deepdiff~=8.6.1", + "azure-cli-diff-tool~=0.1.1", + "packaging", + "pkginfo", + "tomli; python_version < '3.11'", + "tqdm", + # setuptools >= 78.1.1 to address CVE-2025-47273 (path traversal). + # No upper bound: azdev does not call the `setup.py develop` / `easy_install` + # commands deprecated in setuptools 80. Editable installs go through + # `pip install -e --config-settings editable_mode=compat --no-build-isolation` + # (0.2.12), and the remaining `setup.py` invocations against *target* repos + # (bdist_wheel, egg_info, sdist) are verified against current setuptools. + "setuptools>=78.1.1", + # `azdev extension build` runs `setup.py bdist_wheel` against extension repos that + # have not migrated to pyproject.toml; keep the wheel command available even when + # the target environment's setuptools predates the bundled bdist_wheel (< 70.1). + "wheel", + "microsoft-security-utilities-secret-masker~=1.0.0b4", +] + +[project.urls] +Homepage = "https://github.com/Azure/azure-cli-dev-tools" + +[project.scripts] +azdev = "azdev.__main__:main" + +[tool.setuptools.dynamic] +version = { attr = "azdev.__VERSION__" } +readme = { file = ["README.rst", "HISTORY.rst"], content-type = "text/x-rst" } + +[tool.setuptools] +include-package-data = true +packages = [ + "azdev", + "azdev.config", + "azdev.operations", + "azdev.mod_templates", + "azdev.operations.help", + "azdev.operations.help.refdoc", + "azdev.operations.linter", + "azdev.operations.linter.rules", + "azdev.operations.linter.pylint_checkers", + "azdev.operations.testtool", + "azdev.operations.extensions", + "azdev.operations.statistics", + "azdev.operations.command_change", + "azdev.operations.breaking_change", + "azdev.operations.cmdcov", + "azdev.utilities", +] + +[tool.setuptools.package-data] +"azdev.config" = ["*.*", "cli_pylintrc", "ext_pylintrc"] +"azdev.mod_templates" = ["*.*"] +"azdev.operations.linter.rules" = ["ci_exclusions.yml"] +"azdev.operations.linter" = ["data/*"] +"azdev.operations.cmdcov" = ["*.*"] +"azdev.operations.breaking_change" = ["*.*"] diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 424ee57bd..bac626754 100644 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -8,6 +8,5 @@ set -ev cd "${BUILD_SOURCESDIRECTORY}" echo "Build azdev" -pip install -U pip setuptools wheel -python setup.py bdist_wheel -d "${BUILD_STAGINGDIRECTORY}" -python setup.py sdist -d "${BUILD_STAGINGDIRECTORY}" +pip install -U pip build +python -m build --outdir "${BUILD_STAGINGDIRECTORY}" diff --git a/scripts/ci/build_cli_diff_tool.sh b/scripts/ci/build_cli_diff_tool.sh index 5728378bd..e5a360de8 100644 --- a/scripts/ci/build_cli_diff_tool.sh +++ b/scripts/ci/build_cli_diff_tool.sh @@ -9,6 +9,5 @@ cd "${BUILD_SOURCESDIRECTORY}" cd ./azure-cli-diff-tool echo "Build azure cli diff tool" -pip install -U pip setuptools wheel -python setup.py bdist_wheel -d "${BUILD_STAGINGDIRECTORY}" -python setup.py sdist -d "${BUILD_STAGINGDIRECTORY}" +pip install -U pip build +python -m build --outdir "${BUILD_STAGINGDIRECTORY}" diff --git a/scripts/ci/extract_version.sh b/scripts/ci/extract_version.sh index cf4bfc76c..a54baea29 100644 --- a/scripts/ci/extract_version.sh +++ b/scripts/ci/extract_version.sh @@ -3,6 +3,9 @@ # Extract the version of the CLI from azdev's __init__.py file. : "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set}" -ver=`cat azdev/__init__.py | grep __VERSION__ | sed s/' '//g | sed s/'__VERSION__='// | sed s/\"//g` -echo $ver > $BUILD_STAGINGDIRECTORY/version -echo $ver > $BUILD_STAGINGDIRECTORY/azdev-${ver}.txt \ No newline at end of file +# __VERSION__ may be quoted with either ' or ", so strip both. +ver=$(sed -nE "s/^__VERSION__[[:space:]]*=[[:space:]]*['\"]([^'\"]+)['\"].*/\1/p" azdev/__init__.py) +: "${ver:?unable to extract __VERSION__ from azdev/__init__.py}" + +echo "$ver" > "$BUILD_STAGINGDIRECTORY/version" +echo "$ver" > "$BUILD_STAGINGDIRECTORY/azdev-${ver}.txt" diff --git a/setup.py b/setup.py deleted file mode 100644 index a3e5fea25..000000000 --- a/setup.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python - -# ----------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ----------------------------------------------------------------------------- - -"""Azure Developer Tools package that can be installed using setuptools""" - -from codecs import open -import os -import re -from setuptools import setup, find_packages - - -azdev_path = os.path.dirname(os.path.realpath(__file__)) -with open(os.path.join(azdev_path, 'azdev', '__init__.py'), 'r') as version_file: - __VERSION__ = re.search(r'^__VERSION__\s*=\s*[\'"]([^\'"]*)[\'"]', - version_file.read(), re.MULTILINE).group(1) - -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() -with open('HISTORY.rst', 'r', encoding='utf-8') as f: - HISTORY = f.read() - -setup( - name='azdev', - version=__VERSION__, - description='Microsoft Azure CLI Developer Tools', - long_description=README + '\n\n' + HISTORY, - url='https://github.com/Azure/azure-cli-dev-tools', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - license='MIT', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Build Tools', - 'Environment :: Console', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14' - ], - keywords='azure', - python_requires='>=3.10', - packages=[ - 'azdev', - 'azdev.config', - 'azdev.operations', - 'azdev.mod_templates', - 'azdev.operations.help', - 'azdev.operations.help.refdoc', - 'azdev.operations.linter', - 'azdev.operations.linter.rules', - 'azdev.operations.linter.pylint_checkers', - 'azdev.operations.testtool', - 'azdev.operations.extensions', - 'azdev.operations.statistics', - 'azdev.operations.command_change', - 'azdev.operations.breaking_change', - 'azdev.operations.cmdcov', - 'azdev.utilities', - ], - install_requires=[ - 'azure-multiapi-storage', - 'docutils', - 'flake8', - 'gitpython', - 'jinja2', - 'knack', - 'pylint>=4,<5', - 'pytest-xdist', # depends on pytest-forked - 'pytest-forked', - 'pytest>=5.0.0', - 'pyyaml', - 'requests', - 'sphinx==1.6.7', - 'tox', - 'jsbeautifier~=1.14.7', - 'deepdiff~=8.6.1', - 'azure-cli-diff-tool~=0.1.1', - 'packaging', - 'pkginfo', - 'tqdm', - # setuptools >= 78.1.1 to address CVE-2025-47273 (path traversal). - # Upper bound < 80 keeps legacy `setup.py develop` invocations working - # for extensions that haven't migrated to pyproject.toml yet. - 'setuptools>=78.1.1,<80', - # `azdev extension build` runs `setup.py bdist_wheel`; keep the wheel command available even - # when the environment's setuptools predates the bundled bdist_wheel (< 70.1). Modern, unpinned - # wheel for building only -- not the `wheel==0.30.0` metadata pin dropped in 0.2.12b1. - 'wheel', - 'microsoft-security-utilities-secret-masker~=1.0.0b4' - ], - package_data={ - 'azdev.config': ['*.*', 'cli_pylintrc', 'ext_pylintrc'], - 'azdev.mod_templates': ['*.*'], - 'azdev.operations.linter.rules': ['ci_exclusions.yml'], - 'azdev.operations.linter': ["data/*"], - 'azdev.operations.cmdcov': ['*.*'], - 'azdev.operations.breaking_change': ['*.*'], - }, - include_package_data=True, - entry_points={ - 'console_scripts': ['azdev=azdev.__main__:main'] - } -) diff --git a/tox.ini b/tox.ini index 8d6fe6af0..6be515fe8 100644 --- a/tox.ini +++ b/tox.ini @@ -10,8 +10,12 @@ envlist = whitelist_externals = pylint flake8 +deps = + build + twine commands= python ./scripts/license_verify.py - python setup.py check -r -s + python -m build --outdir {envtmpdir}/dist + python -m twine check {envtmpdir}/dist/* pylint azdev --rcfile=.pylintrc -r n flake8 --statistics --append-config=.flake8 azdev