Migrate azdev and azure-cli-diff-tool to pyproject.toml

HISTORY.rst

@@ -2,6 +2,16 @@
 
 Release History
 ===============
+0.2.14b1
+++++++++
+* Migrate azdev and ``azure-cli-diff-tool`` to PEP 621 ``pyproject.toml``. Both ``setup.py`` files are gone; wheel contents are unchanged.
+* Build with ``python -m build`` in CI, and check metadata with ``twine check`` instead of ``setup.py check -r -s``.
+* Drop the ``setuptools<80`` cap. It guarded ``setup.py develop``, which azdev stopped calling in 0.2.12. The ``>=78.1.1`` floor for CVE-2025-47273 stays.
+* Fix ``azdev cli check-versions`` running ``python -m setup.py bdist_wheel``, which could never work.
+* Fix ``azure-cli-diff-tool``'s sdist missing ``HISTORY.rst``, which broke building a wheel from it.
+* Fix ``scripts/ci/extract_version.sh`` producing an empty version.
+* Set ``Description-Content-Type`` and stop reading the long description through ``codecs.open``, which wrote ``\r\r\n`` line endings on Windows.
+
 0.2.13
 ++++++
 * `azdev extension add` / `azdev setup`: regenerate `*.egg-info` after the editable install so development extensions stay discoverable. `--no-build-isolation` (0.2.12) means pip no longer leaves an `*.egg-info` in the source tree, which made `azdev extension add` a silent no-op for discovery. (#554)

azdev.pyproj

@@ -71,7 +71,6 @@
     </Compile>
     <Compile Include="azdev\__init__.py" />
     <Compile Include="azdev\__main__.py" />
-    <Compile Include="setup.py" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="azdev\operations\" />

azdev/__init__.py

@@ -4,4 +4,4 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-__VERSION__ = '0.2.13'
+__VERSION__ = '0.2.14b1'

azdev/operations/code_gen.py

@@ -30,7 +30,6 @@ def _ensure_dir(path):
 
 
 def _generate_files(env, generation_kwargs, file_list, dest_path):
-
     # allow sending a single item
     if not isinstance(file_list, list):
         file_list = [file_list]

azdev/operations/extensions/__init__.py

@@ -301,9 +301,8 @@ def update_extension_index(extensions):
         except IndexError:
             raise CLIError('unable to parse extension name')
 
-        # TODO: Update this!
-        extensions_dir = tempfile.mkdtemp()
-        ext_dir = tempfile.mkdtemp(dir=extensions_dir)
+        # Create temporary directories for extraction and caching
+        ext_dir = tempfile.mkdtemp()
         whl_cache_dir = tempfile.mkdtemp()
         whl_cache = {}
         ext_file = get_whl_from_url(ext_path, ext_path.split("/")[-1], whl_cache_dir, whl_cache)

azdev/operations/latest_index.py

@@ -17,10 +17,7 @@
 
 
 def _resolve_cli_repo_path(cli_path):
-    if cli_path:
-        resolved = os.path.abspath(os.path.expanduser(cli_path))
-    else:
-        resolved = get_cli_repo_path()
+    resolved = os.path.abspath(os.path.expanduser(cli_path)) if cli_path else get_cli_repo_path()
 
     if not resolved or resolved == '_NONE_':
         raise CLIError('Azure CLI repo path is not configured. Specify `--cli` or run `azdev setup`.')

azdev/operations/pypi.py

@@ -7,23 +7,22 @@
 import os
 import re
 import sys
+import tempfile
 
 from docutils import core, io
 
 from knack.log import get_logger
 from knack.util import CLIError
 
 from azdev.utilities import (
-    display, heading, subheading, cmd, py_cmd, get_path_table,
-    pip_cmd, COMMAND_MODULE_PREFIX, require_azure_cli, find_files)
+    build_package_wheel, display, heading, subheading, cmd, get_package_config,
+    get_path_table, pip_cmd, COMMAND_MODULE_PREFIX, require_azure_cli)
 
 logger = get_logger(__name__)
 
 
 HISTORY_NAME = 'HISTORY.rst'
 RELEASE_HISTORY_TITLE = 'Release History'
-SETUP_PY_NAME = 'setup.py'
-
 # modules which should not be included in setup.py
 # because they aren't on PyPI
 EXCLUDED_MODULES = ['azure-cli-testsdk']
@@ -59,7 +58,20 @@ def check_history():
     display('OK')
 
 
-def _check_history_headings(mod_path):
+def _get_built_package_metadata(mod_path):
+    from pkginfo import Wheel
+
+    with tempfile.TemporaryDirectory(prefix='azdev-package-metadata-') as output_dir:
+        wheel_path = build_package_wheel(mod_path, output_dir)
+        metadata = Wheel(wheel_path)
+        return {
+            'description': metadata.description,
+            'description_content_type': metadata.description_content_type,
+            'version': metadata.version,
+        }
+
+
+def _check_history_headings(mod_path, actual_version=None):
     history_path = os.path.join(mod_path, HISTORY_NAME)
 
     source_path = None
@@ -93,41 +105,57 @@ def _check_history_headings(mod_path):
                           "line after the 'Release History' heading.".format(history_path))
 
         first_version_history = all_versions[0]
-        actual_version = cmd(sys.executable + ' setup.py --version', cwd=mod_path)
-        # command can output warnings as well, so we just want the last line, which should have the version
-        actual_version = actual_version.result.splitlines()[-1].strip()
+        if actual_version is None:
+            actual_version_result = cmd(sys.executable + ' setup.py --version', cwd=mod_path)
+            # command can output warnings as well, so we just want the last line, which should have the version
+            actual_version = actual_version_result.result.splitlines()[-1].strip()
         if first_version_history != actual_version:
-            errors.append("The topmost version in {} does not match version {} defined in setup.py.".format(
+            errors.append("The topmost version in {} does not match package version {}.".format(
                 history_path, actual_version))
     return errors
 
 
 def _check_readme_render(mod_path):
     errors = []
-    result = cmd(sys.executable + ' setup.py check -r -s', cwd=mod_path)
-    if result.exit_code:
-        # this outputs some warnings we don't care about
-        error_lines = []
-        target_line = 'The following syntax errors were detected'
-        suppress = True
-        logger.debug(result.error.output)
-
-        # TODO: Checks for syntax errors but potentially not other things
-        for line in result.error.output.splitlines():
-            line = str(line).strip()
-            if not suppress and line:
-                error_lines.append(line)
-            if target_line in line:
-                suppress = False
-        errors.append(os.linesep.join(error_lines))
-    errors += _check_history_headings(mod_path)
+    package_config = get_package_config(mod_path)
+    if package_config and os.path.basename(package_config) == 'setup.py':
+        result = cmd(sys.executable + ' setup.py check -r -s', cwd=mod_path)
+        if result.exit_code:
+            # this outputs some warnings we don't care about
+            error_lines = []
+            target_line = 'The following syntax errors were detected'
+            suppress = True
+            logger.debug(result.error.output)
+
+            # TODO: Checks for syntax errors but potentially not other things
+            for line in result.error.output.splitlines():
+                line = str(line).strip()
+                if not suppress and line:
+                    error_lines.append(line)
+                if target_line in line:
+                    suppress = False
+            errors.append(os.linesep.join(error_lines))
+        errors += _check_history_headings(mod_path)
+        return errors
+
+    metadata = _get_built_package_metadata(mod_path)
+    content_type = (metadata.get('description_content_type') or 'text/x-rst').split(';', 1)[0].strip().lower()
+    if content_type == 'text/x-rst' and metadata.get('description'):
+        _, publication = core.publish_programmatically(
+            source_class=io.StringInput, source=metadata['description'],
+            source_path=None, destination_class=io.NullOutput, destination=None,
+            destination_path=None, reader=None, reader_name='standalone', parser=None,
+            parser_name='restructuredtext', writer=None, writer_name='null', settings=None,
+            settings_spec=None, settings_overrides={}, config_section=None, enable_exit_status=None)
+        if publication.writer.document.reporter.max_level >= 2:
+            errors.append('The package long description contains reStructuredText warnings or errors.')
+    errors += _check_history_headings(mod_path, actual_version=metadata['version'])
     return errors
 # endregion
 
 
 # region verify PyPI versions
 def verify_versions():
-    import tempfile
     import shutil
 
     require_azure_cli()
@@ -174,25 +202,6 @@ def verify_versions():
         display('OK!')
 
 
-def _get_module_versions(results, modules):
-
-    version_pattern = re.compile(r'.*(?P<ver>\d+.\d+.\d+).*')
-
-    for mod, mod_path in modules:
-        if not mod.startswith(COMMAND_MODULE_PREFIX) and mod != 'azure-cli':
-            mod = '{}{}'.format(COMMAND_MODULE_PREFIX, mod)
-
-        setup_path = find_files(mod_path, 'setup.py')
-        with open(setup_path[0], 'r') as f:
-            local_version = 'Unknown'
-            for line in f.readlines():
-                if line.strip().startswith('VERSION'):
-                    local_version = version_pattern.match(line).group('ver')
-                    break
-            results[mod]['local_version'] = local_version
-    return results
-
-
 # pylint: disable=too-many-statements
 def _compare_module_against_pypi(results, root_dir, mod, mod_path):
     import zipfile
@@ -230,10 +239,7 @@ def _compare_module_against_pypi(results, root_dir, mod, mod_path):
     # build from source and extract the version
     setup_path = os.path.normpath(mod_path.strip())
     os.chdir(setup_path)
-    py_cmd('setup.py bdist_wheel -d {}'.format(build_dir))
-    if len(os.listdir(build_dir)) != 1:
-        raise CLIError('Unexpectedly found multiple build files found in {}.'.format(build_dir))
-    build_path = os.path.join(build_dir, os.listdir(build_dir)[0])
+    build_path = build_package_wheel(setup_path, build_dir)
     build_version = version_pattern.match(build_path).group(1)
 
     results[mod].update({

azdev/operations/tests/test_pypi_packaging.py

@@ -0,0 +1,51 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import os
+import unittest
+from unittest.mock import patch
+
+from knack.util import CommandResultItem
+
+from azdev.operations.pypi import _check_readme_render
+
+
+class TestPackageHistoryValidation(unittest.TestCase):
+
+    @patch('azdev.operations.pypi._check_history_headings', return_value=[])
+    @patch('azdev.operations.pypi.cmd')
+    @patch('azdev.operations.pypi.get_package_config')
+    def test_legacy_package_keeps_setup_py_check(
+            self, get_package_config_mock, cmd_mock, check_history_mock):
+        package_dir = os.path.join('repo', 'src', 'legacy')
+        get_package_config_mock.return_value = os.path.join(package_dir, 'setup.py')
+        cmd_mock.return_value = CommandResultItem('', exit_code=0, error=None)
+
+        self.assertEqual([], _check_readme_render(package_dir))
+
+        self.assertTrue(cmd_mock.call_args.args[0].endswith(' setup.py check -r -s'))
+        check_history_mock.assert_called_once_with(package_dir)
+
+    @patch('azdev.operations.pypi._check_history_headings', return_value=[])
+    @patch('azdev.operations.pypi._get_built_package_metadata')
+    @patch('azdev.operations.pypi.get_package_config')
+    def test_pyproject_package_uses_built_metadata(
+            self, get_package_config_mock, get_metadata_mock, check_history_mock):
+        package_dir = os.path.join('repo', 'src', 'modern')
+        get_package_config_mock.return_value = os.path.join(package_dir, 'pyproject.toml')
+        get_metadata_mock.return_value = {
+            'description': None,
+            'description_content_type': None,
+            'version': '2.3.4',
+        }
+
+        self.assertEqual([], _check_readme_render(package_dir))
+
+        check_history_mock.assert_called_once_with(package_dir, actual_version='2.3.4')
+
+
+if __name__ == '__main__':
+    unittest.main()

azdev/utilities/__init__.py

@@ -52,6 +52,11 @@
     get_name_index,
     calc_selected_mod_names
 )
+from .packaging import (
+    build_package_wheel,
+    find_package_configs,
+    get_package_config,
+)
 from .testing import test_cmd
 from .tools import (
     require_virtual_env,
@@ -99,4 +104,7 @@
     'diff_branches_detail',
     'diff_branch_file_patch',
     'calc_selected_mod_names',
+    'build_package_wheel',
+    'find_package_configs',
+    'get_package_config',
 ]