Merge pull request #2145 from webknjaz/maintenance/reproducible-ci-fixes-pr2106-pr2141-pr2142

A combo of CI unblocks via PRs #2106 #2134 #2141 #2142 #2146

Author: webknjaz

.github/workflows/ci.yml

@@ -61,24 +61,15 @@ jobs:
         pip-version: >-
           ${{
             fromJSON(
-              github.job_workflow_sha
+              inputs.cpython-pip-version
               && inputs.cpython-pip-version
-              || '["latest", "previous"]'
+              || '["supported", "lowest"]'
             )
           }}
-        include:
-          - os: Ubuntu
-            python-version: >-
-              ${{
-                github.job_workflow_sha
-                && '3.12-dev'
-                || '3.8'
-              }}
-            pip-version: main
     env:
       TOXENV: >-
         pip${{ matrix.pip-version }}${{
-          !github.job_workflow_sha
+          !inputs.cpython-pip-version
           && '-coverage'
           || ''
         }}
@@ -121,7 +112,7 @@ jobs:
         run: tox --skip-pkg-install
       - name: Upload coverage to Codecov
         if: >-
-          !github.job_workflow_sha
+          !inputs.cpython-pip-version
         uses: codecov/codecov-action@v3
         with:
           files: ./coverage.xml
@@ -148,9 +139,9 @@ jobs:
           - MacOS
           - Windows
         python-version:
-          - pypy-3.8
+          - pypy-3.10
         pip-version:
-          - latest
+          - supported
     env:
       TOXENV: pip${{ matrix.pip-version }}
     steps:

.github/workflows/cron.yml

@@ -11,4 +11,4 @@ jobs:
     uses: ./.github/workflows/ci.yml
     with:
       cpython-pip-version: >-
-        ["main", "latest", "previous"]
+        ["main", "latest", "supported", "lowest"]

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
           - build==1.0.0
           - pyproject_hooks==1.0.0
           - pytest==7.4.2
-        language_version: python3.8
+        language_version: python3.9
   - repo: https://github.com/PyCQA/bandit
     rev: 1.7.8
     hooks:

piptools/build.py

@@ -2,6 +2,7 @@
 
 import collections
 import contextlib
+import os
 import pathlib
 import sys
 import tempfile
@@ -119,6 +120,7 @@ def build_project_metadata(
     src_file: pathlib.Path,
     build_targets: tuple[str, ...],
     *,
+    upgrade_packages: tuple[str, ...] | None = None,
     attempt_static_parse: bool,
     isolated: bool,
     quiet: bool,
@@ -159,7 +161,12 @@ def build_project_metadata(
             return project_metadata
 
     src_dir = src_file.parent
-    with _create_project_builder(src_dir, isolated=isolated, quiet=quiet) as builder:
+    with _create_project_builder(
+        src_dir,
+        upgrade_packages=upgrade_packages,
+        isolated=isolated,
+        quiet=quiet,
+    ) as builder:
         metadata = _build_project_wheel_metadata(builder)
         extras = tuple(metadata.get_all("Provides-Extra") or ())
         requirements = tuple(
@@ -180,9 +187,80 @@ def build_project_metadata(
         )
 
 
+@contextlib.contextmanager
+def _env_var(
+    env_var_name: str,
+    env_var_value: str,
+    /,
+) -> Iterator[None]:
+    sentinel = object()
+    original_pip_constraint = os.getenv(env_var_name, sentinel)
+    pip_constraint_was_unset = original_pip_constraint is sentinel
+
+    os.environ[env_var_name] = env_var_value
+    try:
+        yield
+    finally:
+        if pip_constraint_was_unset:
+            del os.environ[env_var_name]
+            return
+
+        # Assert here is necessary because MyPy can't infer type
+        # narrowing in the complex case.
+        assert isinstance(original_pip_constraint, str)
+        os.environ[env_var_name] = original_pip_constraint
+
+
+@contextlib.contextmanager
+def _temporary_constraints_file_set_for_pip(
+    upgrade_packages: tuple[str, ...],
+) -> Iterator[None]:
+    with tempfile.NamedTemporaryFile(
+        mode="w+t",
+        delete=False,  # FIXME: switch to `delete_on_close` in Python 3.12+
+    ) as tmpfile:
+        # NOTE: `delete_on_close=False` here (or rather `delete=False`,
+        # NOTE: temporarily) is important for cross-platform execution. It is
+        # NOTE: required on Windows so that the underlying `pip install`
+        # NOTE: invocation by pypa/build will be able to access the constraint
+        # NOTE: file via a subprocess and not fail installing it due to a
+        # NOTE: permission error related to this file handle still open in our
+        # NOTE: parent process. To achieve this, we `.close()` the file
+        # NOTE: descriptor before we hand off the control to the build frontend
+        # NOTE: and with `delete_on_close=False`, the
+        # NOTE: `tempfile.NamedTemporaryFile()` context manager does not remove
+        # NOTE: it from disk right away.
+        # NOTE: Due to support of versions below Python 3.12, we are forced to
+        # NOTE: temporarily resort to using `delete=False`, meaning that the CM
+        # NOTE: never attempts removing the file from disk, not even on exit.
+        # NOTE: So we do this manually until we can migrate to using the more
+        # NOTE: ergonomic argument `delete_on_close=False`.
+
+        # Write packages to upgrade to a temporary file to set as
+        # constraints for the installation to the builder environment,
+        # in case build requirements are among them
+        tmpfile.write("\n".join(upgrade_packages))
+
+        # FIXME: replace `delete` with `delete_on_close` in Python 3.12+
+        # FIXME: and replace `.close()` with `.flush()`
+        tmpfile.close()
+
+        try:
+            with _env_var("PIP_CONSTRAINT", tmpfile.name):
+                yield
+        finally:
+            # FIXME: replace `delete` with `delete_on_close` in Python 3.12+
+            # FIXME: and drop this manual deletion
+            os.unlink(tmpfile.name)
+
+
 @contextlib.contextmanager
 def _create_project_builder(
-    src_dir: pathlib.Path, *, isolated: bool, quiet: bool
+    src_dir: pathlib.Path,
+    *,
+    upgrade_packages: tuple[str, ...] | None = None,
+    isolated: bool,
+    quiet: bool,
 ) -> Iterator[build.ProjectBuilder]:
     if quiet:
         runner = pyproject_hooks.quiet_subprocess_runner
@@ -193,7 +271,13 @@ def _create_project_builder(
         yield build.ProjectBuilder(src_dir, runner=runner)
         return
 
-    with build.env.DefaultIsolatedEnv() as env:
+    maybe_pip_constrained_context = (
+        contextlib.nullcontext()
+        if upgrade_packages is None
+        else _temporary_constraints_file_set_for_pip(upgrade_packages)
+    )
+
+    with maybe_pip_constrained_context, build.env.DefaultIsolatedEnv() as env:
         builder = build.ProjectBuilder.from_isolated_env(env, src_dir, runner)
         env.install(builder.build_system_requires)
         env.install(builder.get_requires_for_build("wheel"))

piptools/scripts/compile.py

@@ -370,6 +370,7 @@ def cli(
                 metadata = build_project_metadata(
                     src_file=Path(src_file),
                     build_targets=build_deps_targets,
+                    upgrade_packages=upgrade_packages,
                     attempt_static_parse=not bool(build_deps_targets),
                     isolated=build_isolation,
                     quiet=log.verbosity <= 0,

tests/test_cli_compile.py

@@ -3431,7 +3431,6 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv
             """
         )
     )
-    (tmp_path / "constraints.txt").write_text("wheel<0.43")
     out = runner.invoke(
         cli,
         [
@@ -3446,15 +3445,82 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv
             "--find-links",
             os.fspath(MINIMAL_WHEELS_PATH),
             os.fspath(tmp_path / "pyproject.toml"),
-            "--constraint",
-            os.fspath(tmp_path / "constraints.txt"),
             "--output-file",
             "-",
         ],
     )
     expected = rf"""foo[footest] @ {tmp_path.as_uri()}
 small-fake-a==0.2
 small-fake-b==0.3
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
+"""
+    try:
+        assert out.exit_code == 0
+        assert expected == out.stdout
+    except Exception:  # pragma: no cover
+        print(out.stdout)
+        print(out.stderr)
+        raise
+
+
+@backtracking_resolver_only
+def test_compile_build_targets_setuptools_no_wheel_dep(
+    runner,
+    tmp_path,
+    current_resolver,
+):
+    """Check that user requests apply to build dependencies.
+
+    This verifies that build deps compilation would not use the latest version
+    of an unconstrained build requirements list, when the user requested
+    restricting them.
+
+    It is implemented against `setuptools < 70.1.0` which is known to inject a
+    dependency on `wheel` (newer `setuptools` vendor it). The idea is that
+    `pyproject.toml` does not have an upper bound for `setuptools` but the CLI
+    arg does. And when this works correctly, the `wheel` entry will be included
+    into the resolved output.
+
+    This is a regression test for
+    https://github.com/jazzband/pip-tools/pull/1681#issuecomment-2212541289.
+    """
+    (tmp_path / "pyproject.toml").write_text(
+        dedent(
+            """
+            [project]
+            name = "foo"
+            version = "0.0.1"
+            dependencies = ["small-fake-a"]
+            """
+        )
+    )
+    (tmp_path / "constraints.txt").write_text("wheel<0.43")
+    out = runner.invoke(
+        cli,
+        [
+            "--build-isolation",
+            "--no-header",
+            "--no-annotate",
+            "--no-emit-options",
+            "--extra",
+            "dev",
+            "--build-deps-for",
+            "wheel",
+            "--find-links",
+            os.fspath(MINIMAL_WHEELS_PATH),
+            os.fspath(tmp_path / "pyproject.toml"),
+            "--constraint",
+            os.fspath(tmp_path / "constraints.txt"),
+            "--upgrade-package",
+            "setuptools < 70.1.0",  # setuptools>=70.1.0 doesn't require wheel any more
+            "--output-file",
+            "-",
+        ],
+        catch_exceptions=True,
+    )
+    expected = r"""small-fake-a==0.2
 wheel==0.42.0
 
 # The following packages are considered to be unsafe in a requirements file:

tox.ini

@@ -1,9 +1,9 @@
 [tox]
 envlist =
     # NOTE: keep this in sync with the env list in .github/workflows/ci.yml.
-    py{38,39,310,311,312,py3}-pip{previous,latest,main}-coverage
-    pip{previous,latest,main}-coverage
-    pip{previous,latest,main}
+    py{38,39,310,311,312,py3}-pip{supported,lowest,latest,main}-coverage
+    pip{supported,lowest,latest,main}-coverage
+    pip{supported,lowest,latest,main}
     checkqa
     readme
 skip_missing_interpreters = True
@@ -14,7 +14,8 @@ extras =
     testing
     coverage: coverage
 deps =
-    pipprevious: pip==22.2.*
+    pipsupported: pip==24.2
+    piplowest: pip==22.2.*
     piplatest: pip
     pipmain: https://github.com/pypa/pip/archive/main.zip
 setenv =