Fix pip-sync --python-executable evaluating markers for the wrong environment

Co-authored-by: chrysle <[email protected]>

Author: gschaffner

piptools/scripts/sync.py

@@ -1,18 +1,22 @@
 from __future__ import annotations
 
 import itertools
+import json
 import os
+import platform
 import shlex
 import shutil
 import sys
 from pathlib import Path
+from subprocess import run  # nosec
 from typing import cast
 
 import click
 from pip._internal.commands import create_command
 from pip._internal.commands.install import InstallCommand
 from pip._internal.index.package_finder import PackageFinder
 from pip._internal.metadata import get_environment
+from pip._vendor.packaging.markers import Environment
 
 from .. import sync
 from .._compat import Distribution, parse_requirements
@@ -100,6 +104,9 @@ def cli(
 
     if python_executable:
         _validate_python_executable(python_executable)
+        environment = _get_environment(python_executable)
+    else:
+        environment = {}
 
     install_command = cast(InstallCommand, create_command("install"))
     options, _ = install_command.parse_args([])
@@ -113,7 +120,9 @@ def cli(
     )
 
     try:
-        merged_requirements = sync.merge(requirements, ignore_conflicts=force)
+        merged_requirements = sync.merge(
+            requirements, ignore_conflicts=force, environment=environment
+        )
     except PipToolsError as e:
         log.error(str(e))
         sys.exit(2)
@@ -128,7 +137,9 @@ def cli(
         local_only=python_executable is None,
         paths=paths,
     )
-    to_install, to_uninstall = sync.diff(merged_requirements, installed_dists)
+    to_install, to_uninstall = sync.diff(
+        merged_requirements, installed_dists, environment
+    )
 
     install_flags = _compose_install_flags(
         finder,
@@ -177,6 +188,40 @@ def _validate_python_executable(python_executable: str) -> None:
         sys.exit(2)
 
 
+def _get_environment(python_executable: str) -> Environment:
+    """
+    Return the marker variables of an environment.
+    """
+    # On all platforms, ``json.loads`` supports only UTF-8, UTF-16, and UTF-32.
+    # Prior to PEP 686, a Python subprocess's ``sys.stdout`` will not always
+    # default to using one of these, so we must set it.
+    if platform.system() != "Windows":
+        # Set it to UTF-8. (This is mostly unnecessary as on most Unix systems
+        # Python will already default to this; see
+        # https://peps.python.org/pep-0686/#backward-compatibility.)
+        subprocess_PYTHONIOENCODING = "utf8"
+    else:
+        # Set it to UTF-16 because:
+        #
+        # * On Windows, in this situation (i.e. when stdout is a pipe)
+        #   ``sys.stdout``'s encoding will default to CP-1252, which is not
+        #   supported by ``json.loads``.
+        # * ``pip`` emits mangled output with UTF-8 on Windows; see
+        #   https://github.com/Textualize/rich/issues/2882.
+        subprocess_PYTHONIOENCODING = "utf16"
+
+    return Environment(
+        **json.loads(
+            run(  # nosec
+                [python_executable, "-m", "pip", "inspect"],
+                env={**os.environ, "PYTHONIOENCODING": subprocess_PYTHONIOENCODING},
+                check=True,
+                capture_output=True,
+            ).stdout
+        )["environment"]
+    )
+
+
 def _compose_install_flags(
     finder: PackageFinder,
     no_index: bool,

piptools/sync.py

@@ -4,6 +4,7 @@
 import os
 import sys
 import tempfile
+from functools import wraps
 from subprocess import run  # nosec
 from typing import Deque, Iterable, Mapping, ValuesView
 
@@ -40,6 +41,38 @@
 ]
 
 
+def patch_match_markers() -> None:
+    """
+    Monkey patches ``pip._internal.req.InstallRequirement.match_markers`` to
+    allow us to pass environment other than "extra".
+    """
+
+    @wraps(InstallRequirement.match_markers)
+    def match_markers(
+        self: InstallRequirement,
+        extras_requested: Iterable[str] | None = None,
+        environment: dict[str, str] = {},
+    ) -> bool:
+        assert "extra" not in environment
+
+        if not extras_requested:
+            # Provide an extra to safely evaluate the markers
+            # without matching any extra
+            extras_requested = ("",)
+        if self.markers is not None:
+            return any(
+                self.markers.evaluate({"extra": extra, **environment})
+                for extra in extras_requested
+            )
+        else:
+            return True
+
+    InstallRequirement.match_markers = match_markers
+
+
+patch_match_markers()
+
+
 def dependency_tree(
     installed_keys: Mapping[str, Distribution], root_key: str
 ) -> set[str]:
@@ -93,15 +126,17 @@ def get_dists_to_ignore(installed: Iterable[Distribution]) -> list[str]:
 
 
 def merge(
-    requirements: Iterable[InstallRequirement], ignore_conflicts: bool
+    requirements: Iterable[InstallRequirement],
+    ignore_conflicts: bool,
+    environment: dict[str, str] = {},
 ) -> ValuesView[InstallRequirement]:
     by_key: dict[str, InstallRequirement] = {}
 
     for ireq in requirements:
         # Limitation: URL requirements are merged by precise string match, so
         # "file:///example.zip#egg=example", "file:///example.zip", and
         # "example==1.0" will not merge with each other
-        if ireq.match_markers():
+        if ireq.match_markers(environment=environment):
             key = key_from_ireq(ireq)
 
             if not ignore_conflicts:
@@ -158,6 +193,7 @@ def diff_key_from_req(req: Distribution) -> str:
 def diff(
     compiled_requirements: Iterable[InstallRequirement],
     installed_dists: Iterable[Distribution],
+    environment: dict[str, str] = {},
 ) -> tuple[set[InstallRequirement], set[str]]:
     """
     Calculate which packages should be installed or uninstalled, given a set
@@ -172,13 +208,15 @@ def diff(
     pkgs_to_ignore = get_dists_to_ignore(installed_dists)
     for dist in installed_dists:
         key = diff_key_from_req(dist)
-        if key not in requirements_lut or not requirements_lut[key].match_markers():
+        if key not in requirements_lut or not requirements_lut[key].match_markers(
+            environment=environment
+        ):
             to_uninstall.add(key)
         elif requirements_lut[key].specifier.contains(dist.version):
             satisfied.add(key)
 
     for key, requirement in requirements_lut.items():
-        if key not in satisfied and requirement.match_markers():
+        if key not in satisfied and requirement.match_markers(environment=environment):
             to_install.add(requirement)
 
     # Make sure to not uninstall any packages that should be ignored