diff --git a/CONSTRUCT.md b/CONSTRUCT.md index 210cc88e9..5d9fb159c 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -658,6 +658,22 @@ Allowed strings / keys: `hash`, `info.json`, `licenses`, `lockfile`, `pkgs_list` Use the standalone binary to perform the uninstallation on Windows. Requires conda-standalone 24.11.0 or newer. +### `freeze_base` + +Protects the conda base environment against modifications by supported package managers. + +Supported package managers: + - `conda`: Protects against conda modifications + +For `conda`, the dictionary is written into the `frozen` marker file. +See CEP-22 for the `frozen` marker file specification. For example: + +``` +freeze_base: + conda: + message: "This base environment is frozen and cannot be modified." +``` + ## Available selectors - `aarch64` diff --git a/constructor/_schema.py b/constructor/_schema.py index a8ae97514..4d61bb65c 100644 --- a/constructor/_schema.py +++ b/constructor/_schema.py @@ -98,6 +98,8 @@ class ExtraEnv(BaseModel): Same as the global option, but for this env. See global option for notes about overrides. """ + freeze_env: dict[Literal["conda"], dict] | None = None + "Same as `freeze_base`, but for this conda environment." class BuildOutputs(StrEnum): @@ -830,6 +832,22 @@ class ConstructorConfiguration(BaseModel): Use the standalone binary to perform the uninstallation on Windows. Requires conda-standalone 24.11.0 or newer. """ + freeze_base: dict[Literal["conda"], dict] | None = None + """ + Protects the conda base environment against modifications by supported package managers. + + Supported package managers: + - `conda`: Protects against conda modifications + + For `conda`, the dictionary is written into the `frozen` marker file. + See CEP-22 for the `frozen` marker file specification. For example: + + ``` + freeze_base: + conda: + message: "This base environment is frozen and cannot be modified." + ``` + """ def fix_descriptions(obj): diff --git a/constructor/data/construct.schema.json b/constructor/data/construct.schema.json index 228b82946..4811ccb8a 100644 --- a/constructor/data/construct.schema.json +++ b/constructor/data/construct.schema.json @@ -124,6 +124,26 @@ "description": "Same as the global option, but for this env. See global option for notes about overrides.", "title": "Exclude" }, + "freeze_env": { + "anyOf": [ + { + "additionalProperties": { + "additionalProperties": true, + "type": "object" + }, + "propertyNames": { + "const": "conda" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Same as `freeze_base`, but for this conda environment.", + "title": "Freeze Env" + }, "menu_packages": { "anyOf": [ { @@ -694,6 +714,26 @@ "title": "Extra Files", "type": "array" }, + "freeze_base": { + "anyOf": [ + { + "additionalProperties": { + "additionalProperties": true, + "type": "object" + }, + "propertyNames": { + "const": "conda" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Protects the conda base environment against modifications by supported package managers.\nSupported package managers: - `conda`: Protects against conda modifications\nFor `conda`, the dictionary is written into the `frozen` marker file. See CEP-22 for the `frozen` marker file specification. For example:\n```\nfreeze_base:\n conda:\n message: \"This base environment is frozen and cannot be modified.\"\n```", + "title": "Freeze Base" + }, "header_image": { "anyOf": [ { diff --git a/constructor/main.py b/constructor/main.py index abf03ad2b..5a8ce9af7 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) -def get_installer_type(info): +def get_installer_type(info: dict): osname, unused_arch = info["_platform"].split("-") os_allowed = {"linux": ("sh",), "osx": ("sh", "pkg"), "win": ("exe",)} @@ -60,7 +60,7 @@ def get_installer_type(info): return (itype,) -def get_output_filename(info): +def get_output_filename(info: dict) -> str: try: return info["installer_filename"] except KeyError: @@ -105,16 +105,81 @@ def _win_install_needs_python_exe(conda_exe: str, conda_exe_type: StandaloneExe return results.returncode == 2 +# Validate frozen environments +def validate_frozen_envs( + info: dict, exe_type: StandaloneExe | None, exe_version: Version | None +) -> bool: + """Validate frozen environments. + + Checks: + - No conflicts between freeze_base/freeze_env and extra_files for same environment + - Conda-standalone version if frozen environments exist + """ + + def get_frozen_env(path: str) -> str | None: + """Extract environment name from frozen marker destination path. + + Returns: + Environment name if frozen marker found in the path, otherwise None. + """ + parts = Path(path).parts + if parts == ("conda-meta", "frozen"): + return "base" + if len(parts) == 4 and parts[0] == "envs" and parts[-2:] == ("conda-meta", "frozen"): + return parts[1] + return None + + # Collect environments using freeze_base/freeze_env + frozen_envs = set() + if info.get("freeze_base", {}).get("conda") is not None: + frozen_envs.add("base") + for env_name, env_config in info.get("extra_envs", {}).items(): + if env_config.get("freeze_env", {}).get("conda") is not None: + frozen_envs.add(env_name) + + # Collect environments using extra_files frozen markers + frozen_envs_extra_files = set() + paths = [] + for file in info.get("extra_files", ()): + if isinstance(file, dict): + paths.extend(file.values()) + elif isinstance(file, str): + paths.append(file) + + for path in paths: + if env := get_frozen_env(path): + frozen_envs_extra_files.add(env) + + if not frozen_envs and not frozen_envs_extra_files: + return + + # Check for conflicts with extra_files + if common_envs := frozen_envs.intersection(frozen_envs_extra_files): + raise RuntimeError( + f"Frozen marker files found from both " + f"'freeze_base / freeze_env' and 'extra_files' for the following environments: {', '.join(sorted(common_envs))}" + ) + + # Conda-standalone version validation + if exe_type == StandaloneExe.CONDA and check_version( + exe_version, min_version="25.5.0", max_version="25.7.0" + ): + raise RuntimeError( + "Error: conda-standalone 25.5.x has known issues with frozen environments. " + "Please use conda-standalone 25.7.0 or newer." + ) + + def main_build( - dir_path, - output_dir=".", - platform=cc_platform, - verbose=True, - cache_dir=DEFAULT_CACHE_DIR, - dry_run=False, - conda_exe="conda.exe", - config_filename="construct.yaml", - debug=False, + dir_path: str, + output_dir: str = ".", + platform: str = cc_platform, + verbose: bool = True, + cache_dir: str = DEFAULT_CACHE_DIR, + dry_run: bool = False, + conda_exe: str = "conda.exe", + config_filename: str = "construct.yaml", + debug: bool = False, ): logger.info("platform: %s", platform) if not os.path.isfile(conda_exe): @@ -194,30 +259,7 @@ def main_build( if isinstance(info[key], str): info[key] = list(yield_lines(join(dir_path, info[key]))) - def has_frozen_file(extra_files: list[str | dict[str, str]]) -> bool: - def is_conda_meta_frozen(path_str: str) -> bool: - path = Path(path_str) - return path.parts == ("conda-meta", "frozen") or ( - len(path.parts) == 4 - and path.parts[0] == "envs" - and path.parts[-2:] == ("conda-meta", "frozen") - ) - - for file in extra_files: - if isinstance(file, str) and is_conda_meta_frozen(file): - return True - elif isinstance(file, dict) and any(is_conda_meta_frozen(val) for val in file.values()): - return True - return False - - if ( - has_frozen_file(info.get("extra_files", [])) - and exe_type == StandaloneExe.CONDA - and check_version(exe_version, min_version="25.5.0", max_version="25.7.0") - ): - sys.exit( - "Error: handling conda-meta/frozen marker files requires conda-standalone newer than 25.7.x" - ) + validate_frozen_envs(info, exe_type, exe_version) # normalize paths to be copied; if they are relative, they must be to # construct.yaml's parent (dir_path) diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index 3ebfe1a07..8adddc336 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1637,6 +1637,12 @@ Section "Install" # channels and retain only the first transaction SetOutPath "{{ env.conda_meta }}" File "{{ env.history_abspath }}" + +{%- if env.frozen_abspath %} + # Add frozen marker file + SetOutPath "{{ env.conda_meta }}" + File "{{ env.frozen_abspath }}" +{%- endif %} {%- endfor %} {%- for condarc in WRITE_CONDARC %} diff --git a/constructor/preconda.py b/constructor/preconda.py index 12201ee77..f928213e8 100644 --- a/constructor/preconda.py +++ b/constructor/preconda.py @@ -56,7 +56,7 @@ ) -def write_index_cache(info, dst_dir, used_packages): +def write_index_cache(info: dict, dst_dir: str, used_packages: list[str]): cache_dir = join(dst_dir, "cache") if not isdir(cache_dir): @@ -147,6 +147,7 @@ def write_files(info: dict, workspace: str): - `conda-meta/initial-state.explicit.txt`: Lockfile to provision the base environment. - `conda-meta/history`: Prepared history file with the right requested specs in input file. + - `conda-meta/frozen`: Frozen marker file used to protect conda environment state. - `pkgs/urls` and `pkgs/urls.txt`: Direct URLs of packages used, with and without MD5 hashes. - `pkgs/cache/*.json`: Trimmed repodata to mock offline channels in use. - `pkgs/channels.txt`: Channels in use. @@ -199,6 +200,9 @@ def write_files(info: dict, workspace: str): # (list of specs/dists to install) write_initial_state_explicit_txt(info, join(workspace, "conda-meta"), final_urls_md5s) + # base environment frozen marker files + write_frozen(info.get("freeze_base"), join(workspace, "conda-meta")) + for fn in files: os.chmod(join(workspace, fn), 0o664) @@ -218,9 +222,16 @@ def write_files(info: dict, workspace: str): write_channels_txt(info, env_pkgs, env_config) # shortcuts write_shortcuts_txt(info, env_pkgs, env_config) + # frozen marker file + write_frozen(env_config.get("freeze_env"), env_conda_meta) -def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None): +def write_conda_meta( + info: dict, + dst_dir: str, + final_urls_md5s: tuple[str, str], + user_requested_specs: list[str] | None = None, +): if user_requested_specs is None: user_requested_specs = info.get("user_requested_specs", info.get("specs", ())) @@ -244,7 +255,15 @@ def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None): fh.write("\n".join(builder)) -def write_repodata_record(info, dst_dir): +def write_frozen(freeze_info: dict | None, dst_dir: str): + if not freeze_info or "conda" not in freeze_info: + return + frozen_path = join(dst_dir, "frozen") + with open(frozen_path, "w") as ff: + json.dump(freeze_info["conda"], ff) + + +def write_repodata_record(info: dict, dst_dir: str): all_dists = info["_dists"].copy() for env_data in info.get("_extra_envs_info", {}).values(): all_dists += env_data["_dists"] @@ -271,7 +290,7 @@ def write_repodata_record(info, dst_dir): json.dump(rr_json, rf, indent=2, sort_keys=True) -def write_initial_state_explicit_txt(info, dst_dir, urls): +def write_initial_state_explicit_txt(info: dict, dst_dir: str, urls: tuple[str, str]): """ urls is an iterable of tuples with url and md5 values """ @@ -293,7 +312,7 @@ def write_initial_state_explicit_txt(info, dst_dir, urls): envf.write(f"{url}#{md5}\n") -def write_channels_txt(info, dst_dir, env_config): +def write_channels_txt(info: dict, dst_dir: str, env_config: dict): env_config = env_config.copy() if "channels" not in env_config: env_config["channels"] = info.get("channels", ()) @@ -304,7 +323,7 @@ def write_channels_txt(info, dst_dir, env_config): f.write(",".join(get_final_channels(env_config))) -def write_shortcuts_txt(info, dst_dir, env_config): +def write_shortcuts_txt(info: dict, dst_dir: str, env_config: dict): if "menu_packages" in env_config: contents = shortcuts_flags(env_config) else: diff --git a/constructor/shar.py b/constructor/shar.py index 745f95c61..a0b631267 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -173,12 +173,20 @@ def create(info, verbose=False): pre_t.addfile(tarinfo=tarfile.TarInfo("conda-meta/history")) post_t.add(join(tmp_dir, "conda-meta", "history"), "conda-meta/history") + if os.path.exists(join(tmp_dir, "conda-meta", "frozen")): + post_t.add(join(tmp_dir, "conda-meta", "frozen"), "conda-meta/frozen") + for env_name in info.get("_extra_envs_info", {}): pre_t.addfile(tarinfo=tarfile.TarInfo(f"envs/{env_name}/conda-meta/history")) post_t.add( join(tmp_dir, "envs", env_name, "conda-meta", "history"), f"envs/{env_name}/conda-meta/history", ) + if os.path.exists(join(tmp_dir, "envs", env_name, "conda-meta", "frozen")): + post_t.add( + join(tmp_dir, "envs", env_name, "conda-meta", "frozen"), + f"envs/{env_name}/conda-meta/frozen", + ) extra_files = copy_extra_files(info.get("extra_files", []), tmp_dir) for path in extra_files: diff --git a/constructor/winexe.py b/constructor/winexe.py index cd1cf05aa..b9c271c7a 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -87,6 +87,9 @@ def setup_envs_commands(info, dir_path): "lockfile_txt_abspath": join(dir_path, "conda-meta", "initial-state.explicit.txt"), "conda_meta": r"$INSTDIR\conda-meta", "history_abspath": join(dir_path, "conda-meta", "history"), + "frozen_abspath": join(dir_path, "conda-meta", "frozen") + if info.get("freeze_base", {}).get("conda") is not None + else "", "final_channels": get_final_channels(info), "shortcuts": shortcuts_flags(info), "register_envs": str(info.get("register_envs", True)).lower(), @@ -115,6 +118,9 @@ def setup_envs_commands(info, dir_path): ), "conda_meta": join("$INSTDIR", "envs", env_name, "conda-meta"), "history_abspath": join(dir_path, "envs", env_name, "conda-meta", "history"), + "frozen_abspath": join(dir_path, "envs", env_name, "conda-meta", "frozen") + if env_info.get("freeze_env", {}).get("conda") is not None + else "", "final_channels": get_final_channels(channel_info), "shortcuts": shortcuts_flags(env_info), "register_envs": str(info.get("register_envs", True)).lower(), diff --git a/docs/source/construct-yaml.md b/docs/source/construct-yaml.md index 210cc88e9..5d9fb159c 100644 --- a/docs/source/construct-yaml.md +++ b/docs/source/construct-yaml.md @@ -658,6 +658,22 @@ Allowed strings / keys: `hash`, `info.json`, `licenses`, `lockfile`, `pkgs_list` Use the standalone binary to perform the uninstallation on Windows. Requires conda-standalone 24.11.0 or newer. +### `freeze_base` + +Protects the conda base environment against modifications by supported package managers. + +Supported package managers: + - `conda`: Protects against conda modifications + +For `conda`, the dictionary is written into the `frozen` marker file. +See CEP-22 for the `frozen` marker file specification. For example: + +``` +freeze_base: + conda: + message: "This base environment is frozen and cannot be modified." +``` + ## Available selectors - `aarch64` diff --git a/examples/protected_base/construct.yaml b/examples/protected_base/construct.yaml index c43044761..7e2e09a49 100644 --- a/examples/protected_base/construct.yaml +++ b/examples/protected_base/construct.yaml @@ -12,13 +12,18 @@ specs: - python - conda +freeze_base: + conda: {} + extra_envs: - default: + env1: specs: - python - pip - conda + freeze_env: + conda: + message: This environment is frozen. extra_files: - - frozen.json: conda-meta/frozen - - frozen.json: envs/default/conda-meta/frozen + - frozen.json: envs/env2/conda-meta/frozen diff --git a/examples/protected_base/frozen.json b/examples/protected_base/frozen.json index 0967ef424..087c29648 100644 --- a/examples/protected_base/frozen.json +++ b/examples/protected_base/frozen.json @@ -1 +1 @@ -{} +{"message": "This env is frozen via extra_files."} diff --git a/news/1149-add-support-for-frozen-envs b/news/1149-add-support-for-frozen-envs new file mode 100644 index 000000000..e57efd64d --- /dev/null +++ b/news/1149-add-support-for-frozen-envs @@ -0,0 +1,19 @@ +### Enhancements + +* Add support for installing [protected (frozen) conda environments](https://conda.org/learn/ceps/cep-0022#specification). (#1149) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_examples.py b/tests/test_examples.py index 2561d9c44..d26b7991a 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -9,6 +9,7 @@ import time import warnings import xml.etree.ElementTree as ET +from contextlib import nullcontext from datetime import timedelta from functools import cache from pathlib import Path @@ -1422,39 +1423,6 @@ def test_output_files(tmp_path): assert files_exist == [] -@pytest.mark.xfail( - condition=( - CONDA_EXE == StandaloneExe.CONDA - and check_version(CONDA_EXE_VERSION, min_version="25.5.0", max_version="25.7.0") - ), - reason="conda-standalone 25.5.x fails with protected environments", - strict=True, -) -def test_frozen_environment(tmp_path, request): - input_path = _example_path("protected_base") - for installer, install_dir in create_installer(input_path, tmp_path): - _run_installer( - input_path, - installer, - install_dir, - request=request, - uninstall=False, - ) - - expected_frozen_paths = { - install_dir / "conda-meta" / "frozen", - install_dir / "envs" / "default" / "conda-meta" / "frozen", - } - - actual_frozen_paths = set() - for env in install_dir.glob("**/conda-meta/history"): - frozen_file = env.parent / "frozen" - if frozen_file.exists(): - actual_frozen_paths.add(frozen_file) - - assert expected_frozen_paths == actual_frozen_paths - - def test_regressions(tmp_path, request): input_path = _example_path("regressions") for installer, install_dir in create_installer(input_path, tmp_path): @@ -1502,3 +1470,57 @@ def test_not_in_installed_menu_list_(tmp_path, request, no_registry): assert is_in_installed_apps_menu == (no_registry == 0), ( f"Unable to find program '{partial_name}' in the 'Installed apps' menu" ) + + +@pytest.mark.xfail( + condition=( + CONDA_EXE == StandaloneExe.CONDA + and check_version(CONDA_EXE_VERSION, min_version="25.5.0", max_version="25.7.0") + ), + reason="conda-standalone 25.5.x fails with protected environments", + strict=True, +) +@pytest.mark.parametrize( + "has_conflict", + ( + pytest.param(True, id="with-conflict"), + pytest.param(False, id="without-conflict"), + ), +) +def test_frozen_environment(tmp_path, request, has_conflict): + example_path = _example_path("protected_base") + input_path = tmp_path / "input" + + context = pytest.raises(subprocess.CalledProcessError) if has_conflict else nullcontext() + + shutil.copytree(str(example_path), str(input_path)) + + yaml = YAML() + with open(input_path / "construct.yaml") as f: + config = yaml.load(f) + + if has_conflict: + config.setdefault("extra_files", []).append({"frozen.json": "conda-meta/frozen"}) + with open(input_path / "construct.yaml", "w") as f: + yaml.dump(config, f) + + with context as c: + for installer, install_dir in create_installer(input_path, tmp_path): + _run_installer(input_path, installer, install_dir, request=request, uninstall=False) + + expected_frozen = { + install_dir / "conda-meta" / "frozen": config["freeze_base"]["conda"], + install_dir / "envs" / "env1" / "conda-meta" / "frozen": config["extra_envs"][ + "env1" + ]["freeze_env"]["conda"], + } + + for frozen_path, expected_content in expected_frozen.items(): + assert frozen_path.is_file() + assert json.loads(frozen_path.read_text()) == expected_content + + if has_conflict: + assert all( + s in c.value.stderr + for s in ("RuntimeError", "freeze_base / freeze_env", "extra_files", "base") + )