diff --git a/e2e/conftest.py b/e2e/conftest.py index 80f3ca76..0eecf7ee 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -13,8 +13,8 @@ from questionpy.form import FormModel, text_input from questionpy_common.api.qtype import QuestionTypeInterface from questionpy_common.environment import PackageInitFunction +from questionpy_common.package_location import FunctionPackageLocation from questionpy_sdk.webserver.server import WebServer -from questionpy_server.worker.runtime.package_location import FunctionPackageLocation @pytest.fixture diff --git a/examples/dep-consumer/qpy_config.yml b/examples/dep-consumer/qpy_config.yml index 35df5161..f022206b 100644 --- a/examples/dep-consumer/qpy_config.yml +++ b/examples/dep-consumer/qpy_config.yml @@ -9,4 +9,4 @@ name: languages: [de, en] dependencies: qpy: - - ../../local-dep-0.1.0.qpy + - "@local/dep" diff --git a/frontend/src/types/Manifest.generated.ts b/frontend/src/types/Manifest.generated.ts index 36037514..9e4f42bf 100644 --- a/frontend/src/types/Manifest.generated.ts +++ b/frontend/src/types/Manifest.generated.ts @@ -9,6 +9,10 @@ export type PackageType = 'LIBRARY' | 'QUESTIONTYPE' | 'QUESTION' export type EnvironmentVariableName = string +export type DistQPyDependency = DistStaticQPyDependency | DistDynamicQPyDependency +export type Version = string | null +export type DependencyLockStrategy = 'required' | 'preferred-no-downgrade' | 'preferred-allow-downgrade' +export type Qpy = DistQPyDependency[] /** * Represents a package manifest. @@ -64,9 +68,27 @@ export interface PackageFile { size: number } export interface DistDependencies { - qpy: DistStaticQPyDependency[] + qpy: Qpy } export interface DistStaticQPyDependency { - dir_name: string + namespace: string + short_name: string + version: string + dependencies: DistDependencies1 hash: string } +export interface DistDependencies1 { + qpy: Qpy +} +export interface DistDynamicQPyDependency { + namespace: string + short_name: string + version: Version + include_prereleases: boolean + locked: LockedDependencyInfo | null +} +export interface LockedDependencyInfo { + strategy: DependencyLockStrategy + locked_version: string + locked_hash: string +} diff --git a/poetry.lock b/poetry.lock index 575a2eeb..66765bdc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1696,7 +1696,7 @@ files = [ [[package]] name = "questionpy-server" -version = "0.8.0" +version = "0.9.0" description = "QuestionPy application server" optional = false python-versions = ">=3.12,<4.0" @@ -1711,14 +1711,15 @@ psutil = ">=7.0.0,<8.0.0" pydantic = ">=2.11.4,<3.0.0" pydantic-settings = ">=2.9.1,<3.0.0" pyyaml = ">=6.0.2,<7.0.0" +resolvelib = ">=1.2.1,<2.0.0" semver = ">=3.0.4,<4.0.0" watchdog = ">=6.0.0,<7.0.0" [package.source] type = "git" url = "https://github.com/questionpy-org/questionpy-server.git" -reference = "32833396b7547d7a616cfc6249f2174153181bbd" -resolved_reference = "32833396b7547d7a616cfc6249f2174153181bbd" +reference = "8507508fc346470c8aac06e4cd58359f0b04998a" +resolved_reference = "8507508fc346470c8aac06e4cd58359f0b04998a" [[package]] name = "requests" @@ -1742,6 +1743,23 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "resolvelib" +version = "1.2.1" +description = "Resolve abstract dependencies into concrete ones" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "resolvelib-1.2.1-py3-none-any.whl", hash = "sha256:fb06b66c8da04172d9e72a21d7d06186d8919e32ae5ab5cdf5b9d920be805ac2"}, + {file = "resolvelib-1.2.1.tar.gz", hash = "sha256:7d08a2022f6e16ce405d60b68c390f054efcfd0477d4b9bd019cc941c28fad1c"}, +] + +[package.extras] +lint = ["mypy", "ruff", "types-requests"] +release = ["build", "towncrier", "twine"] +test = ["packaging", "pytest"] + [[package]] name = "ruff" version = "0.11.12" @@ -2028,4 +2046,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = ">=3.12, <4.0" -content-hash = "c79c59d730c9352fc1162cb195235875f81f4de272e0171814b497f5c0c954cc" +content-hash = "225f303e38383db250f1b66976f2959fb015afed3852ccbd30fcd99ba729d9a2" diff --git a/pyproject.toml b/pyproject.toml index 42cfd4c2..3215d523 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "questionpy-sdk" description = "Library and toolset for the development of QuestionPy packages" license = "MIT" urls = { homepage = "https://questionpy.org" } -version = "0.6.0" +version = "0.7.0" authors = [ { name = "TU Berlin innoCampus" }, { email = "info@isis.tu-berlin.de" } @@ -18,7 +18,7 @@ dependencies = [ "aiohttp >=3.11.18, <4.0.0", "pydantic >=2.11.4, <3.0.0", "PyYAML >=6.0.2, <7.0.0", - "questionpy-server @ git+https://github.com/questionpy-org/questionpy-server.git@32833396b7547d7a616cfc6249f2174153181bbd", + "questionpy-server @ git+https://github.com/questionpy-org/questionpy-server.git@8507508fc346470c8aac06e4cd58359f0b04998a", "jinja2 >=3.1.6, <4.0.0", "aiohttp-jinja2 >=1.6, <2.0", "lxml[html-clean] >=5.4.0, <5.5.0", diff --git a/questionpy_sdk/_dependency_resolver.py b/questionpy_sdk/_dependency_resolver.py new file mode 100644 index 00000000..18e4ad54 --- /dev/null +++ b/questionpy_sdk/_dependency_resolver.py @@ -0,0 +1,66 @@ +import asyncio +import bisect +import logging +from collections.abc import Iterable, Sequence +from itertools import chain +from pathlib import Path + +from semver import Version + +from questionpy_common import PackageNamespaceAndShortName +from questionpy_common.package_location import PackageLocation, ZipPackageLocation +from questionpy_common.version_specifiers import QPyDependencyVersionSpecifier +from questionpy_server.dependencies import AvailablePackageVersion, DynamicDependencyResolver, NoPackageWithHashError +from questionpy_server.hash import calculate_hash +from questionpy_server.utils.manifest import ManifestError, read_manifest_from_zip + + +class SdkDynamicDependencyResolver(DynamicDependencyResolver): + def __init__(self, dirs: Sequence[Path]) -> None: + self._dirs = dirs + + self._packages: dict[PackageNamespaceAndShortName, list[AvailablePackageVersion]] = {} + self._locations: dict[str, PackageLocation] = {} + + qpy_file: Path + for qpy_file in chain.from_iterable(dir_.glob("*.qpy") for dir_ in self._dirs): + hash_ = calculate_hash(qpy_file) + + try: + manifest = asyncio.run(read_manifest_from_zip(qpy_file)) + except ManifestError: + _log.warning("Failed to read manifest of package '%s'", qpy_file, exc_info=True) + continue + + apv = AvailablePackageVersion(manifest, hash_, Version.parse(manifest.version)) + + versions = self._packages.setdefault(apv.manifest.nssn, []) + bisect.insort(versions, apv, key=lambda p: p.manifest.version) + if apv.hash not in self._locations: + self._locations[apv.hash] = ZipPackageLocation(qpy_file.absolute(), hash_) + + def get_matching_versions( + self, + nssn: PackageNamespaceAndShortName, + version_spec: QPyDependencyVersionSpecifier | None, + *, + include_prereleases: bool, + ) -> Iterable[AvailablePackageVersion]: + versions_for_nssn = self._packages.get(nssn, ()) + + # We could speed this up with bisections, but it's probably not worth the effort. + return [ + package_version + for package_version in versions_for_nssn + if (include_prereleases or package_version.version.prerelease is None) + and (version_spec is None or version_spec.allows(package_version.version)) + ] + + async def get_package_location(self, hash_: str) -> PackageLocation: + try: + return self._locations[hash_] + except KeyError as e: + raise NoPackageWithHashError(hash_) from e + + +_log = logging.getLogger(__name__) diff --git a/questionpy_sdk/_package/_builder.py b/questionpy_sdk/_package/_builder.py index 6d28ab8a..0c3b1db9 100644 --- a/questionpy_sdk/_package/_builder.py +++ b/questionpy_sdk/_package/_builder.py @@ -1,3 +1,4 @@ +import asyncio import inspect import logging import os @@ -5,7 +6,7 @@ import subprocess import tempfile import zipfile -from collections.abc import Iterator +from collections.abc import Mapping, Sequence from mimetypes import guess_type from pathlib import Path from typing import ClassVar @@ -16,8 +17,16 @@ import questionpy from questionpy import i18n -from questionpy_common.constants import DIST_DIR, MANIFEST_FILENAME -from questionpy_common.manifest import DistStaticQPyDependency, Manifest, PackageFile +from questionpy_common import PackageNamespaceAndShortName +from questionpy_common.constants import MANIFEST_FILENAME +from questionpy_common.manifest import ( + DistDynamicQPyDependency, + DistQPyDependency, + DistStaticQPyDependency, + LockedDependencyInfo, + Manifest, + PackageFile, +) from questionpy_sdk._i18n_utils import bcp47_to_posix from questionpy_sdk._package._ignores import create_ignore_spec from questionpy_sdk._package._targets import BuildTarget, DirBuildTarget @@ -28,36 +37,36 @@ ) from questionpy_sdk._package.errors import PackageBuildError from questionpy_sdk._package.source import PackageSource -from questionpy_sdk.models import BuildHookName, SourceStaticQPyDependency +from questionpy_sdk.models import ( + AbstractDynamicQPyDependency, + BuildHookName, + SourceDynamicQPyDependency, + SourceStaticQPyDependency, +) +from questionpy_server.dependencies import ( + DynamicDependencyResolver, + NoopDependencyResolver, + resolve_dependency_tree, +) from questionpy_server.hash import calculate_hash +from questionpy_server.utils.manifest import read_manifest_from_zip _log = logging.getLogger(__name__) _PYTHON_TEMP_PATHS = GitIgnoreSpec.from_lines(("*.pyc", "__pycache__")) -def _iterate_recursive_dependencies( - package_root: zipfile.Path, - *, - stack: tuple[str, ...] = (), - manifest: Manifest | None = None, -) -> Iterator[tuple[str, ...]]: - if not manifest: - manifest = Manifest.model_validate_json((package_root / DIST_DIR / MANIFEST_FILENAME).read_text()) - - stack = (*stack, manifest.identifier) - yield stack - - for dep in manifest.dependencies.qpy: - yield from _iterate_recursive_dependencies( - package_root / DIST_DIR / "dependencies" / "qpy" / dep.dir_name, stack=stack - ) - - class PackageBuilder: STATIC_FILE_GLOBS: ClassVar[set[str]] = {"css/**/*", "js/**/*", "assets/**/*"} - def __init__(self, source: PackageSource, target: BuildTarget, *, copy_sources: bool) -> None: + def __init__( + self, + source: PackageSource, + target: BuildTarget, + *, + copy_sources: bool, + dependency_resolver: DynamicDependencyResolver, + ) -> None: self._source = source self._target = target @@ -70,6 +79,8 @@ def __init__(self, source: PackageSource, target: BuildTarget, *, copy_sources: self._ignore_spec = create_ignore_spec(self._source.path, self._source.config.ignore) + self._dynamic_dep_resolver = dependency_resolver + def write_package(self) -> None: """Writes the package to the filesystem. @@ -79,7 +90,8 @@ def write_package(self) -> None: self._run_build_hooks("pre") self._install_questionpy() self._install_requirements() - self._install_static_qpy_dependencies() + static_deps = self._install_static_qpy_dependencies() + self._lock_dynamic_dependencies(static_deps) self._write_package_files() self._compile_pos() if self._copy_sources: @@ -142,7 +154,7 @@ def _install_requirements(self) -> None: msg = f"Failed to install requirements: {exc.stderr.decode()}" raise PackageBuildError(msg) from exc - def _install_static_qpy_dependencies(self) -> None: + def _install_static_qpy_dependencies(self) -> dict[Path, DistStaticQPyDependency]: # Ignoring duplicates, we collect all the dependencies in the dependencies directory and those explicitly # listed. dependency_paths = set((self._source.path / "dependencies").glob("*.qpy")) @@ -150,45 +162,72 @@ def _install_static_qpy_dependencies(self) -> None: if not isinstance(dep, SourceStaticQPyDependency): continue - dep_path = dep.path if dep.path.is_absolute() else (self._source.path / dep.path) - if not dep_path.exists(): - msg = f"The specified dependency '{dep.path}' does not exist." + dep_path_abs = (self._source.path / dep.path).absolute() + if not dep_path_abs.exists(): + msg = f"The specified dependency '{dep_path_abs}' does not exist." raise PackageBuildError(msg) - dependency_paths.add(dep_path) + dependency_paths.add(dep_path_abs) - installed_deps: dict[str, tuple[str, ...]] = {} + installed_deps: dict[Path, DistStaticQPyDependency] = {} # Then, we copy all of their contents into a directory in dist/dependencies/qpy. - for dep_path in dependency_paths: - _log.info("Copying static QPy dependency '%s'", dep_path) + for dep_path_abs in dependency_paths: + _log.info("Copying static QPy dependency '%s'", dep_path_abs) - with dep_path.open("rb+") as dep_package_file, zipfile.ZipFile(dep_package_file) as dep_package_zf: + with dep_path_abs.open("rb+") as dep_package_file, zipfile.ZipFile(dep_package_file) as dep_package_zf: dep_hash = calculate_hash(dep_package_file) - dep_manifest = Manifest.model_validate_json(dep_package_zf.read(f"{DIST_DIR}/{MANIFEST_FILENAME}")) - - # Any duplicate dependencies in the tree, even if the same version, would lead to an error at runtime, - # so we prevent them here. - for nested_dep_stack in _iterate_recursive_dependencies( - zipfile.Path(dep_package_zf), manifest=dep_manifest, stack=(self._source.config.identifier,) - ): - already_installed_through = installed_deps.get(nested_dep_stack[-1]) - if already_installed_through is not None: - msg = ( - f"The dependency '{nested_dep_stack[-1]}' is required through " - f"'{' -> '.join(nested_dep_stack)}', but has already been installed though " - f"'{' -> '.join(already_installed_through)}'. This is not allowed even when both " - f"require the same version." - ) - raise PackageBuildError(msg) - - installed_deps[nested_dep_stack[-1]] = nested_dep_stack + dep_manifest = asyncio.run(read_manifest_from_zip(dep_package_zf)) dep_dir_name = f"{dep_manifest.namespace}-{dep_manifest.short_name}-{dep_manifest.version}" dest_path = self._target.dist / "dependencies" / "qpy" / dep_dir_name dep_package_zf.extractall(dest_path) - self._manifest.dependencies.qpy.append(DistStaticQPyDependency(dir_name=dep_dir_name, hash=dep_hash)) + dist_dep = DistStaticQPyDependency( + namespace=dep_manifest.namespace, + short_name=dep_manifest.short_name, + version=str(dep_manifest.version), + dependencies=dep_manifest.dependencies, + hash=dep_hash, + ) + + installed_deps[dep_path_abs] = dist_dep + self._manifest.dependencies.qpy.append(dist_dep) + + return installed_deps + + def _lock_dynamic_dependencies(self, static_dependencies: dict[Path, DistStaticQPyDependency]) -> None: + if not self._source.config.dependencies.qpy: + return + + # Even when no dynamic dependencies use locking, resolve_dependency_tree checks the tree for consistency. + root_deps = _convert_source_deps(self._source, static_dependencies) + resolution = resolve_dependency_tree(self._source.config, root_deps, self._dynamic_dep_resolver) + + for source_dep in self._source.config.dependencies.qpy: + if not isinstance(source_dep, SourceDynamicQPyDependency): + continue + + lock_strategy = source_dep.lock + if lock_strategy is None: + lock_strategy = self._source.config.lock_dependencies + + if not lock_strategy: + lock_info = None + else: + chosen_version = resolution[PackageNamespaceAndShortName(source_dep.namespace, source_dep.short_name)] + lock_info = LockedDependencyInfo( + strategy=lock_strategy, + locked_version=str(chosen_version.version), + locked_hash=chosen_version.hash, + ) + + self._manifest.dependencies.qpy.append( + DistDynamicQPyDependency( + **AbstractDynamicQPyDependency.model_dump(source_dep), + locked=lock_info, + ) + ) def _write_package_files(self) -> None: """Writes custom package files.""" @@ -322,10 +361,35 @@ def _copy_glob( self._add_to_static_files(dest_path) -def build_qpy_package(source: PackageSource, target: BuildTarget | None = None, *, copy_sources: bool = True) -> None: +def build_qpy_package( + source: PackageSource, + target: BuildTarget | None = None, + *, + copy_sources: bool = True, + dependency_resolver: DynamicDependencyResolver | None = None, +) -> None: + if not dependency_resolver: + dependency_resolver = NoopDependencyResolver() + if not target: target = DirBuildTarget.in_source(source) with target: - builder = PackageBuilder(source, target, copy_sources=copy_sources) + builder = PackageBuilder(source, target, copy_sources=copy_sources, dependency_resolver=dependency_resolver) builder.write_package() + + +def _convert_source_deps( + source: PackageSource, source_static_deps: Mapping[Path, DistStaticQPyDependency] +) -> Sequence[DistQPyDependency]: + """Converts `SourceQPyDependency` models from the `PackageConfig` into `DistQPyDependency` models. + + `DistStaticQPyDependency` models were already built when the static dependencies were installed, and we can build + `DistDynamicQPyDependency` by just copying the `SourceDynamicQPyDependency`. + """ + return [ + source_static_deps[(source.path / dep.path).absolute()] + if isinstance(dep, SourceStaticQPyDependency) + else DistDynamicQPyDependency(**dep.model_dump()) + for dep in source.config.dependencies.qpy + ] diff --git a/questionpy_sdk/commands/_helper.py b/questionpy_sdk/commands/_helper.py index 9b26f6dc..c46defdf 100644 --- a/questionpy_sdk/commands/_helper.py +++ b/questionpy_sdk/commands/_helper.py @@ -9,15 +9,16 @@ from pydantic import ValidationError from questionpy_common.constants import DIST_DIR, MANIFEST_FILENAME -from questionpy_sdk._package import build_qpy_package -from questionpy_sdk._package.errors import PackageBuildError, PackageSourceValidationError -from questionpy_sdk._package.source import PackageSource -from questionpy_server.hash import calculate_hash -from questionpy_server.worker.runtime.package_location import ( +from questionpy_common.package_location import ( DirPackageLocation, PackageLocation, ZipPackageLocation, ) +from questionpy_sdk._package import build_qpy_package +from questionpy_sdk._package.errors import PackageBuildError, PackageSourceValidationError +from questionpy_sdk._package.source import PackageSource +from questionpy_server.dependencies import DynamicDependencyResolver +from questionpy_server.hash import calculate_hash def _get_dir_package_location(source_path: Path) -> DirPackageLocation: @@ -28,14 +29,16 @@ def _get_dir_package_location(source_path: Path) -> DirPackageLocation: raise click.ClickException(msg) from exc -def _get_dir_package_location_from_source(pkg_string: str, source_path: Path) -> DirPackageLocation: +def _get_dir_package_location_from_source( + pkg_string: str, source_path: Path, dependency_resolver: DynamicDependencyResolver +) -> DirPackageLocation: # Always rebuild package. try: package_source = PackageSource(source_path) except PackageSourceValidationError as exc: raise click.ClickException(str(exc)) from exc try: - build_qpy_package(package_source) + build_qpy_package(package_source, dependency_resolver=dependency_resolver) click.echo(f"Successfully built package '{pkg_string}'.") except PackageBuildError as exc: msg = f"Failed to build package: {exc}" @@ -44,13 +47,15 @@ def _get_dir_package_location_from_source(pkg_string: str, source_path: Path) -> return _get_dir_package_location(source_path / DIST_DIR) -def get_package_location(pkg_string: str, pkg_path: Path) -> PackageLocation: +def get_package_location( + pkg_string: str, pkg_path: Path, dependency_resolver: DynamicDependencyResolver +) -> PackageLocation: if pkg_path.is_dir(): # dist dir if (pkg_path / MANIFEST_FILENAME).is_file(): return _get_dir_package_location(pkg_path) # source dir - return _get_dir_package_location_from_source(pkg_string, pkg_path) + return _get_dir_package_location_from_source(pkg_string, pkg_path, dependency_resolver=dependency_resolver) if zipfile.is_zipfile(pkg_path): return ZipPackageLocation(pkg_path, calculate_hash(pkg_path)) diff --git a/questionpy_sdk/commands/package.py b/questionpy_sdk/commands/package.py index e61ec89d..d2b5ef82 100644 --- a/questionpy_sdk/commands/package.py +++ b/questionpy_sdk/commands/package.py @@ -1,15 +1,18 @@ # This file is part of the QuestionPy SDK. (https://questionpy.org) # The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md. # (c) Technische Universität Berlin, innoCampus +from collections.abc import Sequence from pathlib import Path import click from questionpy_common.constants import DIST_DIR +from questionpy_sdk._dependency_resolver import SdkDynamicDependencyResolver from questionpy_sdk._package import ZipBuildTarget, build_qpy_package from questionpy_sdk._package.errors import PackageBuildError, PackageSourceValidationError from questionpy_sdk._package.source import PackageSource from questionpy_sdk.commands._helper import confirm_overwrite +from questionpy_server.dependencies import DynamicDependencyResolver def validate_out_path(context: click.Context, _parameter: click.Parameter, value: Path | None) -> Path | None: @@ -44,11 +47,19 @@ def validate_out_path(context: click.Context, _parameter: click.Parameter, value help="Don't copy package sources into the .qpy file.", ) @click.option("--force", "-f", "allow_overwrite", is_flag=True, help="Force overwriting of output file.") +@click.option( + "--local-deps-from", + "-L", + type=click.Path(exists=True, file_okay=False, path_type=Path), + help="Path to a local directory to search for dependencies in.", + multiple=True, +) @click.pass_context def package( ctx: click.Context, source: Path, out_path: Path | None, + local_deps_from: Sequence[Path], *, allow_overwrite: bool, development: bool, @@ -69,8 +80,10 @@ def package( msg = f"The options {param_name} and --dev are mutually exclusive." raise click.UsageError(msg, ctx=ctx) + dependency_resolver = SdkDynamicDependencyResolver(local_deps_from) + if development: - create_dist(package_source) + create_dist(package_source, dependency_resolver=dependency_resolver) else: if not out_path: @@ -81,13 +94,18 @@ def package( out_path /= package_source.normalized_filename create_qpy_package( - ctx, package_source, out_path, allow_overwrite=allow_overwrite, without_sources=without_sources + ctx, + package_source, + out_path, + allow_overwrite=allow_overwrite, + without_sources=without_sources, + dependency_resolver=dependency_resolver, ) -def create_dist(package_source: PackageSource) -> None: +def create_dist(package_source: PackageSource, *, dependency_resolver: DynamicDependencyResolver) -> None: try: - build_qpy_package(package_source) + build_qpy_package(package_source, dependency_resolver=dependency_resolver) except PackageBuildError as exc: msg = f"Failed to build package: {exc}" raise click.ClickException(msg) from exc @@ -96,7 +114,13 @@ def create_dist(package_source: PackageSource) -> None: def create_qpy_package( - ctx: click.Context, package_source: PackageSource, out_path: Path, *, allow_overwrite: bool, without_sources: bool + ctx: click.Context, + package_source: PackageSource, + out_path: Path, + *, + allow_overwrite: bool, + without_sources: bool, + dependency_resolver: DynamicDependencyResolver, ) -> None: if out_path.exists() and not allow_overwrite: if not ctx.obj["no_interaction"]: @@ -107,7 +131,10 @@ def create_qpy_package( try: build_qpy_package( - package_source, ZipBuildTarget(out_path, allow_overwrite=allow_overwrite), copy_sources=not without_sources + package_source, + ZipBuildTarget(out_path, allow_overwrite=allow_overwrite), + copy_sources=not without_sources, + dependency_resolver=dependency_resolver, ) except PackageBuildError as exc: msg = f"Failed to build package: {exc}" diff --git a/questionpy_sdk/commands/repo/_helper.py b/questionpy_sdk/commands/repo/_helper.py index ed58d8ab..9adcc36f 100644 --- a/questionpy_sdk/commands/repo/_helper.py +++ b/questionpy_sdk/commands/repo/_helper.py @@ -8,13 +8,15 @@ from pathlib import Path from zipfile import ZipFile +import semver + from questionpy_common.constants import DIST_DIR, MANIFEST_FILENAME +from questionpy_common.manifest import Manifest from questionpy_server.hash import calculate_hash from questionpy_server.repository.models import RepoMeta, RepoPackageIndex, RepoPackageVersion, RepoPackageVersions -from questionpy_server.utils.manifest import ComparableManifest -def get_manifest(path: Path) -> ComparableManifest: +def get_manifest(path: Path) -> Manifest: """Reads the manifest of a package. Args: @@ -25,7 +27,7 @@ def get_manifest(path: Path) -> ComparableManifest: """ with ZipFile(path) as zip_file: raw_manifest = zip_file.read(f"{DIST_DIR}/{MANIFEST_FILENAME}") - return ComparableManifest.model_validate_json(raw_manifest) + return Manifest.model_validate_json(raw_manifest) class IndexCreator: @@ -36,7 +38,7 @@ def __init__(self, root: Path): self._root.mkdir(parents=True, exist_ok=True) self._packages: dict[str, RepoPackageVersions] = {} - def add(self, path: Path, manifest: ComparableManifest) -> None: + def add(self, path: Path, manifest: Manifest) -> None: """Adds a package to the repository index. Args: @@ -45,7 +47,7 @@ def add(self, path: Path, manifest: ComparableManifest) -> None: """ # Create RepoPackageVersion. version = RepoPackageVersion( - version=str(manifest.version), + version=semver.Version.parse(manifest.version), api_version=manifest.api_version, path=str(path.relative_to(self._root)), size=path.stat().st_size, diff --git a/questionpy_sdk/commands/run.py b/questionpy_sdk/commands/run.py index 37353cf5..08f2660d 100644 --- a/questionpy_sdk/commands/run.py +++ b/questionpy_sdk/commands/run.py @@ -3,11 +3,14 @@ # (c) Technische Universität Berlin, innoCampus import asyncio +from collections.abc import Sequence from pathlib import Path from typing import TYPE_CHECKING, Literal import click +from questionpy_common.package_location import DirPackageLocation +from questionpy_sdk._dependency_resolver import SdkDynamicDependencyResolver from questionpy_sdk.commands._helper import get_package_location from questionpy_sdk.constants import DEFAULT_STATE_STORAGE_PATH from questionpy_sdk.watcher import Watcher @@ -16,7 +19,6 @@ from questionpy_sdk.webserver.server import WebServerArgs from questionpy_server.worker.impl.subprocess import SubprocessWorker from questionpy_server.worker.impl.thread import ThreadWorker -from questionpy_server.worker.runtime.package_location import DirPackageLocation if TYPE_CHECKING: from collections.abc import Coroutine @@ -58,11 +60,19 @@ async def async_run(webserver_args: WebServerArgs) -> None: show_default=True, help="The worker implementation to use. Thread workers offer no isolation but may improve debugging experience.", ) +@click.option( + "--local-deps-from", + "-L", + type=click.Path(exists=True, file_okay=False, path_type=Path), + help="Path to a local directory to search for dependencies in.", + multiple=True, +) def run( package: str, state_storage_path: Path, host: str, port: int, + local_deps_from: Sequence[Path], *, watch: bool, worker: Literal["subprocess", "thread"], @@ -75,8 +85,10 @@ def run( - a dist directory, or - a source directory (built on-the-fly). """ # noqa: D301 + dependency_resolver = SdkDynamicDependencyResolver(local_deps_from) + pkg_path = Path(package).resolve() - pkg_location = get_package_location(package, pkg_path) + pkg_location = get_package_location(package, pkg_path, dependency_resolver=dependency_resolver) coro: Coroutine webserver_args = WebServerArgs( @@ -85,6 +97,7 @@ def run( host=host, port=port, worker_class=ThreadWorker if worker == "thread" else SubprocessWorker, + dependency_resolver=dependency_resolver, ) if watch: diff --git a/questionpy_sdk/models.py b/questionpy_sdk/models.py index 4d247bab..ac9c607d 100644 --- a/questionpy_sdk/models.py +++ b/questionpy_sdk/models.py @@ -1,17 +1,21 @@ # This file is part of the QuestionPy SDK. (https://questionpy.org) # The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md. # (c) Technische Universität Berlin, innoCampus - +import re from collections.abc import Mapping from pathlib import Path -from typing import Literal, Self +from typing import Any, Literal, Self, cast -from pydantic import BaseModel, ModelWrapValidatorHandler, model_validator +from pydantic import BaseModel, Field, ModelWrapValidatorHandler, field_validator, model_validator +from pydantic_core.core_schema import ValidationInfo -from questionpy_common.manifest import SourceManifest +from questionpy_common.manifest import AbstractDynamicQPyDependency, DependencyLockStrategy, PackageType, SourceManifest +from questionpy_common.version_specifiers import QPyDependencyVersionSpecifier BuildHookName = Literal["pre", "post"] +_SHORT_DYN_DEPENDENCY_PATTERN = re.compile(r"^@(?P[a-z\d_]+)/(?P[a-z\d_]+)\s*") + class SourceStaticQPyDependency(BaseModel): path: Path @@ -21,14 +25,66 @@ class SourceStaticQPyDependency(BaseModel): def _validate(cls, data: object, handler: ModelWrapValidatorHandler[Self]) -> Self: if isinstance(data, str): if data.startswith("@"): - msg = f"The dependency '{data}' looks like a dynamic dependency such as @myns/mypackage:myver." + msg = f"The dependency '{data}' looks like a dynamic dependency such as @myns/mypackage:1.2.3." raise ValueError(msg) return cls(path=Path(data)) return handler(data) -type SourceQPyDependency = SourceStaticQPyDependency +class SourceDynamicQPyDependency(AbstractDynamicQPyDependency): + lock: DependencyLockStrategy | Literal[False] | None = None + """Whether and how to lock this dependency when packaging. + + The default is based on the package type: + - `QUESTION` and `QUESTIONTYPE` default to `preferred-no-downgrade`. + - `LIBRARY` default to `False`, i.e. no locking. + """ + + @model_validator(mode="wrap") + @classmethod + def _validate_from_str(cls, data: object, handler: ModelWrapValidatorHandler[Self]) -> Self: + if isinstance(data, str): + data = data.strip() + + match = _SHORT_DYN_DEPENDENCY_PATTERN.match(data) + if not match: + msg = ( + f"The dependency '{data}' doesn't look like a dynamic dependency, which should take the form of " + f"'@myns/mypackage ^= 1.2.3' or '@myns/mypackage == 1.2.3-rc.2'." + ) + raise ValueError(msg) + + if match.group(0) == data: + # There is nothing more (i.e., no version specifier) in the dependency string. + # This means to use the newest version of the dependency. + version_specifier = None + else: + # There's more content, which we expect to be a version specifier. + version_specifier = QPyDependencyVersionSpecifier.from_string(data[match.span(0)[1] :]) + + return cls( + namespace=match.group("ns"), + short_name=match.group("sn"), + version=version_specifier, + include_prereleases=False, + lock=None, + ) + + return handler(data) + + @model_validator(mode="after") + def _validate_prereleases(self) -> Self: + if self.version: + for clause in self.version.clauses: + if "-" in clause.operator: + msg = f"include_prereleases is not set, yet '{clause}' compares to a prerelease." + raise ValueError(msg) + + return self + + +type SourceQPyDependency = SourceStaticQPyDependency | SourceDynamicQPyDependency class SourceDependencies(BaseModel): @@ -44,8 +100,21 @@ class PackageConfig(SourceManifest): build_hooks: Mapping[BuildHookName, str | list[str]] = {} ignore: list[str] = [] + lock_dependencies: DependencyLockStrategy | Literal[False] = Field(default=cast("Any", None), validate_default=True) dependencies: SourceDependencies = SourceDependencies() def to_manifest(self) -> SourceManifest: """Creates [`SourceManifest`][questionpy_common.manifest.SourceManifest] from config model.""" return SourceManifest.model_validate(dict(self)) + + @field_validator("lock_dependencies", mode="before") + @classmethod + def _default_lock_strategy(cls, value: object, info: ValidationInfo) -> object: + if value is None: + match info.data["type"]: + case PackageType.QUESTION | PackageType.QUESTIONTYPE: + return "preferred-no-downgrade" + case PackageType.LIBRARY: + return False + + return value diff --git a/questionpy_sdk/webserver/manifest.py b/questionpy_sdk/webserver/manifest.py index ce829456..304cf69a 100644 --- a/questionpy_sdk/webserver/manifest.py +++ b/questionpy_sdk/webserver/manifest.py @@ -10,7 +10,7 @@ from questionpy_common.constants import DIST_DIR, MANIFEST_FILENAME, MAX_MANIFEST_SIZE from questionpy_common.manifest import Manifest -from questionpy_server.worker.runtime.package_location import ( +from questionpy_common.package_location import ( DirPackageLocation, FunctionPackageLocation, PackageLocation, diff --git a/questionpy_sdk/webserver/server.py b/questionpy_sdk/webserver/server.py index 7fc26657..54cefeac 100644 --- a/questionpy_sdk/webserver/server.py +++ b/questionpy_sdk/webserver/server.py @@ -12,16 +12,17 @@ from questionpy_common.environment import PackagePermissions from questionpy_common.manifest import Manifest +from questionpy_common.package_location import PackageLocation from questionpy_sdk.webserver.middlewares.controller import inject_controller_middleware from questionpy_sdk.webserver.middlewares.error import api_error_middleware, error_middleware from questionpy_sdk.webserver.routes import api_routes from questionpy_sdk.webserver.routes.frontend import routes as frontend_routes from questionpy_sdk.webserver.state import FilesystemStateManager, StateManager from questionpy_server import WorkerPool +from questionpy_server.dependencies import DynamicDependencyResolver, NoopDependencyResolver from questionpy_server.settings import CompletePackagePermissions from questionpy_server.worker import Worker from questionpy_server.worker.impl.subprocess import SubprocessWorker -from questionpy_server.worker.runtime.package_location import PackageLocation from .constants import API_PATH_PREFIX, USE_VITE_DEV_SERVER, WEBSERVER_KEY from .errors import EnvironmentVariablesMissingError @@ -36,6 +37,7 @@ class WebServerArgs(TypedDict): host: NotRequired[str] port: NotRequired[int] worker_class: NotRequired[type[Worker]] + dependency_resolver: NotRequired[DynamicDependencyResolver] class WebServer: @@ -47,6 +49,7 @@ def __init__(self, **kwargs: Unpack[WebServerArgs]) -> None: self._host = kwargs.get("host", "localhost") self._port = kwargs.get("port", 8080) self._worker_class = kwargs.get("worker_class", self.DEFAULT_WORKER_CLASS) + self._dependency_resolver = kwargs.get("dependency_resolver") or NoopDependencyResolver() self._app: web.Application self._api_app: web.Application @@ -74,7 +77,10 @@ async def __aenter__(self) -> Self: # Add worker pool self._worker_pool = await WorkerPool( - max_workers=1, max_memory=self._package_permissions.memory, worker_type=self._worker_class + max_workers=1, + max_memory=self._package_permissions.memory, + worker_type=self._worker_class, + dependency_resolver=self._dependency_resolver, ).__aenter__() # Initialize state manager diff --git a/tests/questionpy_sdk/package/test_builder.py b/tests/questionpy_sdk/package/test_builder.py index 3715eb9f..4e7ac3bc 100644 --- a/tests/questionpy_sdk/package/test_builder.py +++ b/tests/questionpy_sdk/package/test_builder.py @@ -87,7 +87,13 @@ def test_installs_static_dep(tmp_path: Path, source_path: Path, qpy_pkg_path: Pa config_path = source_path / PACKAGE_CONFIG_FILENAME with config_path.open("r") as f: config = yaml.safe_load(f) + + # This is pretty ugly, but we need to change the short name of one of the packages, otherwise we just have a package + # requiring itself. + old_python_dir = source_path / "python" / config["namespace"] / config["short_name"] + config["short_name"] = "consumer_package" config["dependencies"] = {"qpy": [str(qpy_pkg_path)]} + old_python_dir.rename(old_python_dir.parent / "consumer_package") with config_path.open("w") as f: yaml.dump(config, f) diff --git a/tests/questionpy_sdk/webserver/controllers/test_base.py b/tests/questionpy_sdk/webserver/controllers/test_base.py index 86e7a3fc..885a7503 100644 --- a/tests/questionpy_sdk/webserver/controllers/test_base.py +++ b/tests/questionpy_sdk/webserver/controllers/test_base.py @@ -7,9 +7,9 @@ import pytest +from questionpy_common.package_location import FunctionPackageLocation from questionpy_sdk.webserver import WebServer from questionpy_sdk.webserver.controllers.base import BaseController -from questionpy_server.worker.runtime.package_location import FunctionPackageLocation @pytest.mark.parametrize( diff --git a/tests/questionpy_sdk/webserver/test_webserver.py b/tests/questionpy_sdk/webserver/test_webserver.py index e219750e..225e76dd 100644 --- a/tests/questionpy_sdk/webserver/test_webserver.py +++ b/tests/questionpy_sdk/webserver/test_webserver.py @@ -15,17 +15,17 @@ from questionpy.form import FormModel from questionpy_common.api.qtype import QuestionTypeInterface from questionpy_common.constants import DIST_DIR -from questionpy_sdk._package import build_qpy_package -from questionpy_sdk._package._helper import create_normalized_filename -from questionpy_sdk._package.source import PackageSource -from questionpy_sdk.webserver.server import WebServer -from questionpy_server.hash import calculate_hash -from questionpy_server.worker.runtime.package_location import ( +from questionpy_common.package_location import ( DirPackageLocation, FunctionPackageLocation, PackageLocation, ZipPackageLocation, ) +from questionpy_sdk._package import build_qpy_package +from questionpy_sdk._package._helper import create_normalized_filename +from questionpy_sdk._package.source import PackageSource +from questionpy_sdk.webserver.server import WebServer +from questionpy_server.hash import calculate_hash @pytest.fixture