diff --git a/.github/make_ci_environ.py b/.github/make_ci_environ.py new file mode 100644 index 0000000..002ef6b --- /dev/null +++ b/.github/make_ci_environ.py @@ -0,0 +1,88 @@ +import tomllib +from pathlib import Path +import yaml +import os + +def parse_pyproject(path: str, optional_sections_to_skip=None): + with open(path, "rb") as f: + data = tomllib.load(f) + + deps = set() + + # project.dependencies (PEP 621) + for dep in data.get("project", {}).get("dependencies", []): + if "numpy" in dep: + # numpy is also listed in build requirements with a higher version number + # so we skip it here to avoid conflicts. + continue + deps.add(dep) + + # optional dependencies (PEP 621) + if optional_sections_to_skip is None: + optional_sections_to_skip = [] + for group, group_deps in data.get("project", {}).get("optional-dependencies", {}).items(): + if group in optional_sections_to_skip: + print("Skipping optional dependency group:", group) + continue + deps.update(group_deps) + + deps.discard("geoana[all]") + deps.discard("geoana[doc,all]") + deps.discard("geoana[plot,extras,jittable]") + + if "matplotlib" in deps: + deps.remove("matplotlib") + deps.add("matplotlib-base") + return deps + +def create_env_yaml(deps, name="env", python_version=None, free_threaded=False): + conda_pkgs = [] + pip_pkgs = [] + + for dep in deps: + # crude split: try to detect conda vs pip-only packages + if any(dep.startswith(pip_only) for pip_only in ["git+", "http:", "https:", "file:"]): + pip_pkgs.append(dep) + else: + conda_pkgs.append(dep) + + dependencies = conda_pkgs + if pip_pkgs: + dependencies.append({"pip": pip_pkgs}) + + if python_version: + if free_threaded: + dependencies.insert(0, f"python-freethreading={python_version}") + else: + dependencies.insert(0, f"python={python_version}") + + return { + "name": name, + "channels": ["conda-forge"], + "dependencies": dependencies, + } + +if __name__ == "__main__": + pyproject_path = Path("pyproject.toml") + + py_vers = os.environ.get("PYTHON_VERSION", "3.11") + mkl_vers = os.environ.get("MKL_VERSION", "2024") + is_free_threaded = os.environ.get("FREE_THREADED", "false").lower() == "true" + is_coverage = os.environ.get("DO_COVERAGE", "false").lower() == "true" + env_name = os.environ.get("ENV_NAME", "pydiso-ci") + + optional_to_skip = [] + if not is_coverage: + optional_to_skip.append("coverage") + + deps = parse_pyproject(pyproject_path, optional_sections_to_skip=optional_to_skip) + deps.add("mkl-devel") + deps.add("pkg-config") + deps.add(f"mkl={mkl_vers}") + env_data = create_env_yaml(deps, name=env_name, python_version=py_vers, free_threaded=is_free_threaded) + + out_name = "environment_ci.yml" + with open(out_name, "w") as f: + yaml.safe_dump(env_data, f, sort_keys=False) + + print("Generated environment_ci.yml with", len(deps), "dependencies") \ No newline at end of file diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index dc9a96e..81529ce 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -1,20 +1,13 @@ name: Testing -on: - push: - branches: - - '*' - tags: - - 'v*' - pull_request: - branches: - - '*' - schedule: - - cron: "0 13 * * 1" +on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: build-and-test: - name: Testing (Python ${{ matrix.python-version }}, on ${{ matrix.os }}, with MKL ${{ matrix.mkl-version }}) + name: Testing (Python ${{ matrix.python-version }}${{ matrix.free-threaded && 't' || '' }}, on ${{ matrix.os }}, with MKL ${{ matrix.mkl-version }}) runs-on: ${{ matrix.os }} defaults: run: @@ -22,26 +15,61 @@ jobs: strategy: fail-fast: False matrix: - os: [ubuntu-latest, macos-12, windows-latest] - python-version: ["3.10", "3.11", "3.12", "3.13"] - mkl-version: ['2023', '2024'] + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.12", "3.13", "3.14"] + mkl-version: ['2024', '2025'] + free-threaded: [true, false] include: - os: ubuntu-latest python-version: "3.12" - coverage: ${{ true }} - exclude: - - os: macos-12 mkl-version: "2024" + coverage: true + - os: macos-15-intel + python-version: "3.11" + mkl-version: "2023" + free-threaded: + - os: macos-15-intel + python-version: "3.12" + mkl-version: "2023" + free-threaded: + exclude: + - python-version: "3.11" + free-threaded: true + - python-version: "3.12" + free-threaded: true + - python-version: "3.13" + free-threaded: true + mkl-version: "2025" + - python-version: "3.14" + mkl-version: "2025" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.13" + + - name: Create Conda environment file + run: | + python -m pip install pyyaml + python .github/make_ci_environ.py + env: + PYTHON_VERSION: ${{ matrix.python-version}} + MKL_VERSION: ${{ matrix.mkl-version }} + FREE_THREADED: ${{ matrix.free-threaded && 'true' || 'false' }} + DO_COVERAGE: ${{ matrix.coverage && 'true' || 'false' }} + ENV_NAME: pydiso-ci + - name: Setup Conda uses: conda-incubator/setup-miniconda@v3 with: - python-version: ${{ matrix.python-version }} - channels: conda-forge, defaults - channel-priority: true - activate-environment: dev + auto-update-conda: true + environment-file: environment_ci.yml + activate-environment: pydiso-ci - name: Conda information run: | @@ -49,14 +77,6 @@ jobs: conda list conda config --show - - name: Create environment - run: | - conda install --quiet --yes -c conda-forge \ - pip numpy scipy cython mkl=${{ matrix.mkl-version }} pytest \ - mkl-devel pkg-config meson-python meson ninja setuptools_scm \ - ${{ matrix.coverage && 'coverage' || ''}} \ - ${{ matrix.os == 'windows-latest' && '"libblas=*=*mkl"' || ''}} - - name: Install Our Package run: | python -m pip install --no-build-isolation --verbose --editable . \ @@ -64,8 +84,6 @@ jobs: ${{ matrix.coverage && '--config-settings=setup-args="-Db_coverage=true"' || ''}} \ ${{ matrix.os == 'windows-latest' && '--config-settings=setup-args="-Dvsenv=true"' || ''}} - conda list - - name: Run Tests run: | ${{ matrix.coverage && 'coverage run -m' || '' }} pytest -s -v diff --git a/.gitignore b/.gitignore index 72fc51b..9861146 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ mkl_solver.c.dep coverage.xml .idea/ + +.vscode/ + +environment_ci.yml diff --git a/pydiso/_mkl_solver.pyx.in b/pydiso/_mkl_solver.in.pyx similarity index 85% rename from pydiso/_mkl_solver.pyx.in rename to pydiso/_mkl_solver.in.pyx index e7449b2..4750905 100644 --- a/pydiso/_mkl_solver.pyx.in +++ b/pydiso/_mkl_solver.in.pyx @@ -1,13 +1,8 @@ -#cython: language_level=3 +# cython: language_level=3 +# cython: embedsignature=True, language_level=3 +# cython: freethreading_compatible=True cimport numpy as np import cython -from cpython.pythread cimport ( - PyThread_type_lock, - PyThread_allocate_lock, - PyThread_acquire_lock, - PyThread_release_lock, - PyThread_free_lock -) import numpy as np import os @@ -79,7 +74,6 @@ class PardisoError(Exception): class PardisoWarning(UserWarning): pass - #call pardiso (pt, maxfct, mnum, mtype, phase, n, a, ia, ja, perm, nrhs, iparm, msglvl, b, x, error) cdef int mkl_progress(int *thread, int* step, char* stage, int stage_len) nogil: # must be a nogil process to pass to mkl pardiso progress reporting @@ -175,7 +169,7 @@ ctypedef fused real_or_complex: {{for int_type in ["int_t", "long_t"]}} cdef class _PardisoHandle_{{int_type}}: cdef _MKL_DSS_HANDLE_t handle[64] - cdef PyThread_type_lock lock + cdef cython.pymutex lock cdef {{int_type}} n, maxfct, mnum, msglvl cdef public {{int_type}} matrix_type @@ -184,7 +178,6 @@ cdef class _PardisoHandle_{{int_type}}: @cython.boundscheck(False) def __cinit__(self, A_dat_dtype, n, matrix_type, maxfct, mnum, msglvl): - self.lock = PyThread_allocate_lock() np_int_dtype = np.dtype(f"i{sizeof({{int_type}})}") @@ -197,11 +190,13 @@ cdef class _PardisoHandle_{{int_type}}: self.mnum = mnum self.msglvl = msglvl - if self.msglvl: - #for reporting factorization progress via python's `print` - mkl_set_progress(mkl_progress) - else: - mkl_set_progress(mkl_no_progress) + + with self.lock: + if self.msglvl: + #for reporting factorization progress via python's `print` + mkl_set_progress(mkl_progress) + else: + mkl_set_progress(mkl_no_progress) is_single_precision = np.issubdtype(A_dat_dtype, np.single) or np.issubdtype(A_dat_dtype, np.csingle) @@ -264,14 +259,13 @@ cdef class _PardisoHandle_{{int_type}}: cdef {{int_type}} error, nrhs with nogil: nrhs = rhs.shape[1] - PyThread_acquire_lock(self.lock, mode=1) - pardiso{{if int_type == "long_t"}}_64{{endif}}( - self.handle, &self.maxfct, &self.mnum, &self.matrix_type, &phase, &self.n, - &a_data[0], &a_indptr[0], &a_indices[0], &self.perm[0], - &nrhs, self.iparm, &self.msglvl, - &rhs[0, 0], &out[0, 0], &error - ) - PyThread_release_lock(self.lock) + with self.lock: + pardiso{{if int_type == "long_t"}}_64{{endif}}( + self.handle, &self.maxfct, &self.mnum, &self.matrix_type, &phase, &self.n, + &a_data[0], &a_indptr[0], &a_indices[0], &self.perm[0], + &nrhs, self.iparm, &self.msglvl, + &rhs[0, 0], &out[0, 0], &error + ) return error @cython.boundscheck(False) @@ -280,20 +274,15 @@ cdef class _PardisoHandle_{{int_type}}: cdef {{int_type}} phase = -1, nrhs = 0, error = 0 with nogil: - PyThread_acquire_lock(self.lock, mode=1) - if self._initialized(): - pardiso{{if int_type == "long_t"}}_64{{endif}}( - self.handle, &self.maxfct, &self.mnum, &self.matrix_type, - &phase, &self.n, NULL, NULL, NULL, NULL, &nrhs, self.iparm, - &self.msglvl, NULL, NULL, &error) - if error == 0: - for i in range(64): - self.handle[i] = NULL - PyThread_release_lock(self.lock) + with self.lock: + if self._initialized(): + pardiso{{if int_type == "long_t"}}_64{{endif}}( + self.handle, &self.maxfct, &self.mnum, &self.matrix_type, + &phase, &self.n, NULL, NULL, NULL, NULL, &nrhs, self.iparm, + &self.msglvl, NULL, NULL, &error) + if error == 0: + for i in range(64): + self.handle[i] = NULL if error != 0: raise MemoryError("Pardiso Memory release error: " + _err_messages[error]) - if self.lock: - #deallocate the lock - PyThread_free_lock(self.lock) - self.lock = NULL {{endfor}} \ No newline at end of file diff --git a/pydiso/meson.build b/pydiso/meson.build index 5bdd8a9..1b2893a 100644 --- a/pydiso/meson.build +++ b/pydiso/meson.build @@ -1,5 +1,5 @@ cython_file = custom_target( - input: '_mkl_solver.pyx.in', + input: '_mkl_solver.in.pyx', output: '_mkl_solver.pyx', command: [py, '-c', @@ -60,6 +60,11 @@ else endif +cython_args = [] +if cy.version().version_compare('>=3.1.0') + cython_args += ['-Xfreethreading_compatible=True'] +endif + c_undefined_ok = ['-Wno-maybe-uninitialized'] cython_c_args = [numpy_nodepr_api, '-DCYTHON_CCOMPLEX=0'] @@ -68,6 +73,7 @@ module_path = 'pydiso' py.extension_module( '_mkl_solver', cython_file, + cython_args: cython_args, c_args: cython_c_args, install: true, subdir: module_path, diff --git a/pyproject.toml b/pyproject.toml index f06abe0..de934e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ build-backend = 'mesonpy' requires = [ "meson-python>=0.15.0", - "Cython>=3.0.8", + "Cython>=3.1.0", "setuptools_scm[toml]>=6.2", "numpy>=2.0.0rc1", ] @@ -13,7 +13,7 @@ name = 'pydiso' dynamic = ["version"] description = "Wrapper for intel's pardiso implementation in the MKL" readme = 'README.md' -requires-python = '>=3.10' +requires-python = '>=3.11' authors = [ {name = 'SimPEG developers', email = 'josephrcapriotti@gmail.com'}, ] @@ -28,7 +28,7 @@ dependencies = [ # TODO: update to "pin-compatible" once possible, see # https://github.com/mesonbuild/meson-python/issues/29 "numpy>=1.22.4", - "scipy>=1.8", + "scipy>=1.12", ] classifiers = [ "Development Status :: 4 - Beta", @@ -53,15 +53,18 @@ file = 'LICENSE' [project.optional-dependencies] test = [ "pytest", - "pytest-cov", +] + +coverage = [ + "coverage", ] build = [ - "meson-python>=0.14.0", + "meson-python>=0.15.0", "meson", "ninja", - "numpy>=1.22.4", - "cython>=0.29.35", + "numpy>=2.0.0rc1", + "cython>=3.1.0", "setuptools_scm", ] @@ -74,7 +77,6 @@ Repository = 'https://github.com/simpeg/pydiso.git' [tool.coverage.run] branch = true -plugins = ["Cython.Coverage"] source = ["pydiso", "tests"] [tool.coverage.report]