Massive updates to python 3.12, pymc 5.20, numpy 1.26, and assorted lower-importance version updates (#138)

* + update makefile MAMBADL (miniforge) from 23.3.1-1 to 24.11.3-0
+ make uninstall mamba and reinstall mamba
+ make uninstall-env and install-env
+ make uninstall-env and dev

* + update makefile and envrc to use mamba not conda
+ update versions in pyproject
+ make test-dev-env
+ let's try for a release

* + update 3.11 to 3.12 throughout

* + perhaps foolishly updated seaborn to 0.13.2, lets see what breaks

* + updated boxplots to use showfliers not sym

* + remove target_accept from Metropolis builder
+ small updates to plots as reqd by seaborn 13

* + update violinplots

* + update re.sub("", ..) to re.sub(r"", ...)

* + minor improvements to plots

Author: jonsedar

.envrc

@@ -1,3 +1,2 @@
-# new protocol reqd per https://github.com/direnv/direnv/issues/326#issuecomment-574779485
-eval "$(conda shell.zsh hook)"
-conda activate oreum_core
+eval "$(mamba shell hook --shell zsh)"
+mamba activate oreum_core

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ jobs:
     strategy:
       matrix:
         platform: ["ubuntu-latest"]
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     steps:
       - uses: actions/checkout@v4
       - name: Setup Python ${{ matrix.python-version }}

.github/workflows/publish.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         platform: ["ubuntu-latest"]
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -31,10 +31,10 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
-      - name: Setup Python 3.11
+      - name: Setup Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Build and Publish package to TestPyPI
         env:
           FLIT_USERNAME: __token__
@@ -55,10 +55,10 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
-      - name: Setup Python 3.11
+      - name: Setup Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Build and Publish package to PyPI
         env:
           FLIT_USERNAME: __token__

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 # .pre-commit-config.yaml
 default_language_version:
-  python: python3.11
+  python: python3.12
 default_stages: [pre-commit, pre-push]
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks  # general checks
@@ -31,13 +31,13 @@ repos:
       - id: no-print-statements
         files: ^oreum_core/
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.0
+    rev: v0.9.3
     hooks:
       - id: ruff   # lint
         args: [ --fix]
       - id: ruff-format  # formatter
   - repo: https://github.com/pycqa/bandit  # basic security checks for python code
-    rev: 1.8.0
+    rev: 1.8.2
     hooks:
       - id: bandit
         files: ^oreum_core/

LICENSES_THIRD_PARTY.md

@@ -7,7 +7,7 @@
 	    test-dev-env test-dl-ins uninstall-env uninstall-mamba
 .SILENT: build dev help install-env install-mamba lint pre-build pub test-pub\
 	    test-dev-env test-dl-ins uninstall-env uninstall-mamba
-MAMBADL := https://github.com/conda-forge/miniforge/releases/download/23.3.1-1
+MAMBADL := https://github.com/conda-forge/miniforge/releases/download/24.11.3-0
 MAMBAV := Miniforge3-MacOSX-arm64.sh
 MAMBARCMSG := Please create file $(MAMBARC), importantly set `platform: osx-arm64`
 MAMBARC := $(HOME)/.mambarc
@@ -28,9 +28,8 @@ build:  ## build package oreum_core (actually more of an "assemble" than a compi
 dev:  ## create env for local dev
 	make install-env
 	export PATH=$(MAMBADIR)/envs/oreum_core/bin:$$PATH; \
-		export CONDA_ENV_PATH=$(MAMBADIR)/envs/oreum_core/bin; \
-		export CONDA_DEFAULT_ENV=oreum_core; \
-		export CONDA_SUBDIR=osx-arm64; \
+		export MAMBA_EXE='$(MAMBADIR)/bin/mamba'; \
+		export MAMBA_ROOT_PREFIX='$(MAMBADIR)'; \
 		$(PYTHON_ENV) -m pip install -e ".[all]"; \
 		$(PYTHON_ENV) -c "import numpy as np; np.__config__.show()" > dev/install_log/blas_info.txt; \
 		pipdeptree -a > dev/install_log/pipdeptree.txt; \
@@ -43,7 +42,8 @@ install-env:  ## create mamba (conda) environment
 	export PATH=$(MAMBADIR)/bin:$$PATH; \
 		if which mamba; then echo "mamba ready"; else make install-mamba; fi
 	export PATH=$(MAMBADIR)/bin:$$PATH; \
-		export CONDA_SUBDIR=osx-arm64; \
+		export MAMBA_EXE='$(MAMBADIR)/bin/mamba'; \
+		export MAMBA_ROOT_PREFIX='$(MAMBADIR)'; \
 		mamba update -n base mamba; \
 		mamba env create --file condaenv_oreum_core.yml -y;
 
@@ -53,7 +53,7 @@ install-mamba:  ## get mamba via miniforge, explicitly use bash
 	chmod 755 $(HOME)/miniforge.sh
 	bash $(HOME)/miniforge.sh -b -p $(MAMBADIR)
 	export PATH=$(MAMBADIR)/bin:$$PATH; \
-		conda init zsh;
+		mamba shell init;
 	rm $(HOME)/miniforge.sh
 
 lint: ## run code linters and static security (checks only)
@@ -74,7 +74,6 @@ help:
 	@echo "  test-dl-ins   test dl & install from testpypi"
 	@echo "  uninstall-env remove env (use from parent dir \make -C oreum_core ...)"
 
-
 pre-build:  ## setup env for flit build or flit publish
 	rm -rf dist
 	$(PYTHON) -m pip install flit keyring
@@ -90,16 +89,13 @@ test-pub:  ## all-in-one build and publish to testpypi
 	export FLIT_INDEX_URL=https://test.pypi.org/legacy/; \
 		$(PYTHON) -m flit publish
 
-test-dev-env:  ## test the dev machine install of critial numeric packages
+test-dev-env:  ## test the dev machine install of critical numeric packages
 	export PATH=$(MAMBADIR)/bin:$$PATH; \
-		export PATH=$(MAMBADIR)/envs/oreum_core/bin:$$PATH; \
-		export CONDA_ENV_PATH=$(MAMBADIR)/envs/oreum_core/bin; \
-		export CONDA_DEFAULT_ENV=oreum_core; \
+		export MAMBA_EXE='$(MAMBADIR)/bin/mamba'; \
+		export MAMBA_ROOT_PREFIX='$(MAMBADIR)'; \
 		$(PYTHON_ENV) -c "import numpy as np; np.test()" > dev/install_log/tests_numpy.txt; \
 		$(PYTHON_ENV) -c "import scipy as sp; sp.test()" > dev/install_log/tests_scipy.txt;
 
-# $(PYTHON_ENV) -c "import pymc as pm; pm.test()" > dev/install_log/pymc.txt; \
-
 test-dl-ins:  ## test dl & install from testpypi, set env var or pass in VERSION
 	$(PYTHON) -m pip uninstall -y oreum_core
 	$(PYTHON) -m pip index versions --pre -i https://test.pypi.org/simple/ oreum_core
@@ -108,13 +104,13 @@ test-dl-ins:  ## test dl & install from testpypi, set env var or pass in VERSION
 
 uninstall-env: ## remove mamba env
 	export PATH=$(MAMBADIR)/bin:$$PATH; \
-		export CONDA_SUBDIR=osx-arm64; \
+		export MAMBA_ROOT_PREFIX='$(MAMBADIR)'; \
 		mamba env remove --name oreum_core -y; \
-		conda clean --all -afy;
-		# mamba clean -afy  # 2023-12-10 see: https://github.com/mamba-org/mamba/issues/3044
+		mamba clean -ay
 
-uninstall-mamba:  ## last ditch per https://github.com/conda-forge/miniforge#uninstallation
-	conda init zsh --reverse
+uninstall-mamba:  ## nuclear option https://github.com/conda-forge/miniforge?tab=readme-ov-file#uninstall
+	mamba shell deinit
 	rm -rf $(MAMBADIR)
-	rm -rf $(HOME)/.conda
-	rm -f $(HOME)/.condarc
+	rm -rf $(HOME)/.mamba
+	rm -f $(HOME)/.mambarc
+	rm -f $(HOME)/.mambarc_x86

Makefile

@@ -1,6 +1,6 @@
 # Oreum Core Tools `oreum_core`
 
-[![Python](https://img.shields.io/badge/python-3.11-blue)](https://www.python.org)
+[![Python](https://img.shields.io/badge/python-3.12-blue)](https://www.python.org)
 [![License](https://img.shields.io/badge/license-Apache2.0-blue.svg)](https://choosealicense.com/licenses/apache-2.0/)
 [![GitHub Release](https://img.shields.io/github/v/release/oreum-industries/oreum_core?display_name=tag&sort=semver)](https://github.com/oreum-industries/oreum_core/releases)
 [![PyPI](https://img.shields.io/pypi/v/oreum_core)](https://pypi.org/project/oreum_core)
@@ -56,8 +56,16 @@ This package **is not**:
   + Source code repo on [GitHub](https://github.com/oreum-industries/oreum_core)
   + Source code release on [GitHub](https://github.com/oreum-industries/oreum_core/releases)
   + Package release on [PyPi](https://pypi.org/project/oreum_core)
-
-
++ Implementation:
+  + This project is enabled by a modern, open-source, advanced software stack
+    for data curation, statistical analysis and predictive modelling
+  + Specifically we use an open-source Python-based suite of software packages,
+    the core of which is often known as the Scientific Python stack, supported
+    by [NumFOCUS](https://numfocus.org)
+  + Once installed (see section 2), see `LICENSES_THIRD_PARTY.md` for full
+    details of all package licences
++ Environments: this project was originally developed on a Macbook Air M2
+  (Apple Silicon ARM64) running MacOS 15.2 (Sequoia) using `osx-arm64` Accelerate
 
 ## 2. Instructions to Create Dev Environment
 

README.md

@@ -1,5 +1,5 @@
 # Manually created as-at 2021-12-01
-# Last updated as-at 2023-12-10
+# Last updated as-at 2025-01-24
 # NOTES:
 #  + Creates a virtual env for project development
 #  + Require running on Apple Silicon ARM64 (NOT Intel x86 AMD64 CPU via Rosetta2)
@@ -10,19 +10,14 @@
 #    also see https://numpy.org/install/ and https://github.com/numpy/numpy/pull/24053
 #  + libblas, libcblas, liblapack, not strictly needed here but we want to be
 #    explicit about forcing BLAS to use Accelerate
-#  + pooch and scipy-tests required because conda-forge scipy (brought in by pymc)
-#    doesnt contain tests
 name: oreum_core
 channels:
   - conda-forge
   # - defaults
 dependencies:
-- conda-forge::ipykernel>=6.23.1
-- conda-forge::numpy>=1.25.2   #>=1.26.2  # see NOTES
+- conda-forge::ipykernel>=6.29.0
+- conda-forge::numpy>=1.26.0  # see NOTES
 - conda-forge::libblas=*[build=*accelerate]  # see NOTES
 - conda-forge::libcblas=*[build=*accelerate]  # see NOTES
 - conda-forge::liblapack=*[build=*accelerate]  # see NOTES
-- conda-forge::python==3.11.*
-- conda-forge::pymc==5.9.*  # >=5.10.1
-- conda-forge::scipy-tests  # see NOTES
-- conda-forge::pooch  # see NOTES
+- conda-forge::python==3.12.*

condaenv_oreum_core.yml

@@ -3,6 +3,8 @@
 # + Intended for dev install on MacOS x64 (Intel via Rosetta 2)
 # + On MacOS, the miniforge.sh installer is too clever and complains
 #   that "$(uname -m)" != "x86_64", but we ignore that: Rosetta2 will run x86_64
+# NOTE as-at 2025-01-24 this is largely untested since 2023, so if/when needed,
+# expect to need to do some updating / troubleshooting
 .PHONY: build create-env dev lint help mamba pre-build pub test-pub test-dev-env test-dl-ins uninstall uninstall-mamba
 .SILENT: build create-env dev lint help mamba pre-build pub test-pub test-dev-env test-dl-ins uninstall uninstall-mamba
 MAMBADL := https://github.com/conda-forge/miniforge/releases/download/23.3.1-1

dev/Makefile_osx-64

@@ -1,5 +1,5 @@
 # Manually created as-at 2021-12-01
-# Last updated as-at 2023-10-04
+# Last updated as-at 2025-01-24
 # NOTE:
 #  + Creates a virtual env for project usage
 #  + Require running on Intel x86 AMD64 CPU (or Rosetta2 on MacOS)
@@ -15,12 +15,11 @@ channels:
   - conda-forge
   # - defaults
 dependencies:
-- pkgs/main::numpy==1.24.3  # force numpy MKL see NOTE
-- conda-forge::ipykernel>=6.23.1
+- pkgs/main::numpy==1.26.0  # force numpy MKL see NOTE
+- conda-forge::ipykernel>=6.29.0
 - conda-forge::libblas=*[build=*mkl]  # force BLAS with MKL see NOTE
 - conda-forge::libcblas=*[build=*mkl]  # force BLAS with MKL see NOTE
 - conda-forge::liblapack=*[build=*mkl]  # force BLAS with MKL see NOTE
 - conda-forge::mkl==2021.4.*  # force MKL version see NOTE
 - conda-forge::mkl-service==2.4.*
-- conda-forge::python==3.11.*
-- conda-forge::pymc>=5.9.0
+- conda-forge::python==3.12.*