Upgrade Tron to Python 3.10 - TRON-2435

.github/workflows/ci.yml

@@ -15,12 +15,12 @@ jobs:
       fail-fast: false
       matrix:
         toxenv:
-          - py38,docs
+          - py310,docs
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: '3.10'
       # GHA won't setup tox for us
       - run: pip install tox==3.2
         # there are no pre-built wheels for bsddb3, so we need to install
@@ -44,7 +44,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: '3.10'
       # Update package lists to ensure we have the latest information
       - run: sudo apt-get update
       # the container provided by GitHub doesn't include utilities

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 ---
 default_language_version:
-    python: python3.8
+    python: python3.10
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.5.0
@@ -27,10 +27,10 @@ repos:
       - id: reorder-python-imports
         args: [--py3-plus]
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+    rev: v3.20.0
     hooks:
       - id: pyupgrade
-        args: [--py38-plus]
+        args: [--py39-plus]
   - repo: local
     hooks:
       - id: patch-enforce-autospec
@@ -44,4 +44,4 @@ repos:
     rev: 22.3.0
     hooks:
       - id: black
-        args: [--target-version, py38]
+        args: [--target-version, py310]

.readthedocs.yaml

@@ -8,7 +8,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.8"
+    python: "3.10"
     # You can also specify other tool versions:
     # nodejs: "20"
     # rust: "1.70"

Makefile

@@ -60,10 +60,10 @@ coffee_%: docker_%
 	'
 
 test:
-	tox -e py38
+	tox -e py310
 
 test_in_docker_%: docker_%
-	$(DOCKER_RUN) tron-builder-$* python3.8 -m tox -vv -e py38
+	$(DOCKER_RUN) tron-builder-$* python3.10 -m tox -vv -e py310
 
 tox_%:
 	tox -e $*
@@ -78,13 +78,13 @@ itest_%: debitest_%
 	@echo "itest $* OK"
 
 dev:
-	SSH_AUTH_SOCK=$(SSH_AUTH_SOCK) .tox/py38/bin/trond --debug --working-dir=dev -l logging.conf --host=0.0.0.0
+	SSH_AUTH_SOCK=$(SSH_AUTH_SOCK) .tox/py310/bin/trond --debug --working-dir=dev -l logging.conf --host=0.0.0.0
 
 example_cluster:
 	tox -e example-cluster
 
 yelpy:
-	.tox/py38/bin/pip install -r yelp_package/extra_requirements_yelp.txt
+	.tox/py310/bin/pip install -r yelp_package/extra_requirements_yelp.txt
 
 
 # 1. Bump version at the top of this file

bin/tronctl

@@ -11,12 +11,9 @@ import logging
 import pprint
 import sys
 from collections import defaultdict
+from collections.abc import Callable
+from collections.abc import Generator
 from typing import Any
-from typing import Callable
-from typing import Dict
-from typing import Generator
-from typing import Optional
-from typing import Tuple
 from urllib.parse import urljoin
 
 import argcomplete  # type: ignore
@@ -238,7 +235,7 @@ def parse_cli():
     return args
 
 
-def request(url: str, data: Dict[str, Any], headers=None, method=None) -> bool:
+def request(url: str, data: dict[str, Any], headers=None, method=None) -> bool:
     # We want every tronctl request to be attributable
     response = client.request(url, data=data, headers=headers, method=method, user_attribution=True)
     if response.error:
@@ -264,7 +261,7 @@ def event_discard(args):
         )
 
 
-def _get_triggers_for_action(server: str, action_identifier: str) -> Optional[Tuple[str, ...]]:
+def _get_triggers_for_action(server: str, action_identifier: str) -> tuple[str, ...] | None:
     try:
         namespace, job_name, run_number, action_name = action_identifier.split(".")
     except ValueError:
@@ -452,7 +449,7 @@ def tron_version(args):
     yield True
 
 
-COMMANDS: Dict[str, Callable[[argparse.Namespace], Generator[bool, None, None]]] = defaultdict(
+COMMANDS: dict[str, Callable[[argparse.Namespace], Generator[bool, None, None]]] = defaultdict(
     lambda: control_objects,
     publish=event_publish,
     discard=event_discard,

contrib/mock_patch_checker.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.8
+#!/usr/bin/env python3.10
 import ast
 import sys
 

contrib/patch-config-loggers.diff

@@ -1,5 +1,5 @@
---- a/debian/tron/opt/venvs/tron/lib/python3.8/site-packages/kubernetes/client/configuration.py
-+++ b/debian/tron/opt/venvs/tron/lib/python3.8/site-packages/kubernetes/client/configuration.py
+--- a/debian/tron/opt/venvs/tron/lib/python3.10/site-packages/kubernetes/client/configuration.py
++++ b/debian/tron/opt/venvs/tron/lib/python3.10/site-packages/kubernetes/client/configuration.py
 @@ -71,11 +71,11 @@
      """
 

debian/control

@@ -2,13 +2,13 @@ Source: tron
 Section: admin
 Priority: optional
 Maintainer: Daniel Nephin <[email protected]>
-Build-Depends: debhelper (>= 7), python3.8-dev, libdb5.3-dev, libyaml-dev, libssl-dev, libffi-dev, dh-virtualenv
+Build-Depends: debhelper (>= 7), python3.10-dev, libdb5.3-dev, libyaml-dev, libssl-dev, libffi-dev, dh-virtualenv
 Standards-Version: 3.8.3
 
 Package: tron
 Architecture: all
 Homepage: http://github.com/yelp/Tron
-Depends: bsdutils, python3.8, libdb5.3, libyaml-0-2, ${shlibs:Depends}, ${misc:Depends}
+Depends: bsdutils, python3.10, libdb5.3, libyaml-0-2, ${shlibs:Depends}, ${misc:Depends}
 Description: Tron is a job scheduling, running and monitoring package.
   Designed to replace Cron for complex scheduling and dependencies.
   Provides:

debian/rules

@@ -22,11 +22,11 @@ override_dh_virtualenv:
 	dh_virtualenv --index-url $(PIP_INDEX_URL) \
 	            --extra-pip-arg  --trusted-host=169.254.255.254 \
 	            --extra-pip-arg  --only-binary=cryptography \
-                --python=/usr/bin/python3.8 \
+                --python=/usr/bin/python3.10 \
                 --preinstall cython==0.29.36 \
                 --preinstall pip==24.3.1 \
                 --preinstall setuptools==65.5.1
 	@echo patching k8s client lib for configuration class
-	patch debian/tron/opt/venvs/tron/lib/python3.8/site-packages/kubernetes/client/configuration.py contrib/patch-config-loggers.diff
+	patch debian/tron/opt/venvs/tron/lib/python3.10/site-packages/kubernetes/client/configuration.py contrib/patch-config-loggers.diff
 override_dh_installinit:
 	dh_installinit --noscripts

dev/config/MASTER.yaml

@@ -17,7 +17,7 @@ nodes:
 # action_runner:
 #   runner_type: "subprocess"
 #   remote_status_path: "pg/tron/status"
-#   remote_exec_path: "pg/tron/.tox/py38/bin"
+#   remote_exec_path: "pg/tron/.tox/py310/bin"
 
 jobs:
   testjob0:

mypy.ini

@@ -1,14 +1,14 @@
 [mypy]
-python_version = 3.8
+python_version = 3.10
 # TODO: we'd like to be as strict as we are internally, but we need to fully type Tron first
 # disallow_any_generics = true
-disallow_incomplete_defs = true
+disallow_incomplete_defs = True
 # disallow_untyped_calls = true
-disallow_untyped_decorators = true
+disallow_untyped_decorators = True
 # disallow_untyped_defs = true
 
 show_column_numbers = True
-show_error_codes = true
+show_error_codes = True
 show_error_context = True
 
 warn_incomplete_stub = True

pyproject.toml

@@ -1,3 +1,3 @@
 [tool.black]
 line-length = 120
-target_version = ['py38']
+target_version = ['py310']

requirements-dev-minimal.txt

@@ -5,7 +5,6 @@ flake8
 mock
 mypy
 pre-commit
-pylint
 pytest
 pytest-asyncio
 requirements-tools

requirements-dev.txt

@@ -1,17 +1,13 @@
-astroid==2.13.3
 asynctest==0.12.0
 boto3-stubs==1.35.63
 botocore-stubs==1.38.19
 cfgv==2.0.1
 debugpy==1.8.1
-dill==0.3.6
 distlib==0.3.6
 filelock==3.4.1
 flake8==5.0.4
 identify==2.5.5
 iniconfig==1.1.1
-isort==4.3.18
-lazy-object-proxy==1.9.0
 mccabe==0.7.0
 mypy==1.9.0
 mypy-extensions==1.0.0
@@ -23,14 +19,12 @@ pre-commit==2.20.0
 py==1.10.0
 pycodestyle==2.9.0
 pyflakes==2.5.0
-pylint==2.15.10
 pyparsing==2.4.2
-pytest==6.2.2
+pytest==7.0.1
 pytest-asyncio==0.14.0
-requirements-tools==1.2.1
+requirements-tools==2.1.0
 toml==0.10.2
 tomli==2.0.1
-tomlkit==0.11.6
 types-awscrt==0.27.2
 types-boto3==1.0.2
 types-cachetools==5.5.0.20240820

requirements-minimal.txt

@@ -11,6 +11,7 @@ ipdb
 ipython
 Jinja2>=3.1.2
 lockfile
+mock==3.0.5
 moto
 prometheus-client
 psutil

requirements.txt

@@ -3,27 +3,20 @@ argcomplete==1.9.5
 asttokens==2.2.1
 attrs==19.3.0
 Automat==20.2.0
-aws-sam-translator==1.15.1
-aws-xray-sdk==2.4.2
 backcall==0.1.0
-boto==2.49.0
 boto3==1.34.80
 botocore==1.34.80
 bsddb3==6.2.7
 cachetools==4.2.1
 certifi==2022.12.7
-cffi==1.12.3
-cfn-lint==0.24.4
+cffi==1.15.0
 charset-normalizer==2.0.12
 constantly==15.1.0
 cryptography==41.0.5
 dataclasses==0.6
-DateTime==4.3
 decorator==4.4.0
-docker==4.1.0
 ecdsa==0.13.3
 executing==1.2.0
-future==0.18.3
 google-auth==1.23.0
 http-parser==0.9.0
 humanize==4.10.0
@@ -36,17 +29,12 @@ ipython-genutils==0.2.0
 jedi==0.16.0
 Jinja2==3.1.2
 jmespath==0.9.4
-jsondiff==1.1.2
-jsonpatch==1.24
-jsonpickle==1.2
-jsonpointer==2.0
-jsonschema==3.2.0
 kubernetes==26.1.0
 lockfile==0.12.2
 MarkupSafe==2.1.1
 matplotlib-inline==0.1.3
 mock==3.0.5
-moto==1.3.13
+moto==4.1.0
 oauthlib==3.1.0
 parso==0.7.0
 pexpect==4.7.0
@@ -68,18 +56,16 @@ pyrsistent==0.15.4
 pysensu-yelp==1.0.3
 PyStaticConfiguration==0.11.1
 python-dateutil==2.8.1
-python-jose==3.0.1
 pytimeparse==1.1.8
 pytz==2019.3
 PyYAML==6.0.1
 requests==2.27.1
 requests-oauthlib==1.2.0
-responses==0.10.6
+responses==0.13.0
 rsa==4.9
 s3transfer==0.10.1
 setuptools==65.5.1
 six==1.15.0
-sshpubkeys==3.1.0
 stack-data==0.6.2
 task-processing==1.3.5
 traitlets==5.0.0
@@ -89,6 +75,5 @@ urllib3==1.25.10
 wcwidth==0.1.7
 websocket-client==0.56.0
 Werkzeug==2.2.3
-wrapt==1.11.2
 xmltodict==0.12.0
-zope.interface==5.1.0
+zope.interface==7.2

tests/kubernetes_test.py

@@ -1,5 +1,4 @@
 from typing import Any
-from typing import Dict
 from unittest import mock
 
 import pytest
@@ -66,7 +65,7 @@ def mock_event_factory(
     task_id: str,
     platform_type: str,
     message: str = None,
-    raw: Dict[str, Any] = None,
+    raw: dict[str, Any] = None,
     success: bool = False,
     terminal: bool = False,
 ) -> Event:

tests/serialize/runstate/dynamodb_state_store_test.py

@@ -4,8 +4,8 @@
 import boto3
 import pytest
 import staticconf.testing
-from moto import mock_dynamodb2
-from moto.dynamodb2.responses import dynamo_json_dump
+from moto import mock_dynamodb
+from moto.dynamodb.responses import dynamo_json_dump
 
 from testifycompat import assert_equal
 from tron.serialize.runstate.dynamodb_state_store import DynamoDBStateStore
@@ -61,10 +61,10 @@ def update_item(item):
 @pytest.fixture(autouse=True)
 def store():
     with mock.patch(
-        "moto.dynamodb2.responses.DynamoHandler.transact_write_items",
+        "moto.dynamodb.responses.DynamoHandler.transact_write_items",
         new=mock_transact_write_items,
         create=True,
-    ), mock_dynamodb2():
+    ), mock_dynamodb():
         dynamodb = boto3.resource("dynamodb", region_name="us-west-2")
         table_name = "tmp"
         store = DynamoDBStateStore(table_name, "us-west-2", stopping=True)
@@ -125,7 +125,7 @@ def large_object():
         "runs": [],
         "cleanup_run": None,
         "manual": False,
-        "large_data": [i for i in range(1_000_000)],
+        "large_data": [i for i in range(10000)],
     }
 
 

tests/serialize/runstate/statemanager_test.py

@@ -87,7 +87,7 @@ def test_restore(self):
         with mock.patch.object(self.manager, "_restore_dicts", autospec=True,) as mock_restore_dicts, mock.patch.object(
             self.manager,
             "_restore_runs_for_job",
-            autospect=True,
+            autospec=True,
         ) as mock_restore_runs:
             mock_restore_dicts.side_effect = [
                 # _restore_dicts for JOB_STATE