Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions .github/workflows/generate-pickle-files.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Generate static pickle files

on: pull_request
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed, I think we should not run this at each PR. Let's use a manual trigger instead: https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow


jobs:
build:
strategy:
matrix:
python_version: [3.6, 3.7, 3.8, 3.9, "pypy3"]
cloudpickle_version: [1.5.0]

runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2

- name: Set up Python ${{ matrix.python_version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python_version }}

- name: Install project and dependencies
shell: bash
run: python -m pip install cloudpickle==${{ matrix.cloudpickle_version }}

- name: Display Python version
shell: bash
run: python -c "import sys; print(sys.version)"

- name: List software environment
shell: bash
run: python -m pip list

- name: Generate pickle files
shell: bash
run: |
cd tests
rm -rf old_pickles
python generate_old_pickles.py
cd ../
- name: Upload pickle files
uses: actions/upload-artifact@v2
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IIUC, Storing pickles as artifacts implies that these pickles will only be available for 90 days (see https://github.com/actions/upload-artifact#retention-period), so we need maintainers to re-run this workflow every 90 days. Are people fine with it? @ogrisel @jrbourbeau

with:
name: old_pickles
path: tests/old_pickles
66 changes: 19 additions & 47 deletions cloudpickle/cloudpickle.py
Original file line number Diff line number Diff line change
Expand Up @@ -645,23 +645,13 @@ def parametrized_type_hint_getinitargs(obj):
return initargs


# Tornado support

def is_tornado_coroutine(func):
"""
Return whether *func* is a Tornado coroutine function.
Running coroutines are not supported.
"""
if 'tornado.gen' not in sys.modules:
return False
gen = sys.modules['tornado.gen']
if not hasattr(gen, "is_coroutine_function"):
# Tornado version is too old
return False
return gen.is_coroutine_function(func)


def _rebuild_tornado_coroutine(func):
# This function is deprecated and should be removed in cloudpickle 2.1
warnings.warn(
"A pickle file created using an old version of cloudpickle "
"is currently being loaded. This is no longer supported by cloudpickle and "
"will be removed in cloudpickle 2.1", category=FutureWarning
)
from tornado import gen
return gen.coroutine(func)

Expand All @@ -688,10 +678,22 @@ def dynamic_subimport(name, vars):


def _gen_ellipsis():
# this function is deprecated and should be removed in cloudpickle 2.1
warnings.warn(
"a pickle file created using an old version of cloudpickle "
"is currently being loaded. this is no longer supported by cloudpickle and "
"will be removed in cloudpickle 2.1", category=FutureWarning
)
return Ellipsis


def _gen_not_implemented():
# This function is deprecated and should be removed in cloudpickle 2.1
warnings.warn(
"A pickle file created using an old version of cloudpickle "
"is currently being loaded. This is no longer supported by cloudpickle and "
"will be removed in cloudpickle 2.1", category=FutureWarning
)
return NotImplemented


Expand Down Expand Up @@ -729,10 +731,7 @@ def __reduce__(cls):


def _fill_function(*args):
"""Fills in the rest of function data into the skeleton function object

The skeleton itself is create by _make_skel_func().
"""
"""Fills in the rest of function data into the skeleton function object"""
if len(args) == 2:
func = args[0]
state = args[1]
Expand Down Expand Up @@ -809,33 +808,6 @@ def _make_cell(value=_empty_cell_value):
return cell


def _make_skel_func(code, cell_count, base_globals=None):
""" Creates a skeleton function object that contains just the provided
code and the correct number of cells in func_closure. All other
func attributes (e.g. func_globals) are empty.
"""
# This function is deprecated and should be removed in cloudpickle 1.7
warnings.warn(
"A pickle file created using an old (<=1.4.1) version of cloudpickle "
"is currently being loaded. This is not supported by cloudpickle and "
"will break in cloudpickle 1.7", category=UserWarning
)
# This is backward-compatibility code: for cloudpickle versions between
# 0.5.4 and 0.7, base_globals could be a string or None. base_globals
# should now always be a dictionary.
if base_globals is None or isinstance(base_globals, str):
base_globals = {}

base_globals['__builtins__'] = __builtins__

closure = (
tuple(_make_empty_cell() for _ in range(cell_count))
if cell_count >= 0 else
None
)
return types.FunctionType(code, base_globals, None, None, closure)


def _make_skeleton_class(type_constructor, name, bases, type_kwargs,
class_tracker_id, extra):
"""Build dynamic class with an empty __dict__ to be filled once memoized
Expand Down
Binary file removed tests/old_pickles/cpython_35/class_with_type_hints.pkl
Binary file not shown.
Binary file not shown.
Binary file removed tests/old_pickles/cpython_35/nested_function.pkl
Binary file not shown.
Binary file removed tests/old_pickles/cpython_35/simple_class.pkl
Binary file not shown.
Binary file removed tests/old_pickles/cpython_35/simple_enum.pkl
Binary file not shown.
Binary file removed tests/old_pickles/cpython_35/simple_func.pkl
Binary file not shown.
Binary file removed tests/old_pickles/cpython_35/simple_module.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_36/class_with_type_hints.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_36/function_with_type_hints.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_36/nested_function.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_36/simple_class.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_36/simple_enum.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_36/simple_func.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_36/simple_module.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_37/class_with_type_hints.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_37/function_with_type_hints.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_37/nested_function.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_37/simple_class.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_37/simple_enum.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_37/simple_func.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_37/simple_module.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_38/class_with_type_hints.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_38/function_with_type_hints.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_38/nested_function.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_38/simple_class.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_38/simple_enum.pkl
Binary file not shown.
Binary file modified tests/old_pickles/cpython_38/simple_func.pkl
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file added tests/old_pickles/cpython_39/nested_function.pkl
Binary file not shown.
Binary file added tests/old_pickles/cpython_39/simple_class.pkl
Binary file not shown.
Binary file added tests/old_pickles/cpython_39/simple_enum.pkl
Binary file not shown.
Binary file added tests/old_pickles/cpython_39/simple_func.pkl
Binary file not shown.
Binary file added tests/old_pickles/cpython_39/simple_module.pkl
Binary file not shown.
Binary file modified tests/old_pickles/pypy_36/class_with_type_hints.pkl
Binary file not shown.
Binary file modified tests/old_pickles/pypy_36/function_with_type_hints.pkl
Binary file not shown.
Binary file added tests/old_pickles/pypy_36/nested_function.pkl
Binary file not shown.
Binary file modified tests/old_pickles/pypy_36/simple_class.pkl
Binary file not shown.
Binary file modified tests/old_pickles/pypy_36/simple_enum.pkl
Binary file not shown.
Binary file modified tests/old_pickles/pypy_36/simple_func.pkl
Binary file not shown.
Binary file modified tests/old_pickles/pypy_36/simple_module.pkl
Binary file not shown.
10 changes: 3 additions & 7 deletions tests/test_backward_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,7 @@
from .generate_old_pickles import PICKLE_DIRECTORY


def load_obj(filename, check_deprecation_warning='auto'):
if check_deprecation_warning == 'auto':
# pickles files generated with cloudpickle_fast.py on old versions of
# coudpickle with Python < 3.8 use non-deprecated reconstructors.
check_deprecation_warning = (sys.version_info < (3, 8))
def load_obj(filename, check_deprecation_warning=False):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since _make_skel_func has been removed we no longer raise a warning in most cases for the tests in this module. Because of this I decided to change check_deprecation_warning to be False by default, so checking deprecation warnings will need to be done explicitly. Happy to try something else out though

pickle_filepath = PICKLE_DIRECTORY / filename
if not pickle_filepath.exists():
pytest.skip("Could not find {}".format(str(pickle_filepath)))
Expand Down Expand Up @@ -61,13 +57,13 @@ def test_dynamic_module():


def test_simple_enum():
enum = load_obj("simple_enum.pkl", check_deprecation_warning=False)
enum = load_obj("simple_enum.pkl")
assert hasattr(enum, "RED")
assert enum.RED == 1
assert enum.BLUE == 2

# test enum tracking feature
new_enum = load_obj("simple_enum.pkl", check_deprecation_warning=False)
new_enum = load_obj("simple_enum.pkl")
assert new_enum is enum


Expand Down