From c26a32ec2195ffe0dab8f70b781a0bd910c72b20 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 1 Oct 2025 13:49:41 +0200 Subject: [PATCH] Remove contrib test suite --- .github/workflows/contrib-tests.yml | 50 -------- .github/workflows/python-quality.yml | 7 +- Makefile | 45 +------- contrib/README.md | 70 ----------- contrib/__init__.py | 0 contrib/conftest.py | 52 --------- contrib/sentence_transformers/__init__.py | 0 .../sentence_transformers/requirements.txt | 1 - .../test_sentence_transformers.py | 37 ------ contrib/spacy/__init__.py | 0 contrib/spacy/requirements.txt | 1 - contrib/spacy/test_spacy.py | 48 -------- contrib/timm/__init__.py | 0 contrib/timm/requirements.txt | 3 - contrib/timm/test_timm.py | 20 ---- contrib/utils.py | 56 --------- utils/check_contrib_list.py | 109 ------------------ 17 files changed, 5 insertions(+), 494 deletions(-) delete mode 100644 .github/workflows/contrib-tests.yml delete mode 100644 contrib/README.md delete mode 100644 contrib/__init__.py delete mode 100644 contrib/conftest.py delete mode 100644 contrib/sentence_transformers/__init__.py delete mode 100644 contrib/sentence_transformers/requirements.txt delete mode 100644 contrib/sentence_transformers/test_sentence_transformers.py delete mode 100644 contrib/spacy/__init__.py delete mode 100644 contrib/spacy/requirements.txt delete mode 100644 contrib/spacy/test_spacy.py delete mode 100644 contrib/timm/__init__.py delete mode 100644 contrib/timm/requirements.txt delete mode 100644 contrib/timm/test_timm.py delete mode 100644 contrib/utils.py delete mode 100644 utils/check_contrib_list.py diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml deleted file mode 100644 index d294b7b530..0000000000 --- a/.github/workflows/contrib-tests.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Contrib tests - -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * 6' # Run once a week, Saturday midnight - push: - branches: - - ci_contrib_* - pull_request: - types: [assigned, opened, synchronize, reopened] - paths: - - contrib/** - -jobs: - build: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - contrib: [ - "sentence_transformers", - "spacy", - "timm", - ] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - # Install pip - - name: Install pip - run: pip install --upgrade pip - - # Install downstream library and its specific dependencies - - name: Install ${{ matrix.contrib }} - run: pip install -r contrib/${{ matrix.contrib }}/requirements.txt - - # Install huggingface_hub from source code + testing extras - - name: Install `huggingface_hub` - run: | - pip uninstall -y huggingface_hub - pip install .[testing] - - # Run tests - - name: Run tests - run: pytest contrib/${{ matrix.contrib }} diff --git a/.github/workflows/python-quality.yml b/.github/workflows/python-quality.yml index d9594a5435..a29c577e75 100644 --- a/.github/workflows/python-quality.yml +++ b/.github/workflows/python-quality.yml @@ -35,9 +35,8 @@ jobs: - name: Install dependencies run: uv pip install "huggingface_hub[dev] @ ." - - run: .venv/bin/ruff check tests src contrib # linter - - run: .venv/bin/ruff format --check tests src contrib # formatter - - run: .venv/bin/python utils/check_contrib_list.py + - run: .venv/bin/ruff check tests src # linter + - run: .venv/bin/ruff format --check tests src # formatter - run: .venv/bin/python utils/check_inference_input_params.py - run: .venv/bin/python utils/check_static_imports.py - run: .venv/bin/python utils/check_all_variable.py @@ -50,4 +49,4 @@ jobs: - run: .venv/bin/mypy src/huggingface_hub/__init__.py --follow-imports=silent --show-traceback # Run mypy on full package - - run: .venv/bin/mypy src \ No newline at end of file + - run: .venv/bin/mypy src diff --git a/Makefile b/Makefile index faa2a63500..35128faf2b 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,13 @@ -.PHONY: contrib quality style test +.PHONY: quality style test -check_dirs := contrib src tests utils setup.py +check_dirs := src tests utils setup.py quality: ruff check $(check_dirs) # linter ruff format --check $(check_dirs) # formatter python utils/check_inference_input_params.py - python utils/check_contrib_list.py python utils/check_static_imports.py python utils/check_all_variable.py python utils/generate_async_inference_client.py @@ -18,7 +17,6 @@ quality: style: ruff format $(check_dirs) # formatter ruff check --fix $(check_dirs) # linter - python utils/check_contrib_list.py --update python utils/check_static_imports.py --update python utils/check_all_variable.py --update python utils/generate_async_inference_client.py --update @@ -38,42 +36,3 @@ repocard: test: pytest ./tests/ - -# Taken from https://stackoverflow.com/a/12110773 -# Commands: -# make contrib_setup_timm : setup tests for timm -# make contrib_test_timm : run tests for timm -# make contrib_timm : setup and run tests for timm -# make contrib_clear_timm : delete timm virtual env -# -# make contrib_setup : setup ALL tests -# make contrib_test : run ALL tests -# make contrib : setup and run ALL tests -# make contrib_clear : delete all virtual envs -# Use -j4 flag to run jobs in parallel. -CONTRIB_LIBS := sentence_transformers spacy timm -CONTRIB_JOBS := $(addprefix contrib_,${CONTRIB_LIBS}) -CONTRIB_CLEAR_JOBS := $(addprefix contrib_clear_,${CONTRIB_LIBS}) -CONTRIB_SETUP_JOBS := $(addprefix contrib_setup_,${CONTRIB_LIBS}) -CONTRIB_TEST_JOBS := $(addprefix contrib_test_,${CONTRIB_LIBS}) - -contrib_clear_%: - rm -rf contrib/$*/.venv - -contrib_setup_%: - python3 -m venv contrib/$*/.venv - ./contrib/$*/.venv/bin/pip install -r contrib/$*/requirements.txt - ./contrib/$*/.venv/bin/pip uninstall -y huggingface_hub - ./contrib/$*/.venv/bin/pip install -e .[testing] - -contrib_test_%: - ./contrib/$*/.venv/bin/python -m pytest contrib/$* - -contrib_%: - make contrib_setup_$* - make contrib_test_$* - -contrib: ${CONTRIB_JOBS}; -contrib_clear: ${CONTRIB_CLEAR_JOBS}; echo "Successful contrib tests." -contrib_setup: ${CONTRIB_SETUP_JOBS}; echo "Successful contrib setup." -contrib_test: ${CONTRIB_TEST_JOBS}; echo "Successful contrib tests." diff --git a/contrib/README.md b/contrib/README.md deleted file mode 100644 index 05db2d705b..0000000000 --- a/contrib/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Contrib test suite - -The contrib folder contains simple end-to-end scripts to test integration of `huggingface_hub` in downstream libraries. The main goal is to proactively notice breaking changes and deprecation warnings. - -## Add tests for a new library - -To add another contrib lib, one must: -1. Create a subfolder with the lib name. Example: `./contrib/transformers` -2. Create a `requirements.txt` file specific to this lib. Example `./contrib/transformers/requirements.txt` -3. Implements tests for this lib. Example: `./contrib/transformers/test_push_to_hub.py` -4. Run `make style`. This will edit both `makefile` and `.github/workflows/contrib-tests.yml` to add the lib to list of libs to test. Make sure changes are accurate before committing. - -## Run contrib tests on CI - -Contrib tests can be [manually triggered in GitHub](https://github.com/huggingface/huggingface_hub/actions) with the `Contrib tests` workflow. - -Tests are not run in the default test suite (for each PR) as this would slow down development process. The goal is to notice breaking changes, not to avoid them. In particular, it is interesting to trigger it before a release to make sure it will not cause too much friction. - -## Run contrib tests locally - -Tests must be ran individually for each dependent library. Here is an example to run -`timm` tests. Tests are separated to avoid conflicts between version dependencies. - -### Run all contrib tests - -Before running tests, a virtual env must be setup for each contrib library. To do so, run: - -```sh -# Run setup in parallel to save time -make contrib_setup -j4 -``` - -Then tests can be run - -```sh -# Optional: -j4 to run in parallel. Output will be messy in that case. -make contrib_test -j4 -``` - -Optionally, it is possible to setup and run all tests in a single command. However this -take more time as you don't need to setup the venv each time you run tests. - -```sh -make contrib -j4 -``` - -Finally, it is possible to delete all virtual envs to get a fresh start for contrib tests. -After running this command, `contrib_setup` will have to re-download/re-install all dependencies. - -``` -make contrib_clear -``` - -### Run contrib tests for a single lib - -Instead of running tests for all contrib libraries, you can run a specific lib: - -```sh -# Setup timm tests -make contrib_setup_timm - -# Run timm tests -make contrib_test_timm - -# (or) Setup and run timm tests at once -make contrib_timm - -# Delete timm virtualenv if corrupted -make contrib_clear_timm -``` diff --git a/contrib/__init__.py b/contrib/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/contrib/conftest.py b/contrib/conftest.py deleted file mode 100644 index 285139fd69..0000000000 --- a/contrib/conftest.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -import time -import uuid -from typing import Generator - -import pytest - -from huggingface_hub import delete_repo - - -@pytest.fixture(scope="session") -def token() -> str: - # Not critical, only usable on the sandboxed CI instance. - return "hf_94wBhPGp6KrrTH3KDchhKpRxZwd6dmHWLL" - - -@pytest.fixture(scope="session") -def user() -> str: - return "__DUMMY_TRANSFORMERS_USER__" - - -@pytest.fixture(autouse=True, scope="session") -def login_as_dummy_user(token: str) -> Generator: - """Log in with dummy user token.""" - # Cannot use `monkeypatch` fixture since we want it to be "session-scoped" - old_token = os.environ["HF_TOKEN"] - os.environ["HF_TOKEN"] = token - yield - os.environ["HF_TOKEN"] = old_token - - -@pytest.fixture -def repo_name(request) -> None: - """ - Return a readable pseudo-unique repository name for tests. - - Example: "repo-2fe93f-16599646671840" - """ - prefix = request.module.__name__ # example: `test_timm` - id = uuid.uuid4().hex[:6] - ts = int(time.time() * 10e3) - return f"repo-{prefix}-{id}-{ts}" - - -@pytest.fixture -def cleanup_repo(user: str, repo_name: str) -> None: - """Delete the repo at the end of the tests. - - TODO: Adapt to handle `repo_type` as well - """ - yield # run test - delete_repo(repo_id=f"{user}/{repo_name}") diff --git a/contrib/sentence_transformers/__init__.py b/contrib/sentence_transformers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/contrib/sentence_transformers/requirements.txt b/contrib/sentence_transformers/requirements.txt deleted file mode 100644 index c8c5244b95..0000000000 --- a/contrib/sentence_transformers/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -git+https://github.com/UKPLab/sentence-transformers.git#egg=sentence-transformers diff --git a/contrib/sentence_transformers/test_sentence_transformers.py b/contrib/sentence_transformers/test_sentence_transformers.py deleted file mode 100644 index d1ceeb43dc..0000000000 --- a/contrib/sentence_transformers/test_sentence_transformers.py +++ /dev/null @@ -1,37 +0,0 @@ -import time - -import pytest -from sentence_transformers import SentenceTransformer, util - -from huggingface_hub import model_info - -from ..utils import production_endpoint - - -@pytest.fixture(scope="module") -def multi_qa_model() -> SentenceTransformer: - with production_endpoint(): - return SentenceTransformer("multi-qa-MiniLM-L6-cos-v1") - - -def test_from_pretrained(multi_qa_model: SentenceTransformer) -> None: - # Example taken from https://www.sbert.net/docs/hugging_face.html#using-hugging-face-models. - query_embedding = multi_qa_model.encode("How big is London") - passage_embedding = multi_qa_model.encode( - [ - "London has 9,787,426 inhabitants at the 2011 census", - "London is known for its financial district", - ] - ) - print("Similarity:", util.dot_score(query_embedding, passage_embedding)) - - -def test_push_to_hub(multi_qa_model: SentenceTransformer, repo_name: str, user: str, cleanup_repo: None) -> None: - multi_qa_model.save_to_hub(repo_name, organization=user) - - # Sleep to ensure that model_info isn't called too soon - time.sleep(1) - - # Check model has been pushed properly - model_id = f"{user}/{repo_name}" - assert model_info(model_id).library_name == "sentence-transformers" diff --git a/contrib/spacy/__init__.py b/contrib/spacy/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/contrib/spacy/requirements.txt b/contrib/spacy/requirements.txt deleted file mode 100644 index 6255342454..0000000000 --- a/contrib/spacy/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -git+https://github.com/explosion/spacy-huggingface-hub.git#egg=spacy-huggingface-hub diff --git a/contrib/spacy/test_spacy.py b/contrib/spacy/test_spacy.py deleted file mode 100644 index 00d4c9b671..0000000000 --- a/contrib/spacy/test_spacy.py +++ /dev/null @@ -1,48 +0,0 @@ -import time - -from spacy_huggingface_hub import push - -from huggingface_hub import delete_repo, hf_hub_download, model_info -from huggingface_hub.errors import HfHubHTTPError - -from ..utils import production_endpoint - - -def test_push_to_hub(user: str) -> None: - """Test equivalent of `python -m spacy huggingface-hub push`. - - (0. Delete existing repo on the Hub (if any)) - 1. Download an example file from production - 2. Push the model! - 3. Check model pushed the Hub + as spacy library - (4. Cleanup) - """ - model_id = f"{user}/en_core_web_sm" - _delete_repo(model_id) - - # Download example file from HF Hub (see https://huggingface.co/spacy/en_core_web_sm) - with production_endpoint(): - whl_path = hf_hub_download( - repo_id="spacy/en_core_web_sm", - filename="en_core_web_sm-any-py3-none-any.whl", - ) - - # Push spacy model to Hub - push(whl_path) - - # Sleep to ensure that model_info isn't called too soon - time.sleep(1) - - # Check model has been pushed properly - model_id = f"{user}/en_core_web_sm" - assert model_info(model_id).library_name == "spacy" - - # Cleanup - _delete_repo(model_id) - - -def _delete_repo(model_id: str) -> None: - try: - delete_repo(model_id) - except HfHubHTTPError: - pass diff --git a/contrib/timm/__init__.py b/contrib/timm/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/contrib/timm/requirements.txt b/contrib/timm/requirements.txt deleted file mode 100644 index 33944e7373..0000000000 --- a/contrib/timm/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Timm -git+https://github.com/rwightman/pytorch-image-models.git#egg=timm -safetensors diff --git a/contrib/timm/test_timm.py b/contrib/timm/test_timm.py deleted file mode 100644 index f57788f8c4..0000000000 --- a/contrib/timm/test_timm.py +++ /dev/null @@ -1,20 +0,0 @@ -import timm - -from ..utils import production_endpoint - - -MODEL_ID = "timm/mobilenetv3_large_100.ra_in1k" - - -@production_endpoint() -def test_load_from_hub() -> None: - # Test load only config - _ = timm.models.load_model_config_from_hf(MODEL_ID) - - # Load entire model from Hub - _ = timm.create_model("hf_hub:" + MODEL_ID, pretrained=True) - - -def test_push_to_hub(repo_name: str, cleanup_repo: None) -> None: - model = timm.create_model("mobilenetv3_rw") - timm.models.push_to_hf_hub(model, repo_name) diff --git a/contrib/utils.py b/contrib/utils.py deleted file mode 100644 index e1681cd561..0000000000 --- a/contrib/utils.py +++ /dev/null @@ -1,56 +0,0 @@ -import contextlib -from typing import Generator -from unittest.mock import patch - - -@contextlib.contextmanager -def production_endpoint() -> Generator: - """Patch huggingface_hub to connect to production server in a context manager. - - Ugly way to patch all constants at once. - TODO: refactor when https://github.com/huggingface/huggingface_hub/issues/1172 is fixed. - - Example: - ```py - def test_push_to_hub(): - # Pull from production Hub - with production_endpoint(): - model = ...from_pretrained("modelname") - - # Push to staging Hub - model.push_to_hub() - ``` - """ - PROD_ENDPOINT = "https://huggingface.co" - ENDPOINT_TARGETS = [ - "huggingface_hub.constants", - "huggingface_hub._commit_api", - "huggingface_hub.hf_api", - "huggingface_hub.lfs", - "huggingface_hub.commands.user", - "huggingface_hub.utils._git_credential", - ] - - PROD_URL_TEMPLATE = PROD_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}" - URL_TEMPLATE_TARGETS = [ - "huggingface_hub.constants", - "huggingface_hub.file_download", - ] - - from huggingface_hub.hf_api import api - - patchers = ( - [patch(target + ".ENDPOINT", PROD_ENDPOINT) for target in ENDPOINT_TARGETS] - + [patch(target + ".HUGGINGFACE_CO_URL_TEMPLATE", PROD_URL_TEMPLATE) for target in URL_TEMPLATE_TARGETS] - + [patch.object(api, "endpoint", PROD_URL_TEMPLATE)] - ) - - # Start all patches - for patcher in patchers: - patcher.start() - - yield - - # Stop all patches - for patcher in patchers: - patcher.stop() diff --git a/utils/check_contrib_list.py b/utils/check_contrib_list.py deleted file mode 100644 index 5b89e27a86..0000000000 --- a/utils/check_contrib_list.py +++ /dev/null @@ -1,109 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains a tool to list contrib test suites automatically.""" - -import argparse -import re -from pathlib import Path -from typing import NoReturn - - -ROOT_DIR = Path(__file__).parent.parent -CONTRIB_PATH = ROOT_DIR / "contrib" -MAKEFILE_PATH = ROOT_DIR / "Makefile" -WORKFLOW_PATH = ROOT_DIR / ".github" / "workflows" / "contrib-tests.yml" - -MAKEFILE_REGEX = re.compile(r"^CONTRIB_LIBS := .*$", flags=re.MULTILINE) -WORKFLOW_REGEX = re.compile( - r""" - # First: match "contrib: [" - (?P^\s{8}contrib:\s\[\n) - # Match list of libs - (\s{10}\".*\",\n)* - # Finally: match trailing "]" - (?P^\s{8}\]) - """, - flags=re.MULTILINE | re.VERBOSE, -) - - -def check_contrib_list(update: bool) -> NoReturn: - """List `contrib` test suites. - - Make sure `Makefile` and `.github/workflows/contrib-tests.yml` are consistent with - the list.""" - # List contrib test suites - contrib_list = sorted( - path.name for path in CONTRIB_PATH.glob("*") if path.is_dir() and not path.name.startswith("_") - ) - - # Check Makefile is consistent with list - makefile_content = MAKEFILE_PATH.read_text() - makefile_expected_content = MAKEFILE_REGEX.sub(f"CONTRIB_LIBS := {' '.join(contrib_list)}", makefile_content) - - # Check workflow is consistent with list - workflow_content = WORKFLOW_PATH.read_text() - _substitute = "\n".join(f'{" " * 10}"{lib}",' for lib in contrib_list) - workflow_content_expected = WORKFLOW_REGEX.sub(rf"\g{_substitute}\n\g", workflow_content) - - # - failed = False - if makefile_content != makefile_expected_content: - if update: - print( - "✅ Contrib libs have been updated in `Makefile`." - "\n Please make sure the changes are accurate and commit them." - ) - MAKEFILE_PATH.write_text(makefile_expected_content) - else: - print( - "❌ Expected content mismatch in `Makefile`.\n It is most likely that" - " you added a contrib test and did not update the Makefile.\n Please" - " run `make style` or `python utils/check_contrib_list.py --update`." - ) - failed = True - - if workflow_content != workflow_content_expected: - if update: - print( - f"✅ Contrib libs have been updated in `{WORKFLOW_PATH}`." - "\n Please make sure the changes are accurate and commit them." - ) - WORKFLOW_PATH.write_text(workflow_content_expected) - else: - print( - f"❌ Expected content mismatch in `{WORKFLOW_PATH}`.\n It is most" - " likely that you added a contrib test and did not update the github" - " workflow file.\n Please run `make style` or `python" - " utils/check_contrib_list.py --update`." - ) - failed = True - - if failed: - exit(1) - print("✅ All good! (contrib list)") - exit(0) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--update", - action="store_true", - help="Whether to fix Makefile and github workflow if a new lib is detected.", - ) - args = parser.parse_args() - - check_contrib_list(update=args.update)