diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7ff8c29af3..176b3d1fd1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -26,6 +26,8 @@ env: SCCACHE_AZURE_KEY_PREFIX: genai/ubuntu/22_04/x64 HF_HOME: /mount/caches/huggingface/lin OV_CACHE: /mount/caches/huggingface/.ov_cache/lin + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug OPENVINO_LOG_LEVEL: 5 GENAI_ARCHIVE_NAME: genai.tar.gz GENAI_SAMPLES_NAME: genai_samples.tar.gz diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index bb544bc0cf..0854e96bfd 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -24,6 +24,8 @@ env: CCACHE_MAXSIZE: 500Mi HF_HOME: C:/mount/caches/huggingface/win OV_CACHE: C:/mount/caches/huggingface/.ov_cache/win/ + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug OPENVINO_LOG_LEVEL: 5 ARTIFACTS_SHARE: '/mount/build-artifacts' BASE_PRODUCT_TYPE: public_windows_vs2022 diff --git a/tools/who_what_benchmark/tests/constants.py b/tools/who_what_benchmark/tests/constants.py new file mode 100644 index 0000000000..b3a223b024 --- /dev/null +++ b/tools/who_what_benchmark/tests/constants.py @@ -0,0 +1,26 @@ +from pathlib import Path +import os +import tempfile +from datetime import datetime +from importlib import metadata + + +SHOULD_CLEANUP = os.environ.get("CLEANUP_CACHE", "").lower() in ("1", "true", "yes") + + +def get_wwb_cache_dir(temp_dir=tempfile.TemporaryDirectory()) -> Path: + if "OV_CACHE" in os.environ: + date_subfolder = datetime.now().strftime("%Y%m%d") + ov_cache = os.path.join(os.environ["OV_CACHE"], date_subfolder) + try: + optimum_intel_version = metadata.version("optimum-intel") + transformers_version = metadata.version("transformers") + ov_cache = os.path.join(ov_cache, f"optimum-intel-{optimum_intel_version}_transformers-{transformers_version}") + except metadata.PackageNotFoundError: + pass + else: + ov_cache = temp_dir.name + return Path(ov_cache) / "wwb_cache" + + +WWB_CACHE_PATH = get_wwb_cache_dir() diff --git a/tools/who_what_benchmark/tests/test_cli_image.py b/tools/who_what_benchmark/tests/test_cli_image.py index a2fa412c5c..16fa84235e 100644 --- a/tools/who_what_benchmark/tests/test_cli_image.py +++ b/tools/who_what_benchmark/tests/test_cli_image.py @@ -5,14 +5,14 @@ import shutil import pytest import logging -import tempfile import re +from constants import WWB_CACHE_PATH, SHOULD_CLEANUP logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -MODEL_CACHE = tempfile.mkdtemp() +MODEL_CACHE = WWB_CACHE_PATH OV_IMAGE_MODELS = ["echarlaix/tiny-random-stable-diffusion-xl", "yujiepan/stable-diffusion-3-tiny-random", "katuni4ka/tiny-random-flux", @@ -38,13 +38,14 @@ def run_wwb(args): def setup_module(): for model_id in OV_IMAGE_MODELS: - MODEL_PATH = os.path.join(MODEL_CACHE, model_id.replace("/", "--")) + MODEL_PATH = MODEL_CACHE / model_id.replace("/", "--") subprocess.run(["optimum-cli", "export", "openvino", "--model", model_id, MODEL_PATH], capture_output=True, text=True) def teardown_module(): - logger.info("Remove models") - shutil.rmtree(MODEL_CACHE) + if SHOULD_CLEANUP: + logger.info("Removing models") + shutil.rmtree(MODEL_CACHE) def get_similarity(output: str) -> float: @@ -121,11 +122,12 @@ def test_image_model_genai(model_id, model_type, tmp_path): pytest.xfail("Ticket 173169") GT_FILE = tmp_path / "gt.csv" - MODEL_PATH = os.path.join(MODEL_CACHE, model_id.replace("/", "--")) + MODEL_PATH = MODEL_CACHE / model_id.replace("/", "--") + MODEL_PATH = MODEL_PATH if MODEL_PATH.exists() else model_id run_wwb([ "--base-model", - model_id, + MODEL_PATH, "--num-samples", "1", "--gt-data", diff --git a/tools/who_what_benchmark/tests/test_cli_text.py b/tools/who_what_benchmark/tests/test_cli_text.py index 4b8d54c305..a477ca2f88 100644 --- a/tools/who_what_benchmark/tests/test_cli_text.py +++ b/tools/who_what_benchmark/tests/test_cli_text.py @@ -1,12 +1,13 @@ import os import shutil -import tempfile import pandas as pd import pytest import logging import json import sys +from constants import WWB_CACHE_PATH, SHOULD_CLEANUP + from transformers import AutoTokenizer from optimum.intel.openvino import OVModelForCausalLM, OVWeightQuantizationConfig @@ -18,9 +19,9 @@ model_id = "facebook/opt-125m" -tmp_dir = tempfile.mkdtemp() -base_model_path = os.path.join(tmp_dir, "opt125m") -target_model_path = os.path.join(tmp_dir, "opt125m_int8") +cache_dir = WWB_CACHE_PATH +base_model_path = cache_dir / "opt125m" +target_model_path = cache_dir / "opt125m_int8" gptq_model_id = "ybelkada/opt-125m-gptq-4bit" awq_model_id = "TitanML/tiny-mixtral-AWQ-4bit" @@ -29,24 +30,30 @@ def setup_module(): from optimum.exporters.openvino.convert import export_tokenizer - logger.info("Create models") - tokenizer = AutoTokenizer.from_pretrained(model_id) - base_model = OVModelForCausalLM.from_pretrained(model_id) - base_model.save_pretrained(base_model_path) - tokenizer.save_pretrained(base_model_path) + if not os.path.exists(base_model_path): + logger.info("Create models") + tokenizer = AutoTokenizer.from_pretrained(model_id) + base_model = OVModelForCausalLM.from_pretrained(model_id) + base_model.save_pretrained(base_model_path) + tokenizer.save_pretrained(base_model_path) + else: + tokenizer = AutoTokenizer.from_pretrained(base_model_path) export_tokenizer(tokenizer, base_model_path) - target_model = OVModelForCausalLM.from_pretrained( - model_id, quantization_config=OVWeightQuantizationConfig(bits=8) - ) - target_model.save_pretrained(target_model_path) - tokenizer.save_pretrained(target_model_path) + if not os.path.exists(target_model_path): + target_model = OVModelForCausalLM.from_pretrained( + model_id, quantization_config=OVWeightQuantizationConfig(bits=8) + ) + tokenizer = AutoTokenizer.from_pretrained(model_id) + target_model.save_pretrained(target_model_path) + tokenizer.save_pretrained(target_model_path) export_tokenizer(tokenizer, target_model_path) def teardown_module(): - logger.info("Remove models") - shutil.rmtree(tmp_dir) + if SHOULD_CLEANUP: + logger.info("Removing models") + shutil.rmtree(cache_dir) @pytest.mark.skipif((sys.platform == "darwin"), reason='173169') diff --git a/tools/who_what_benchmark/tests/test_cli_vlm.py b/tools/who_what_benchmark/tests/test_cli_vlm.py index 1d459c1ab5..2fa76cdf11 100644 --- a/tools/who_what_benchmark/tests/test_cli_vlm.py +++ b/tools/who_what_benchmark/tests/test_cli_vlm.py @@ -2,6 +2,7 @@ import pytest import logging import sys +from constants import WWB_CACHE_PATH from test_cli_image import run_wwb, get_similarity @@ -13,17 +14,18 @@ def run_test(model_id, model_type, optimum_threshold, genai_threshold, tmp_path) if sys.platform == 'darwin': pytest.xfail("Ticket 173169") GT_FILE = tmp_path / "gt.csv" - MODEL_PATH = tmp_path / model_id.replace("/", "--") + MODEL_PATH = WWB_CACHE_PATH / model_id.replace("/", "--") - result = subprocess.run(["optimum-cli", "export", - "openvino", "-m", model_id, - MODEL_PATH, "--task", - "image-text-to-text", - "--trust-remote-code"], - capture_output=True, - text=True, - ) - assert result.returncode == 0 + if not MODEL_PATH.exists(): + result = subprocess.run(["optimum-cli", "export", + "openvino", "-m", model_id, + MODEL_PATH, "--task", + "image-text-to-text", + "--trust-remote-code"], + capture_output=True, + text=True, + ) + assert result.returncode == 0 # Collect reference with HF model run_wwb([ @@ -71,7 +73,7 @@ def run_test(model_id, model_type, optimum_threshold, genai_threshold, tmp_path) model_type, "--genai", "--output", - tmp_path, + WWB_CACHE_PATH, ]) if genai_threshold is not None: similarity = get_similarity(output) @@ -80,7 +82,7 @@ def run_test(model_id, model_type, optimum_threshold, genai_threshold, tmp_path) # test w/o models run_wwb([ "--target-data", - tmp_path / "target.csv", + WWB_CACHE_PATH / "target.csv", "--num-samples", "1", "--gt-data",