diff --git a/tests/python_tests/data/models.py b/tests/python_tests/data/models.py index ab5eb44167..18f13147e4 100644 --- a/tests/python_tests/data/models.py +++ b/tests/python_tests/data/models.py @@ -6,7 +6,7 @@ def get_models_list() -> tuple[str, ...]: model_ids: list[str] = [ - "katuni4ka/tiny-random-phi3", + "optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM", ] if pytest.selected_model_ids: model_ids = [model_id for model_id in model_ids if model_id in pytest.selected_model_ids.split(' ')] diff --git a/tests/python_tests/data/tokenizer_configs.py b/tests/python_tests/data/tokenizer_configs.py index cca90673e3..ab969ba2a5 100644 --- a/tests/python_tests/data/tokenizer_configs.py +++ b/tests/python_tests/data/tokenizer_configs.py @@ -670,7 +670,7 @@ def get_tokenizer_configs(): "unk_token": "", "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>'+ '\n' + message['content'] + '\n'}}{% elif message['role'] == 'user' %}{{'<|user|>' + '\n' + message['content'] + '\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>' + '\n' + message['content'] + '<|endoftext|>' + ('' if loop.last else '\n')}}{% endif %}{% endfor %}" }, - "katuni4ka/tiny-random-phi3": { + "optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM": { "bos_token": "", "eos_token": "<|endoftext|>", "pad_token": "<|endoftext|>", @@ -712,7 +712,7 @@ def get_tokenizer_configs(): "unk_token": "", "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '### User:\n' + message['content'] }}\n{% elif message['role'] == 'system' %}\n{{ '### System:\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '### Assistant:\n' + message['content'] }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '### Assistant:' }}\n{% endif %}\n{% endfor %}" }, - "katuni4ka/tiny-random-minicpm": { + "optimum-intel-internal-testing/tiny-random-minicpm": { "bos_token": "", "eos_token": "", "pad_token": None, diff --git a/tests/python_tests/samples/conftest.py b/tests/python_tests/samples/conftest.py index 3aa71fcb3c..e157ddf7e1 100644 --- a/tests/python_tests/samples/conftest.py +++ b/tests/python_tests/samples/conftest.py @@ -94,7 +94,7 @@ "convert_args": ['--trust-remote-code', '--weight-format', 'fp16'] }, "tiny-random-minicpmv-2_6": { - "name": "katuni4ka/tiny-random-minicpmv-2_6", + "name": "optimum-intel-internal-testing/tiny-random-minicpmv-2_6", "convert_args": ['--trust-remote-code', "--task", "image-text-to-text"] }, "InternVL2-1B": { @@ -118,7 +118,7 @@ "convert_args": ["--task", "text-generation-with-past", "--weight-format", "int8"] }, "tiny-random-latent-consistency": { - "name": "echarlaix/tiny-random-latent-consistency", + "name": "optimum-intel-internal-testing/tiny-random-latent-consistency", "convert_args": ['--trust-remote-code', '--weight-format', 'fp16'] }, "tiny-random-latent-consistency-lora": { @@ -126,7 +126,7 @@ "convert_args": [] }, "tiny-random-llava": { - "name": "katuni4ka/tiny-random-llava", + "name": "optimum-intel-internal-testing/tiny-random-llava", "convert_args": ["--trust-remote-code", "--task", "image-text-to-text"] }, "tiny-random-qwen2vl": { diff --git a/tests/python_tests/test_llm_pipeline.py b/tests/python_tests/test_llm_pipeline.py index c26afe1dfa..407a0c0f0c 100644 --- a/tests/python_tests/test_llm_pipeline.py +++ b/tests/python_tests/test_llm_pipeline.py @@ -202,14 +202,14 @@ def test_batch_string_inputs( ) -@pytest.mark.parametrize("llm_model", ['katuni4ka/tiny-random-phi3'], indirect=True) +@pytest.mark.parametrize("llm_model", ["optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"], indirect=True) def test_batch_size_switch(ov_pipe: ov_genai.LLMPipeline) -> None: ov_pipe.generate(["a"], max_new_tokens=2) ov_pipe.generate(["1", "2"], max_new_tokens=2) ov_pipe.generate(["a"], max_new_tokens=2) -@pytest.mark.parametrize("llm_model", ['katuni4ka/tiny-random-phi3'], indirect=True) +@pytest.mark.parametrize("llm_model", ["optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"], indirect=True) def test_empty_encoded_inputs_throw(ov_pipe: ov_genai.LLMPipeline) -> None: with pytest.raises(RuntimeError): ov_pipe.generate(ov.Tensor(np.array([[]], dtype=np.int64)), max_new_tokens=2) @@ -698,29 +698,35 @@ def test_pipeline_validates_generation_config(ov_pipe: ov_genai.LLMPipeline) -> # Work with Unicode in Python API # -@pytest.mark.parametrize("llm_model", MODELS_LIST, indirect=True) -def test_unicode_pybind_decoding_one_string(ov_pipe: ov_genai.LLMPipeline) -> None: - res_str = ov_pipe.generate(',', max_new_tokens=4, apply_chat_template=False) +# Model, prompt and max_new_tokens that generate unfinished utf-8 string +UNICODE_PYBIND_DECODING_TEST_CASES: list[tuple[str, str, int]] = [ + ("optimum-intel-internal-testing/tiny-random-PhiForCausalLM", ",", 3) +] + + +@pytest.mark.parametrize("llm_model,prompt,max_new_tokens", UNICODE_PYBIND_DECODING_TEST_CASES, indirect=["llm_model"]) +def test_unicode_pybind_decoding_one_string(ov_pipe: ov_genai.LLMPipeline, prompt: str, max_new_tokens: int) -> None: + res_str = ov_pipe.generate(prompt, max_new_tokens=max_new_tokens, apply_chat_template=False) assert '�' == res_str[-1] -@pytest.mark.parametrize("llm_model", MODELS_LIST, indirect=True) -def test_unicode_pybind_decoding_batched(ov_pipe: ov_genai.LLMPipeline) -> None: - res_str = ov_pipe.generate([","], max_new_tokens=4, apply_chat_template=False) +@pytest.mark.parametrize("llm_model,prompt,max_new_tokens", UNICODE_PYBIND_DECODING_TEST_CASES, indirect=["llm_model"]) +def test_unicode_pybind_decoding_batched(ov_pipe: ov_genai.LLMPipeline, prompt: str, max_new_tokens: int) -> None: + res_str = ov_pipe.generate([prompt], max_new_tokens=max_new_tokens, apply_chat_template=False) assert '�' == res_str.texts[0][-1] -@pytest.mark.parametrize("llm_model", MODELS_LIST, indirect=True) -def test_unicode_pybind_decoding_one_string_streamer(ov_pipe: ov_genai.LLMPipeline) -> None: +@pytest.mark.parametrize("llm_model,prompt,max_new_tokens", UNICODE_PYBIND_DECODING_TEST_CASES, indirect=["llm_model"]) +def test_unicode_pybind_decoding_one_string_streamer(ov_pipe: ov_genai.LLMPipeline, prompt: str, max_new_tokens: int) -> None: res_str = [] - ov_pipe.generate(",", max_new_tokens=4, apply_chat_template=False, streamer=lambda x: res_str.append(x)) + ov_pipe.generate(prompt, max_new_tokens=max_new_tokens, apply_chat_template=False, streamer=lambda x: res_str.append(x)) assert '�' == ''.join(res_str)[-1] # # Perf metrics # -@pytest.mark.parametrize("llm_model", ['katuni4ka/tiny-random-gemma2'], indirect=True) +@pytest.mark.parametrize("llm_model", ["optimum-intel-internal-testing/tiny-random-gemma2"], indirect=True) @pytest.mark.parametrize("generation_config,prompt", PERF_METRICS_TEST_CASES) @pytest.mark.parametrize("pipeline_type", [PipelineType.STATEFUL, PipelineType.PAGED_ATTENTION]) def test_perf_metrics( @@ -812,7 +818,7 @@ def test_perf_metrics( assert len(raw_metrics.m_durations) == num_generated_tokens - 1 -@pytest.mark.parametrize("llm_model", ['katuni4ka/tiny-random-gemma2'], indirect=True) +@pytest.mark.parametrize("llm_model", ["optimum-intel-internal-testing/tiny-random-gemma2"], indirect=True) @pytest.mark.parametrize("generation_config,prompt", PERF_METRICS_STRUCTURED_OUTPUT_TEST_CASES) def test_perf_metrics_with_structured_output( ov_pipe: ov_genai.LLMPipeline, diff --git a/tests/python_tests/test_parsers.py b/tests/python_tests/test_parsers.py index 7828eb77a7..5c21b25749 100644 --- a/tests/python_tests/test_parsers.py +++ b/tests/python_tests/test_parsers.py @@ -25,7 +25,7 @@ def hf_ov_genai_models(request, tmp_path_factory): @pytest.mark.parametrize( "hf_ov_genai_models", - ["katuni4ka/tiny-random-phi3"], # this tokenizer is used as a stub only + ["optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"], # this tokenizer is used as a stub only indirect=True ) @pytest.mark.parametrize("answer", [ @@ -64,7 +64,7 @@ def write(self, message): @pytest.mark.parametrize( "hf_ov_genai_models", - ["katuni4ka/tiny-random-phi3"], # this tokenizer is used as a stub only + ["optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"], # this tokenizer is used as a stub only indirect=True ) def test_incremental_phi4_reason_integer_token_ids(hf_ov_genai_models): @@ -92,7 +92,7 @@ def write(self, message): @pytest.mark.parametrize( "hf_ov_genai_models", - ["katuni4ka/tiny-random-phi3"], # this tokenizer is used as a stub only + ["optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"], # this tokenizer is used as a stub only indirect=True ) def test_incremental_integer_token_ids(hf_ov_genai_models): @@ -144,7 +144,7 @@ def write(self, message): @pytest.mark.parametrize( "hf_ov_genai_models", - ["katuni4ka/tiny-random-phi3"], + ["optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"], indirect=True ) @pytest.mark.parametrize("split_answer", [ @@ -202,7 +202,7 @@ def test_incremental_phi4_reason_parser_nostreamer(answer): @pytest.mark.parametrize("do_reset", [False]) @pytest.mark.parametrize( "hf_ov_genai_models", - ["katuni4ka/tiny-random-phi3"], # this tokenizer is used as a stub only + ["optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"], # this tokenizer is used as a stub only indirect=True ) @pytest.mark.parametrize("answer", [ @@ -266,7 +266,7 @@ def test_incremental_deepseek_parser(): @pytest.mark.parametrize( "hf_ov_genai_models", - ["katuni4ka/tiny-random-phi3"], + ["optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"], indirect=True ) def test_custom_incremental_parser(hf_ov_genai_models): @@ -308,7 +308,7 @@ def write(self, message): @pytest.mark.parametrize( "hf_ov_genai_models", - ["katuni4ka/tiny-random-phi3"], + ["optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"], indirect=True ) def test_final_parser_llama_32_json(hf_ov_genai_models): diff --git a/tests/python_tests/test_sampling.py b/tests/python_tests/test_sampling.py index a89dbb3030..47efd2f775 100644 --- a/tests/python_tests/test_sampling.py +++ b/tests/python_tests/test_sampling.py @@ -21,8 +21,8 @@ def model_facebook_opt_125m() -> OVConvertedModelSchema: @pytest.fixture(scope="module") -def model_katuni4ka_tiny_random_phi3() -> OVConvertedModelSchema: - model_id : str = "katuni4ka/tiny-random-phi3" +def model_tiny_random_phi3() -> OVConvertedModelSchema: + model_id : str = "optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM" return download_and_convert_model(model_id) @@ -50,11 +50,11 @@ def model_tinyllama_1_1b_chat() -> OVConvertedModelSchema: ] ) def test_basic_stop_criteria( - model_katuni4ka_tiny_random_phi3: OVConvertedModelSchema, + model_tiny_random_phi3: OVConvertedModelSchema, generation_config: GenerationConfig, prompt ): - generate_and_compare(model_katuni4ka_tiny_random_phi3, [prompt], generation_config) + generate_and_compare(model_tiny_random_phi3, [prompt], generation_config) @pytest.mark.parametrize( @@ -159,14 +159,14 @@ def test_stop_strings_tinyllama( 'I have an interview about product speccing with the company Weekend Health. Give me an example of a question they might ask with regards about a new feature' ]) def test_greedy( - model_katuni4ka_tiny_random_phi3: OVConvertedModelSchema, + model_tiny_random_phi3: OVConvertedModelSchema, generation_config, prompt ): prompt = prompt.decode('unicode_escape') if isinstance(prompt, bytes) else prompt generate_and_compare( - model_katuni4ka_tiny_random_phi3, + model_tiny_random_phi3, prompt, generation_config ) diff --git a/tests/python_tests/test_stateful_speculative_decoding.py b/tests/python_tests/test_stateful_speculative_decoding.py index 7f0247ba6e..a300537341 100644 --- a/tests/python_tests/test_stateful_speculative_decoding.py +++ b/tests/python_tests/test_stateful_speculative_decoding.py @@ -70,7 +70,7 @@ def test_string_inputs(main_model, main_device, draft_model, draft_device, promp def test_perf_metrics(): import time start_time = time.perf_counter() - model_id = 'katuni4ka/tiny-random-gemma2' + model_id = "optimum-intel-internal-testing/tiny-random-gemma2" model_path = download_and_convert_model(model_id).models_path # Create OpenVINO GenAI pipeline: diff --git a/tests/python_tests/test_structured_output.py b/tests/python_tests/test_structured_output.py index 60e83c36e0..71429b846f 100644 --- a/tests/python_tests/test_structured_output.py +++ b/tests/python_tests/test_structured_output.py @@ -35,7 +35,7 @@ class RESTAPIResponse(BaseModel): structured_id_models = [ "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "katuni4ka/tiny-random-phi3", + "optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM", ] diff --git a/tests/python_tests/test_text_streamer.py b/tests/python_tests/test_text_streamer.py index a8df543da9..128dbe715e 100644 --- a/tests/python_tests/test_text_streamer.py +++ b/tests/python_tests/test_text_streamer.py @@ -88,7 +88,7 @@ def test_text_prompts(tmp_path, prompt, model_id): # '\n\n# 利用re.sub()方法,�' with UTF8 invalid for "microsoft/phi-1_5" [198, 198, 2, 10263, 230, 102, 18796, 101, 260, 13], - # '룅튜룅튜�' causes error on "openbmb/MiniCPM-o-2_6" / "katuni4ka/tiny-random-minicpmv-2_6" + # '룅튜룅튜�' causes error on "openbmb/MiniCPM-o-2_6" / "optimum-intel-internal-testing/tiny-random-minicpmv-2_6" [167, 96, 227, 169, 232, 250, 167, 96, 227, 169, 232, 250, 167] ] @pytest.mark.parametrize("model_id", tokenizer_model_ids) diff --git a/tests/python_tests/test_tokenizer.py b/tests/python_tests/test_tokenizer.py index 8714b3f712..b680436d08 100644 --- a/tests/python_tests/test_tokenizer.py +++ b/tests/python_tests/test_tokenizer.py @@ -296,7 +296,7 @@ def test_set_chat_template(ov_hf_tokenizers): @pytest.mark.parametrize( "ov_hf_tokenizers", [ - "katuni4ka/tiny-random-phi3", + "optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM", "TinyLlama/TinyLlama-1.1B-Chat-v1.0", # ("black-forest-labs/FLUX.1-dev", dict(subfolder="tokenizer")), # FLUX.1-dev has tokenizer in subfolder ], @@ -433,10 +433,10 @@ def hf_ov_genai_models(request, tmp_path_factory): @pytest.mark.parametrize( "hf_ov_genai_models", [ - ("katuni4ka/tiny-random-phi3", {"padding_side": None}), + ("optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM", {"padding_side": None}), ("TinyLlama/TinyLlama-1.1B-Chat-v1.0", {"padding_side": None}), - ("katuni4ka/tiny-random-llava-next", {"padding_side": "right"}), - ("katuni4ka/tiny-random-llava-next", {"padding_side": "left"}), + ("optimum-intel-internal-testing/tiny-random-llava-next", {"padding_side": "right"}), + ("optimum-intel-internal-testing/tiny-random-llava-next", {"padding_side": "left"}), ( "BAAI/bge-small-en-v1.5", {"padding_side": None}, @@ -506,8 +506,8 @@ def test_padding( base_models_for_paired_input_test = [ ("answerdotai/ModernBERT-base", {"padding_side": None}), ("TinyLlama/TinyLlama-1.1B-Chat-v1.0", {"padding_side": None}), - ("katuni4ka/tiny-random-llava-next", {"padding_side": "right"}), - ("katuni4ka/tiny-random-llava-next", {"padding_side": "left"}), + ("optimum-intel-internal-testing/tiny-random-llava-next", {"padding_side": "right"}), + ("optimum-intel-internal-testing/tiny-random-llava-next", {"padding_side": "left"}), ] def make_model_params(): diff --git a/tests/python_tests/test_vlm_pipeline.py b/tests/python_tests/test_vlm_pipeline.py index 8168ad86cf..b2d5cedfe6 100644 --- a/tests/python_tests/test_vlm_pipeline.py +++ b/tests/python_tests/test_vlm_pipeline.py @@ -82,20 +82,20 @@ class VlmModelInfo: VIDEO_MODEL_IDS = [ - "katuni4ka/tiny-random-llava-next-video", - "katuni4ka/tiny-random-qwen2vl", - "katuni4ka/tiny-random-qwen2.5-vl" + "optimum-intel-internal-testing/tiny-random-llava-next-video", + "optimum-intel-internal-testing/tiny-random-qwen2vl", + "optimum-intel-internal-testing/tiny-random-qwen2.5-vl" ] MODEL_IDS: list[str] = [ - "katuni4ka/tiny-random-minicpmv-2_6", - "katuni4ka/tiny-random-phi3-vision", - "katuni4ka/tiny-random-phi-4-multimodal", - "katuni4ka/tiny-random-llava", - "katuni4ka/tiny-random-llava-next", - "katuni4ka/tiny-random-internvl2", - "katuni4ka/tiny-random-gemma3", + "optimum-intel-internal-testing/tiny-random-minicpmv-2_6", + "optimum-intel-internal-testing/tiny-random-phi3-vision", + "optimum-intel-internal-testing/tiny-random-phi-4-multimodal", + "optimum-intel-internal-testing/tiny-random-llava", + "optimum-intel-internal-testing/tiny-random-llava-next", + "optimum-intel-internal-testing/tiny-random-internvl2", + "optimum-intel-internal-testing/tiny-random-gemma3", "qnguyen3/nanoLLaVA", *VIDEO_MODEL_IDS, ] @@ -108,28 +108,28 @@ class VlmModelInfo: TAG_GENERATOR_BY_MODEL: dict[str, Callable[[int], str]] = { - "katuni4ka/tiny-random-llava": lambda idx: "", - "katuni4ka/tiny-random-llava-next": lambda idx: "", - "katuni4ka/tiny-random-qwen2vl": lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", - "katuni4ka/tiny-random-qwen2.5-vl": lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", - "katuni4ka/tiny-random-gemma3": lambda idx: "", - "katuni4ka/tiny-random-internvl2": lambda idx: "\n", - "katuni4ka/tiny-random-minicpmv-2_6": lambda idx: "./\n", - "katuni4ka/tiny-random-phi3-vision": lambda idx: f"<|image_{idx + 1}|>\n", - "katuni4ka/tiny-random-llava-next-video": lambda idx: "\n", + "optimum-intel-internal-testing/tiny-random-llava": lambda idx: "", + "optimum-intel-internal-testing/tiny-random-llava-next": lambda idx: "", + "optimum-intel-internal-testing/tiny-random-qwen2vl": lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", + "optimum-intel-internal-testing/tiny-random-qwen2.5-vl": lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", + "optimum-intel-internal-testing/tiny-random-gemma3": lambda idx: "", + "optimum-intel-internal-testing/tiny-random-internvl2": lambda idx: "\n", + "optimum-intel-internal-testing/tiny-random-minicpmv-2_6": lambda idx: "./\n", + "optimum-intel-internal-testing/tiny-random-phi3-vision": lambda idx: f"<|image_{idx + 1}|>\n", + "optimum-intel-internal-testing/tiny-random-llava-next-video": lambda idx: "\n", "qnguyen3/nanoLLaVA": lambda idx: "\n", } RESOLUTION_BY_MODEL: dict[str, int | None] = { - "katuni4ka/tiny-random-gemma3": 32, + "optimum-intel-internal-testing/tiny-random-gemma3": 32, "qnguyen3/nanoLLaVA": 384, - "katuni4ka/tiny-random-llava-next-video": 336, + "optimum-intel-internal-testing/tiny-random-llava-next-video": 336, } RESOLUTION_BY_VIDEO_MODEL: dict[str, int | None] = { - "katuni4ka/tiny-random-llava-next-video": 32, + "optimum-intel-internal-testing/tiny-random-llava-next-video": 32, } @@ -154,8 +154,8 @@ class VlmModelInfo: NPU_UNSUPPORTED_MODELS = { - "katuni4ka/tiny-random-internvl2", - "katuni4ka/tiny-random-gemma3", + "optimum-intel-internal-testing/tiny-random-internvl2", + "optimum-intel-internal-testing/tiny-random-gemma3", } @@ -180,9 +180,9 @@ def _setup_generation_config( def _get_ov_model(model_id: str) -> str: - if model_id in {"katuni4ka/tiny-random-phi-4-multimodal", "qnguyen3/nanoLLaVA"}: + if model_id in {"optimum-intel-internal-testing/tiny-random-phi-4-multimodal", "qnguyen3/nanoLLaVA"}: pytest.skip("ValueError: The current version of Transformers does not allow for the export of the model. Maximum required is 4.53.3, got: 4.55.4") - if "katuni4ka/tiny-random-phi3-vision" == model_id: + if "optimum-intel-internal-testing/tiny-random-phi3-vision" == model_id: pytest.xfail("AttributeError: 'DynamicCache' object has no attribute 'get_usable_length'. Ticket CVS-175110") ov_cache_converted_dir = get_ov_cache_converted_models_dir() dir_name = str(model_id).replace(os.sep, "_") @@ -210,10 +210,10 @@ def convert_to_temp(temp_dir: Path) -> None: export=True, load_in_8bit=False, trust_remote_code=model_id in { - "katuni4ka/tiny-random-minicpmv-2_6", - "katuni4ka/tiny-random-internvl2", - "katuni4ka/tiny-random-phi3-vision", - "katuni4ka/tiny-random-phi-4-multimodal", + "optimum-intel-internal-testing/tiny-random-minicpmv-2_6", + "optimum-intel-internal-testing/tiny-random-internvl2", + "optimum-intel-internal-testing/tiny-random-phi3-vision", + "optimum-intel-internal-testing/tiny-random-phi-4-multimodal", "qnguyen3/nanoLLaVA", }, ) @@ -809,7 +809,7 @@ def test_perf_metrics( max_new_tokens = DEFAULT_MAX_NEW_TOKENS # Using non-cached model to get more accurate load time - model_path = _get_ov_model("katuni4ka/tiny-random-minicpmv-2_6") + model_path = _get_ov_model("optimum-intel-internal-testing/tiny-random-minicpmv-2_6") start_time = perf_counter_ns() pipe = VLMPipeline(model_path, "CPU", ATTENTION_BACKEND=backend) start_generate = perf_counter_ns() @@ -1141,24 +1141,24 @@ def conversation_requests( TAG_INSERTED_BY_TEMPLATE = [ - ("katuni4ka/tiny-random-llava", "PA"), - ("katuni4ka/tiny-random-llava-next", "PA"), - ("katuni4ka/tiny-random-qwen2vl", "PA"), - ("katuni4ka/tiny-random-qwen2.5-vl", "PA"), - ("katuni4ka/tiny-random-gemma3", "SDPA"), + ("optimum-intel-internal-testing/tiny-random-llava", "PA"), + ("optimum-intel-internal-testing/tiny-random-llava-next", "PA"), + ("optimum-intel-internal-testing/tiny-random-qwen2vl", "PA"), + ("optimum-intel-internal-testing/tiny-random-qwen2.5-vl", "PA"), + ("optimum-intel-internal-testing/tiny-random-gemma3", "SDPA"), ("qnguyen3/nanoLLaVA", "PA"), - ("katuni4ka/tiny-random-llava-next-video", "PA"), + ("optimum-intel-internal-testing/tiny-random-llava-next-video", "PA"), ] IMAGE_ID_IGNORANT_MODELS_TO_TAG = TAG_INSERTED_BY_TEMPLATE + [ - ("katuni4ka/tiny-random-internvl2", "PA"), + ("optimum-intel-internal-testing/tiny-random-internvl2", "PA"), ] MODELS_TO_TAG = IMAGE_ID_IGNORANT_MODELS_TO_TAG + [ - ("katuni4ka/tiny-random-minicpmv-2_6", "PA"), - ("katuni4ka/tiny-random-phi3-vision", "PA"), + ("optimum-intel-internal-testing/tiny-random-minicpmv-2_6", "PA"), + ("optimum-intel-internal-testing/tiny-random-phi3-vision", "PA"), ] @@ -1380,32 +1380,32 @@ def test_model_tags_missing_native(ov_pipe_model: VlmModelInfo): @pytest.mark.parametrize( "ov_pipe_model,has_image,has_video", [ - pytest.param(("katuni4ka/tiny-random-qwen2vl","SDPA"), True, False, id="qwen2vl/SDPA/image"), - pytest.param(("katuni4ka/tiny-random-qwen2vl", "PA"), True, False, id="qwen2vl/PA/image"), - pytest.param(("katuni4ka/tiny-random-qwen2vl","SDPA"), False, True, id="qwen2vl/SDPA/video"), - pytest.param(("katuni4ka/tiny-random-qwen2vl", "PA"), False, True, id="qwen2vl/PA/video"), - pytest.param(("katuni4ka/tiny-random-qwen2vl", "SDPA"), True, True, id="qwen2vl/PA/image+video"), - pytest.param(("katuni4ka/tiny-random-qwen2vl", "PA"), True, True, id="qwen2vl/PA/image+video"), - pytest.param(("katuni4ka/tiny-random-qwen2.5-vl", "SDPA"), True, False, id="qwen2.5-vl/SDPA/image"), - pytest.param(("katuni4ka/tiny-random-qwen2.5-vl", "PA"), True, False, id="qwen2.5-vl/PA/image", marks=pytest.mark.xfail(reason="CVS-167316")), - pytest.param(("katuni4ka/tiny-random-qwen2.5-vl", "SDPA"), False, True, id="qwen2.5-vl/SDPA/video"), - pytest.param(("katuni4ka/tiny-random-qwen2.5-vl", "PA"), False, True, id="qwen2.5-vl/PA/video", marks=pytest.mark.xfail(reason="CVS-167316")), - pytest.param(("katuni4ka/tiny-random-qwen2.5-vl", "SDPA"), True, True, id="qwen2.5-vl/SDPA/image+video"), - pytest.param(("katuni4ka/tiny-random-qwen2.5-vl", "PA"), True, True, id="qwen2.5-vl/PA/image+video", marks=pytest.mark.xfail(reason="CVS-167316")), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2vl","SDPA"), True, False, id="qwen2vl/SDPA/image"), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2vl", "PA"), True, False, id="qwen2vl/PA/image"), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2vl","SDPA"), False, True, id="qwen2vl/SDPA/video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2vl", "PA"), False, True, id="qwen2vl/PA/video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2vl", "SDPA"), True, True, id="qwen2vl/PA/image+video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2vl", "PA"), True, True, id="qwen2vl/PA/image+video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2.5-vl", "SDPA"), True, False, id="qwen2.5-vl/SDPA/image"), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2.5-vl", "PA"), True, False, id="qwen2.5-vl/PA/image", marks=pytest.mark.xfail(reason="CVS-167316")), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2.5-vl", "SDPA"), False, True, id="qwen2.5-vl/SDPA/video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2.5-vl", "PA"), False, True, id="qwen2.5-vl/PA/video", marks=pytest.mark.xfail(reason="CVS-167316")), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2.5-vl", "SDPA"), True, True, id="qwen2.5-vl/SDPA/image+video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-qwen2.5-vl", "PA"), True, True, id="qwen2.5-vl/PA/image+video", marks=pytest.mark.xfail(reason="CVS-167316")), ( - pytest.param(("katuni4ka/tiny-random-gemma3", "SDPA"), True, False, id="gemma3/SDPA/image", marks=pytest.mark.xfail(reason=GEMMA3_MACOS_XFAIL_REASON)) + pytest.param(("optimum-intel-internal-testing/tiny-random-gemma3", "SDPA"), True, False, id="gemma3/SDPA/image", marks=pytest.mark.xfail(reason=GEMMA3_MACOS_XFAIL_REASON)) if sys.platform == "darwin" - else pytest.param(("katuni4ka/tiny-random-gemma3", "SDPA"), True, False, id="gemma3/SDPA/image") + else pytest.param(("optimum-intel-internal-testing/tiny-random-gemma3", "SDPA"), True, False, id="gemma3/SDPA/image") ), - pytest.param(("katuni4ka/tiny-random-gemma3", "PA"), True, False, id="gemma3/PA/image", marks=pytest.mark.xfail(reason="CVS-171180")), + pytest.param(("optimum-intel-internal-testing/tiny-random-gemma3", "PA"), True, False, id="gemma3/PA/image", marks=pytest.mark.xfail(reason="CVS-171180")), pytest.param(("qnguyen3/nanoLLaVA", "SDPA"), True, False, id="nanoLLaVA/SDPA/image"), pytest.param(("qnguyen3/nanoLLaVA", "PA"), True, False, id="nanoLLaVA/PA/image"), - pytest.param(("katuni4ka/tiny-random-llava-next-video", "SDPA"), True, False, id="llava-next-video/SDPA/image"), - pytest.param(("katuni4ka/tiny-random-llava-next-video", "PA"), True, False, id="llava-next-video/PA/image"), - pytest.param(("katuni4ka/tiny-random-llava-next-video", "SDPA"), False, True, id="llava-next-video/SDPA/video"), - pytest.param(("katuni4ka/tiny-random-llava-next-video", "PA"), False, True, id="llava-next-video/PA/video"), - pytest.param(("katuni4ka/tiny-random-llava-next-video", "SDPA"), True, True, id="llava-next-video/SDPA/image+video"), - pytest.param(("katuni4ka/tiny-random-llava-next-video", "PA"), True, True, id="llava-next-video/PA/image+video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-llava-next-video", "SDPA"), True, False, id="llava-next-video/SDPA/image"), + pytest.param(("optimum-intel-internal-testing/tiny-random-llava-next-video", "PA"), True, False, id="llava-next-video/PA/image"), + pytest.param(("optimum-intel-internal-testing/tiny-random-llava-next-video", "SDPA"), False, True, id="llava-next-video/SDPA/video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-llava-next-video", "PA"), False, True, id="llava-next-video/PA/video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-llava-next-video", "SDPA"), True, True, id="llava-next-video/SDPA/image+video"), + pytest.param(("optimum-intel-internal-testing/tiny-random-llava-next-video", "PA"), True, True, id="llava-next-video/PA/image+video"), ], indirect=["ov_pipe_model"], ) @@ -1456,7 +1456,10 @@ def get_nanollava_processor(): # For QWen-VL series models, in GenAI VLM implementation, video is placed before image in chat template, # but in Optimum, this order depends only on the image and video order in the "conversation". # So just reverse here in order to keep align. - if has_image and has_video and model_id in ["katuni4ka/tiny-random-qwen2.5-vl", "katuni4ka/tiny-random-qwen2vl"]: + if has_image and has_video and model_id in [ + "optimum-intel-internal-testing/tiny-random-qwen2.5-vl", + "optimum-intel-internal-testing/tiny-random-qwen2vl" + ]: media_content.reverse() conversation[0]["content"] = media_content + conversation[0]["content"] diff --git a/tools/who_what_benchmark/tests/test_cli_image.py b/tools/who_what_benchmark/tests/test_cli_image.py index bf8c688b00..47df0352af 100644 --- a/tools/who_what_benchmark/tests/test_cli_image.py +++ b/tools/who_what_benchmark/tests/test_cli_image.py @@ -13,10 +13,10 @@ logger = logging.getLogger(__name__) MODEL_CACHE = tempfile.mkdtemp() -OV_IMAGE_MODELS = ["echarlaix/tiny-random-stable-diffusion-xl", - "yujiepan/stable-diffusion-3-tiny-random", - "katuni4ka/tiny-random-flux", - "katuni4ka/tiny-random-flux-fill"] +OV_IMAGE_MODELS = ["optimum-intel-internal-testing/tiny-random-stable-diffusion-xl", + "optimum-intel-internal-testing/stable-diffusion-3-tiny-random", + "optimum-intel-internal-testing/tiny-random-flux", + "optimum-intel-internal-testing/tiny-random-flux-fill"] def run_wwb(args): @@ -111,7 +111,7 @@ def test_image_model_genai(model_id, model_type, tmp_path): pytest.skip(reason="FLUX-Fill is supported as inpainting only") if model_type == "image-inpainting": pytest.xfail("Segfault. Ticket 170877") - if model_id == "katuni4ka/tiny-random-flux" and model_type == "image-to-image": + if model_id == "optimum-intel-internal-testing/tiny-random-flux" and model_type == "image-to-image": pytest.xfail("Randomly wwb died with . Ticket 170878") mac_arm64_skip = any(substring in model_id for substring in ('stable-diffusion-xl', diff --git a/tools/who_what_benchmark/tests/test_cli_vlm.py b/tools/who_what_benchmark/tests/test_cli_vlm.py index e215d901cc..f0ff2103b2 100644 --- a/tools/who_what_benchmark/tests/test_cli_vlm.py +++ b/tools/who_what_benchmark/tests/test_cli_vlm.py @@ -96,7 +96,7 @@ def run_test(model_id, model_type, optimum_threshold, genai_threshold, tmp_path) @pytest.mark.parametrize( ("model_id", "model_type"), [ - ("katuni4ka/tiny-random-llava", "visual-text"), + ("optimum-intel-internal-testing/tiny-random-llava", "visual-text"), ], ) def test_vlm_basic(model_id, model_type, tmp_path):