fix checking tokenizers version

src/cpp/src/tokenizer/tokenizer_impl.cpp

@@ -273,7 +273,9 @@ void Tokenizer::TokenizerImpl::setup_tokenizer(const std::filesystem::path& mode
     std::shared_ptr<ov::Model> ov_tokenizer = nullptr;
     std::shared_ptr<ov::Model> ov_detokenizer = nullptr;
     auto [filtered_properties, enable_save_ov_model] = utils::extract_gguf_properties(properties);
-    if (is_gguf_model(models_path)) {
+
+    is_gguf_model = ov::genai::is_gguf_model(models_path);
+    if (is_gguf_model) {
         std::map<std::string, GGUFMetaData> tokenizer_config{};
         std::tie(ov_tokenizer, ov_detokenizer, tokenizer_config) =
             create_tokenizer_from_config(m_shared_object_ov_tokenizers, models_path);
@@ -374,6 +376,9 @@ void Tokenizer::TokenizerImpl::setup_tokenizer(const std::pair<std::shared_ptr<o
 
     // Saving IR version was added only in 24.5, so if it's missing, then it's older than 24.5
     m_older_than_24_5 = !(ov_tokenizer ? ov_tokenizer : ov_detokenizer)->has_rt_info("openvino_tokenizers_version");
+
+    // gguf models are always newer than 24.5 despite the fact they don't hold rt_info with 'openvino_tokenizers_version'
+    m_older_than_24_5 &= !is_gguf_model;
 
     if (ov_tokenizer) {
         ov::pass::Manager manager;

src/cpp/src/tokenizer/tokenizer_impl.hpp

@@ -35,6 +35,7 @@ class Tokenizer::TokenizerImpl {
     std::shared_ptr<void> m_shared_object_ov_tokenizers = nullptr;
     bool is_paired_input = false;
     bool m_older_than_24_5 = false;
+    bool is_gguf_model = false;
     int64_t m_pad_token_id = -1;
     int64_t m_bos_token_id = -1;
     int64_t m_eos_token_id = -1;

tests/python_tests/test_gguf_reader.py

@@ -97,6 +97,10 @@ def test_full_gguf_pipeline(pipeline_type, model_ids, enable_save_ov_model):
     gguf_full_path = download_gguf_model(gguf_model_id, gguf_filename)
     ov_pipe_gguf = create_ov_pipeline(gguf_full_path, pipeline_type=pipeline_type, enable_save_ov_model=enable_save_ov_model, dynamic_quantization_group_size=dynamic_quantization_group_size)
     res_string_input_2 = ov_pipe_gguf.generate(prompt, generation_config=ov_generation_config)
+
+    ov_pipe_gguf.get_tokenizer().get_eos_token() == hf_tokenizer.eos_token
+    ov_pipe_gguf.get_tokenizer().get_bos_token() == hf_tokenizer.bos_token
+
     del ov_pipe_gguf
     gc.collect()