|
| 1 | +--- |
| 2 | +layout: model |
| 3 | +title: Qwen2-VL-2B-Instruct (Q4 GGUF Quantized) |
| 4 | +author: John Snow Labs |
| 5 | +name: qwen2_vl_2b_instruct_q4_gguf |
| 6 | +date: 2025-08-11 |
| 7 | +tags: [qwen2_vl, image_to_text, multimodal, conversational, instruct, q4, 2b, en, open_source, llamacpp] |
| 8 | +task: Image Captioning |
| 9 | +language: en |
| 10 | +edition: Spark NLP 6.1.1 |
| 11 | +spark_version: 3.0 |
| 12 | +supported: true |
| 13 | +engine: llamacpp |
| 14 | +annotator: AutoGGUFVisionModel |
| 15 | +article_header: |
| 16 | + type: cover |
| 17 | +use_language_switcher: "Python-Scala-Java" |
| 18 | +--- |
| 19 | + |
| 20 | +## Description |
| 21 | + |
| 22 | +Qwen2-VL-2B-Instruct is a 2-billion-parameter vision-language model fine-tuned for following instructions across text, image, and video inputs, enabling tasks like captioning, visual question answering, and multimodal dialogue. |
| 23 | + |
| 24 | +Originally from [Qwen/Qwen2-VL-2B-Instruct](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct) |
| 25 | + |
| 26 | +{:.btn-box} |
| 27 | +<button class="button button-orange" disabled>Live Demo</button> |
| 28 | +<button class="button button-orange" disabled>Open in Colab</button> |
| 29 | +[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/qwen2_vl_2b_instruct_q4_gguf_en_6.1.1_3.0_1754924858631.zip){:.button.button-orange.button-orange-trans.arr.button-icon} |
| 30 | +[Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/qwen2_vl_2b_instruct_q4_gguf_en_6.1.1_3.0_1754924858631.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} |
| 31 | + |
| 32 | +## How to use |
| 33 | + |
| 34 | + |
| 35 | + |
| 36 | +<div class="tabs-box" markdown="1"> |
| 37 | +{% include programmingLanguageSelectScalaPythonNLU.html %} |
| 38 | +```python |
| 39 | +from sparknlp.base import DocumentAssembler, ImageAssembler |
| 40 | +from sparknlp.annotator import AutoGGUFVisionModel |
| 41 | +from pyspark.sql.functions import lit |
| 42 | +from pyspark.ml import Pipeline |
| 43 | + |
| 44 | +images_path = "path/to/images/folder" |
| 45 | +prompt = "Caption this image." |
| 46 | + |
| 47 | +data = ImageAssembler.loadImagesAsBytes(spark, images_path) |
| 48 | +data = data.withColumn("caption", lit(prompt)) |
| 49 | + |
| 50 | +document_assembler = ( |
| 51 | + DocumentAssembler() |
| 52 | + .setInputCol("caption") |
| 53 | + .setOutputCol("caption_document") |
| 54 | +) |
| 55 | + |
| 56 | +image_assembler = ( |
| 57 | + ImageAssembler() |
| 58 | + .setInputCol("image") |
| 59 | + .setOutputCol("image_assembler") |
| 60 | +) |
| 61 | + |
| 62 | +qwen_chat_template = """<|im_start|>user |
| 63 | +{prompt}<|im_end|> |
| 64 | +<|im_start|>assistant |
| 65 | +""" |
| 66 | + |
| 67 | +autoGGUFVisionModel = ( |
| 68 | + AutoGGUFVisionModel.pretrained("qwen2_vl_2b_instruct_q4_gguf") |
| 69 | + .setInputCols(["caption_document", "image_assembler"]) |
| 70 | + .setOutputCol("completions") |
| 71 | + .setChatTemplate(qwen_chat_template) |
| 72 | + .setBatchSize(4) |
| 73 | + .setNGpuLayers(32) |
| 74 | + .setNCtx(4096) |
| 75 | + .setMinKeep(0) |
| 76 | + .setMinP(0.05) |
| 77 | + .setNPredict(64) |
| 78 | + .setNProbs(0) |
| 79 | + .setPenalizeNl(False) |
| 80 | + .setRepeatLastN(256) |
| 81 | + .setRepeatPenalty(1.1) |
| 82 | + .setStopStrings(["</s>", "<|im_end|>", "User:"]) |
| 83 | + .setTemperature(0.2) |
| 84 | + .setTfsZ(1) |
| 85 | + .setTypicalP(1) |
| 86 | + .setTopK(40) |
| 87 | + .setTopP(0.95) |
| 88 | +) |
| 89 | + |
| 90 | +pipeline = Pipeline().setStages([ |
| 91 | + document_assembler, |
| 92 | + image_assembler, |
| 93 | + autoGGUFVisionModel |
| 94 | +]) |
| 95 | + |
| 96 | +model = pipeline.fit(data) |
| 97 | +result = model.transform(data) |
| 98 | + |
| 99 | +result.selectExpr( |
| 100 | + "reverse(split(image.origin, '/'))[0] as image_name", |
| 101 | + "completions.result" |
| 102 | +).show(truncate=False) |
| 103 | + |
| 104 | +``` |
| 105 | +```scala |
| 106 | +import com.johnsnowlabs.nlp.base._ |
| 107 | +import com.johnsnowlabs.nlp.annotators._ |
| 108 | +import org.apache.spark.sql.functions.lit |
| 109 | +import org.apache.spark.ml.Pipeline |
| 110 | + |
| 111 | +val images_path = "path/to/images/folder" |
| 112 | +val prompt = "Caption this image." |
| 113 | + |
| 114 | +var data = ImageAssembler.loadImagesAsBytes(spark, images_path) |
| 115 | +data = data.withColumn("caption", lit(prompt)) |
| 116 | + |
| 117 | +val document_assembler = new DocumentAssembler() |
| 118 | + .setInputCol("caption") |
| 119 | + .setOutputCol("caption_document") |
| 120 | + |
| 121 | +val image_assembler = new ImageAssembler() |
| 122 | + .setInputCol("image") |
| 123 | + .setOutputCol("image_assembler") |
| 124 | + |
| 125 | +val qwen_chat_template = """<|im_start|>user |
| 126 | +{prompt}<|im_end|> |
| 127 | +<|im_start|>assistant |
| 128 | +""" |
| 129 | + |
| 130 | +val autoGGUFVisionModel = AutoGGUFVisionModel.pretrained("qwen2_vl_2b_instruct_q4_gguf") |
| 131 | + .setInputCols(Array("caption_document", "image_assembler")) |
| 132 | + .setOutputCol("completions") |
| 133 | + .setChatTemplate(qwen_chat_template) |
| 134 | + .setBatchSize(4) |
| 135 | + .setNGpuLayers(32) |
| 136 | + .setNCtx(4096) |
| 137 | + .setMinKeep(0) |
| 138 | + .setMinP(0.05) |
| 139 | + .setNPredict(64) |
| 140 | + .setNProbs(0) |
| 141 | + .setPenalizeNl(false) |
| 142 | + .setRepeatLastN(256) |
| 143 | + .setRepeatPenalty(1.1) |
| 144 | + .setStopStrings(Array("</s>", "<|im_end|>", "User:")) |
| 145 | + .setTemperature(0.2) |
| 146 | + .setTfsZ(1) |
| 147 | + .setTypicalP(1) |
| 148 | + .setTopK(40) |
| 149 | + .setTopP(0.95) |
| 150 | + |
| 151 | +val pipeline = new Pipeline().setStages(Array( |
| 152 | + document_assembler, |
| 153 | + image_assembler, |
| 154 | + autoGGUFVisionModel |
| 155 | +)) |
| 156 | + |
| 157 | +val model = pipeline.fit(data) |
| 158 | +val result = model.transform(data) |
| 159 | + |
| 160 | +result.selectExpr( |
| 161 | + "reverse(split(image.origin, '/'))[0] as image_name", |
| 162 | + "completions.result" |
| 163 | +).show(false) |
| 164 | + |
| 165 | +``` |
| 166 | +</div> |
| 167 | + |
| 168 | +## Results |
| 169 | + |
| 170 | +```bash |
| 171 | + |
| 172 | ++---------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------+ |
| 173 | +|image_name |result | |
| 174 | ++---------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------+ |
| 175 | +|[prescription_02.png, images, content, file:]|["Outpatient Summary: Rheumatology Consultation for Systemic Lupus Erythematosus and Scleroderma Overlap with Interstitial Lung Disease"]| |
| 176 | +|[prescription_01.png, images, content, file:]|["Medical prescription for treatment of fever and headache with medication details."] | |
| 177 | ++---------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------+ |
| 178 | + |
| 179 | +``` |
| 180 | + |
| 181 | +{:.model-param} |
| 182 | +## Model Information |
| 183 | + |
| 184 | +{:.table-model} |
| 185 | +|---|---| |
| 186 | +|Model Name:|qwen2_vl_2b_instruct_q4_gguf| |
| 187 | +|Compatibility:|Spark NLP 6.1.1+| |
| 188 | +|License:|Open Source| |
| 189 | +|Edition:|Official| |
| 190 | +|Input Labels:|[caption_document, image_assembler]| |
| 191 | +|Output Labels:|[completions]| |
| 192 | +|Language:|en| |
| 193 | +|Size:|1.6 GB| |
0 commit comments