Upload folder using huggingface_hub

README.md

@@ -1,11 +1,17 @@
 ---
 tags:
 - unsloth
-base_model:
-- Qwen/Qwen3-VL-4B-Instruct-FP8
 license: apache-2.0
 pipeline_tag: image-text-to-text
+library_name: transformers
+base_model:
+- Qwen/Qwen3-VL-4B-Instruct-FP8
+base_model_relation: quantized
 ---
+> [!NOTE]
+>  Includes Unsloth **chat template fixes**! <br> For `llama.cpp`, use `--jinja`
+>
+
 <div>
 <p style="margin-top: 0;margin-bottom: 0;">
     <em><a href="https://docs.unsloth.ai/basics/unsloth-dynamic-v2.0-gguf">Unsloth Dynamic 2.0</a> achieves superior accuracy & outperforms other leading quants.</em>
@@ -29,7 +35,7 @@ pipeline_tag: image-text-to-text
 
 # Qwen3-VL-4B-Instruct-FP8
 
-> This repository contains an FP8 quantized version of the [Qwen3-VL-4B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct) model. The quantization method is fine-grained fp8 quantization with block size of 128, and its performance metrics are nearly identical to those of the original BF16 model. Enjoy!
+> This repository contains an FP8 quantized version of the [Qwen3-VL-4B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct) model. The quantization method is fine-grained fp8 quantization with block size of 128, and its performance metrics are nearly identical to those of the original BF16 model. Enjoy!
 
 
 Meet Qwen3-VL — the most powerful vision-language model in the Qwen series to date.
@@ -81,11 +87,10 @@ This is the weight repository for Qwen3-VL-4B-Instruct-FP8.
 
 **Multimodal performance**
 
-![](https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_vl-4b.jpg)
+![](https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/qwen3vl_4b_8b_vl_instruct.jpg)
 
 **Pure text performance**
-![](https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text-4b.jpg)
-
+![](https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/qwen3vl_4b_8b_text_instruct.jpg)
 
 ## Quickstart
 
@@ -246,6 +251,28 @@ if __name__ == "__main__":
 ```
 
 
+### Generation Hyperparameters
+#### VL
+```bash
+export greedy='false'
+export top_p=0.8
+export top_k=20
+export temperature=0.7
+export repetition_penalty=1.0
+export presence_penalty=1.5
+export out_seq_length=16384
+```
+
+#### Text
+```bash
+export greedy='false'
+export top_p=1.0
+export top_k=40
+export repetition_penalty=1.0
+export presence_penalty=2.0
+export temperature=1.0
+export out_seq_length=32768
+```
 
 
 ## Citation

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {%- if messages[0].content is string %}\n            {{- messages[0].content }}\n        {%- else %}\n            {%- for content in messages[0].content %}\n                {%- if 'text' in content %}\n                    {{- content.text }}\n                {%- endif %}\n            {%- endfor %}\n        {%- endif %}\n        {{- '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' }}\n        {%- if messages[0].content is string %}\n            {{- messages[0].content }}\n        {%- else %}\n            {%- for content in messages[0].content %}\n                {%- if 'text' in content %}\n                    {{- content.text }}\n                {%- endif %}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- for message in messages %}\n    {%- if message.role == \"user\" %}\n        {{- '<|im_start|>' + message.role + '\\n' }}\n        {%- if message.content is string %}\n            {{- message.content }}\n        {%- else %}\n            {%- for content in message.content %}\n                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n                    {%- set image_count.value = image_count.value + 1 %}\n                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n                    <|vision_start|><|image_pad|><|vision_end|>\n                {%- elif content.type == 'video' or 'video' in content %}\n                    {%- set video_count.value = video_count.value + 1 %}\n                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n                    <|vision_start|><|video_pad|><|vision_end|>\n                {%- elif 'text' in content %}\n                    {{- content.text }}\n                {%- endif %}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role + '\\n' }}\n        {%- if message.content is string %}\n            {{- message.content }}\n        {%- else %}\n            {%- for content_item in message.content %}\n                {%- if 'text' in content_item %}\n                    {{- content_item.text }}\n                {%- endif %}\n            {%- endfor %}\n        {%- endif %}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and message.content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {%- if message.content is string %}\n            {{- message.content }}\n        {%- else %}\n            {%- for content in message.content %}\n                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n                    {%- set image_count.value = image_count.value + 1 %}\n                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n                    <|vision_start|><|image_pad|><|vision_end|>\n                {%- elif content.type == 'video' or 'video' in content %}\n                    {%- set video_count.value = video_count.value + 1 %}\n                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n                    <|vision_start|><|video_pad|><|vision_end|>\n                {%- elif 'text' in content %}\n                    {{- content.text }}\n                {%- endif %}\n            {%- endfor %}\n        {%- endif %}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n"
+}

config.json

@@ -2,13 +2,259 @@
   "architectures": [
     "Qwen3VLForConditionalGeneration"
   ],
-  "torch_dtype": "bfloat16",
-  "eos_token_id": 151645,
   "image_token_id": 151655,
   "model_type": "qwen3_vl",
   "pad_token_id": 151654,
   "quantization_config": {
     "activation_scheme": "dynamic",
+    "fmt": "e4m3",
+    "ignored_layers": [
+      "lm_head",
+      "model.visual.merger.linear_fc1",
+      "model.visual.merger.linear_fc2",
+      "model.visual.merger.norm",
+      "model.visual.patch_embed.proj",
+      "model.visual.pos_embed",
+      "visual.merger.linear_fc1",
+      "visual.merger.linear_fc2",
+      "visual.merger.norm",
+      "visual.patch_embed.proj",
+      "visual.pos_embed",
+      "model.visual.blocks.0.attn.proj",
+      "model.visual.blocks.0.attn.qkv",
+      "model.visual.blocks.0.mlp.linear_fc1",
+      "model.visual.blocks.0.mlp.linear_fc2",
+      "visual.blocks.0.attn.proj",
+      "visual.blocks.0.attn.qkv_proj",
+      "visual.blocks.0.mlp.linear_fc1",
+      "visual.blocks.0.mlp.linear_fc2",
+      "model.visual.blocks.1.attn.proj",
+      "model.visual.blocks.1.attn.qkv",
+      "model.visual.blocks.1.mlp.linear_fc1",
+      "model.visual.blocks.1.mlp.linear_fc2",
+      "visual.blocks.1.attn.proj",
+      "visual.blocks.1.attn.qkv_proj",
+      "visual.blocks.1.mlp.linear_fc1",
+      "visual.blocks.1.mlp.linear_fc2",
+      "model.visual.blocks.2.attn.proj",
+      "model.visual.blocks.2.attn.qkv",
+      "model.visual.blocks.2.mlp.linear_fc1",
+      "model.visual.blocks.2.mlp.linear_fc2",
+      "visual.blocks.2.attn.proj",
+      "visual.blocks.2.attn.qkv_proj",
+      "visual.blocks.2.mlp.linear_fc1",
+      "visual.blocks.2.mlp.linear_fc2",
+      "model.visual.blocks.3.attn.proj",
+      "model.visual.blocks.3.attn.qkv",
+      "model.visual.blocks.3.mlp.linear_fc1",
+      "model.visual.blocks.3.mlp.linear_fc2",
+      "visual.blocks.3.attn.proj",
+      "visual.blocks.3.attn.qkv_proj",
+      "visual.blocks.3.mlp.linear_fc1",
+      "visual.blocks.3.mlp.linear_fc2",
+      "model.visual.blocks.4.attn.proj",
+      "model.visual.blocks.4.attn.qkv",
+      "model.visual.blocks.4.mlp.linear_fc1",
+      "model.visual.blocks.4.mlp.linear_fc2",
+      "visual.blocks.4.attn.proj",
+      "visual.blocks.4.attn.qkv_proj",
+      "visual.blocks.4.mlp.linear_fc1",
+      "visual.blocks.4.mlp.linear_fc2",
+      "model.visual.blocks.5.attn.proj",
+      "model.visual.blocks.5.attn.qkv",
+      "model.visual.blocks.5.mlp.linear_fc1",
+      "model.visual.blocks.5.mlp.linear_fc2",
+      "visual.blocks.5.attn.proj",
+      "visual.blocks.5.attn.qkv_proj",
+      "visual.blocks.5.mlp.linear_fc1",
+      "visual.blocks.5.mlp.linear_fc2",
+      "model.visual.blocks.6.attn.proj",
+      "model.visual.blocks.6.attn.qkv",
+      "model.visual.blocks.6.mlp.linear_fc1",
+      "model.visual.blocks.6.mlp.linear_fc2",
+      "visual.blocks.6.attn.proj",
+      "visual.blocks.6.attn.qkv_proj",
+      "visual.blocks.6.mlp.linear_fc1",
+      "visual.blocks.6.mlp.linear_fc2",
+      "model.visual.blocks.7.attn.proj",
+      "model.visual.blocks.7.attn.qkv",
+      "model.visual.blocks.7.mlp.linear_fc1",
+      "model.visual.blocks.7.mlp.linear_fc2",
+      "visual.blocks.7.attn.proj",
+      "visual.blocks.7.attn.qkv_proj",
+      "visual.blocks.7.mlp.linear_fc1",
+      "visual.blocks.7.mlp.linear_fc2",
+      "model.visual.blocks.8.attn.proj",
+      "model.visual.blocks.8.attn.qkv",
+      "model.visual.blocks.8.mlp.linear_fc1",
+      "model.visual.blocks.8.mlp.linear_fc2",
+      "visual.blocks.8.attn.proj",
+      "visual.blocks.8.attn.qkv_proj",
+      "visual.blocks.8.mlp.linear_fc1",
+      "visual.blocks.8.mlp.linear_fc2",
+      "model.visual.blocks.9.attn.proj",
+      "model.visual.blocks.9.attn.qkv",
+      "model.visual.blocks.9.mlp.linear_fc1",
+      "model.visual.blocks.9.mlp.linear_fc2",
+      "visual.blocks.9.attn.proj",
+      "visual.blocks.9.attn.qkv_proj",
+      "visual.blocks.9.mlp.linear_fc1",
+      "visual.blocks.9.mlp.linear_fc2",
+      "model.visual.blocks.10.attn.proj",
+      "model.visual.blocks.10.attn.qkv",
+      "model.visual.blocks.10.mlp.linear_fc1",
+      "model.visual.blocks.10.mlp.linear_fc2",
+      "visual.blocks.10.attn.proj",
+      "visual.blocks.10.attn.qkv_proj",
+      "visual.blocks.10.mlp.linear_fc1",
+      "visual.blocks.10.mlp.linear_fc2",
+      "model.visual.blocks.11.attn.proj",
+      "model.visual.blocks.11.attn.qkv",
+      "model.visual.blocks.11.mlp.linear_fc1",
+      "model.visual.blocks.11.mlp.linear_fc2",
+      "visual.blocks.11.attn.proj",
+      "visual.blocks.11.attn.qkv_proj",
+      "visual.blocks.11.mlp.linear_fc1",
+      "visual.blocks.11.mlp.linear_fc2",
+      "model.visual.blocks.12.attn.proj",
+      "model.visual.blocks.12.attn.qkv",
+      "model.visual.blocks.12.mlp.linear_fc1",
+      "model.visual.blocks.12.mlp.linear_fc2",
+      "visual.blocks.12.attn.proj",
+      "visual.blocks.12.attn.qkv_proj",
+      "visual.blocks.12.mlp.linear_fc1",
+      "visual.blocks.12.mlp.linear_fc2",
+      "model.visual.blocks.13.attn.proj",
+      "model.visual.blocks.13.attn.qkv",
+      "model.visual.blocks.13.mlp.linear_fc1",
+      "model.visual.blocks.13.mlp.linear_fc2",
+      "visual.blocks.13.attn.proj",
+      "visual.blocks.13.attn.qkv_proj",
+      "visual.blocks.13.mlp.linear_fc1",
+      "visual.blocks.13.mlp.linear_fc2",
+      "model.visual.blocks.14.attn.proj",
+      "model.visual.blocks.14.attn.qkv",
+      "model.visual.blocks.14.mlp.linear_fc1",
+      "model.visual.blocks.14.mlp.linear_fc2",
+      "visual.blocks.14.attn.proj",
+      "visual.blocks.14.attn.qkv_proj",
+      "visual.blocks.14.mlp.linear_fc1",
+      "visual.blocks.14.mlp.linear_fc2",
+      "model.visual.blocks.15.attn.proj",
+      "model.visual.blocks.15.attn.qkv",
+      "model.visual.blocks.15.mlp.linear_fc1",
+      "model.visual.blocks.15.mlp.linear_fc2",
+      "visual.blocks.15.attn.proj",
+      "visual.blocks.15.attn.qkv_proj",
+      "visual.blocks.15.mlp.linear_fc1",
+      "visual.blocks.15.mlp.linear_fc2",
+      "model.visual.blocks.16.attn.proj",
+      "model.visual.blocks.16.attn.qkv",
+      "model.visual.blocks.16.mlp.linear_fc1",
+      "model.visual.blocks.16.mlp.linear_fc2",
+      "visual.blocks.16.attn.proj",
+      "visual.blocks.16.attn.qkv_proj",
+      "visual.blocks.16.mlp.linear_fc1",
+      "visual.blocks.16.mlp.linear_fc2",
+      "model.visual.blocks.17.attn.proj",
+      "model.visual.blocks.17.attn.qkv",
+      "model.visual.blocks.17.mlp.linear_fc1",
+      "model.visual.blocks.17.mlp.linear_fc2",
+      "visual.blocks.17.attn.proj",
+      "visual.blocks.17.attn.qkv_proj",
+      "visual.blocks.17.mlp.linear_fc1",
+      "visual.blocks.17.mlp.linear_fc2",
+      "model.visual.blocks.18.attn.proj",
+      "model.visual.blocks.18.attn.qkv",
+      "model.visual.blocks.18.mlp.linear_fc1",
+      "model.visual.blocks.18.mlp.linear_fc2",
+      "visual.blocks.18.attn.proj",
+      "visual.blocks.18.attn.qkv_proj",
+      "visual.blocks.18.mlp.linear_fc1",
+      "visual.blocks.18.mlp.linear_fc2",
+      "model.visual.blocks.19.attn.proj",
+      "model.visual.blocks.19.attn.qkv",
+      "model.visual.blocks.19.mlp.linear_fc1",
+      "model.visual.blocks.19.mlp.linear_fc2",
+      "visual.blocks.19.attn.proj",
+      "visual.blocks.19.attn.qkv_proj",
+      "visual.blocks.19.mlp.linear_fc1",
+      "visual.blocks.19.mlp.linear_fc2",
+      "model.visual.blocks.20.attn.proj",
+      "model.visual.blocks.20.attn.qkv",
+      "model.visual.blocks.20.mlp.linear_fc1",
+      "model.visual.blocks.20.mlp.linear_fc2",
+      "visual.blocks.20.attn.proj",
+      "visual.blocks.20.attn.qkv_proj",
+      "visual.blocks.20.mlp.linear_fc1",
+      "visual.blocks.20.mlp.linear_fc2",
+      "model.visual.blocks.21.attn.proj",
+      "model.visual.blocks.21.attn.qkv",
+      "model.visual.blocks.21.mlp.linear_fc1",
+      "model.visual.blocks.21.mlp.linear_fc2",
+      "visual.blocks.21.attn.proj",
+      "visual.blocks.21.attn.qkv_proj",
+      "visual.blocks.21.mlp.linear_fc1",
+      "visual.blocks.21.mlp.linear_fc2",
+      "model.visual.blocks.22.attn.proj",
+      "model.visual.blocks.22.attn.qkv",
+      "model.visual.blocks.22.mlp.linear_fc1",
+      "model.visual.blocks.22.mlp.linear_fc2",
+      "visual.blocks.22.attn.proj",
+      "visual.blocks.22.attn.qkv_proj",
+      "visual.blocks.22.mlp.linear_fc1",
+      "visual.blocks.22.mlp.linear_fc2",
+      "model.visual.blocks.23.attn.proj",
+      "model.visual.blocks.23.attn.qkv",
+      "model.visual.blocks.23.mlp.linear_fc1",
+      "model.visual.blocks.23.mlp.linear_fc2",
+      "visual.blocks.23.attn.proj",
+      "visual.blocks.23.attn.qkv_proj",
+      "visual.blocks.23.mlp.linear_fc1",
+      "visual.blocks.23.mlp.linear_fc2",
+      "model.visual.blocks.24.attn.proj",
+      "model.visual.blocks.24.attn.qkv",
+      "model.visual.blocks.24.mlp.linear_fc1",
+      "model.visual.blocks.24.mlp.linear_fc2",
+      "visual.blocks.24.attn.proj",
+      "visual.blocks.24.attn.qkv_proj",
+      "visual.blocks.24.mlp.linear_fc1",
+      "visual.blocks.24.mlp.linear_fc2",
+      "model.visual.blocks.25.attn.proj",
+      "model.visual.blocks.25.attn.qkv",
+      "model.visual.blocks.25.mlp.linear_fc1",
+      "model.visual.blocks.25.mlp.linear_fc2",
+      "visual.blocks.25.attn.proj",
+      "visual.blocks.25.attn.qkv_proj",
+      "visual.blocks.25.mlp.linear_fc1",
+      "visual.blocks.25.mlp.linear_fc2",
+      "model.visual.blocks.26.attn.proj",
+      "model.visual.blocks.26.attn.qkv",
+      "model.visual.blocks.26.mlp.linear_fc1",
+      "model.visual.blocks.26.mlp.linear_fc2",
+      "visual.blocks.26.attn.proj",
+      "visual.blocks.26.attn.qkv_proj",
+      "visual.blocks.26.mlp.linear_fc1",
+      "visual.blocks.26.mlp.linear_fc2",
+      "model.visual.deepstack_merger_list.0.linear_fc1",
+      "model.visual.deepstack_merger_list.0.linear_fc2",
+      "model.visual.deepstack_merger_list.0.norm",
+      "visual.deepstack_merger_list.0.linear_fc1",
+      "visual.deepstack_merger_list.0.linear_fc2",
+      "visual.deepstack_merger_list.0.norm",
+      "model.visual.deepstack_merger_list.1.linear_fc1",
+      "model.visual.deepstack_merger_list.1.linear_fc2",
+      "model.visual.deepstack_merger_list.1.norm",
+      "visual.deepstack_merger_list.1.linear_fc1",
+      "visual.deepstack_merger_list.1.linear_fc2",
+      "visual.deepstack_merger_list.1.norm",
+      "model.visual.deepstack_merger_list.2.linear_fc1",
+      "model.visual.deepstack_merger_list.2.linear_fc2",
+      "model.visual.deepstack_merger_list.2.norm",
+      "visual.deepstack_merger_list.2.linear_fc1",
+      "visual.deepstack_merger_list.2.linear_fc2",
+      "visual.deepstack_merger_list.2.norm"
+    ],
     "modules_to_not_convert": [
       "lm_head",
       "model.visual"
@@ -51,7 +297,7 @@
     "vocab_size": 151936
   },
   "tie_word_embeddings": false,
-  "transformers_version": "4.57.0",
+  "transformers_version": "4.57.1",
   "unsloth_fixed": true,
   "video_token_id": 151656,
   "vision_config": {
@@ -61,7 +307,6 @@
       17
     ],
     "depth": 24,
-    "torch_dtype": "bfloat16",
     "hidden_act": "gelu_pytorch_tanh",
     "hidden_size": 1024,
     "in_channels": 3,
@@ -77,4 +322,4 @@
   },
   "vision_end_token_id": 151653,
   "vision_start_token_id": 151652
-}
+}

generation_config.json

@@ -1,13 +1,14 @@
 {
   "bos_token_id": 151643,
+  "pad_token_id": 151643,
   "do_sample": true,
   "eos_token_id": [
     151645,
     151643
   ],
-  "pad_token_id": 151654,
-  "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.57.0"
-}
+  "repetition_penalty": 1.0,
+  "temperature": 0.7,
+  "transformers_version": "4.56.0"
+}

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ef095ca05f3a6698c40c555a7954befb770ecc8f84489e68705b3450b2266c1
-size 4991149856
+oid sha256:39c3e38447b0fe8b94c95a01a6b3f1171f1db2a4a6e03ccf55e3b6c0c5f07a44
+size 5366863440

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db00b554724d6077f4117c1ba6bdb2806933d4ece49fc58111debe35eaffe74c
-size 251731288
+oid sha256:ece22ccb44f97514a08b5bf10513530e0928f2e9bc93c7dc0efe8e18796fe4d9
+size 654372016