modify qwq-32b deploy

Author: 11zhouxuan

src/emd/models/engines.py

@@ -135,6 +135,13 @@ class KtransformersEngine(OpenAICompitableEngine):
             "default_cli_args": " --chat-template emd/models/chat_templates/qwq_32b_add_prefill_chat_template.jinja --max_model_len 16000  --max_num_seq 10 --disable-log-stats --enable-auto-tool-choice --tool-call-parser hermes"
 })
 
+vllm_qwq_engine082 = VllmEngine(**{
+            **vllm_qwen25vl72b_engine073.model_dump(),
+            "engine_dockerfile_config": {"VERSION":"v0.8.2"},
+            "environment_variables": "export VLLM_ATTENTION_BACKEND=FLASHINFER && export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
+            "default_cli_args": " --chat-template emd/models/chat_templates/qwq_32b_add_prefill_chat_template.jinja --max_model_len 16000  --max_num_seq 10 --disable-log-stats --enable-auto-tool-choice --tool-call-parser hermes  --enable-reasoning --reasoning-parser deepseek_r1"
+})
+
 
 vllm_internvl2d5_76b_engine064 = VllmEngine(**{
              **vllm_engine064.model_dump(),

src/emd/models/llms/qwen.py

@@ -7,7 +7,8 @@
     tgi_qwen2d5_on_inf2,
     tgi_qwen2d5_72b_on_inf2,
     vllm_qwen2d5_72b_engine064,
-    vllm_qwq_engine073
+    vllm_qwq_engine073,
+    vllm_qwq_engine082
 )
 from ..services import (
     sagemaker_service,
@@ -471,7 +472,7 @@
 Model.register(
     dict(
         model_id = "QwQ-32B",
-        supported_engines=[vllm_qwq_engine073],
+        supported_engines=[vllm_qwq_engine082],
         supported_instances=[
             g5d12xlarge_instance,
             g5d24xlarge_instance,

src/emd/models/services.py

@@ -91,6 +91,7 @@
         "ServiceType":"service_type",
         "EngineType":"engine_type",
         "Region": "region",
+        "DesiredCapacity": "desired_capacity",
         "ContainerCpu": "container_cpu",
         "ContainerMemory": "container_memory",
         "ContainerGpu":"instance_gpu_num"

src/pipeline/backend/backend.py

@@ -134,9 +134,10 @@ def start_server(self, server_start_command):
         logger.info(f"Starting {self.engine_type} server with command: {server_start_command}")
         t = threading.Thread(target=os.system,args=(server_start_command,),daemon=True)
         t.start()
-        t2 = threading.Thread(target=self.check_model_serve_ready,args=(t, "127.0.0.1", self.server_port),daemon=True)
-        t2.start()
-        t2.join()
+        self.check_model_serve_ready(t, "127.0.0.1", self.server_port)
+        logger.info(f"Server started successfully.")
+        # t2.start()
+        # t2.join()
         return