diff --git a/python/sglang/srt/entrypoints/grpc_server.py b/python/sglang/srt/entrypoints/grpc_server.py index 90ac9ca5fe5..7adbd309d9b 100644 --- a/python/sglang/srt/entrypoints/grpc_server.py +++ b/python/sglang/srt/entrypoints/grpc_server.py @@ -14,6 +14,7 @@ from typing import AsyncIterator, Dict, Optional, Tuple import grpc +from google.protobuf.json_format import MessageToDict from grpc_reflection.v1alpha import reflection from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode @@ -483,28 +484,52 @@ def _convert_sampling_params( elif grpc_params.HasField("structural_tag"): structural_tag = grpc_params.structural_tag + # Handle optional parameters conversion + custom_params = ( + MessageToDict(grpc_params.custom_params) + if grpc_params.HasField("custom_params") + else None + ) + max_new_tokens = ( + grpc_params.max_new_tokens + if grpc_params.HasField("max_new_tokens") + else None + ) + stream_interval = ( + grpc_params.stream_interval + if grpc_params.HasField("stream_interval") + else None + ) + logit_bias = dict(grpc_params.logit_bias) if grpc_params.logit_bias else None + stop = list(grpc_params.stop) if grpc_params.stop else None + stop_token_ids = ( + list(grpc_params.stop_token_ids) if grpc_params.stop_token_ids else None + ) + return SGLSamplingParams( - temperature=grpc_params.temperature or 1.0, - top_p=grpc_params.top_p or 1.0, - top_k=grpc_params.top_k or -1, - min_p=grpc_params.min_p or 0.0, - frequency_penalty=grpc_params.frequency_penalty or 0.0, - presence_penalty=grpc_params.presence_penalty or 0.0, - repetition_penalty=grpc_params.repetition_penalty or 1.0, - max_new_tokens=grpc_params.max_new_tokens or 128, - min_new_tokens=grpc_params.min_new_tokens or 0, - stop=list(grpc_params.stop) if grpc_params.stop else [], - stop_token_ids=( - list(grpc_params.stop_token_ids) if grpc_params.stop_token_ids else [] - ), + temperature=grpc_params.temperature, + top_p=grpc_params.top_p, + top_k=grpc_params.top_k, + min_p=grpc_params.min_p, + frequency_penalty=grpc_params.frequency_penalty, + presence_penalty=grpc_params.presence_penalty, + repetition_penalty=grpc_params.repetition_penalty, + max_new_tokens=max_new_tokens, + min_new_tokens=grpc_params.min_new_tokens, + stop=stop, + stop_token_ids=stop_token_ids, skip_special_tokens=grpc_params.skip_special_tokens, spaces_between_special_tokens=grpc_params.spaces_between_special_tokens, + no_stop_trim=grpc_params.no_stop_trim, regex=regex, json_schema=json_schema, ebnf=ebnf_grammar, structural_tag=structural_tag, - n=grpc_params.n or 1, + n=grpc_params.n, ignore_eos=grpc_params.ignore_eos, + stream_interval=stream_interval, + logit_bias=logit_bias, + custom_params=custom_params, ) def _convert_output_logprobs_to_proto( diff --git a/python/sglang/srt/grpc/sglang_scheduler.proto b/python/sglang/srt/grpc/sglang_scheduler.proto index 152e65cb533..d3f19587a0b 100644 --- a/python/sglang/srt/grpc/sglang_scheduler.proto +++ b/python/sglang/srt/grpc/sglang_scheduler.proto @@ -27,6 +27,11 @@ service SglangScheduler { // ===================== // Sampling parameters matching SGLang's SamplingParams +// +// IMPORTANT: Do not use SamplingParams::default() directly! +// The proto3 defaults (0 for numeric fields) do NOT match the semantic defaults +// (temperature=1.0, top_p=1.0, top_k=-1, etc.). Always construct with explicit values +// or use the conversion functions in sglang_scheduler.rs / grpc_server.py. message SamplingParams { float temperature = 1; float top_p = 2; @@ -50,24 +55,18 @@ message SamplingParams { string structural_tag = 16; } - // LoRA adapter - string lora_path = 17; - // Speculative decoding - int32 n = 18; // Number of samples - - // Token healing - bool token_healing = 19; + int32 n = 17; // Number of samples // Additional parameters - int32 min_new_tokens = 20; - bool ignore_eos = 21; - bool no_stop_trim = 22; - int32 stream_interval = 23; - map logit_bias = 24; + int32 min_new_tokens = 18; + bool ignore_eos = 19; + bool no_stop_trim = 20; + optional int32 stream_interval = 21; + map logit_bias = 22; // Custom parameters for extensibility - google.protobuf.Struct custom_params = 25; + google.protobuf.Struct custom_params = 23; } diff --git a/python/sglang/srt/grpc/sglang_scheduler_pb2.py b/python/sglang/srt/grpc/sglang_scheduler_pb2.py index 10127567b2e..1bb2bbc3936 100644 --- a/python/sglang/srt/grpc/sglang_scheduler_pb2.py +++ b/python/sglang/srt/grpc/sglang_scheduler_pb2.py @@ -29,7 +29,7 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16sglang_scheduler.proto\x12\x15sglang.grpc.scheduler\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xe1\x05\n\x0eSamplingParams\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_p\x18\x02 \x01(\x02\x12\r\n\x05top_k\x18\x03 \x01(\x05\x12\r\n\x05min_p\x18\x04 \x01(\x02\x12\x19\n\x11\x66requency_penalty\x18\x05 \x01(\x02\x12\x18\n\x10presence_penalty\x18\x06 \x01(\x02\x12\x1a\n\x12repetition_penalty\x18\x07 \x01(\x02\x12\x1b\n\x0emax_new_tokens\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12\x0c\n\x04stop\x18\t \x03(\t\x12\x16\n\x0estop_token_ids\x18\n \x03(\r\x12\x1b\n\x13skip_special_tokens\x18\x0b \x01(\x08\x12%\n\x1dspaces_between_special_tokens\x18\x0c \x01(\x08\x12\x0f\n\x05regex\x18\r \x01(\tH\x00\x12\x15\n\x0bjson_schema\x18\x0e \x01(\tH\x00\x12\x16\n\x0c\x65\x62nf_grammar\x18\x0f \x01(\tH\x00\x12\x18\n\x0estructural_tag\x18\x10 \x01(\tH\x00\x12\x11\n\tlora_path\x18\x11 \x01(\t\x12\t\n\x01n\x18\x12 \x01(\x05\x12\x15\n\rtoken_healing\x18\x13 \x01(\x08\x12\x16\n\x0emin_new_tokens\x18\x14 \x01(\x05\x12\x12\n\nignore_eos\x18\x15 \x01(\x08\x12\x14\n\x0cno_stop_trim\x18\x16 \x01(\x08\x12\x17\n\x0fstream_interval\x18\x17 \x01(\x05\x12H\n\nlogit_bias\x18\x18 \x03(\x0b\x32\x34.sglang.grpc.scheduler.SamplingParams.LogitBiasEntry\x12.\n\rcustom_params\x18\x19 \x01(\x0b\x32\x17.google.protobuf.Struct\x1a\x30\n\x0eLogitBiasEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\nconstraintB\x11\n\x0f_max_new_tokens\"]\n\x13\x44isaggregatedParams\x12\x16\n\x0e\x62ootstrap_host\x18\x01 \x01(\t\x12\x16\n\x0e\x62ootstrap_port\x18\x02 \x01(\x05\x12\x16\n\x0e\x62ootstrap_room\x18\x03 \x01(\x05\"\xe2\x04\n\x0fGenerateRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x04 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x16\n\x0ereturn_logprob\x18\x05 \x01(\x08\x12\x19\n\x11logprob_start_len\x18\x06 \x01(\x05\x12\x18\n\x10top_logprobs_num\x18\x07 \x01(\x05\x12\x19\n\x11token_ids_logprob\x18\x08 \x03(\r\x12\x1c\n\x14return_hidden_states\x18\t \x01(\x08\x12H\n\x14\x64isaggregated_params\x18\n \x01(\x0b\x32*.sglang.grpc.scheduler.DisaggregatedParams\x12\x1e\n\x16\x63ustom_logit_processor\x18\x0b \x01(\t\x12-\n\ttimestamp\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x13\n\x0blog_metrics\x18\r \x01(\x08\x12\x14\n\x0cinput_embeds\x18\x0e \x03(\x02\x12\x0f\n\x07lora_id\x18\x0f \x01(\t\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x10 \x01(\x05\x12\x0e\n\x06stream\x18\x11 \x01(\x08\":\n\x0eTokenizedInput\x12\x15\n\roriginal_text\x18\x01 \x01(\t\x12\x11\n\tinput_ids\x18\x02 \x03(\r\"\xd3\x01\n\x10MultimodalInputs\x12\x12\n\nimage_urls\x18\x01 \x03(\t\x12\x12\n\nvideo_urls\x18\x02 \x03(\t\x12\x12\n\naudio_urls\x18\x03 \x03(\t\x12\x33\n\x12processed_features\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x12\n\nimage_data\x18\x05 \x03(\x0c\x12\x12\n\nvideo_data\x18\x06 \x03(\x0c\x12\x12\n\naudio_data\x18\x07 \x03(\x0c\x12\x12\n\nmodalities\x18\x08 \x03(\t\"\xe3\x01\n\x10GenerateResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12;\n\x05\x63hunk\x18\x02 \x01(\x0b\x32*.sglang.grpc.scheduler.GenerateStreamChunkH\x00\x12;\n\x08\x63omplete\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.GenerateCompleteH\x00\x12\x35\n\x05\x65rror\x18\x04 \x01(\x0b\x32$.sglang.grpc.scheduler.GenerateErrorH\x00\x42\n\n\x08response\"\x95\x02\n\x13GenerateStreamChunk\x12\x11\n\ttoken_ids\x18\x01 \x03(\r\x12\x15\n\rprompt_tokens\x18\x02 \x01(\x05\x12\x19\n\x11\x63ompletion_tokens\x18\x03 \x01(\x05\x12\x15\n\rcached_tokens\x18\x04 \x01(\x05\x12>\n\x0foutput_logprobs\x18\x05 \x01(\x0b\x32%.sglang.grpc.scheduler.OutputLogProbs\x12\x15\n\rhidden_states\x18\x06 \x03(\x02\x12<\n\x0einput_logprobs\x18\x07 \x01(\x0b\x32$.sglang.grpc.scheduler.InputLogProbs\x12\r\n\x05index\x18\x08 \x01(\r\"\x9b\x03\n\x10GenerateComplete\x12\x12\n\noutput_ids\x18\x01 \x03(\r\x12\x15\n\rfinish_reason\x18\x02 \x01(\t\x12\x15\n\rprompt_tokens\x18\x03 \x01(\x05\x12\x19\n\x11\x63ompletion_tokens\x18\x04 \x01(\x05\x12\x15\n\rcached_tokens\x18\x05 \x01(\x05\x12>\n\x0foutput_logprobs\x18\x06 \x01(\x0b\x32%.sglang.grpc.scheduler.OutputLogProbs\x12>\n\x11\x61ll_hidden_states\x18\x07 \x03(\x0b\x32#.sglang.grpc.scheduler.HiddenStates\x12\x1a\n\x10matched_token_id\x18\x08 \x01(\rH\x00\x12\x1a\n\x10matched_stop_str\x18\t \x01(\tH\x00\x12<\n\x0einput_logprobs\x18\n \x01(\x0b\x32$.sglang.grpc.scheduler.InputLogProbs\x12\r\n\x05index\x18\x0b \x01(\rB\x0e\n\x0cmatched_stop\"K\n\rGenerateError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x18\n\x10http_status_code\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"u\n\x0eOutputLogProbs\x12\x16\n\x0etoken_logprobs\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x38\n\x0ctop_logprobs\x18\x03 \x03(\x0b\x32\".sglang.grpc.scheduler.TopLogProbs\"\x9e\x01\n\rInputLogProbs\x12@\n\x0etoken_logprobs\x18\x01 \x03(\x0b\x32(.sglang.grpc.scheduler.InputTokenLogProb\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x38\n\x0ctop_logprobs\x18\x03 \x03(\x0b\x32\".sglang.grpc.scheduler.TopLogProbs\"1\n\x11InputTokenLogProb\x12\x12\n\x05value\x18\x01 \x01(\x02H\x00\x88\x01\x01\x42\x08\n\x06_value\"0\n\x0bTopLogProbs\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\"?\n\x0cHiddenStates\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05layer\x18\x02 \x01(\x05\x12\x10\n\x08position\x18\x03 \x01(\x05\"\xca\x02\n\x0c\x45mbedRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x04 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x05 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x13\n\x0blog_metrics\x18\x06 \x01(\x08\x12\x16\n\x0etoken_type_ids\x18\x07 \x03(\x05\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x08 \x01(\x05\x12\x18\n\x10is_cross_encoder\x18\t \x01(\x08\x12\r\n\x05texts\x18\n \x03(\t\"\x9d\x01\n\rEmbedResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\x08\x63omplete\x18\x02 \x01(\x0b\x32$.sglang.grpc.scheduler.EmbedCompleteH\x00\x12\x32\n\x05\x65rror\x18\x03 \x01(\x0b\x32!.sglang.grpc.scheduler.EmbedErrorH\x00\x42\n\n\x08response\"\xa3\x01\n\rEmbedComplete\x12\x11\n\tembedding\x18\x01 \x03(\x02\x12\x15\n\rprompt_tokens\x18\x02 \x01(\x05\x12\x15\n\rcached_tokens\x18\x03 \x01(\x05\x12\x15\n\rembedding_dim\x18\x04 \x01(\x05\x12:\n\x10\x62\x61tch_embeddings\x18\x05 \x03(\x0b\x32 .sglang.grpc.scheduler.Embedding\"*\n\tEmbedding\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05index\x18\x02 \x01(\x05\"<\n\nEmbedError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"N\n\x12HealthCheckRequest\x12\x38\n\ttokenized\x18\x01 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\"7\n\x13HealthCheckResponse\x12\x0f\n\x07healthy\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"2\n\x0c\x41\x62ortRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\"1\n\rAbortResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"I\n\x0fLoadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\x12\x14\n\x0c\x61\x64\x61pter_path\x18\x02 \x01(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\"H\n\x10LoadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x12\n\nadapter_id\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"\'\n\x11UnloadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\"6\n\x12UnloadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"w\n\x14UpdateWeightsRequest\x12\x13\n\tdisk_path\x18\x01 \x01(\tH\x00\x12\x15\n\x0btensor_data\x18\x02 \x01(\x0cH\x00\x12\x14\n\nremote_url\x18\x03 \x01(\tH\x00\x12\x13\n\x0bweight_name\x18\x04 \x01(\tB\x08\n\x06source\"9\n\x15UpdateWeightsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"-\n\x17GetInternalStateRequest\x12\x12\n\nstate_keys\x18\x01 \x03(\t\"B\n\x18GetInternalStateResponse\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"A\n\x17SetInternalStateRequest\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"<\n\x18SetInternalStateResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2\xfe\x02\n\x0fSglangScheduler\x12]\n\x08Generate\x12&.sglang.grpc.scheduler.GenerateRequest\x1a\'.sglang.grpc.scheduler.GenerateResponse0\x01\x12R\n\x05\x45mbed\x12#.sglang.grpc.scheduler.EmbedRequest\x1a$.sglang.grpc.scheduler.EmbedResponse\x12\x64\n\x0bHealthCheck\x12).sglang.grpc.scheduler.HealthCheckRequest\x1a*.sglang.grpc.scheduler.HealthCheckResponse\x12R\n\x05\x41\x62ort\x12#.sglang.grpc.scheduler.AbortRequest\x1a$.sglang.grpc.scheduler.AbortResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16sglang_scheduler.proto\x12\x15sglang.grpc.scheduler\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xd0\x05\n\x0eSamplingParams\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_p\x18\x02 \x01(\x02\x12\r\n\x05top_k\x18\x03 \x01(\x05\x12\r\n\x05min_p\x18\x04 \x01(\x02\x12\x19\n\x11\x66requency_penalty\x18\x05 \x01(\x02\x12\x18\n\x10presence_penalty\x18\x06 \x01(\x02\x12\x1a\n\x12repetition_penalty\x18\x07 \x01(\x02\x12\x1b\n\x0emax_new_tokens\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12\x0c\n\x04stop\x18\t \x03(\t\x12\x16\n\x0estop_token_ids\x18\n \x03(\r\x12\x1b\n\x13skip_special_tokens\x18\x0b \x01(\x08\x12%\n\x1dspaces_between_special_tokens\x18\x0c \x01(\x08\x12\x0f\n\x05regex\x18\r \x01(\tH\x00\x12\x15\n\x0bjson_schema\x18\x0e \x01(\tH\x00\x12\x16\n\x0c\x65\x62nf_grammar\x18\x0f \x01(\tH\x00\x12\x18\n\x0estructural_tag\x18\x10 \x01(\tH\x00\x12\t\n\x01n\x18\x11 \x01(\x05\x12\x16\n\x0emin_new_tokens\x18\x12 \x01(\x05\x12\x12\n\nignore_eos\x18\x13 \x01(\x08\x12\x14\n\x0cno_stop_trim\x18\x14 \x01(\x08\x12\x1c\n\x0fstream_interval\x18\x15 \x01(\x05H\x02\x88\x01\x01\x12H\n\nlogit_bias\x18\x16 \x03(\x0b\x32\x34.sglang.grpc.scheduler.SamplingParams.LogitBiasEntry\x12.\n\rcustom_params\x18\x17 \x01(\x0b\x32\x17.google.protobuf.Struct\x1a\x30\n\x0eLogitBiasEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\nconstraintB\x11\n\x0f_max_new_tokensB\x12\n\x10_stream_interval\"]\n\x13\x44isaggregatedParams\x12\x16\n\x0e\x62ootstrap_host\x18\x01 \x01(\t\x12\x16\n\x0e\x62ootstrap_port\x18\x02 \x01(\x05\x12\x16\n\x0e\x62ootstrap_room\x18\x03 \x01(\x05\"\xe2\x04\n\x0fGenerateRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x04 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x16\n\x0ereturn_logprob\x18\x05 \x01(\x08\x12\x19\n\x11logprob_start_len\x18\x06 \x01(\x05\x12\x18\n\x10top_logprobs_num\x18\x07 \x01(\x05\x12\x19\n\x11token_ids_logprob\x18\x08 \x03(\r\x12\x1c\n\x14return_hidden_states\x18\t \x01(\x08\x12H\n\x14\x64isaggregated_params\x18\n \x01(\x0b\x32*.sglang.grpc.scheduler.DisaggregatedParams\x12\x1e\n\x16\x63ustom_logit_processor\x18\x0b \x01(\t\x12-\n\ttimestamp\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x13\n\x0blog_metrics\x18\r \x01(\x08\x12\x14\n\x0cinput_embeds\x18\x0e \x03(\x02\x12\x0f\n\x07lora_id\x18\x0f \x01(\t\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x10 \x01(\x05\x12\x0e\n\x06stream\x18\x11 \x01(\x08\":\n\x0eTokenizedInput\x12\x15\n\roriginal_text\x18\x01 \x01(\t\x12\x11\n\tinput_ids\x18\x02 \x03(\r\"\xd3\x01\n\x10MultimodalInputs\x12\x12\n\nimage_urls\x18\x01 \x03(\t\x12\x12\n\nvideo_urls\x18\x02 \x03(\t\x12\x12\n\naudio_urls\x18\x03 \x03(\t\x12\x33\n\x12processed_features\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x12\n\nimage_data\x18\x05 \x03(\x0c\x12\x12\n\nvideo_data\x18\x06 \x03(\x0c\x12\x12\n\naudio_data\x18\x07 \x03(\x0c\x12\x12\n\nmodalities\x18\x08 \x03(\t\"\xe3\x01\n\x10GenerateResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12;\n\x05\x63hunk\x18\x02 \x01(\x0b\x32*.sglang.grpc.scheduler.GenerateStreamChunkH\x00\x12;\n\x08\x63omplete\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.GenerateCompleteH\x00\x12\x35\n\x05\x65rror\x18\x04 \x01(\x0b\x32$.sglang.grpc.scheduler.GenerateErrorH\x00\x42\n\n\x08response\"\x95\x02\n\x13GenerateStreamChunk\x12\x11\n\ttoken_ids\x18\x01 \x03(\r\x12\x15\n\rprompt_tokens\x18\x02 \x01(\x05\x12\x19\n\x11\x63ompletion_tokens\x18\x03 \x01(\x05\x12\x15\n\rcached_tokens\x18\x04 \x01(\x05\x12>\n\x0foutput_logprobs\x18\x05 \x01(\x0b\x32%.sglang.grpc.scheduler.OutputLogProbs\x12\x15\n\rhidden_states\x18\x06 \x03(\x02\x12<\n\x0einput_logprobs\x18\x07 \x01(\x0b\x32$.sglang.grpc.scheduler.InputLogProbs\x12\r\n\x05index\x18\x08 \x01(\r\"\x9b\x03\n\x10GenerateComplete\x12\x12\n\noutput_ids\x18\x01 \x03(\r\x12\x15\n\rfinish_reason\x18\x02 \x01(\t\x12\x15\n\rprompt_tokens\x18\x03 \x01(\x05\x12\x19\n\x11\x63ompletion_tokens\x18\x04 \x01(\x05\x12\x15\n\rcached_tokens\x18\x05 \x01(\x05\x12>\n\x0foutput_logprobs\x18\x06 \x01(\x0b\x32%.sglang.grpc.scheduler.OutputLogProbs\x12>\n\x11\x61ll_hidden_states\x18\x07 \x03(\x0b\x32#.sglang.grpc.scheduler.HiddenStates\x12\x1a\n\x10matched_token_id\x18\x08 \x01(\rH\x00\x12\x1a\n\x10matched_stop_str\x18\t \x01(\tH\x00\x12<\n\x0einput_logprobs\x18\n \x01(\x0b\x32$.sglang.grpc.scheduler.InputLogProbs\x12\r\n\x05index\x18\x0b \x01(\rB\x0e\n\x0cmatched_stop\"K\n\rGenerateError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x18\n\x10http_status_code\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"u\n\x0eOutputLogProbs\x12\x16\n\x0etoken_logprobs\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x38\n\x0ctop_logprobs\x18\x03 \x03(\x0b\x32\".sglang.grpc.scheduler.TopLogProbs\"\x9e\x01\n\rInputLogProbs\x12@\n\x0etoken_logprobs\x18\x01 \x03(\x0b\x32(.sglang.grpc.scheduler.InputTokenLogProb\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x38\n\x0ctop_logprobs\x18\x03 \x03(\x0b\x32\".sglang.grpc.scheduler.TopLogProbs\"1\n\x11InputTokenLogProb\x12\x12\n\x05value\x18\x01 \x01(\x02H\x00\x88\x01\x01\x42\x08\n\x06_value\"0\n\x0bTopLogProbs\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\"?\n\x0cHiddenStates\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05layer\x18\x02 \x01(\x05\x12\x10\n\x08position\x18\x03 \x01(\x05\"\xca\x02\n\x0c\x45mbedRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x04 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x05 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x13\n\x0blog_metrics\x18\x06 \x01(\x08\x12\x16\n\x0etoken_type_ids\x18\x07 \x03(\x05\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x08 \x01(\x05\x12\x18\n\x10is_cross_encoder\x18\t \x01(\x08\x12\r\n\x05texts\x18\n \x03(\t\"\x9d\x01\n\rEmbedResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\x08\x63omplete\x18\x02 \x01(\x0b\x32$.sglang.grpc.scheduler.EmbedCompleteH\x00\x12\x32\n\x05\x65rror\x18\x03 \x01(\x0b\x32!.sglang.grpc.scheduler.EmbedErrorH\x00\x42\n\n\x08response\"\xa3\x01\n\rEmbedComplete\x12\x11\n\tembedding\x18\x01 \x03(\x02\x12\x15\n\rprompt_tokens\x18\x02 \x01(\x05\x12\x15\n\rcached_tokens\x18\x03 \x01(\x05\x12\x15\n\rembedding_dim\x18\x04 \x01(\x05\x12:\n\x10\x62\x61tch_embeddings\x18\x05 \x03(\x0b\x32 .sglang.grpc.scheduler.Embedding\"*\n\tEmbedding\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05index\x18\x02 \x01(\x05\"<\n\nEmbedError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"N\n\x12HealthCheckRequest\x12\x38\n\ttokenized\x18\x01 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\"7\n\x13HealthCheckResponse\x12\x0f\n\x07healthy\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"2\n\x0c\x41\x62ortRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\"1\n\rAbortResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"I\n\x0fLoadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\x12\x14\n\x0c\x61\x64\x61pter_path\x18\x02 \x01(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\"H\n\x10LoadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x12\n\nadapter_id\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"\'\n\x11UnloadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\"6\n\x12UnloadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"w\n\x14UpdateWeightsRequest\x12\x13\n\tdisk_path\x18\x01 \x01(\tH\x00\x12\x15\n\x0btensor_data\x18\x02 \x01(\x0cH\x00\x12\x14\n\nremote_url\x18\x03 \x01(\tH\x00\x12\x13\n\x0bweight_name\x18\x04 \x01(\tB\x08\n\x06source\"9\n\x15UpdateWeightsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"-\n\x17GetInternalStateRequest\x12\x12\n\nstate_keys\x18\x01 \x03(\t\"B\n\x18GetInternalStateResponse\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"A\n\x17SetInternalStateRequest\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"<\n\x18SetInternalStateResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2\xfe\x02\n\x0fSglangScheduler\x12]\n\x08Generate\x12&.sglang.grpc.scheduler.GenerateRequest\x1a\'.sglang.grpc.scheduler.GenerateResponse0\x01\x12R\n\x05\x45mbed\x12#.sglang.grpc.scheduler.EmbedRequest\x1a$.sglang.grpc.scheduler.EmbedResponse\x12\x64\n\x0bHealthCheck\x12).sglang.grpc.scheduler.HealthCheckRequest\x1a*.sglang.grpc.scheduler.HealthCheckResponse\x12R\n\x05\x41\x62ort\x12#.sglang.grpc.scheduler.AbortRequest\x1a$.sglang.grpc.scheduler.AbortResponseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -39,73 +39,73 @@ _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._loaded_options = None _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_options = b'8\001' _globals['_SAMPLINGPARAMS']._serialized_start=113 - _globals['_SAMPLINGPARAMS']._serialized_end=850 - _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_start=769 - _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_end=817 - _globals['_DISAGGREGATEDPARAMS']._serialized_start=852 - _globals['_DISAGGREGATEDPARAMS']._serialized_end=945 - _globals['_GENERATEREQUEST']._serialized_start=948 - _globals['_GENERATEREQUEST']._serialized_end=1558 - _globals['_TOKENIZEDINPUT']._serialized_start=1560 - _globals['_TOKENIZEDINPUT']._serialized_end=1618 - _globals['_MULTIMODALINPUTS']._serialized_start=1621 - _globals['_MULTIMODALINPUTS']._serialized_end=1832 - _globals['_GENERATERESPONSE']._serialized_start=1835 - _globals['_GENERATERESPONSE']._serialized_end=2062 - _globals['_GENERATESTREAMCHUNK']._serialized_start=2065 - _globals['_GENERATESTREAMCHUNK']._serialized_end=2342 - _globals['_GENERATECOMPLETE']._serialized_start=2345 - _globals['_GENERATECOMPLETE']._serialized_end=2756 - _globals['_GENERATEERROR']._serialized_start=2758 - _globals['_GENERATEERROR']._serialized_end=2833 - _globals['_OUTPUTLOGPROBS']._serialized_start=2835 - _globals['_OUTPUTLOGPROBS']._serialized_end=2952 - _globals['_INPUTLOGPROBS']._serialized_start=2955 - _globals['_INPUTLOGPROBS']._serialized_end=3113 - _globals['_INPUTTOKENLOGPROB']._serialized_start=3115 - _globals['_INPUTTOKENLOGPROB']._serialized_end=3164 - _globals['_TOPLOGPROBS']._serialized_start=3166 - _globals['_TOPLOGPROBS']._serialized_end=3214 - _globals['_HIDDENSTATES']._serialized_start=3216 - _globals['_HIDDENSTATES']._serialized_end=3279 - _globals['_EMBEDREQUEST']._serialized_start=3282 - _globals['_EMBEDREQUEST']._serialized_end=3612 - _globals['_EMBEDRESPONSE']._serialized_start=3615 - _globals['_EMBEDRESPONSE']._serialized_end=3772 - _globals['_EMBEDCOMPLETE']._serialized_start=3775 - _globals['_EMBEDCOMPLETE']._serialized_end=3938 - _globals['_EMBEDDING']._serialized_start=3940 - _globals['_EMBEDDING']._serialized_end=3982 - _globals['_EMBEDERROR']._serialized_start=3984 - _globals['_EMBEDERROR']._serialized_end=4044 - _globals['_HEALTHCHECKREQUEST']._serialized_start=4046 - _globals['_HEALTHCHECKREQUEST']._serialized_end=4124 - _globals['_HEALTHCHECKRESPONSE']._serialized_start=4126 - _globals['_HEALTHCHECKRESPONSE']._serialized_end=4181 - _globals['_ABORTREQUEST']._serialized_start=4183 - _globals['_ABORTREQUEST']._serialized_end=4233 - _globals['_ABORTRESPONSE']._serialized_start=4235 - _globals['_ABORTRESPONSE']._serialized_end=4284 - _globals['_LOADLORAREQUEST']._serialized_start=4286 - _globals['_LOADLORAREQUEST']._serialized_end=4359 - _globals['_LOADLORARESPONSE']._serialized_start=4361 - _globals['_LOADLORARESPONSE']._serialized_end=4433 - _globals['_UNLOADLORAREQUEST']._serialized_start=4435 - _globals['_UNLOADLORAREQUEST']._serialized_end=4474 - _globals['_UNLOADLORARESPONSE']._serialized_start=4476 - _globals['_UNLOADLORARESPONSE']._serialized_end=4530 - _globals['_UPDATEWEIGHTSREQUEST']._serialized_start=4532 - _globals['_UPDATEWEIGHTSREQUEST']._serialized_end=4651 - _globals['_UPDATEWEIGHTSRESPONSE']._serialized_start=4653 - _globals['_UPDATEWEIGHTSRESPONSE']._serialized_end=4710 - _globals['_GETINTERNALSTATEREQUEST']._serialized_start=4712 - _globals['_GETINTERNALSTATEREQUEST']._serialized_end=4757 - _globals['_GETINTERNALSTATERESPONSE']._serialized_start=4759 - _globals['_GETINTERNALSTATERESPONSE']._serialized_end=4825 - _globals['_SETINTERNALSTATEREQUEST']._serialized_start=4827 - _globals['_SETINTERNALSTATEREQUEST']._serialized_end=4892 - _globals['_SETINTERNALSTATERESPONSE']._serialized_start=4894 - _globals['_SETINTERNALSTATERESPONSE']._serialized_end=4954 - _globals['_SGLANGSCHEDULER']._serialized_start=4957 - _globals['_SGLANGSCHEDULER']._serialized_end=5339 + _globals['_SAMPLINGPARAMS']._serialized_end=833 + _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_start=732 + _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_end=780 + _globals['_DISAGGREGATEDPARAMS']._serialized_start=835 + _globals['_DISAGGREGATEDPARAMS']._serialized_end=928 + _globals['_GENERATEREQUEST']._serialized_start=931 + _globals['_GENERATEREQUEST']._serialized_end=1541 + _globals['_TOKENIZEDINPUT']._serialized_start=1543 + _globals['_TOKENIZEDINPUT']._serialized_end=1601 + _globals['_MULTIMODALINPUTS']._serialized_start=1604 + _globals['_MULTIMODALINPUTS']._serialized_end=1815 + _globals['_GENERATERESPONSE']._serialized_start=1818 + _globals['_GENERATERESPONSE']._serialized_end=2045 + _globals['_GENERATESTREAMCHUNK']._serialized_start=2048 + _globals['_GENERATESTREAMCHUNK']._serialized_end=2325 + _globals['_GENERATECOMPLETE']._serialized_start=2328 + _globals['_GENERATECOMPLETE']._serialized_end=2739 + _globals['_GENERATEERROR']._serialized_start=2741 + _globals['_GENERATEERROR']._serialized_end=2816 + _globals['_OUTPUTLOGPROBS']._serialized_start=2818 + _globals['_OUTPUTLOGPROBS']._serialized_end=2935 + _globals['_INPUTLOGPROBS']._serialized_start=2938 + _globals['_INPUTLOGPROBS']._serialized_end=3096 + _globals['_INPUTTOKENLOGPROB']._serialized_start=3098 + _globals['_INPUTTOKENLOGPROB']._serialized_end=3147 + _globals['_TOPLOGPROBS']._serialized_start=3149 + _globals['_TOPLOGPROBS']._serialized_end=3197 + _globals['_HIDDENSTATES']._serialized_start=3199 + _globals['_HIDDENSTATES']._serialized_end=3262 + _globals['_EMBEDREQUEST']._serialized_start=3265 + _globals['_EMBEDREQUEST']._serialized_end=3595 + _globals['_EMBEDRESPONSE']._serialized_start=3598 + _globals['_EMBEDRESPONSE']._serialized_end=3755 + _globals['_EMBEDCOMPLETE']._serialized_start=3758 + _globals['_EMBEDCOMPLETE']._serialized_end=3921 + _globals['_EMBEDDING']._serialized_start=3923 + _globals['_EMBEDDING']._serialized_end=3965 + _globals['_EMBEDERROR']._serialized_start=3967 + _globals['_EMBEDERROR']._serialized_end=4027 + _globals['_HEALTHCHECKREQUEST']._serialized_start=4029 + _globals['_HEALTHCHECKREQUEST']._serialized_end=4107 + _globals['_HEALTHCHECKRESPONSE']._serialized_start=4109 + _globals['_HEALTHCHECKRESPONSE']._serialized_end=4164 + _globals['_ABORTREQUEST']._serialized_start=4166 + _globals['_ABORTREQUEST']._serialized_end=4216 + _globals['_ABORTRESPONSE']._serialized_start=4218 + _globals['_ABORTRESPONSE']._serialized_end=4267 + _globals['_LOADLORAREQUEST']._serialized_start=4269 + _globals['_LOADLORAREQUEST']._serialized_end=4342 + _globals['_LOADLORARESPONSE']._serialized_start=4344 + _globals['_LOADLORARESPONSE']._serialized_end=4416 + _globals['_UNLOADLORAREQUEST']._serialized_start=4418 + _globals['_UNLOADLORAREQUEST']._serialized_end=4457 + _globals['_UNLOADLORARESPONSE']._serialized_start=4459 + _globals['_UNLOADLORARESPONSE']._serialized_end=4513 + _globals['_UPDATEWEIGHTSREQUEST']._serialized_start=4515 + _globals['_UPDATEWEIGHTSREQUEST']._serialized_end=4634 + _globals['_UPDATEWEIGHTSRESPONSE']._serialized_start=4636 + _globals['_UPDATEWEIGHTSRESPONSE']._serialized_end=4693 + _globals['_GETINTERNALSTATEREQUEST']._serialized_start=4695 + _globals['_GETINTERNALSTATEREQUEST']._serialized_end=4740 + _globals['_GETINTERNALSTATERESPONSE']._serialized_start=4742 + _globals['_GETINTERNALSTATERESPONSE']._serialized_end=4808 + _globals['_SETINTERNALSTATEREQUEST']._serialized_start=4810 + _globals['_SETINTERNALSTATEREQUEST']._serialized_end=4875 + _globals['_SETINTERNALSTATERESPONSE']._serialized_start=4877 + _globals['_SETINTERNALSTATERESPONSE']._serialized_end=4937 + _globals['_SGLANGSCHEDULER']._serialized_start=4940 + _globals['_SGLANGSCHEDULER']._serialized_end=5322 # @@protoc_insertion_point(module_scope) diff --git a/python/sglang/srt/grpc/sglang_scheduler_pb2.pyi b/python/sglang/srt/grpc/sglang_scheduler_pb2.pyi index 53559ebfdfe..62cfb21a95e 100644 --- a/python/sglang/srt/grpc/sglang_scheduler_pb2.pyi +++ b/python/sglang/srt/grpc/sglang_scheduler_pb2.pyi @@ -11,7 +11,7 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class SamplingParams(_message.Message): - __slots__ = ("temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "max_new_tokens", "stop", "stop_token_ids", "skip_special_tokens", "spaces_between_special_tokens", "regex", "json_schema", "ebnf_grammar", "structural_tag", "lora_path", "n", "token_healing", "min_new_tokens", "ignore_eos", "no_stop_trim", "stream_interval", "logit_bias", "custom_params") + __slots__ = ("temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "max_new_tokens", "stop", "stop_token_ids", "skip_special_tokens", "spaces_between_special_tokens", "regex", "json_schema", "ebnf_grammar", "structural_tag", "n", "min_new_tokens", "ignore_eos", "no_stop_trim", "stream_interval", "logit_bias", "custom_params") class LogitBiasEntry(_message.Message): __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] @@ -35,9 +35,7 @@ class SamplingParams(_message.Message): JSON_SCHEMA_FIELD_NUMBER: _ClassVar[int] EBNF_GRAMMAR_FIELD_NUMBER: _ClassVar[int] STRUCTURAL_TAG_FIELD_NUMBER: _ClassVar[int] - LORA_PATH_FIELD_NUMBER: _ClassVar[int] N_FIELD_NUMBER: _ClassVar[int] - TOKEN_HEALING_FIELD_NUMBER: _ClassVar[int] MIN_NEW_TOKENS_FIELD_NUMBER: _ClassVar[int] IGNORE_EOS_FIELD_NUMBER: _ClassVar[int] NO_STOP_TRIM_FIELD_NUMBER: _ClassVar[int] @@ -60,16 +58,14 @@ class SamplingParams(_message.Message): json_schema: str ebnf_grammar: str structural_tag: str - lora_path: str n: int - token_healing: bool min_new_tokens: int ignore_eos: bool no_stop_trim: bool stream_interval: int logit_bias: _containers.ScalarMap[str, float] custom_params: _struct_pb2.Struct - def __init__(self, temperature: _Optional[float] = ..., top_p: _Optional[float] = ..., top_k: _Optional[int] = ..., min_p: _Optional[float] = ..., frequency_penalty: _Optional[float] = ..., presence_penalty: _Optional[float] = ..., repetition_penalty: _Optional[float] = ..., max_new_tokens: _Optional[int] = ..., stop: _Optional[_Iterable[str]] = ..., stop_token_ids: _Optional[_Iterable[int]] = ..., skip_special_tokens: bool = ..., spaces_between_special_tokens: bool = ..., regex: _Optional[str] = ..., json_schema: _Optional[str] = ..., ebnf_grammar: _Optional[str] = ..., structural_tag: _Optional[str] = ..., lora_path: _Optional[str] = ..., n: _Optional[int] = ..., token_healing: bool = ..., min_new_tokens: _Optional[int] = ..., ignore_eos: bool = ..., no_stop_trim: bool = ..., stream_interval: _Optional[int] = ..., logit_bias: _Optional[_Mapping[str, float]] = ..., custom_params: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ... + def __init__(self, temperature: _Optional[float] = ..., top_p: _Optional[float] = ..., top_k: _Optional[int] = ..., min_p: _Optional[float] = ..., frequency_penalty: _Optional[float] = ..., presence_penalty: _Optional[float] = ..., repetition_penalty: _Optional[float] = ..., max_new_tokens: _Optional[int] = ..., stop: _Optional[_Iterable[str]] = ..., stop_token_ids: _Optional[_Iterable[int]] = ..., skip_special_tokens: bool = ..., spaces_between_special_tokens: bool = ..., regex: _Optional[str] = ..., json_schema: _Optional[str] = ..., ebnf_grammar: _Optional[str] = ..., structural_tag: _Optional[str] = ..., n: _Optional[int] = ..., min_new_tokens: _Optional[int] = ..., ignore_eos: bool = ..., no_stop_trim: bool = ..., stream_interval: _Optional[int] = ..., logit_bias: _Optional[_Mapping[str, float]] = ..., custom_params: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ... class DisaggregatedParams(_message.Message): __slots__ = ("bootstrap_host", "bootstrap_port", "bootstrap_room") diff --git a/sgl-router/src/grpc_client/sglang_scheduler.rs b/sgl-router/src/grpc_client/sglang_scheduler.rs index 1097a18f13d..5de461d3d2a 100644 --- a/sgl-router/src/grpc_client/sglang_scheduler.rs +++ b/sgl-router/src/grpc_client/sglang_scheduler.rs @@ -202,6 +202,7 @@ impl SglangSchedulerClient { stop: stop_sequences, stop_token_ids: request.stop_token_ids.clone().unwrap_or_default(), skip_special_tokens, + spaces_between_special_tokens: true, // Default from Python SamplingParams ignore_eos: request.ignore_eos, no_stop_trim: request.no_stop_trim, n: request.n.unwrap_or(1) as i32, @@ -301,6 +302,8 @@ impl SglangSchedulerClient { top_k: -1, repetition_penalty: 1.0, n: 1, + skip_special_tokens: true, + spaces_between_special_tokens: true, ..Default::default() }; @@ -444,10 +447,24 @@ mod tests { #[test] fn test_sampling_params_defaults() { let params = proto::SamplingParams::default(); + // Numeric fields have proto defaults (0) assert_eq!(params.temperature, 0.0); - assert_eq!(params.max_new_tokens, None); assert_eq!(params.top_p, 0.0); assert_eq!(params.top_k, 0); + assert_eq!(params.repetition_penalty, 0.0); + assert_eq!(params.n, 0); + // Bool fields have proto defaults (false) + assert!(!params.skip_special_tokens); + assert!(!params.spaces_between_special_tokens); + assert!(!params.ignore_eos); + assert!(!params.no_stop_trim); + // Optional int fields should be None + assert_eq!(params.max_new_tokens, None); + assert_eq!(params.stream_interval, None); + // Other non-optional fields + assert_eq!(params.min_p, 0.0); + assert_eq!(params.frequency_penalty, 0.0); + assert_eq!(params.presence_penalty, 0.0); assert!(params.stop.is_empty()); } diff --git a/sgl-router/src/proto/sglang_scheduler.proto b/sgl-router/src/proto/sglang_scheduler.proto index 152e65cb533..d3f19587a0b 100644 --- a/sgl-router/src/proto/sglang_scheduler.proto +++ b/sgl-router/src/proto/sglang_scheduler.proto @@ -27,6 +27,11 @@ service SglangScheduler { // ===================== // Sampling parameters matching SGLang's SamplingParams +// +// IMPORTANT: Do not use SamplingParams::default() directly! +// The proto3 defaults (0 for numeric fields) do NOT match the semantic defaults +// (temperature=1.0, top_p=1.0, top_k=-1, etc.). Always construct with explicit values +// or use the conversion functions in sglang_scheduler.rs / grpc_server.py. message SamplingParams { float temperature = 1; float top_p = 2; @@ -50,24 +55,18 @@ message SamplingParams { string structural_tag = 16; } - // LoRA adapter - string lora_path = 17; - // Speculative decoding - int32 n = 18; // Number of samples - - // Token healing - bool token_healing = 19; + int32 n = 17; // Number of samples // Additional parameters - int32 min_new_tokens = 20; - bool ignore_eos = 21; - bool no_stop_trim = 22; - int32 stream_interval = 23; - map logit_bias = 24; + int32 min_new_tokens = 18; + bool ignore_eos = 19; + bool no_stop_trim = 20; + optional int32 stream_interval = 21; + map logit_bias = 22; // Custom parameters for extensibility - google.protobuf.Struct custom_params = 25; + google.protobuf.Struct custom_params = 23; }