fix broken generate in 4.26.1

Author: mayank31398

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
         name: isort (python)
   - repo: https://github.com/psf/black
-    rev: 22.8.0
+    rev: 23.1.0
     hooks:
       - id: black
         args: [--line-length=119,--target-version=py35]

bloom-inference-scripts/bloom-accelerate-inference.py

@@ -61,12 +61,14 @@ def print_rank0(*msg):
     device_map="auto",
 )
 
+
 def get_world_size() -> int:
     if dist.is_initialized():
         return dist.get_world_size()
     else:
         return 1
 
+
 # balanced_low_0 - because it allows a larger batch size with multiple GPUs
 if get_world_size() > 1:
     kwargs["device_map"] = "balanced_low_0"

inference_server/model_handler/grpc_utils/pb/generation_pb2.py

@@ -10,24 +10,24 @@ class BaseResponse(BaseModel):
 
 class GenerateRequest(BaseModel):
     text: List[str] = None
-    min_length: int = None
-    do_sample: bool = None
-    early_stopping: bool = None
-    temperature: float = None
-    top_k: int = None
-    top_p: float = None
-    typical_p: float = None
-    repetition_penalty: float = None
+    min_length: int = 0
+    do_sample: bool = False
+    early_stopping: bool = False
+    temperature: float = 1
+    top_k: int = 50
+    top_p: float = 1
+    typical_p: float = 1
+    repetition_penalty: float = 1
     bos_token_id: int = None
     pad_token_id: int = None
     eos_token_id: int = None
-    length_penalty: float = None
-    no_repeat_ngram_size: int = None
-    encoder_no_repeat_ngram_size: int = None
+    length_penalty: float = 1
+    no_repeat_ngram_size: int = 0
+    encoder_no_repeat_ngram_size: int = 0
     max_time: float = None
     max_new_tokens: int = None
     decoder_start_token_id: int = None
-    diversity_penalty: float = None
+    diversity_penalty: float = 0
     forced_bos_token_id: int = None
     forced_eos_token_id: int = None
     exponential_decay_length_penalty: float = None
@@ -89,32 +89,51 @@ def parse_field(kwargs: dict, field: str, dtype: type, default_value: Any = None
 
 def create_generate_request(text: List[str], generate_kwargs: dict) -> GenerateRequest:
     # get user generate_kwargs as json and parse it
+    default_request = GenerateRequest()
+
     return GenerateRequest(
         text=text,
-        min_length=parse_field(generate_kwargs, "min_length", int),
-        do_sample=parse_field(generate_kwargs, "do_sample", bool),
-        early_stopping=parse_field(generate_kwargs, "early_stopping", bool),
-        num_beams=parse_field(generate_kwargs, "num_beams", int),
-        temperature=parse_field(generate_kwargs, "temperature", float),
-        top_k=parse_field(generate_kwargs, "top_k", int),
-        top_p=parse_field(generate_kwargs, "top_p", float),
-        typical_p=parse_field(generate_kwargs, "typical_p", float),
-        repetition_penalty=parse_field(generate_kwargs, "repetition_penalty", float),
-        bos_token_id=parse_field(generate_kwargs, "bos_token_id", int),
-        pad_token_id=parse_field(generate_kwargs, "pad_token_id", int),
-        eos_token_id=parse_field(generate_kwargs, "eos_token_id", int),
-        length_penalty=parse_field(generate_kwargs, "length_penalty", float),
-        no_repeat_ngram_size=parse_field(generate_kwargs, "no_repeat_ngram_size", int),
-        encoder_no_repeat_ngram_size=parse_field(generate_kwargs, "encoder_no_repeat_ngram_size", int),
-        max_time=parse_field(generate_kwargs, "max_time", float),
-        max_new_tokens=parse_field(generate_kwargs, "max_new_tokens", int),
-        decoder_start_token_id=parse_field(generate_kwargs, "decoder_start_token_id", int),
-        num_beam_group=parse_field(generate_kwargs, "num_beam_group", int),
-        diversity_penalty=parse_field(generate_kwargs, "diversity_penalty", float),
-        forced_bos_token_id=parse_field(generate_kwargs, "forced_bos_token_id", int),
-        forced_eos_token_id=parse_field(generate_kwargs, "forced_eos_token_id", int),
-        exponential_decay_length_penalty=parse_field(generate_kwargs, "exponential_decay_length_penalty", float),
-        remove_input_from_output=parse_field(generate_kwargs, "remove_input_from_output", bool, False),
+        min_length=parse_field(generate_kwargs, "min_length", int, default_request.min_length),
+        do_sample=parse_field(generate_kwargs, "do_sample", bool, default_request.do_sample),
+        early_stopping=parse_field(generate_kwargs, "early_stopping", bool, default_request.early_stopping),
+        temperature=parse_field(generate_kwargs, "temperature", float, default_request.temperature),
+        top_k=parse_field(generate_kwargs, "top_k", int, default_request.top_k),
+        top_p=parse_field(generate_kwargs, "top_p", float, default_request.top_p),
+        typical_p=parse_field(generate_kwargs, "typical_p", float, default_request.typical_p),
+        repetition_penalty=parse_field(
+            generate_kwargs, "repetition_penalty", float, default_request.repetition_penalty
+        ),
+        bos_token_id=parse_field(generate_kwargs, "bos_token_id", int, default_request.bos_token_id),
+        pad_token_id=parse_field(generate_kwargs, "pad_token_id", int, default_request.pad_token_id),
+        eos_token_id=parse_field(generate_kwargs, "eos_token_id", int, default_request.eos_token_id),
+        length_penalty=parse_field(generate_kwargs, "length_penalty", float, default_request.length_penalty),
+        no_repeat_ngram_size=parse_field(
+            generate_kwargs, "no_repeat_ngram_size", int, default_request.no_repeat_ngram_size
+        ),
+        encoder_no_repeat_ngram_size=parse_field(
+            generate_kwargs, "encoder_no_repeat_ngram_size", int, default_request.encoder_no_repeat_ngram_size
+        ),
+        max_time=parse_field(generate_kwargs, "max_time", float, default_request.max_time),
+        max_new_tokens=parse_field(generate_kwargs, "max_new_tokens", int, default_request.max_new_tokens),
+        decoder_start_token_id=parse_field(
+            generate_kwargs, "decoder_start_token_id", int, default_request.decoder_start_token_id
+        ),
+        diversity_penalty=parse_field(generate_kwargs, "diversity_penalty", float, default_request.diversity_penalty),
+        forced_bos_token_id=parse_field(
+            generate_kwargs, "forced_bos_token_id", int, default_request.forced_bos_token_id
+        ),
+        forced_eos_token_id=parse_field(
+            generate_kwargs, "forced_eos_token_id", int, default_request.forced_eos_token_id
+        ),
+        exponential_decay_length_penalty=parse_field(
+            generate_kwargs,
+            "exponential_decay_length_penalty",
+            float,
+            default_request.exponential_decay_length_penalty,
+        ),
+        remove_input_from_output=parse_field(
+            generate_kwargs, "remove_input_from_output", bool, default_request.remove_input_from_output
+        ),
     )