drop duplicate html files (#56)

* drop duplicate html and js
* fix model load path
* update dockerfile

Author: mayank31398

Dockerfile

@@ -26,17 +26,17 @@ RUN conda install -c anaconda cmake -y
 
 # necessary stuff
 RUN pip install torch==1.12.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 \
-    transformers==4.25.1 \
+    transformers==4.26.1 \
     deepspeed==0.7.6 \
-    accelerate==0.15.0 \
+    accelerate==0.16.0 \
     gunicorn==20.1.0 \
     flask \
     flask_api \
     fastapi==0.89.1 \
     uvicorn==0.19.0 \
     jinja2==3.1.2 \
     pydantic==1.10.2 \
-    huggingface_hub==0.10.1 \
+    huggingface_hub==0.12.1 \
 	grpcio-tools==1.50.0 \
     --no-cache-dir
 
@@ -64,5 +64,4 @@ CMD git clone https://github.com/huggingface/transformers-bloom-inference.git &&
     cd transformers-bloom-inference && \
     # install grpc and compile protos
     make gen-proto && \
-    make ui model_name=bigscience/bloom-560m && \
     make bloom-560m

Makefile

@@ -10,8 +10,13 @@ gen-proto:
 
 	rm -rf inference_server/model_handler/grpc_utils/pb/*.py-e
 
+ui:
+	python -m ui --ui_host 127.0.0.1 --ui_port 5001 --generation_backend_host 127.0.0.1 --generation_backend_port 5000 &
+
 # ------------------------- DS inference -------------------------
 bloom-176b:
+	make ui
+
 	TOKENIZERS_PARALLELISM=false \
 	MODEL_NAME=bigscience/bloom \
 	MODEL_CLASS=AutoModelForCausalLM \
@@ -24,6 +29,8 @@ bloom-176b:
 
 # loads faster than the above one
 microsoft-bloom-176b:
+	make ui
+
 	TOKENIZERS_PARALLELISM=false \
 	MODEL_NAME=microsoft/bloom-deepspeed-inference-fp16 \
 	MODEL_CLASS=AutoModelForCausalLM \
@@ -35,6 +42,8 @@ microsoft-bloom-176b:
 	gunicorn -t 0 -w 1 -b 127.0.0.1:5000 inference_server.server:app --access-logfile - --access-logformat '%(h)s %(t)s "%(r)s" %(s)s %(b)s'
 
 bloomz-176b:
+	make ui
+
 	TOKENIZERS_PARALLELISM=false \
 	MODEL_NAME=bigscience/bloomz \
 	MODEL_CLASS=AutoModelForCausalLM \
@@ -46,6 +55,8 @@ bloomz-176b:
 	gunicorn -t 0 -w 1 -b 127.0.0.1:5000 inference_server.server:app --access-logfile - --access-logformat '%(h)s %(t)s "%(r)s" %(s)s %(b)s'
 
 bloom-176b-int8:
+	make ui
+
 	TOKENIZERS_PARALLELISM=false \
 	MODEL_NAME=microsoft/bloom-deepspeed-inference-int8 \
 	MODEL_CLASS=AutoModelForCausalLM \
@@ -58,6 +69,8 @@ bloom-176b-int8:
 
 # ------------------------- HF accelerate -------------------------
 bloom-560m:
+	make ui
+
 	TOKENIZERS_PARALLELISM=false \
 	MODEL_NAME=bigscience/bloom-560m \
 	MODEL_CLASS=AutoModelForCausalLM \
@@ -69,6 +82,8 @@ bloom-560m:
 	gunicorn -t 0 -w 1 -b 127.0.0.1:5000 inference_server.server:app --access-logfile - --access-logformat '%(h)s %(t)s "%(r)s" %(s)s %(b)s'
 
 flan-t5-xxl:
+	make ui
+
 	TOKENIZERS_PARALLELISM=false \
 	MODEL_NAME=google/flan-t5-xxl \
 	MODEL_CLASS=AutoModelForSeq2SeqLM \
@@ -80,6 +95,8 @@ flan-t5-xxl:
 	gunicorn -t 0 -w 1 -b 127.0.0.1:5000 inference_server.server:app --access-logfile - --access-logformat '%(h)s %(t)s "%(r)s" %(s)s %(b)s'
 
 ul2:
+	make ui
+
 	TOKENIZERS_PARALLELISM=false \
 	MODEL_NAME=google/ul2 \
 	MODEL_CLASS=AutoModelForSeq2SeqLM \
@@ -91,6 +108,8 @@ ul2:
 	gunicorn -t 0 -w 1 -b 127.0.0.1:5000 inference_server.server:app --access-logfile - --access-logformat '%(h)s %(t)s "%(r)s" %(s)s %(b)s'
 
 codegen-mono:
+	make ui
+
 	TOKENIZERS_PARALLELISM=false \
 	MODEL_NAME=Salesforce/codegen-16B-mono \
 	MODEL_CLASS=AutoModelForCausalLM \
@@ -103,22 +122,23 @@ codegen-mono:
 
 # ------------------------- HF CPU -------------------------
 bloom-560m-cpu:
+	make ui
+
 	MODEL_NAME=bigscience/bloom-560m \
 	MODEL_CLASS=AutoModelForCausalLM \
 	DEPLOYMENT_FRAMEWORK=hf_cpu \
-	DTYPE=bf16 \
+	DTYPE=fp32 \
 	MAX_INPUT_LENGTH=2048 \
 	MAX_BATCH_SIZE=32 \
 	gunicorn -t 0 -w 1 -b 127.0.0.1:5000 inference_server.server:app --access-logfile - --access-logformat '%(h)s %(t)s "%(r)s" %(s)s %(b)s'
 
 flan-t5-base-cpu:
+	make ui
+
 	MODEL_NAME=google/flan-t5-base \
 	MODEL_CLASS=AutoModelForSeq2SeqLM \
 	DEPLOYMENT_FRAMEWORK=hf_cpu \
 	DTYPE=bf16 \
 	MAX_INPUT_LENGTH=2048 \
 	MAX_BATCH_SIZE=32 \
 	gunicorn -t 0 -w 1 -b 127.0.0.1:5000 inference_server.server:app --access-logfile - --access-logformat '%(h)s %(t)s "%(r)s" %(s)s %(b)s'
-
-ui:
-	python -m ui --model_name $(model_name) &

inference_server/benchmark.py

@@ -14,7 +14,7 @@
     get_dummy_batch,
     get_world_size,
     parse_args,
-    print_rank_n,
+    print_rank_0,
     run_and_log_time,
 )
 
@@ -49,18 +49,18 @@ def benchmark_end_to_end(args: argparse.Namespace) -> None:
 
     request = create_generate_request(get_dummy_batch(args.batch_size), args.generate_kwargs)
 
-    print_rank_n(f"generate_kwargs = {args.generate_kwargs}")
-    print_rank_n(f"batch_size = {args.batch_size}")
+    print_rank_0(f"generate_kwargs = {args.generate_kwargs}")
+    print_rank_0(f"batch_size = {args.batch_size}")
 
     # warmup is a must if measuring speed as it's when all the optimizations are performed
     # e.g. on 8x80 a100 the first pass of 100 tokens takes 23sec, and the next one is 4secs
     response = model.generate(request=request)
 
     for i, (o, _) in zip(request.text, zip(response.text, response.num_generated_tokens)):
-        print_rank_n(f"{'-' * 60}\nin = {i}\nout = {o}\n")
+        print_rank_0(f"{'-' * 60}\nin = {i}\nout = {o}\n")
 
     if args.benchmark_cycles > 0:
-        print_rank_n("*** Running benchmark")
+        print_rank_0("*** Running benchmark")
 
         torch.cuda.empty_cache()
         gc.collect()
@@ -78,7 +78,7 @@ def benchmark_end_to_end(args: argparse.Namespace) -> None:
         if args.deployment_framework == DS_ZERO:
             total_new_tokens_generated *= get_world_size()
 
-        print_rank_n(
+        print_rank_0(
             get_benchmark_results(
                 benchmark_time, initialization_time, total_new_tokens_generated, args.batch_size, args.benchmark_cycles
             )

inference_server/cli.py

@@ -3,7 +3,7 @@
 import sys
 
 from .model_handler import ModelDeployment
-from .utils import get_argument_parser, parse_args, print_rank_n
+from .utils import get_argument_parser, parse_args, print_rank_0
 
 
 def get_args() -> argparse.Namespace:
@@ -35,8 +35,8 @@ def main() -> None:
 
         response = model.generate(text=[input_text], generate_kwargs=generate_kwargs)
 
-        print_rank_n("Output text:", response.text[0])
-        print_rank_n("Generated tokens:", response.num_generated_tokens[0])
+        print_rank_0("Output text:", response.text[0])
+        print_rank_0("Generated tokens:", response.num_generated_tokens[0])
 
 
 if __name__ == "__main__":

inference_server/model_handler/deployment.py

@@ -21,7 +21,7 @@
     get_cuda_visible_devices,
     get_str_dtype,
     get_world_size,
-    print_rank_n,
+    print_rank_0,
 )
 from .grpc_utils.pb import generation_pb2, generation_pb2_grpc
 
@@ -53,7 +53,7 @@ def __init__(self, args: argparse.Namespace, grpc_allowed: bool = False):
         else:
             self.model = get_model_class(args.deployment_framework)(args)
 
-        print_rank_n("model loaded")
+        print_rank_0("model loaded")
 
     def should_use_grpc(self, deployment_framework: str, grpc_allowed: bool) -> bool:
         if grpc_allowed and get_world_size() > 1:
@@ -94,8 +94,8 @@ def _wait_until_server_is_live(self):
             if not process_alive:
                 raise RuntimeError("server crashed for some reason, unable to proceed")
             time.sleep(4)
-            print_rank_n("waiting for server to start...")
-        print_rank_n(f"server has started on {self.ports[0]}")
+            print_rank_0("waiting for server to start...")
+        print_rank_0(f"server has started on {self.ports[0]}")
 
     def dict_to_proto(self, generate_kwargs: dict) -> dict:
         result = {}
@@ -229,9 +229,3 @@ def tokenize(self, request: TokenizeRequest) -> TokenizeResponse:
             response = self.model.tokenize(request)
 
         return response
-
-    def _request_response(self):
-        raise NotImplementedError("This method should not be implemented")
-
-    def query(self):
-        raise NotImplementedError("This method should not be implemented")

inference_server/model_handler/grpc_utils/generation_server.py

@@ -7,7 +7,7 @@
 
 # from ...constants import GRPC_MAX_MSG_SIZE
 from ...models import Model
-from ...utils import ForwardRequest, create_generate_request, print_rank_n
+from ...utils import ForwardRequest, TokenizeRequest, create_generate_request, print_rank_0
 from .pb import generation_pb2, generation_pb2_grpc
 
 
@@ -33,10 +33,14 @@ def Generate(self, request, context):
 
         if isinstance(response, Exception):
             # if exception occurs, we don't this subprocess to crash
-            response = generation_pb2.GenerationResponseProto(error=str(response))
+            response = generation_pb2.GenerationResponseProto(
+                error=str(response), is_encoder_decoder=response.is_encoder_decoder
+            )
         else:
             response = generation_pb2.GenerationResponseProto(
-                texts=response.text, num_generated_tokens=response.num_generated_tokens
+                texts=response.text,
+                num_generated_tokens=response.num_generated_tokens,
+                is_encoder_decoder=response.is_encoder_decoder,
             )
 
         return response
@@ -55,9 +59,13 @@ def Forward(self, request, context):
 
         if isinstance(response, Exception):
             # if exception occurs, we don't this subprocess to crash
-            response = generation_pb2.ForwardResponseProto(error=str(response))
+            response = generation_pb2.ForwardResponseProto(
+                error=str(response), is_encoder_decoder=response.is_encoder_decoder
+            )
         else:
-            response = generation_pb2.ForwardResponseProto(nll=response.nll)
+            response = generation_pb2.ForwardResponseProto(
+                nll=response.nll, is_encoder_decoder=response.is_encoder_decoder
+            )
 
         return response
 
@@ -72,7 +80,7 @@ def serve(inference_pipeline, port):
     )
     generation_pb2_grpc.add_GenerationServiceServicer_to_server(GenerationServer(inference_pipeline), server)
     server.add_insecure_port(f"[::]:{port}")
-    print_rank_n("About to start server")
+    print_rank_0("About to start server")
     server.start()
-    print_rank_n("Started")
+    print_rank_0("Started")
     server.wait_for_termination()

inference_server/model_handler/grpc_utils/pb/generation_pb2.py

@@ -24,6 +24,7 @@ message GenerationResponseProto {
     repeated string texts = 1;
     repeated int32 num_generated_tokens = 2;
     string error = 3;
+    bool is_encoder_decoder = 4;
 }
 
 message ForwardRequestProto {
@@ -34,4 +35,5 @@ message ForwardRequestProto {
 message ForwardResponseProto {
     float nll = 1;
     string error = 2;
+    bool is_encoder_decoder = 3;
 }

inference_server/model_handler/grpc_utils/proto/generation.proto

@@ -74,14 +74,15 @@ def __init__(self, model_path: str):
         self.tmp_file = os.path.join(self.tmp_directory, "checkpoints.json")
         self.model_path = model_path
 
-    def write_checkpoints_json(self, model_path: str) -> None:
+    def write_checkpoints_json(self) -> None:
+        print(self.model_path)
         with io.open(self.tmp_file, "w", encoding="utf-8") as f:
-            data = {"type": "BLOOM", "checkpoints": glob.glob(f"{model_path}/*.bin"), "version": 1.0}
+            data = {"type": "BLOOM", "checkpoints": glob.glob(f"{self.model_path}/*.bin"), "version": 1.0}
             json.dump(data, f)
 
     def __enter__(self):
-        run_rank_n(partial(os.makedirs, name=self.tmp_directory, exist_ok=True))
-        run_rank_n(partial(self.write_checkpoints_json, model_path=self.model_path), barrier=True)
+        run_rank_n(os.makedirs, barrier=True)(self.tmp_directory, exist_ok=True)
+        run_rank_n(self.write_checkpoints_json, barrier=True)()
         return self.tmp_file
 
     def __exit__(self, type, value, traceback):
@@ -98,7 +99,7 @@ def get_model_path(model_name: str):
         if config_path is None:
             # treat the model name as an explicit model path
             return model_name
-        elif os.path.isfile(os.path.join(model_name, config_file)):
+        else:
             return os.path.dirname(config_path)
     except:
         # treat the model name as an explicit model path

inference_server/models/ds_inference.py

@@ -92,7 +92,11 @@ def generate(self, request: GenerateRequest) -> Union[GenerateResponse, Exceptio
                 else:
                     generated_text = self.tokenizer.batch_decode(output_tokens, skip_special_tokens=True)
 
-            return GenerateResponse(text=generated_text, num_generated_tokens=num_generated_tokens)
+            return GenerateResponse(
+                text=generated_text,
+                num_generated_tokens=num_generated_tokens,
+                is_encoder_decoder=self.is_encoder_decoder,
+            )
         except Exception as exception:
             return exception
 
@@ -140,13 +144,15 @@ def pad(arrays: list, padding: int, max_length: int = None):
 
             loss = self.model(**input_tokens).loss
 
-            return ForwardResponse(nll=loss.item())
+            return ForwardResponse(nll=loss.item(), is_encoder_decoder=self.is_encoder_decoder)
         except Exception as exception:
             return exception
 
     def tokenize(self, request: TokenizeRequest) -> TokenizeResponse:
-        response = self.tokenizer(request.text, padding=request.padding)
-        return TokenizeResponse(token_ids=response.input_ids, attention_mask=response.attention_mask)
+        return TokenizeResponse(
+            token_ids=self.tokenizer(request.text).input_ids,
+            is_encoder_decoder=self.is_encoder_decoder,
+        )
 
 
 def check_max_input_length(input_token_length: int, max_input_length: int) -> None: