From 8b2c7700d4a8303b1ef1179f38869a64fce149aa Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 15:37:15 +0200 Subject: [PATCH 01/17] first try gen script --- evaluation/generation/generate.py | 57 +++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 evaluation/generation/generate.py diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py new file mode 100644 index 00000000..8bde5bc3 --- /dev/null +++ b/evaluation/generation/generate.py @@ -0,0 +1,57 @@ +import argparse +import json +import datetime + +from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint", type=str, help="Checkpoint path", required=True) + parser.add_argument("--parallelize", action="store_true") + parser.add_argument("--global-step", type=str, default=None) + parser.add_argument("--generate-max-length", type=int, default=50, help="max generation length") + parser.add_argument("--greedy", action="store_true") + parser.add_argument("--top-k", type=int, default=0) + + return parser.parse_args() + +def generate_from_text(model, text, tokenizer, max_length=200, greedy=False, top_k=0): + input_ids = tokenizer.encode(text, return_tensors='pt').to("cuda:0") + max_length = input_ids.size(-1) + max_length + + greedy_output = model.generate( + input_ids.to('cuda:0'), + max_length=max_length, + do_sample=not greedy, + top_k=None if greedy else top_k, + ) + return { + "inputs": text, + "outputs": tokenizer.decode(greedy_output, skip_special_tokens=True) + } + +def main(args): + print(f"Loading model", flush=True) + + tokenizer = AutoTokenizer.from_pretrained("bigscience/bloom", padding_side="left") + + print("Loaded tokenizer !") + start = datetime.datetime.now() + model = AutoModelForCausalLM.from_pretrained( + args.checkpoint, + device_map="auto" if args.parallelize else None, + torch_dtype=torch.bfloat16, + revision="gs{}".format(args.global_step) if args.global_step else None + ) + model.eval() + print(f"Loaded model in {datetime.datetime.now() - start}") + + while True: + text = '' + while True: + dummy = input('''Enter the paragraph :''')+'\n' + if dummy=='\n': + break + text += dummy + output = generate_from_text(model, text, tokenizer, max_length=args.generate_max_length, greedy=args.greedy, top_k=args.top_k) + print(json.dumps(output, indent=2)) From 62fcebb7eee70b09933f77e1412d3773a7a94021 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 15:39:22 +0200 Subject: [PATCH 02/17] add args --- evaluation/generation/generate.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 8bde5bc3..af9a7f83 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -30,7 +30,8 @@ def generate_from_text(model, text, tokenizer, max_length=200, greedy=False, top "outputs": tokenizer.decode(greedy_output, skip_special_tokens=True) } -def main(args): +def main(): + args = get_args() print(f"Loading model", flush=True) tokenizer = AutoTokenizer.from_pretrained("bigscience/bloom", padding_side="left") @@ -55,3 +56,6 @@ def main(args): text += dummy output = generate_from_text(model, text, tokenizer, max_length=args.generate_max_length, greedy=args.greedy, top_k=args.top_k) print(json.dumps(output, indent=2)) + +if __name__ == "__main__": + main() \ No newline at end of file From 1782c2d89c08c66e78267318c3dd3ed874670562 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 15:40:36 +0200 Subject: [PATCH 03/17] correct tokenizer --- evaluation/generation/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index af9a7f83..7329769b 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -34,7 +34,7 @@ def main(): args = get_args() print(f"Loading model", flush=True) - tokenizer = AutoTokenizer.from_pretrained("bigscience/bloom", padding_side="left") + tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, padding_side="left") print("Loaded tokenizer !") start = datetime.datetime.now() From 80df231ec7e3af4f38f228c4d6190f49edb5c6d7 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 15:41:13 +0200 Subject: [PATCH 04/17] add import torch --- evaluation/generation/generate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 7329769b..68b0374a 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -2,6 +2,7 @@ import json import datetime +import torch from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline def get_args(): From 0d11810d3eacceb884022c4bf30bb6f4ff5e69c7 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 15:44:25 +0200 Subject: [PATCH 05/17] few nits --- evaluation/generation/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 68b0374a..ef88ba8d 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -28,7 +28,7 @@ def generate_from_text(model, text, tokenizer, max_length=200, greedy=False, top ) return { "inputs": text, - "outputs": tokenizer.decode(greedy_output, skip_special_tokens=True) + "outputs": tokenizer.decode(greedy_output[0], skip_special_tokens=True) } def main(): @@ -51,7 +51,7 @@ def main(): while True: text = '' while True: - dummy = input('''Enter the paragraph :''')+'\n' + dummy = input('''Enter the paragraph :''') if dummy=='\n': break text += dummy From 5899ed18b5cc4d1743a4fb1a9f3ab38108920fd1 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 15:58:51 +0200 Subject: [PATCH 06/17] new line --- evaluation/generation/generate.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index ef88ba8d..48cca78a 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -48,15 +48,15 @@ def main(): model.eval() print(f"Loaded model in {datetime.datetime.now() - start}") + text = '' while True: - text = '' - while True: - dummy = input('''Enter the paragraph :''') - if dummy=='\n': - break + try: + dummy = input('''Enter the paragraph (Enter for new line and Ctrl-c to end the prompt):''')+'\n' text += dummy - output = generate_from_text(model, text, tokenizer, max_length=args.generate_max_length, greedy=args.greedy, top_k=args.top_k) - print(json.dumps(output, indent=2)) + except KeyboardInterrupt: + output = generate_from_text(model, text, tokenizer, max_length=args.generate_max_length, greedy=args.greedy, top_k=args.top_k) + print(json.dumps(output, indent=2)) + text = '' if __name__ == "__main__": main() \ No newline at end of file From 609b880fea8079a2d842bfdcea7fe5a99b341a13 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 16:05:23 +0200 Subject: [PATCH 07/17] print decoded output --- evaluation/generation/generate.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 48cca78a..669c6716 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -26,10 +26,7 @@ def generate_from_text(model, text, tokenizer, max_length=200, greedy=False, top do_sample=not greedy, top_k=None if greedy else top_k, ) - return { - "inputs": text, - "outputs": tokenizer.decode(greedy_output[0], skip_special_tokens=True) - } + return tokenizer.decode(greedy_output[0], skip_special_tokens=True) def main(): args = get_args() @@ -55,7 +52,7 @@ def main(): text += dummy except KeyboardInterrupt: output = generate_from_text(model, text, tokenizer, max_length=args.generate_max_length, greedy=args.greedy, top_k=args.top_k) - print(json.dumps(output, indent=2)) + print(output) text = '' if __name__ == "__main__": From 877f76e789e8ca4f5fb3427ea3c427bd41645405 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 16:10:18 +0200 Subject: [PATCH 08/17] add small nit --- evaluation/generation/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 669c6716..214b7688 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -48,8 +48,9 @@ def main(): text = '' while True: try: - dummy = input('''Enter the paragraph (Enter for new line and Ctrl-c to end the prompt):''')+'\n' + dummy = input('''Enter the paragraph (Enter for new line and Ctrl-c to end the prompt):''') text += dummy + text += '\n' except KeyboardInterrupt: output = generate_from_text(model, text, tokenizer, max_length=args.generate_max_length, greedy=args.greedy, top_k=args.top_k) print(output) From 560b3fc91ebef0058044e4452268776a68413e46 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 16:14:00 +0200 Subject: [PATCH 09/17] fix small nit --- evaluation/generation/generate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 214b7688..669c6716 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -48,9 +48,8 @@ def main(): text = '' while True: try: - dummy = input('''Enter the paragraph (Enter for new line and Ctrl-c to end the prompt):''') + dummy = input('''Enter the paragraph (Enter for new line and Ctrl-c to end the prompt):''')+'\n' text += dummy - text += '\n' except KeyboardInterrupt: output = generate_from_text(model, text, tokenizer, max_length=args.generate_max_length, greedy=args.greedy, top_k=args.top_k) print(output) From 5e2d2a291ff460e26b7670a0f2c0f156349bf9fb Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 16:15:54 +0200 Subject: [PATCH 10/17] forward contrib credits Co-authored-by: Thomas Wang <24695242+thomasw21@users.noreply.github.com> Co-authored-by: Narsil --- evaluation/generation/test.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 evaluation/generation/test.py diff --git a/evaluation/generation/test.py b/evaluation/generation/test.py new file mode 100644 index 00000000..2f0f3c05 --- /dev/null +++ b/evaluation/generation/test.py @@ -0,0 +1,8 @@ +text = '' +while True: + try: + dummy = raw_input("""Enter the paragraph :""") + text += dummy + except KeyboardInterrupt: + print(text) + text = '' From 7c6c171ed5bd392500d88b3f1c1aa90cac436b39 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 16:17:54 +0200 Subject: [PATCH 11/17] rm useless file --- evaluation/generation/test.py | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 evaluation/generation/test.py diff --git a/evaluation/generation/test.py b/evaluation/generation/test.py deleted file mode 100644 index 2f0f3c05..00000000 --- a/evaluation/generation/test.py +++ /dev/null @@ -1,8 +0,0 @@ -text = '' -while True: - try: - dummy = raw_input("""Enter the paragraph :""") - text += dummy - except KeyboardInterrupt: - print(text) - text = '' From d6dd6760686272293de6fb047c1e9c028d37f62d Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 10 Jun 2022 16:38:52 +0200 Subject: [PATCH 12/17] small nits - remove eval - remove pipeline & json from import --- evaluation/generation/generate.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 669c6716..d6699732 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -1,9 +1,8 @@ import argparse -import json import datetime import torch -from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline +from transformers import AutoTokenizer, AutoModelForCausalLM def get_args(): parser = argparse.ArgumentParser() @@ -30,7 +29,7 @@ def generate_from_text(model, text, tokenizer, max_length=200, greedy=False, top def main(): args = get_args() - print(f"Loading model", flush=True) + print(f"Loading model") tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, padding_side="left") @@ -42,7 +41,6 @@ def main(): torch_dtype=torch.bfloat16, revision="gs{}".format(args.global_step) if args.global_step else None ) - model.eval() print(f"Loaded model in {datetime.datetime.now() - start}") text = '' From 21eca5000d06f8a0b0dc13c703e14e9f79dfb87b Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 27 Jun 2022 10:26:21 +0200 Subject: [PATCH 13/17] Update evaluation/generation/generate.py Co-authored-by: Niklas Muennighoff --- evaluation/generation/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index d6699732..28c2ff7c 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -29,7 +29,7 @@ def generate_from_text(model, text, tokenizer, max_length=200, greedy=False, top def main(): args = get_args() - print(f"Loading model") + print("Loading model") tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, padding_side="left") From 839622c6112e6369f4109a6c99ad8a01f19a3407 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 27 Jun 2022 10:26:36 +0200 Subject: [PATCH 14/17] Update evaluation/generation/generate.py Co-authored-by: Niklas Muennighoff --- evaluation/generation/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 28c2ff7c..38bc4300 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -33,7 +33,7 @@ def main(): tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, padding_side="left") - print("Loaded tokenizer !") + print("Loaded tokenizer!") start = datetime.datetime.now() model = AutoModelForCausalLM.from_pretrained( args.checkpoint, From ac16dd718587f470d390c7447e36b87bcfa65cf6 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 28 Jun 2022 11:07:00 +0200 Subject: [PATCH 15/17] commit suggestions --- evaluation/generation/generate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 38bc4300..215c4c30 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -12,6 +12,7 @@ def get_args(): parser.add_argument("--generate-max-length", type=int, default=50, help="max generation length") parser.add_argument("--greedy", action="store_true") parser.add_argument("--top-k", type=int, default=0) + parser.add_argument("--offload_folder", type=str, help="offload folder for accelerate", default="./offload") return parser.parse_args() @@ -40,6 +41,7 @@ def main(): device_map="auto" if args.parallelize else None, torch_dtype=torch.bfloat16, revision="gs{}".format(args.global_step) if args.global_step else None + offload_floder=args.offload_folder, ) print(f"Loaded model in {datetime.datetime.now() - start}") From 7191802779367a729354f29c7191a3963e78b546 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 28 Jun 2022 11:12:22 +0200 Subject: [PATCH 16/17] Update evaluation/generation/generate.py --- evaluation/generation/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index 215c4c30..fde023bf 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -41,7 +41,7 @@ def main(): device_map="auto" if args.parallelize else None, torch_dtype=torch.bfloat16, revision="gs{}".format(args.global_step) if args.global_step else None - offload_floder=args.offload_folder, + offload_folder=args.offload_folder is args.parallelize else None, ) print(f"Loaded model in {datetime.datetime.now() - start}") From 5759518eead6af9f7baf3bb736d733f769c5e66d Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 8 Jul 2022 14:01:30 +0200 Subject: [PATCH 17/17] fix small accelerate nits --- evaluation/generation/generate.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/evaluation/generation/generate.py b/evaluation/generation/generate.py index fde023bf..ca45b643 100644 --- a/evaluation/generation/generate.py +++ b/evaluation/generation/generate.py @@ -13,6 +13,8 @@ def get_args(): parser.add_argument("--greedy", action="store_true") parser.add_argument("--top-k", type=int, default=0) parser.add_argument("--offload_folder", type=str, help="offload folder for accelerate", default="./offload") + parser.add_argument("--max_memory", type=str, help="max memory per GPU", default="30GB") + parser.add_argument("--max_cpu_memory", type=str, help="max memory on CPU", default="300GB") return parser.parse_args() @@ -40,9 +42,12 @@ def main(): args.checkpoint, device_map="auto" if args.parallelize else None, torch_dtype=torch.bfloat16, - revision="gs{}".format(args.global_step) if args.global_step else None - offload_folder=args.offload_folder is args.parallelize else None, + revision="gs{}".format(args.global_step) if args.global_step else None, + max_memory=args.max_memory if args.parallelize else None, + max_cpu_memory=args.max_cpu_memory if args.parallelize else None, + offload_folder=args.offload_folder if args.parallelize else None, ) + print(f"Loaded model in {datetime.datetime.now() - start}") text = ''