Sync with upstream

Author: vthorsteinsson

README.md

@@ -86,7 +86,6 @@ mkdir -p $DATA_DIR $TMP_DIR $TRAIN_DIR
 t2t-datagen \
   --data_dir=$DATA_DIR \
   --tmp_dir=$TMP_DIR \
-  --num_shards=100 \
   --problem=$PROBLEM
 
 # Train

setup.py

@@ -5,13 +5,14 @@
 
 setup(
     name='tensor2tensor',
-    version='1.1.1',
+    version='1.1.3',
     description='Tensor2Tensor',
     author='Google Inc.',
     author_email='no-reply@google.com',
     url='http://github.com/tensorflow/tensor2tensor',
     license='Apache 2.0',
     packages=find_packages(),
+    package_data={'tensor2tensor.data_generators': ['test_data/*']},
     scripts=[
         'tensor2tensor/bin/t2t-trainer',
         'tensor2tensor/bin/t2t-datagen',
@@ -26,6 +27,8 @@
         'tensorflow': ['tensorflow>=1.2.0rc1'],
         'tensorflow_gpu': ['tensorflow-gpu>=1.2.0rc1'],
     },
+    tests_require=['nose'],
+    test_suite='nose.collector',
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',

tensor2tensor/__init__.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

tensor2tensor/bin/t2t-datagen

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -42,7 +43,6 @@ from tensor2tensor.data_generators import audio
 from tensor2tensor.data_generators import generator_utils
 from tensor2tensor.data_generators import image
 from tensor2tensor.data_generators import lm1b
-from tensor2tensor.data_generators import ptb
 from tensor2tensor.data_generators import snli
 from tensor2tensor.data_generators import wiki
 from tensor2tensor.data_generators import wmt
@@ -62,10 +62,12 @@ flags.DEFINE_string("problem", "",
                     "The name of the problem to generate data for.")
 flags.DEFINE_string("exclude_problems", "",
                     "Comma-separates list of problems to exclude.")
-flags.DEFINE_integer("num_shards", 10, "How many shards to use.")
+flags.DEFINE_integer("num_shards", 0, "How many shards to use. Ignored for "
+                     "registered Problems.")
 flags.DEFINE_integer("max_cases", 0,
                      "Maximum number of cases to generate (unbounded if 0).")
 flags.DEFINE_integer("random_seed", 429459, "Random seed to use.")
+flags.DEFINE_integer("task_id", -1, "For distributed data generation.")
 flags.DEFINE_string("t2t_usr_dir", "",
                     "Path to a Python module that will be imported. The "
                     "__init__.py file should include the necessary imports. "
@@ -103,6 +105,10 @@ _SUPPORTED_PROBLEM_GENERATORS = {
         lambda: lm1b.generator(FLAGS.tmp_dir, True),
         lambda: lm1b.generator(FLAGS.tmp_dir, False)
     ),
+    "lm1b_characters": (
+        lambda: lm1b.generator(FLAGS.tmp_dir, True, characters=True),
+        lambda: lm1b.generator(FLAGS.tmp_dir, False, characters=True)
+    ),
     "wiki_32k": (
         lambda: wiki.generator(FLAGS.tmp_dir, True),
         1000
@@ -164,12 +170,6 @@ _SUPPORTED_PROBLEM_GENERATORS = {
         lambda: audio.timit_generator(
             FLAGS.data_dir, FLAGS.tmp_dir, False, 626,
             vocab_filename="vocab.endefr.%d" % 2**15, vocab_size=2**15)),
-    "lmptb_10k": (
-        lambda: ptb.train_generator(
-            FLAGS.tmp_dir,
-            FLAGS.data_dir,
-            False),
-        ptb.valid_generator),
 }
 
 # pylint: enable=g-long-lambda
@@ -241,7 +241,7 @@ def generate_data_for_problem(problem):
   if isinstance(dev_gen, int):
     # The dev set and test sets are generated as extra shards using the
     # training generator.  The integer specifies the number of training
-    # shards.  FLAGS.num_shards is ignored.
+    # shards. FLAGS.num_shards is ignored.
     num_training_shards = dev_gen
     tf.logging.info("Generating data for %s.", problem)
     all_output_files = generator_utils.combined_data_filenames(
@@ -252,10 +252,11 @@ def generate_data_for_problem(problem):
   else:
     # usual case - train data and dev data are generated using separate
     # generators.
+    num_shards = FLAGS.num_shards or 10
     tf.logging.info("Generating training data for %s.", problem)
     train_output_files = generator_utils.train_data_filenames(
         problem + generator_utils.UNSHUFFLED_SUFFIX, FLAGS.data_dir,
-        FLAGS.num_shards)
+        num_shards)
     generator_utils.generate_files(training_gen(), train_output_files,
                                    FLAGS.max_cases)
     tf.logging.info("Generating development data for %s.", problem)
@@ -270,10 +271,14 @@ def generate_data_for_problem(problem):
 
 
 def generate_data_for_registered_problem(problem_name):
+  tf.logging.info("Generating training data for %s.", problem_name)
+  if FLAGS.num_shards:
+    raise ValueError("--num_shards should not be set for registered Problem.")
   problem = registry.problem(problem_name)
+  task_id = None if FLAGS.task_id < 0 else FLAGS.task_id
   problem.generate_data(os.path.expanduser(FLAGS.data_dir),
                         os.path.expanduser(FLAGS.tmp_dir),
-                        FLAGS.num_shards)
+                        task_id=task_id)
 
 
 if __name__ == "__main__":

tensor2tensor/bin/t2t-make-tf-configs

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

tensor2tensor/bin/t2t-trainer

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

tensor2tensor/data_generators/__init__.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

tensor2tensor/data_generators/algorithmic.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -65,10 +66,7 @@ def dev_size(self):
   def num_shards(self):
     return 10
 
-  def generate_data(self, data_dir, _, num_shards=None):
-    if num_shards is None:
-      num_shards = self.num_shards
-
+  def generate_data(self, data_dir, _, task_id=-1):
     def generator_eos(generator):
       """Shift by NUM_RESERVED_IDS and append EOS token."""
       for case in generator:
@@ -86,7 +84,7 @@ def generator_eos(generator):
 
     utils.generate_dataset_and_shuffle(
         train_generator_eos(),
-        self.training_filepaths(data_dir, num_shards, shuffled=True),
+        self.training_filepaths(data_dir, self.num_shards, shuffled=True),
         dev_generator_eos(),
         self.dev_filepaths(data_dir, 1, shuffled=True),
         shuffle=False)
@@ -253,7 +251,7 @@ def zipf_distribution(nbr_symbols, alpha):
 
 
 def zipf_random_sample(distr_map, sample_len):
-  """Helper function: Generate a random Zipf sample of given lenght.
+  """Helper function: Generate a random Zipf sample of given length.
 
   Args:
     distr_map: list of float, Zipf's distribution over nbr_symbols.
@@ -286,7 +284,7 @@ def reverse_generator_nlplike(nbr_symbols,
     max_length: integer, maximum length of sequences to generate.
     nbr_cases: the number of cases to generate.
     scale_std_dev: float, Normal distribution's standard deviation scale factor
-      used to draw the lenght of sequence. Default = 1% of the max_length.
+      used to draw the length of sequence. Default = 1% of the max_length.
     alpha: float, Zipf's Law Distribution parameter. Default = 1.5.
       Usually for modelling natural text distribution is in
       the range [1.1-1.6].

tensor2tensor/data_generators/algorithmic_math.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

tensor2tensor/data_generators/algorithmic_math_test.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");