Sync with upstream

Author: vthorsteinsson

README.md

@@ -89,7 +89,7 @@ t2t-datagen \
   --problem=$PROBLEM
 
 # Train
-# *  If you run out of memory, add --hparams='batch_size=2048' or even 1024.
+# *  If you run out of memory, add --hparams='batch_size=1024'.
 t2t-trainer \
   --data_dir=$DATA_DIR \
   --problems=$PROBLEM \
@@ -166,7 +166,7 @@ python -c "from tensor2tensor.models.transformer import Transformer"
   with `Modality` objects, which are specified per-feature in the dataset/task
   specification.
 * Support for multi-GPU machines and synchronous (1 master, many workers) and
-  asynchrounous (independent workers synchronizing through a parameter server)
+  asynchronous (independent workers synchronizing through a parameter server)
   [distributed training](https://github.com/tensorflow/tensor2tensor/tree/master/docs/distributed_training.md).
 * Easily swap amongst datasets and models by command-line flag with the data
   generation script `t2t-datagen` and the training script `t2t-trainer`.

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='tensor2tensor',
-    version='1.1.3',
+    version='1.1.6',
     description='Tensor2Tensor',
     author='Google Inc.',
     author_email='no-reply@google.com',
@@ -20,6 +20,7 @@
     ],
     install_requires=[
         'numpy',
+        'requests',
         'sympy',
         'six',
     ],

tensor2tensor/bin/t2t-datagen

@@ -16,14 +16,15 @@
 
 """Produces the training and dev data for --problem into --data_dir.
 
-generator.py produces sharded and shuffled TFRecord files of tensorflow.Example
-protocol buffers for a variety of datasets registered in this file.
-
-All datasets are registered in _SUPPORTED_PROBLEM_GENERATORS. Each entry maps a
-string name (selectable on the command-line with --problem) to a function that
-takes 2 arguments - input_directory and mode (one of "train" or "dev") - and
-yields for each training example a dictionary mapping string feature names to
-lists of {string, int, float}. The generator will be run once for each mode.
+Produces sharded and shuffled TFRecord files of tensorflow.Example protocol
+buffers for a variety of registered datasets.
+
+All Problems are registered with @registry.register_problem or are in
+_SUPPORTED_PROBLEM_GENERATORS in this file. Each entry maps a string name
+(selectable on the command-line with --problem) to a function that takes 2
+arguments - input_directory and mode (one of "train" or "dev") - and yields for
+each training example a dictionary mapping string feature names to lists of
+{string, int, float}. The generator will be run once for each mode.
 """
 from __future__ import absolute_import
 from __future__ import division
@@ -113,40 +114,9 @@ _SUPPORTED_PROBLEM_GENERATORS = {
         lambda: wiki.generator(FLAGS.tmp_dir, True),
         1000
     ),
-    "image_mnist_tune": (
-        lambda: image.mnist_generator(FLAGS.tmp_dir, True, 55000),
-        lambda: image.mnist_generator(FLAGS.tmp_dir, True, 5000, 55000)),
-    "image_mnist_test": (
-        lambda: image.mnist_generator(FLAGS.tmp_dir, True, 60000),
-        lambda: image.mnist_generator(FLAGS.tmp_dir, False, 10000)),
-    "image_cifar10_tune": (
-        lambda: image.cifar10_generator(FLAGS.tmp_dir, True, 48000),
-        lambda: image.cifar10_generator(FLAGS.tmp_dir, True, 2000, 48000)),
-    "image_cifar10_test": (
-        lambda: image.cifar10_generator(FLAGS.tmp_dir, True, 50000),
-        lambda: image.cifar10_generator(FLAGS.tmp_dir, False, 10000)),
-    "image_mscoco_characters_test": (
-        lambda: image.mscoco_generator(
-            FLAGS.data_dir, FLAGS.tmp_dir, True, 80000),
-        lambda: image.mscoco_generator(
-            FLAGS.data_dir, FLAGS.tmp_dir, False, 40000)),
     "image_celeba_tune": (
         lambda: image.celeba_generator(FLAGS.tmp_dir, 162770),
         lambda: image.celeba_generator(FLAGS.tmp_dir, 19867, 162770)),
-    "image_mscoco_tokens_8k_test": (
-        lambda: image.mscoco_generator(
-            FLAGS.data_dir, FLAGS.tmp_dir, True, 80000,
-            vocab_filename="vocab.endefr.%d" % 2**13, vocab_size=2**13),
-        lambda: image.mscoco_generator(
-            FLAGS.data_dir, FLAGS.tmp_dir, False, 40000,
-            vocab_filename="vocab.endefr.%d" % 2**13, vocab_size=2**13)),
-    "image_mscoco_tokens_32k_test": (
-        lambda: image.mscoco_generator(
-            FLAGS.data_dir, FLAGS.tmp_dir, True, 80000,
-            vocab_filename="vocab.endefr.%d" % 2**15, vocab_size=2**15),
-        lambda: image.mscoco_generator(
-            FLAGS.data_dir, FLAGS.tmp_dir, False, 40000,
-            vocab_filename="vocab.endefr.%d" % 2**15, vocab_size=2**15)),
     "snli_32k": (
         lambda: snli.snli_token_generator(FLAGS.tmp_dir, True, 2**15),
         lambda: snli.snli_token_generator(FLAGS.tmp_dir, False, 2**15),
@@ -255,8 +225,7 @@ def generate_data_for_problem(problem):
     num_shards = FLAGS.num_shards or 10
     tf.logging.info("Generating training data for %s.", problem)
     train_output_files = generator_utils.train_data_filenames(
-        problem + generator_utils.UNSHUFFLED_SUFFIX, FLAGS.data_dir,
-        num_shards)
+        problem + generator_utils.UNSHUFFLED_SUFFIX, FLAGS.data_dir, num_shards)
     generator_utils.generate_files(training_gen(), train_output_files,
                                    FLAGS.max_cases)
     tf.logging.info("Generating development data for %s.", problem)
@@ -276,9 +245,10 @@ def generate_data_for_registered_problem(problem_name):
     raise ValueError("--num_shards should not be set for registered Problem.")
   problem = registry.problem(problem_name)
   task_id = None if FLAGS.task_id < 0 else FLAGS.task_id
-  problem.generate_data(os.path.expanduser(FLAGS.data_dir),
-                        os.path.expanduser(FLAGS.tmp_dir),
-                        task_id=task_id)
+  problem.generate_data(
+      os.path.expanduser(FLAGS.data_dir),
+      os.path.expanduser(FLAGS.tmp_dir),
+      task_id=task_id)
 
 
 if __name__ == "__main__":

tensor2tensor/data_generators/README.md

@@ -28,7 +28,7 @@ for an example.
 
 `Problem`s support data generation, training, and decoding.
 
-Data generation is handles by `Problem.generate_data` which should produce 2
+Data generation is handled by `Problem.generate_data` which should produce 2
 datasets, training and dev, which should be named according to
 `Problem.training_filepaths` and `Problem.dev_filepaths`.
 `Problem.generate_data` should also produce any other files that may be required

tensor2tensor/data_generators/all_problems.py

@@ -22,6 +22,8 @@
 from tensor2tensor.data_generators import algorithmic
 from tensor2tensor.data_generators import algorithmic_math
 from tensor2tensor.data_generators import audio
+from tensor2tensor.data_generators import cipher
+from tensor2tensor.data_generators import desc2code
 from tensor2tensor.data_generators import image
 from tensor2tensor.data_generators import lm1b
 from tensor2tensor.data_generators import ptb
@@ -30,6 +32,7 @@
 from tensor2tensor.data_generators import wmt
 from tensor2tensor.data_generators import wsj_parsing
 
+
 # Problem modules that require optional dependencies
 # pylint: disable=g-import-not-at-top
 try: