Add full system exercise to Travis

PiperOrigin-RevId: 170553043

Author: Ryan Sepassi

.travis.yml

@@ -8,9 +8,21 @@ before_install:
 install:
   - pip install tensorflow
   - pip install .[tests]
+env:
+  - T2T_PROBLEM=algorithmic_reverse_binary40_test
+  - T2T_DATA_DIR=/tmp/t2t-data
+  - T2T_TRAIN_DIR=/tmp/t2t-train
 script:
   - pytest --ignore=tensor2tensor/utils/registry_test.py --ignore=tensor2tensor/utils/trainer_utils_test.py --ignore=tensor2tensor/problems_test.py
   - pytest tensor2tensor/utils/registry_test.py
   - pytest tensor2tensor/utils/trainer_utils_test.py
+  - t2t-datagen 2>&1 | grep translate && echo passed
+  - python -c "from tensor2tensor.models import transformer; print(transformer.Transformer.__name__)"
+  - t2t-trainer --registry_help
+  - mkdir $T2T_DATA_DIR
+  - mkdir $T2T_TRAIN_DIR
+  - t2t-datagen --problem=$T2T_PROBLEM --data_dir=$T2T_DATA_DIR
+  - t2t-trainer --problems=$T2T_PROBLEM --data_dir=$T2T_DATA_DIR --model=transformer --hparams_set=transformer_tiny --train_steps=5 --eval_steps=5 --output_dir=$T2T_TRAIN_DIR
+  - t2t-decoder --problems=$T2T_PROBLEM --data_dir=$T2T_DATA_DIR --model=transformer --hparams_set=transformer_tiny --output_dir=$T2T_TRAIN_DIR
 git:
-  depth: 3
+  depth: 3

tensor2tensor/data_generators/algorithmic.py

@@ -62,13 +62,15 @@ def num_shards(self):
     return 10
 
   def generate_data(self, data_dir, _, task_id=-1):
+
     def generator_eos(nbr_symbols, max_length, nbr_cases):
       """Shift by NUM_RESERVED_IDS and append EOS token."""
       for case in self.generator(nbr_symbols, max_length, nbr_cases):
         new_case = {}
         for feature in case:
-          new_case[feature] = [i + text_encoder.NUM_RESERVED_TOKENS
-                               for i in case[feature]] + [text_encoder.EOS_ID]
+          new_case[feature] = [
+              i + text_encoder.NUM_RESERVED_TOKENS for i in case[feature]
+          ] + [text_encoder.EOS_ID]
         yield new_case
 
     utils.generate_dataset_and_shuffle(
@@ -154,10 +156,7 @@ def generator(self, nbr_symbols, max_length, nbr_cases):
     for _ in xrange(nbr_cases):
       l = np.random.randint(max_length) + 1
       inputs = [np.random.randint(nbr_symbols - shift) for _ in xrange(l)]
-      yield {
-          "inputs": inputs,
-          "targets": [i + shift for i in inputs]
-      }
+      yield {"inputs": inputs, "targets": [i + shift for i in inputs]}
 
   @property
   def dev_length(self):
@@ -191,10 +190,7 @@ def generator(self, nbr_symbols, max_length, nbr_cases):
     for _ in xrange(nbr_cases):
       l = np.random.randint(max_length) + 1
       inputs = [np.random.randint(nbr_symbols) for _ in xrange(l)]
-      yield {
-          "inputs": inputs,
-          "targets": list(reversed(inputs))
-      }
+      yield {"inputs": inputs, "targets": list(reversed(inputs))}
 
 
 @registry.register_problem
@@ -272,10 +268,7 @@ def reverse_generator_nlplike(nbr_symbols,
   for _ in xrange(nbr_cases):
     l = int(abs(np.random.normal(loc=max_length / 2, scale=std_dev)) + 1)
     inputs = zipf_random_sample(distr_map, l)
-    yield {
-        "inputs": inputs,
-        "targets": list(reversed(inputs))
-    }
+    yield {"inputs": inputs, "targets": list(reversed(inputs))}
 
 
 @registry.register_problem
@@ -287,8 +280,8 @@ def num_symbols(self):
     return 8000
 
   def generator(self, nbr_symbols, max_length, nbr_cases):
-    return reverse_generator_nlplike(
-        nbr_symbols, max_length, nbr_cases, 10, 1.300)
+    return reverse_generator_nlplike(nbr_symbols, max_length, nbr_cases, 10,
+                                     1.300)
 
   @property
   def train_length(self):
@@ -308,8 +301,8 @@ def num_symbols(self):
     return 32000
 
   def generator(self, nbr_symbols, max_length, nbr_cases):
-    return reverse_generator_nlplike(
-        nbr_symbols, max_length, nbr_cases, 10, 1.050)
+    return reverse_generator_nlplike(nbr_symbols, max_length, nbr_cases, 10,
+                                     1.050)
 
 
 def lower_endian_to_number(l, base):
@@ -431,3 +424,28 @@ class AlgorithmicMultiplicationDecimal40(AlgorithmicMultiplicationBinary40):
   @property
   def num_symbols(self):
     return 10
+
+
+@registry.register_problem
+class AlgorithmicReverseBinary40Test(AlgorithmicReverseBinary40):
+  """Test Problem with tiny dataset."""
+
+  @property
+  def train_length(self):
+    return 10
+
+  @property
+  def dev_length(self):
+    return 10
+
+  @property
+  def train_size(self):
+    return 1000
+
+  @property
+  def dev_size(self):
+    return 100
+
+  @property
+  def num_shards(self):
+    return 1

tensor2tensor/tpu/__init__.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+# Copyright 2017 The Tensor2Tensor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+