delete hparam.force_beginning_resets

PiperOrigin-RevId: 219241227

Author: blazejosinski

tensor2tensor/models/research/rl.py

@@ -137,8 +137,7 @@ def standard_atari_env_spec(env=None, simulated=False):
       simulated_env=simulated,
       reward_range=env.reward_range,
       observation_space=env.observation_space,
-      action_space=env.action_space,
-      force_beginning_resets=True
+      action_space=env.action_space
   )
   if not simulated:
     env_spec.add_hparam("env", env)
@@ -150,7 +149,6 @@ def standard_atari_env_simulated_spec(real_env, **kwargs):
   env_spec = standard_atari_env_spec(real_env, simulated=True)
   for (name, value) in six.iteritems(kwargs):
     env_spec.add_hparam(name, value)
-  env_spec.force_beginning_resets = False
   return env_spec
 
 

tensor2tensor/rl/collect.py

@@ -141,16 +141,12 @@ def initialization_lambda(sess):
     should_reset_var = tf.Variable(True, trainable=False)
     zeros_tensor = tf.zeros(len(batch_env))
 
-  force_beginning_resets = tf.convert_to_tensor(
-      environment_spec.force_beginning_resets
-  )
-
   def reset_ops_group():
     return tf.group(batch_env.reset(tf.range(len(batch_env))),
                     tf.assign(cumulative_rewards, zeros_tensor))
 
   reset_op = tf.cond(
-      tf.logical_or(should_reset_var.read_value(), force_beginning_resets),
+      tf.logical_or(should_reset_var.read_value(), eval_phase_t),
       reset_ops_group, tf.no_op)
 
   with tf.control_dependencies([reset_op]):
@@ -238,18 +234,6 @@ def stop_condition(i, _, resets):
         parallel_iterations=1,
         back_prop=False)
 
-  # We handle force_beginning_resets differently. We assume that all envs are
-  # reseted at the end of episod (though it happens at the beginning of the
-  # next one
-  scores_num = tf.cond(force_beginning_resets,
-                       lambda: scores_num + len(batch_env), lambda: scores_num)
-
-  with tf.control_dependencies([scores_sum]):
-    scores_sum = tf.cond(
-        force_beginning_resets,
-        lambda: scores_sum + tf.reduce_sum(cumulative_rewards.read_value()),
-        lambda: scores_sum)
-
   mean_score = tf.cond(tf.greater(scores_num, 0),
                        lambda: scores_sum / tf.cast(scores_num, tf.float32),
                        lambda: 0.)