def tf_discounted_cumulative_reward(self, terminal, reward, discount=None, final_reward=0.0, horizon=0):
        """
        Creates and returns the TensorFlow operations for calculating the sequence of discounted cumulative rewards
        for a given sequence of single rewards.

        Example:
        single rewards = 2.0 1.0 0.0 0.5 1.0 -1.0
        terminal = False, False, False, False True False
        gamma = 0.95
        final_reward = 100.0 (only matters for last episode (r=-1.0) as this episode has no terminal signal)
        horizon=3
        output = 2.95 1.45 1.38 1.45 1.0 94.0

        Args:
            terminal: Tensor (bool) holding the is-terminal sequence. This sequence may contain more than one
                True value. If its very last element is False (not terminating), the given `final_reward` value
                is assumed to follow the last value in the single rewards sequence (see below).
            reward: Tensor (float) holding the sequence of single rewards. If the last element of `terminal` is False,
                an assumed last reward of the value of `final_reward` will be used.
            discount (float): The discount factor (gamma). By default, take the Model's discount factor.
            final_reward (float): Reward value to use if last episode in sequence does not terminate (terminal sequence
                ends with False). This value will be ignored if horizon == 1 or discount == 0.0.
            horizon (int): The length of the horizon (e.g. for n-step cumulative rewards in continuous tasks
                without terminal signals). Use 0 (default) for an infinite horizon. Note that horizon=1 leads to the
                exact same results as a discount factor of 0.0.

        Returns:
            Discounted cumulative reward tensor with the same shape as `reward`.
        """

        # By default -> take Model's gamma value
        if discount is None:
            discount = self.discount

        # Accumulates discounted (n-step) reward (start new if terminal)
        def cumulate(cumulative, reward_terminal_horizon_subtract):
            rew, is_terminal, is_over_horizon, sub = reward_terminal_horizon_subtract
            return tf.where(
                # If terminal, start new cumulation.
                condition=is_terminal,
                x=rew,
                y=tf.where(
                    # If we are above the horizon length (H) -> subtract discounted value from H steps back.
                    condition=is_over_horizon,
                    x=(rew + cumulative * discount - sub),
                    y=(rew + cumulative * discount)
                )
            )

        # Accumulates length of episodes (starts new if terminal)
        def len_(cumulative, term):
            return tf.where(
                condition=term,
                # Start counting from 1 after is-terminal signal
                x=tf.ones(shape=(), dtype=tf.int32),
                # Otherwise, increase length by 1
                y=cumulative + 1
            )

        # Reverse, since reward cumulation is calculated right-to-left, but tf.scan only works left-to-right.
        reward = tf.reverse(tensor=reward, axis=(0,))
        # e.g. -1.0 1.0 0.5 0.0 1.0 2.0
        terminal = tf.reverse(tensor=terminal, axis=(0,))
        # e.g. F T F F F F

        # Store the steps until end of the episode(s) determined by the input terminal signals (True starts new count).
        lengths = tf.scan(fn=len_, elems=terminal, initializer=0)
        # e.g. 1 1 2 3 4 5
        off_horizon = tf.greater(lengths, tf.fill(dims=tf.shape(lengths), value=horizon))
        # e.g. F F F F T T

        # Calculate the horizon-subtraction value for each step.
        if horizon > 0:
            horizon_subtractions = tf.map_fn(lambda x: (discount ** horizon) * x, reward, dtype=tf.float32)
            # Shift right by size of horizon (fill rest with 0.0).
            horizon_subtractions = tf.concat([np.zeros(shape=(horizon,)), horizon_subtractions], axis=0)
            horizon_subtractions = tf.slice(horizon_subtractions, begin=(0,), size=tf.shape(reward))
            # e.g. 0.0, 0.0, 0.0, -1.0*g^3, 1.0*g^3, 0.5*g^3
        # all 0.0 if infinite horizon (special case: horizon=0)
        else:
            horizon_subtractions = tf.zeros(shape=tf.shape(reward))

        # Now do the scan, each time summing up the previous step (discounted by gamma) and
        # subtracting the respective `horizon_subtraction`.
        reward = tf.scan(
            fn=cumulate,
            elems=(reward, terminal, off_horizon, horizon_subtractions),
            initializer=final_reward if horizon != 1 else 0.0
        )
        # Re-reverse again to match input sequences.
        return tf.reverse(tensor=reward, axis=(0,))

def tf_discounted_cumulative_reward ( self , terminal , reward , discount = None , final_reward = 0.0 , horizon = 0 ) : if discount is None : discount = self . discount def cumulate ( cumulative , reward_terminal_horizon_subtract ) : rew , is_terminal , is_over_horizon , sub = reward_terminal_horizon_subtract return tf . where ( condition = is_terminal , x = rew , y = tf . where ( condition = is_over_horizon , x = ( rew + cumulative * discount - sub ) , y = ( rew + cumulative * discount ) ) ) def len_ ( cumulative , term ) : return tf . where ( condition = term , x = tf . ones ( shape = ( ) , dtype = tf . int32 ) , y = cumulative + 1 ) reward = tf . reverse ( tensor = reward , axis = ( 0 , ) ) terminal = tf . reverse ( tensor = terminal , axis = ( 0 , ) ) lengths = tf . scan ( fn = len_ , elems = terminal , initializer = 0 ) off_horizon = tf . greater ( lengths , tf . fill ( dims = tf . shape ( lengths ) , value = horizon ) ) if horizon > 0 : horizon_subtractions = tf . map_fn ( lambda x : ( discount ** horizon ) * x , reward , dtype = tf . float32 ) horizon_subtractions = tf . concat ( [ np . zeros ( shape = ( horizon , ) ) , horizon_subtractions ] , axis = 0 ) horizon_subtractions = tf . slice ( horizon_subtractions , begin = ( 0 , ) , size = tf . shape ( reward ) ) else : horizon_subtractions = tf . zeros ( shape = tf . shape ( reward ) ) reward = tf . scan ( fn = cumulate , elems = ( reward , terminal , off_horizon , horizon_subtractions ) , initializer = final_reward if horizon != 1 else 0.0 ) return tf . reverse ( tensor = reward , axis = ( 0 , ) )

Creates and returns the TensorFlow operations for calculating the sequence of discounted cumulative rewards for a given sequence of single rewards .


def tf_loss_per_instance(self, states, internals, actions, terminal, reward,
                             next_states, next_internals, update, reference=None):
        """
        Creates the TensorFlow operations for calculating the loss per batch instance.

        Args:
            states: Dict of state tensors.
            internals: Dict of prior internal state tensors.
            actions: Dict of action tensors.
            terminal: Terminal boolean tensor.
            reward: Reward tensor.
            next_states: Dict of successor state tensors.
            next_internals: List of posterior internal state tensors.
            update: Boolean tensor indicating whether this call happens during an update.
            reference: Optional reference tensor(s), in case of a comparative loss.

        Returns:
            Loss per instance tensor.
        """
        raise NotImplementedError

def tf_loss_per_instance ( self , states , internals , actions , terminal , reward , next_states , next_internals , update , reference = None ) : raise NotImplementedError

Creates the TensorFlow operations for calculating the loss per batch instance .


def tf_loss(self, states, internals, actions, terminal, reward, next_states, next_internals, update, reference=None):
        """
        Creates the TensorFlow operations for calculating the full loss of a batch.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            actions: Dict of action tensors.
            terminal: Terminal boolean tensor.
            reward: Reward tensor.
            next_states: Dict of successor state tensors.
            next_internals: List of posterior internal state tensors.
            update: Boolean tensor indicating whether this call happens during an update.
            reference: Optional reference tensor(s), in case of a comparative loss.

        Returns:
            Loss tensor.
        """
        # Mean loss per instance
        loss_per_instance = self.fn_loss_per_instance(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward,
            next_states=next_states,
            next_internals=next_internals,
            update=update,
            reference=reference
        )

        # Returns no-op.
        updated = self.memory.update_batch(loss_per_instance=loss_per_instance)
        with tf.control_dependencies(control_inputs=(updated,)):
            loss = tf.reduce_mean(input_tensor=loss_per_instance, axis=0)

            # Loss without regularization summary.
            if 'losses' in self.summary_labels:
                tf.contrib.summary.scalar(name='loss-without-regularization', tensor=loss)

            # Regularization losses.
            losses = self.fn_regularization_losses(states=states, internals=internals, update=update)
            if len(losses) > 0:
                loss += tf.add_n(inputs=[losses[name] for name in sorted(losses)])
                if 'regularization' in self.summary_labels:
                    for name in sorted(losses):
                        tf.contrib.summary.scalar(name=('regularization/' + name), tensor=losses[name])

            # Total loss summary.
            if 'losses' in self.summary_labels or 'total-loss' in self.summary_labels:
                tf.contrib.summary.scalar(name='total-loss', tensor=loss)

            return loss

def tf_loss ( self , states , internals , actions , terminal , reward , next_states , next_internals , update , reference = None ) : loss_per_instance = self . fn_loss_per_instance ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward , next_states = next_states , next_internals = next_internals , update = update , reference = reference ) updated = self . memory . update_batch ( loss_per_instance = loss_per_instance ) with tf . control_dependencies ( control_inputs = ( updated , ) ) : loss = tf . reduce_mean ( input_tensor = loss_per_instance , axis = 0 ) if 'losses' in self . summary_labels : tf . contrib . summary . scalar ( name = 'loss-without-regularization' , tensor = loss ) losses = self . fn_regularization_losses ( states = states , internals = internals , update = update ) if len ( losses ) > 0 : loss += tf . add_n ( inputs = [ losses [ name ] for name in sorted ( losses ) ] ) if 'regularization' in self . summary_labels : for name in sorted ( losses ) : tf . contrib . summary . scalar ( name = ( 'regularization/' + name ) , tensor = losses [ name ] ) if 'losses' in self . summary_labels or 'total-loss' in self . summary_labels : tf . contrib . summary . scalar ( name = 'total-loss' , tensor = loss ) return loss

Creates the TensorFlow operations for calculating the full loss of a batch .


def optimizer_arguments(self, states, internals, actions, terminal, reward, next_states, next_internals):
        """
        Returns the optimizer arguments including the time, the list of variables to optimize,
        and various functions which the optimizer might require to perform an update step.

        Args:
            states (dict): Dict of state tensors.
            internals (dict): Dict of prior internal state tensors.
            actions (dict): Dict of action tensors.
            terminal: 1D boolean is-terminal tensor.
            reward: 1D (float) rewards tensor.
            next_states (dict): Dict of successor state tensors.
            next_internals (dict): Dict of posterior internal state tensors.

        Returns:
            Optimizer arguments as dict to be used as **kwargs to the optimizer.
        """
        arguments = dict(
            time=self.global_timestep,
            variables=self.get_variables(),
            arguments=dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward,
                next_states=next_states,
                next_internals=next_internals,
                update=tf.constant(value=True)
            ),
            fn_reference=self.fn_reference,
            fn_loss=self.fn_loss
        )
        if self.global_model is not None:
            arguments['global_variables'] = self.global_model.get_variables()
        return arguments

def optimizer_arguments ( self , states , internals , actions , terminal , reward , next_states , next_internals ) : arguments = dict ( time = self . global_timestep , variables = self . get_variables ( ) , arguments = dict ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward , next_states = next_states , next_internals = next_internals , update = tf . constant ( value = True ) ) , fn_reference = self . fn_reference , fn_loss = self . fn_loss ) if self . global_model is not None : arguments [ 'global_variables' ] = self . global_model . get_variables ( ) return arguments

Returns the optimizer arguments including the time the list of variables to optimize and various functions which the optimizer might require to perform an update step .


def tf_optimization(self, states, internals, actions, terminal, reward, next_states=None, next_internals=None):
        """
        Creates the TensorFlow operations for performing an optimization update step based
        on the given input states and actions batch.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            actions: Dict of action tensors.
            terminal: Terminal boolean tensor.
            reward: Reward tensor.
            next_states: Dict of successor state tensors.
            next_internals: List of posterior internal state tensors.

        Returns:
            The optimization operation.
        """
        arguments = self.optimizer_arguments(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward,
            next_states=next_states,
            next_internals=next_internals
        )
        return self.optimizer.minimize(**arguments)

def tf_optimization ( self , states , internals , actions , terminal , reward , next_states = None , next_internals = None ) : arguments = self . optimizer_arguments ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward , next_states = next_states , next_internals = next_internals ) return self . optimizer . minimize ( ** arguments )

Creates the TensorFlow operations for performing an optimization update step based on the given input states and actions batch .


def tf_import_experience(self, states, internals, actions, terminal, reward):
        """
        Imports experiences into the TensorFlow memory structure. Can be used to import
        off-policy data.

        :param states: Dict of state values to import with keys as state names and values as values to set.
        :param internals: Internal values to set, can be fetched from agent via agent.current_internals
            if no values available.
        :param actions: Dict of action values to import with keys as action names and values as values to set.
        :param terminal: Terminal value(s)
        :param reward: Reward value(s)
        """
        return self.memory.store(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )

def tf_import_experience ( self , states , internals , actions , terminal , reward ) : return self . memory . store ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward )

Imports experiences into the TensorFlow memory structure . Can be used to import off - policy data .


def import_experience(self, states, internals, actions, terminal, reward):
        """
        Stores experiences.
        """
        fetches = self.import_experience_output

        feed_dict = self.get_feed_dict(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )

        self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)

def import_experience ( self , states , internals , actions , terminal , reward ) : fetches = self . import_experience_output feed_dict = self . get_feed_dict ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward ) self . monitored_session . run ( fetches = fetches , feed_dict = feed_dict )


def from_spec(spec, kwargs=None):
        """
        Creates a distribution from a specification dict.
        """
        distribution = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.distributions.distributions,
            kwargs=kwargs
        )
        assert isinstance(distribution, Distribution)
        return distribution

def from_spec ( spec , kwargs = None ) : distribution = util . get_object ( obj = spec , predefined_objects = tensorforce . core . distributions . distributions , kwargs = kwargs ) assert isinstance ( distribution , Distribution ) return distribution

Creates a distribution from a specification dict .


def atomic_observe(self, states, actions, internals, reward, terminal):
        """
        Utility method for unbuffered observing where each tuple is inserted into TensorFlow via
        a single session call, thus avoiding race conditions in multi-threaded mode.

        Observe full experience  tuplefrom the environment to learn from. Optionally pre-processes rewards
        Child classes should call super to get the processed reward
        EX: terminal, reward = super()...

        Args:
            states (any): One state (usually a value tuple) or dict of states if multiple states are expected.
            actions (any): One action (usually a value tuple) or dict of states if multiple actions are expected.
            internals (any): Internal list.
            terminal (bool): boolean indicating if the episode terminated after the observation.
            reward (float): scalar reward that resulted from executing the action.
        """
        # TODO probably unnecessary here.
        self.current_terminal = terminal
        self.current_reward = reward
        # print('action = {}'.format(actions))
        if self.unique_state:
            states = dict(state=states)
        if self.unique_action:
            actions = dict(action=actions)

        self.episode = self.model.atomic_observe(
            states=states,
            actions=actions,
            internals=internals,
            terminal=self.current_terminal,
            reward=self.current_reward
        )

def atomic_observe ( self , states , actions , internals , reward , terminal ) : self . current_terminal = terminal self . current_reward = reward if self . unique_state : states = dict ( state = states ) if self . unique_action : actions = dict ( action = actions ) self . episode = self . model . atomic_observe ( states = states , actions = actions , internals = internals , terminal = self . current_terminal , reward = self . current_reward )

Utility method for unbuffered observing where each tuple is inserted into TensorFlow via a single session call thus avoiding race conditions in multi - threaded mode .


def from_spec(spec, kwargs):
        """
        Creates an agent from a specification dict.
        """
        agent = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.agents.agents,
            kwargs=kwargs
        )
        assert isinstance(agent, Agent)
        return agent

def from_spec ( spec , kwargs ) : agent = util . get_object ( obj = spec , predefined_objects = tensorforce . agents . agents , kwargs = kwargs ) assert isinstance ( agent , Agent ) return agent

Creates an agent from a specification dict .


def get_named_tensor(self, name):
        """
        Returns a named tensor if available.

        Returns:
            valid: True if named tensor found, False otherwise
            tensor: If valid, will be a tensor, otherwise None
        """
        if name in self.named_tensors:
            return True, self.named_tensors[name]
        else:
            return False, None

def get_named_tensor ( self , name ) : if name in self . named_tensors : return True , self . named_tensors [ name ] else : return False , None


def from_spec(spec, kwargs=None):
        """
        Creates a network from a specification dict.
        """
        network = util.get_object(
            obj=spec,
            default_object=LayeredNetwork,
            kwargs=kwargs
        )
        assert isinstance(network, Network)
        return network

def from_spec ( spec , kwargs = None ) : network = util . get_object ( obj = spec , default_object = LayeredNetwork , kwargs = kwargs ) assert isinstance ( network , Network ) return network

Creates a network from a specification dict .


def put(self, item, priority=None):
        """
        Stores a transition in replay memory.

        If the memory is full, the oldest entry is replaced.
        """
        if not self._isfull():
            self._memory.append(None)
        position = self._next_position_then_increment()
        old_priority = 0 if self._memory[position] is None \
            else (self._memory[position].priority or 0)
        row = _SumRow(item, priority)
        self._memory[position] = row
        self._update_internal_nodes(
            position, (row.priority or 0) - old_priority)

def put ( self , item , priority = None ) : if not self . _isfull ( ) : self . _memory . append ( None ) position = self . _next_position_then_increment ( ) old_priority = 0 if self . _memory [ position ] is None else ( self . _memory [ position ] . priority or 0 ) row = _SumRow ( item , priority ) self . _memory [ position ] = row self . _update_internal_nodes ( position , ( row . priority or 0 ) - old_priority )


def move(self, external_index, new_priority):
        """
        Change the priority of a leaf node
        """
        index = external_index + (self._capacity - 1)
        return self._move(index, new_priority)

def move ( self , external_index , new_priority ) : index = external_index + ( self . _capacity - 1 ) return self . _move ( index , new_priority )


def _move(self, index, new_priority):
        """
        Change the priority of a leaf node.
        """
        item, old_priority = self._memory[index]
        old_priority = old_priority or 0
        self._memory[index] = _SumRow(item, new_priority)
        self._update_internal_nodes(index, new_priority - old_priority)

def _move ( self , index , new_priority ) : item , old_priority = self . _memory [ index ] old_priority = old_priority or 0 self . _memory [ index ] = _SumRow ( item , new_priority ) self . _update_internal_nodes ( index , new_priority - old_priority )


def _next_position_then_increment(self):
        """
        Similar to position++.
        """
        start = self._capacity - 1
        position = start + self._position
        self._position = (self._position + 1) % self._capacity
        return position

def _next_position_then_increment ( self ) : start = self . _capacity - 1 position = start + self . _position self . _position = ( self . _position + 1 ) % self . _capacity return position


def _sample_with_priority(self, p):
        """
        Sample random element with priority greater than p.
        """
        parent = 0
        while True:
            left = 2 * parent + 1
            if left >= len(self._memory):
                # parent points to a leaf node already.
                return parent

            left_p = self._memory[left] if left < self._capacity - 1 \
                else (self._memory[left].priority or 0)
            if p <= left_p:
                parent = left
            else:
                if left + 1 >= len(self._memory):
                    raise RuntimeError('Right child is expected to exist.')
                p -= left_p
                parent = left + 1

def
_sample_with_priority
(
self
,
p
)
:
parent
=
0
while
True
:
left
=
2
*
parent
+
1
if
left
>=
len
(
self
.
_memory
)
:
return
parent
left_p
=
self
.
_memory
[
left
]
if
left
<
self
.
_capacity
1
else
(
self
.
_memory
[
left
]
.
priority
or
0
)
if
p
<=
left_p
:
parent
=
left
else
:
if
left
+
1
>=
len
(
self
.
_memory
)
:
raise
RuntimeError
(
'Right child is expected to exist.'
)
p
-=
left_p
parent
=
left
+
1

Sample random element with priority greater than p .


def sample_minibatch(self, batch_size):
        """
        Sample minibatch of size batch_size.
        """
        pool_size = len(self)
        if pool_size == 0:
            return []

        delta_p = self._memory[0] / batch_size
        chosen_idx = []
        # if all priorities sum to ~0  choose randomly otherwise random sample
        if abs(self._memory[0]) < util.epsilon:
            chosen_idx = np.random.randint(self._capacity - 1, self._capacity - 1 + len(self), size=batch_size).tolist()
        else:
            for i in xrange(batch_size):
                lower = max(i * delta_p, 0)
                upper = min((i + 1) * delta_p, self._memory[0])
                p = random.uniform(lower, upper)
                chosen_idx.append(self._sample_with_priority(p))
        return [(i, self._memory[i]) for i in chosen_idx]

def sample_minibatch ( self , batch_size ) : pool_size = len ( self ) if pool_size == 0 : return [ ] delta_p = self . _memory [ 0 ] / batch_size chosen_idx = [ ] if abs ( self . _memory [ 0 ] ) < util . epsilon : chosen_idx = np . random . randint ( self . _capacity - 1 , self . _capacity - 1 + len ( self ) , size = batch_size ) . tolist ( ) else : for i in xrange ( batch_size ) : lower = max ( i * delta_p , 0 ) upper = min ( ( i + 1 ) * delta_p , self . _memory [ 0 ] ) p = random . uniform ( lower , upper ) chosen_idx . append ( self . _sample_with_priority ( p ) ) return [ ( i , self . _memory [ i ] ) for i in chosen_idx ]


def update_batch(self, loss_per_instance):
        """
        Computes priorities according to loss.

        Args:
            loss_per_instance:

        """
        if self.batch_indices is None:
            raise TensorForceError("Need to call get_batch before each update_batch call.")
        # if len(loss_per_instance) != len(self.batch_indices):
        #     raise TensorForceError("For all instances a loss value has to be provided.")

        for index, loss in zip(self.batch_indices, loss_per_instance):
            # Sampling priority is proportional to the largest absolute temporal difference error.
            new_priority = (np.abs(loss) + self.prioritization_constant) ** self.prioritization_weight
            self.observations._move(index, new_priority)
            self.none_priority_index += 1


def import_experience(self, experiences):
        """
        Imports experiences.

        Args:
            experiences:
        """
        if isinstance(experiences, dict):
            if self.unique_state:
                experiences['states'] = dict(state=experiences['states'])
            if self.unique_action:
                experiences['actions'] = dict(action=experiences['actions'])

            self.model.import_experience(**experiences)

        else:
            if self.unique_state:
                states = dict(state=list())
            else:
                states = {name: list() for name in experiences[0]['states']}
            internals = [list() for _ in experiences[0]['internals']]
            if self.unique_action:
                actions = dict(action=list())
            else:
                actions = {name: list() for name in experiences[0]['actions']}
            terminal = list()
            reward = list()

            for experience in experiences:
                if self.unique_state:
                    states['state'].append(experience['states'])
                else:
                    for name in sorted(states):
                        states[name].append(experience['states'][name])
                for n, internal in enumerate(internals):
                    internal.append(experience['internals'][n])
                if self.unique_action:
                    actions['action'].append(experience['actions'])
                else:
                    for name in sorted(actions):
                        actions[name].append(experience['actions'][name])
                terminal.append(experience['terminal'])
                reward.append(experience['reward'])

            self.model.import_experience(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )

def import_experience ( self , experiences ) : if isinstance ( experiences , dict ) : if self . unique_state : experiences [ 'states' ] = dict ( state = experiences [ 'states' ] ) if self . unique_action : experiences [ 'actions' ] = dict ( action = experiences [ 'actions' ] ) self . model . import_experience ( ** experiences ) else : if self . unique_state : states = dict ( state = list ( ) ) else : states = { name : list ( ) for name in experiences [ 0 ] [ 'states' ] } internals = [ list ( ) for _ in experiences [ 0 ] [ 'internals' ] ] if self . unique_action : actions = dict ( action = list ( ) ) else : actions = { name : list ( ) for name in experiences [ 0 ] [ 'actions' ] } terminal = list ( ) reward = list ( ) for experience in experiences : if self . unique_state : states [ 'state' ] . append ( experience [ 'states' ] ) else : for name in sorted ( states ) : states [ name ] . append ( experience [ 'states' ] [ name ] ) for n , internal in enumerate ( internals ) : internal . append ( experience [ 'internals' ] [ n ] ) if self . unique_action : actions [ 'action' ] . append ( experience [ 'actions' ] ) else : for name in sorted ( actions ) : actions [ name ] . append ( experience [ 'actions' ] [ name ] ) terminal . append ( experience [ 'terminal' ] ) reward . append ( experience [ 'reward' ] ) self . model . import_experience ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward )

Imports experiences.

        Args:
            experiences:


def disconnect(self):
        """
        Ends our server tcp connection.
        """
        # If we are not connected, return error.
        if not self.socket:
            logging.warning("No active socket to close!")
            return
        # Close our socket.
        self.socket.close()
        self.socket = None

def
disconnect
(
self
)
:
if
not
self
.
socket
:
logging
.
warning
(
"No active socket to close!"
)
return
self
.
socket
.
close
(
)
self
.
socket
=


def recv(self, socket_, encoding=None):
        """
        Receives a message as msgpack-numpy encoded byte-string from the given socket object.
        Blocks until something was received.

        Args:
            socket_: The python socket object to use.
            encoding (str): The encoding to use for unpacking messages from the socket.
        Returns: The decoded (as dict) message received.
        """
        unpacker = msgpack.Unpacker(encoding=encoding)

        # Wait for an immediate response.
        response = socket_.recv(8)  # get the length of the message
        if response == b"":
            raise TensorForceError("No data received by socket.recv in call to method `recv` " +
                                   "(listener possibly closed)!")
        orig_len = int(response)
        received_len = 0
        while True:
            data = socket_.recv(min(orig_len - received_len, self.max_msg_len))
            # There must be a response.
            if not data:
                raise TensorForceError("No data of len {} received by socket.recv in call to method `recv`!".
                                       format(orig_len - received_len))
            data_len = len(data)
            received_len += data_len
            unpacker.feed(data)

            if received_len == orig_len:
                break

        # Get the data.
        for message in unpacker:
            sts = message.get("status", message.get(b"status"))
            if sts:
                if sts == "ok" or sts == b"ok":
                    return message
                else:
                    raise TensorForceError("RemoteEnvironment server error: {}".
                                           format(message.get("message", "not specified")))
            else:
                raise TensorForceError("Message without field 'status' received!")
        raise TensorForceError("No message encoded in data stream (data stream had len={})".
                               format(orig_len))

Receives a message as msgpack - numpy encoded byte - string from the given socket object . Blocks until something was received .


def is_action_available(self, action):
        """Determines whether action is available.
        That is, executing it would change the state.
        """

        temp_state = np.rot90(self._state, action)
        return self._is_action_available_left(temp_state)

def is_action_available ( self , action ) : temp_state = np . rot90 ( self . _state , action ) return self . _is_action_available_left ( temp_state )

Determines whether action is available . That is executing it would change the state .


def _is_action_available_left(self, state):
        """Determines whether action 'Left' is available."""

        # True if any field is 0 (empty) on the left of a tile or two tiles can
        # be merged.
        for row in range(4):
            has_empty = False
            for col in range(4):
                has_empty |= state[row, col] == 0
                if state[row, col] != 0 and has_empty:
                    return True
                if (state[row, col] != 0 and col > 0 and
                        state[row, col] == state[row, col - 1]):
                    return True

        return False

def _is_action_available_left ( self , state ) : for row in range ( 4 ) : has_empty = False for col in range ( 4 ) : has_empty |= state [ row , col ] == 0 if state [ row , col ] != 0 and has_empty : return True if ( state [ row , col ] != 0 and col > 0 and state [ row , col ] == state [ row , col - 1 ] ) : return True return False

Determines whether action Left is available .


def do_action(self, action):
        """Execute action, add a new tile, update the score & return the reward."""

        temp_state = np.rot90(self._state, action)
        reward = self._do_action_left(temp_state)
        self._state = np.rot90(temp_state, -action)
        self._score += reward

        self.add_random_tile()

        return reward

def do_action ( self , action ) : temp_state = np . rot90 ( self . _state , action ) reward = self . _do_action_left ( temp_state ) self . _state = np . rot90 ( temp_state , - action ) self . _score += reward self . add_random_tile ( ) return reward

Execute action add a new tile update the score & return the reward .


def _do_action_left(self, state):
        """Executes action 'Left'."""

        reward = 0

        for row in range(4):
            # Always the rightmost tile in the current row that was already moved
            merge_candidate = -1
            merged = np.zeros((4,), dtype=np.bool)

            for col in range(4):
                if state[row, col] == 0:
                    continue

                if (merge_candidate != -1 and
                        not merged[merge_candidate] and
                        state[row, merge_candidate] == state[row, col]):
                    # Merge tile with merge_candidate
                    state[row, col] = 0
                    merged[merge_candidate] = True
                    state[row, merge_candidate] += 1
                    reward += 2 ** state[row, merge_candidate]

                else:
                    # Move tile to the left
                    merge_candidate += 1
                    if col != merge_candidate:
                        state[row, merge_candidate] = state[row, col]
                        state[row, col] = 0

        return reward

def _do_action_left ( self , state ) : reward = 0 for row in range ( 4 ) : merge_candidate = - 1 merged = np . zeros ( ( 4 , ) , dtype = np . bool ) for col in range ( 4 ) : if state [ row , col ] == 0 : continue if ( merge_candidate != - 1 and not merged [ merge_candidate ] and state [ row , merge_candidate ] == state [ row , col ] ) : state [ row , col ] = 0 merged [ merge_candidate ] = True state [ row , merge_candidate ] += 1 reward += 2 ** state [ row , merge_candidate ] else : merge_candidate += 1 if col != merge_candidate : state [ row , merge_candidate ] = state [ row , col ] state [ row , col ] = 0 return reward


def add_random_tile(self):
        """Adds a random tile to the grid. Assumes that it has empty fields."""

        x_pos, y_pos = np.where(self._state == 0)
        assert len(x_pos) != 0
        empty_index = np.random.choice(len(x_pos))
        value = np.random.choice([1, 2], p=[0.9, 0.1])

        self._state[x_pos[empty_index], y_pos[empty_index]] = value

def add_random_tile ( self ) : x_pos , y_pos = np . where ( self . _state == 0 ) assert len ( x_pos ) != 0 empty_index = np . random . choice ( len ( x_pos ) ) value = np . random . choice ( [ 1 , 2 ] , p = [ 0.9 , 0.1 ] ) self . _state [ x_pos [ empty_index ] , y_pos [ empty_index ] ] = value

Adds a random tile to the grid . Assumes that it has empty fields .


def print_state(self):
        """Prints the current state."""

        def tile_string(value):
            """Concert value to string."""
            if value > 0:
                return '% 5d' % (2 ** value,)
            return "     "

        separator_line = '-' * 25
        print(separator_line)
        for row in range(4):
            print("|" + "|".join([tile_string(v) for v in self._state[row, :]]) + "|")
            print(separator_line)

def
print_state
(
self
)
:
def
tile_string
(
value
)
:
if
value
>
0
:
return
'% 5d'
%
(
2
**
value
,
)
return
"     "
separator_line
=
'-'
*
25
print
(
separator_line
)
for
row
in
range
(
4
)
:
print
(
"|"
+
"|"
.
join
(
[
tile_string
(
v
)
for
v
in
self
.
_state
[
row
,
:
]
]
)
+
"|"
)
print
(
separator_line
)


def setup_saver(self):
        """
        Creates the tf.train.Saver object and stores it in self.saver.
        """
        if self.execution_type == "single":
            global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)
        else:
            global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)

        # global_variables += [self.global_episode, self.global_timestep]

        for c in self.get_savable_components():
            c.register_saver_ops()

        # TensorFlow saver object
        # TODO potentially make other options configurable via saver spec.
        self.saver = tf.train.Saver(
            var_list=global_variables,  # should be given?
            reshape=False,
            sharded=False,
            max_to_keep=5,
            keep_checkpoint_every_n_hours=10000.0,
            name=None,
            restore_sequentially=False,
            saver_def=None,
            builder=None,
            defer_build=False,
            allow_empty=True,
            write_version=tf.train.SaverDef.V2,
            pad_step_number=False,
            save_relative_paths=True
            # filename=None
        )

def setup_saver ( self ) : if self . execution_type == "single" : global_variables = self . get_variables ( include_submodules = True , include_nontrainable = True ) else : global_variables = self . global_model . get_variables ( include_submodules = True , include_nontrainable = True ) for c in self . get_savable_components ( ) : c . register_saver_ops ( ) self . saver = tf . train . Saver ( var_list = global_variables , reshape = False , sharded = False , max_to_keep = 5 , keep_checkpoint_every_n_hours = 10000.0 , name = None , restore_sequentially = False , saver_def = None , builder = None , defer_build = False , allow_empty = True , write_version = tf . train . SaverDef . V2 , pad_step_number = False , save_relative_paths = True )

Creates the tf . train . Saver object and stores it in self . saver .


def setup_scaffold(self):
        """
        Creates the tf.train.Scaffold object and assigns it to self.scaffold.
        Other fields of the Scaffold are generated automatically.
        """
        if self.execution_type == "single":
            global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)
            # global_variables += [self.global_episode, self.global_timestep]
            init_op = tf.variables_initializer(var_list=global_variables)
            if self.summarizer_init_op is not None:
                init_op = tf.group(init_op, self.summarizer_init_op)
            if self.graph_summary is None:
                ready_op = tf.report_uninitialized_variables(var_list=global_variables)
                ready_for_local_init_op = None
                local_init_op = None
            else:
                ready_op = None
                ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)
                local_init_op = self.graph_summary

        else:
            # Global and local variable initializers.
            global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)
            # global_variables += [self.global_episode, self.global_timestep]
            local_variables = self.get_variables(include_submodules=True, include_nontrainable=True)
            init_op = tf.variables_initializer(var_list=global_variables)
            if self.summarizer_init_op is not None:
                init_op = tf.group(init_op, self.summarizer_init_op)
            ready_op = tf.report_uninitialized_variables(var_list=(global_variables + local_variables))
            ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)
            if self.graph_summary is None:
                local_init_op = tf.group(
                    tf.variables_initializer(var_list=local_variables),
                    # Synchronize values of trainable variables.
                    *(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(
                        self.get_variables(include_submodules=True),
                        self.global_model.get_variables(include_submodules=True)
                    ))
                )
            else:
                local_init_op = tf.group(
                    tf.variables_initializer(var_list=local_variables),
                    self.graph_summary,
                    # Synchronize values of trainable variables.
                    *(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(
                        self.get_variables(include_submodules=True),
                        self.global_model.get_variables(include_submodules=True)
                    ))
                )

        def init_fn(scaffold, session):
            if self.saver_spec is not None and self.saver_spec.get('load', True):
                directory = self.saver_spec['directory']
                file = self.saver_spec.get('file')
                if file is None:
                    file = tf.train.latest_checkpoint(
                        checkpoint_dir=directory,
                        latest_filename=None  # Corresponds to argument of saver.save() in Model.save().
                    )
                elif not os.path.isfile(file):
                    file = os.path.join(directory, file)
                if file is not None:
                    try:
                        scaffold.saver.restore(sess=session, save_path=file)
                        session.run(fetches=self.list_buffer_index_reset_op)
                    except tf.errors.NotFoundError:
                        raise TensorForceError("Error: Existing checkpoint could not be loaded! Set \"load\" to false in saver_spec.")

        # TensorFlow scaffold object
        # TODO explain what it does.
        self.scaffold = tf.train.Scaffold(
            init_op=init_op,
            init_feed_dict=None,
            init_fn=init_fn,
            ready_op=ready_op,
            ready_for_local_init_op=ready_for_local_init_op,
            local_init_op=local_init_op,
            summary_op=None,
            saver=self.saver,
            copy_from_scaffold=None
        )

Creates the tf . train . Scaffold object and assigns it to self . scaffold . Other fields of the Scaffold are generated automatically .


def setup_hooks(self):
        """
        Creates and returns a list of hooks to use in a session. Populates self.saver_directory.

        Returns: List of hooks to use in a session.
        """
        hooks = list()

        # Checkpoint saver hook
        if self.saver_spec is not None and (self.execution_type == 'single' or self.distributed_spec['task_index'] == 0):
            self.saver_directory = self.saver_spec['directory']
            hooks.append(tf.train.CheckpointSaverHook(
                checkpoint_dir=self.saver_directory,
                save_secs=self.saver_spec.get('seconds', None if 'steps' in self.saver_spec else 600),
                save_steps=self.saver_spec.get('steps'),  # Either one or the other has to be set.
                saver=None,  # None since given via 'scaffold' argument.
                checkpoint_basename=self.saver_spec.get('basename', 'model.ckpt'),
                scaffold=self.scaffold,
                listeners=None
            ))
        else:
            self.saver_directory = None

        # Stop at step hook
        # hooks.append(tf.train.StopAtStepHook(
        #     num_steps=???,  # This makes more sense, if load and continue training.
        #     last_step=None  # Either one or the other has to be set.
        # ))

        # # Step counter hook
        # hooks.append(tf.train.StepCounterHook(
        #     every_n_steps=counter_config.get('steps', 100),  # Either one or the other has to be set.
        #     every_n_secs=counter_config.get('secs'),  # Either one or the other has to be set.
        #     output_dir=None,  # None since given via 'summary_writer' argument.
        #     summary_writer=summary_writer
        # ))

        # Other available hooks:
        # tf.train.FinalOpsHook(final_ops, final_ops_feed_dict=None)
        # tf.train.GlobalStepWaiterHook(wait_until_step)
        # tf.train.LoggingTensorHook(tensors, every_n_iter=None, every_n_secs=None)
        # tf.train.NanTensorHook(loss_tensor, fail_on_nan_loss=True)
        # tf.train.ProfilerHook(save_steps=None, save_secs=None, output_dir='', show_dataflow=True, show_memory=False)

        return hooks

def setup_hooks ( self ) : hooks = list ( ) if self . saver_spec is not None and ( self . execution_type == 'single' or self . distributed_spec [ 'task_index' ] == 0 ) : self . saver_directory = self . saver_spec [ 'directory' ] hooks . append ( tf . train . CheckpointSaverHook ( checkpoint_dir = self . saver_directory , save_secs = self . saver_spec . get ( 'seconds' , None if 'steps' in self . saver_spec else 600 ) , save_steps = self . saver_spec . get ( 'steps' ) , saver = None , checkpoint_basename = self . saver_spec . get ( 'basename' , 'model.ckpt' ) , scaffold = self . scaffold , listeners = None ) ) else : self . saver_directory = None return hooks

Creates and returns a list of hooks to use in a session . Populates self . saver_directory .


def create_atomic_observe_operations(self, states, actions, internals, terminal, reward, index):
        """
        Returns the tf op to fetch when unbuffered observations are passed in.

        Args:
            states (any): One state (usually a value tuple) or dict of states if multiple states are expected.
            actions (any): One action (usually a value tuple) or dict of states if multiple actions are expected.
            internals (any): Internal list.
            terminal (bool): boolean indicating if the episode terminated after the observation.
            reward (float): scalar reward that resulted from executing the action.

        Returns: Tf op to fetch when `observe()` is called.
        """
        # Increment episode
        num_episodes = tf.count_nonzero(input_tensor=terminal, dtype=util.tf_dtype('int'))
        increment_episode = tf.assign_add(ref=self.episode, value=tf.to_int64(x=num_episodes))
        increment_global_episode = tf.assign_add(ref=self.global_episode, value=tf.to_int64(x=num_episodes))

        with tf.control_dependencies(control_inputs=(increment_episode, increment_global_episode)):
            # Stop gradients
            # Not using buffers here.
            states = util.map_tensors(fn=tf.stop_gradient, tensors=states)
            internals = util.map_tensors(fn=tf.stop_gradient, tensors=internals)
            actions = util.map_tensors(fn=tf.stop_gradient, tensors=actions)
            terminal = tf.stop_gradient(input=terminal)
            reward = tf.stop_gradient(input=reward)

            # Observation
            observation = self.fn_observe_timestep(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )

        with tf.control_dependencies(control_inputs=(observation,)):
            # Trivial operation to enforce control dependency.
            self.unbuffered_episode_output = self.global_episode + 0

def create_atomic_observe_operations ( self , states , actions , internals , terminal , reward , index ) : num_episodes = tf . count_nonzero ( input_tensor = terminal , dtype = util . tf_dtype ( 'int' ) ) increment_episode = tf . assign_add ( ref = self . episode , value = tf . to_int64 ( x = num_episodes ) ) increment_global_episode = tf . assign_add ( ref = self . global_episode , value = tf . to_int64 ( x = num_episodes ) ) with tf . control_dependencies ( control_inputs = ( increment_episode , increment_global_episode ) ) : states = util . map_tensors ( fn = tf . stop_gradient , tensors = states ) internals = util . map_tensors ( fn = tf . stop_gradient , tensors = internals ) actions = util . map_tensors ( fn = tf . stop_gradient , tensors = actions ) terminal = tf . stop_gradient ( input = terminal ) reward = tf . stop_gradient ( input = reward ) observation = self . fn_observe_timestep ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward ) with tf . control_dependencies ( control_inputs = ( observation , ) ) : self . unbuffered_episode_output = self . global_episode + 0

Returns the tf op to fetch when unbuffered observations are passed in .


def get_savable_components(self):
        """
        Returns the list of all of the components this model consists of that can be individually saved and restored.
        For instance the network or distribution.

        Returns:
            List of util.SavableComponent
        """
        components = self.get_components()
        components = [components[name] for name in sorted(components)]
        return set(filter(lambda x: isinstance(x, util.SavableComponent), components))

def get_savable_components ( self ) : components = self . get_components ( ) components = [ components [ name ] for name in sorted ( components ) ] return set ( filter ( lambda x : isinstance ( x , util . SavableComponent ) , components ) )

Returns the list of all of the components this model consists of that can be individually saved and restored . For instance the network or distribution .


def save_component(self, component_name, save_path):
        """
        Saves a component of this model to the designated location.

        Args:
            component_name: The component to save.
            save_path: The location to save to.
        Returns:
            Checkpoint path where the component was saved.
        """
        component = self.get_component(component_name=component_name)
        self._validate_savable(component=component, component_name=component_name)
        return component.save(sess=self.session, save_path=save_path)

def save_component ( self , component_name , save_path ) : component = self . get_component ( component_name = component_name ) self . _validate_savable ( component = component , component_name = component_name ) return component . save ( sess = self . session , save_path = save_path )

Saves a component of this model to the designated location .


def restore_component(self, component_name, save_path):
        """
        Restores a component's parameters from a save location.

        Args:
            component_name: The component to restore.
            save_path: The save location.
        """
        component = self.get_component(component_name=component_name)
        self._validate_savable(component=component, component_name=component_name)
        component.restore(sess=self.session, save_path=save_path)

def restore_component ( self , component_name , save_path ) : component = self . get_component ( component_name = component_name ) self . _validate_savable ( component = component , component_name = component_name ) component . restore ( sess = self . session , save_path = save_path )

Restores a component s parameters from a save location .


def get_component(self, component_name):
        """
        Looks up a component by its name.

        Args:
            component_name: The name of the component to look up.
        Returns:
            The component for the provided name or None if there is no such component.
        """
        mapping = self.get_components()
        return mapping[component_name] if component_name in mapping else None

def get_component ( self , component_name ) : mapping = self . get_components ( ) return mapping [ component_name ] if component_name in mapping else None


def import_demonstrations(self, demonstrations):
        """
        Imports demonstrations, i.e. expert observations. Note that for large numbers of observations,
        set_demonstrations is more appropriate, which directly sets memory contents to an array an expects
        a different layout.

        Args:
            demonstrations: List of observation dicts
        """
        if isinstance(demonstrations, dict):
            if self.unique_state:
                demonstrations['states'] = dict(state=demonstrations['states'])
            if self.unique_action:
                demonstrations['actions'] = dict(action=demonstrations['actions'])

            self.model.import_demo_experience(**demonstrations)

        else:
            if self.unique_state:
                states = dict(state=list())
            else:
                states = {name: list() for name in demonstrations[0]['states']}
            internals = {name: list() for name in demonstrations[0]['internals']}
            if self.unique_action:
                actions = dict(action=list())
            else:
                actions = {name: list() for name in demonstrations[0]['actions']}
            terminal = list()
            reward = list()

            for demonstration in demonstrations:
                if self.unique_state:
                    states['state'].append(demonstration['states'])
                else:
                    for name, state in states.items():
                        state.append(demonstration['states'][name])
                for name, internal in internals.items():
                    internal.append(demonstration['internals'][name])
                if self.unique_action:
                    actions['action'].append(demonstration['actions'])
                else:
                    for name, action in actions.items():
                        action.append(demonstration['actions'][name])
                terminal.append(demonstration['terminal'])
                reward.append(demonstration['reward'])

            self.model.import_demo_experience(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )

def import_demonstrations ( self , demonstrations ) : if isinstance ( demonstrations , dict ) : if self . unique_state : demonstrations [ 'states' ] = dict ( state = demonstrations [ 'states' ] ) if self . unique_action : demonstrations [ 'actions' ] = dict ( action = demonstrations [ 'actions' ] ) self . model . import_demo_experience ( ** demonstrations ) else : if self . unique_state : states = dict ( state = list ( ) ) else : states = { name : list ( ) for name in demonstrations [ 0 ] [ 'states' ] } internals = { name : list ( ) for name in demonstrations [ 0 ] [ 'internals' ] } if self . unique_action : actions = dict ( action = list ( ) ) else : actions = { name : list ( ) for name in demonstrations [ 0 ] [ 'actions' ] } terminal = list ( ) reward = list ( ) for demonstration in demonstrations : if self . unique_state : states [ 'state' ] . append ( demonstration [ 'states' ] ) else : for name , state in states . items ( ) : state . append ( demonstration [ 'states' ] [ name ] ) for name , internal in internals . items ( ) : internal . append ( demonstration [ 'internals' ] [ name ] ) if self . unique_action : actions [ 'action' ] . append ( demonstration [ 'actions' ] ) else : for name , action in actions . items ( ) : action . append ( demonstration [ 'actions' ] [ name ] ) terminal . append ( demonstration [ 'terminal' ] ) reward . append ( demonstration [ 'reward' ] ) self . model . import_demo_experience ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward )

Imports demonstrations i . e . expert observations . Note that for large numbers of observations set_demonstrations is more appropriate which directly sets memory contents to an array an expects a different layout .


def states(self):
        """
        Return the state space. Might include subdicts if multiple states are
        available simultaneously.

        Returns: dict of state properties (shape and type).

        """
        screen = self.env.getScreenRGB()
        return dict(shape=screen.shape, type='int')

def states ( self ) : screen = self . env . getScreenRGB ( ) return dict ( shape = screen . shape , type = 'int' )

Return the state space . Might include subdicts if multiple states are available simultaneously .


def sanity_check_states(states_spec):
    """
    Sanity checks a states dict, used to define the state space for an MDP.
    Throws an error or warns if mismatches are found.

    Args:
        states_spec (Union[None,dict]): The spec-dict to check (or None).

    Returns: Tuple of 1) the state space desc and 2) whether there is only one component in the state space.
    """
    # Leave incoming states dict intact.
    states = copy.deepcopy(states_spec)

    # Unique state shortform.
    is_unique = ('shape' in states)
    if is_unique:
        states = dict(state=states)

    # Normalize states.
    for name, state in states.items():
        # Convert int to unary tuple.
        if isinstance(state['shape'], int):
            state['shape'] = (state['shape'],)

        # Set default type to float.
        if 'type' not in state:
            state['type'] = 'float'

    return states, is_unique

def sanity_check_states ( states_spec ) : states = copy . deepcopy ( states_spec ) is_unique = ( 'shape' in states ) if is_unique : states = dict ( state = states ) for name , state in states . items ( ) : if isinstance ( state [ 'shape' ] , int ) : state [ 'shape' ] = ( state [ 'shape' ] , ) if 'type' not in state : state [ 'type' ] = 'float' return states , is_unique

Sanity checks a states dict used to define the state space for an MDP . Throws an error or warns if mismatches are found .


def sanity_check_actions(actions_spec):
    """
    Sanity checks an actions dict, used to define the action space for an MDP.
    Throws an error or warns if mismatches are found.

    Args:
        actions_spec (Union[None,dict]): The spec-dict to check (or None).

    Returns: Tuple of 1) the action space desc and 2) whether there is only one component in the action space.
    """
    # Leave incoming spec-dict intact.
    actions = copy.deepcopy(actions_spec)

    # Unique action shortform.
    is_unique = ('type' in actions)
    if is_unique:
        actions = dict(action=actions)

    # Normalize actions.
    for name, action in actions.items():
        # Set default type to int
        if 'type' not in action:
            action['type'] = 'int'

        # Check required values
        if action['type'] == 'int':
            if 'num_actions' not in action:
                raise TensorForceError("Action requires value 'num_actions' set!")
        elif action['type'] == 'float':
            if ('min_value' in action) != ('max_value' in action):
                raise TensorForceError("Action requires both values 'min_value' and 'max_value' set!")

        # Set default shape to empty tuple (single-int, discrete action space)
        if 'shape' not in action:
            action['shape'] = ()

        # Convert int to unary tuple
        if isinstance(action['shape'], int):
            action['shape'] = (action['shape'],)

    return actions, is_unique

Sanity checks an actions dict used to define the action space for an MDP . Throws an error or warns if mismatches are found .


def make_game():
  """Builds and returns an Extraterrestrial Marauders game."""
  return ascii_art.ascii_art_to_game(
      GAME_ART, what_lies_beneath=' ',
      sprites=dict(
          [('P', PlayerSprite)] +
          [(c, UpwardLaserBoltSprite) for c in UPWARD_BOLT_CHARS] +
          [(c, DownwardLaserBoltSprite) for c in DOWNWARD_BOLT_CHARS]),
      drapes=dict(X=MarauderDrape,
                  B=BunkerDrape),
      update_schedule=['P', 'B', 'X'] + list(_ALL_BOLT_CHARS))

def
make_game
(
)
:
return
ascii_art
.
ascii_art_to_game
(
GAME_ART
,
what_lies_beneath
=
' '
,
sprites
=
dict
(
[
(
'P'
,
PlayerSprite
)
]
+
[
(
c
,
UpwardLaserBoltSprite
)
for
c
in
UPWARD_BOLT_CHARS
]
+
[
(
c
,
DownwardLaserBoltSprite
)
for
c
in
DOWNWARD_BOLT_CHARS
]
)
,
drapes
=
dict
(
X
=
MarauderDrape
,
B
=
BunkerDrape
)
,
update_schedule
=
[
'P'
,
'B'
,
'X'
]
+
list
(
_ALL_BOLT_CHARS
)
)

Builds and returns an Extraterrestrial Marauders game .


def _fly(self, board, layers, things, the_plot):
    """Handles the behaviour of visible bolts flying toward Marauders."""
    # Disappear if we've hit a Marauder or a bunker.
    if (self.character in the_plot['bunker_hitters'] or
        self.character in the_plot['marauder_hitters']):
      return self._teleport((-1, -1))
    # Otherwise, northward!
    self._north(board, the_plot)

def _fly ( self , board , layers , things , the_plot ) : if ( self . character in the_plot [ 'bunker_hitters' ] or self . character in the_plot [ 'marauder_hitters' ] ) : return self . _teleport ( ( - 1 , - 1 ) ) self . _north ( board , the_plot )

Handles the behaviour of visible bolts flying toward Marauders .


def _fire(self, layers, things, the_plot):
    """Launches a new bolt from the player."""
    # We don't fire if the player fired another bolt just now.
    if the_plot.get('last_player_shot') == the_plot.frame: return
    the_plot['last_player_shot'] = the_plot.frame
    # We start just above the player.
    row, col = things['P'].position
    self._teleport((row-1, col))

def _fire ( self , layers , things , the_plot ) : if the_plot . get ( 'last_player_shot' ) == the_plot . frame : return the_plot [ 'last_player_shot' ] = the_plot . frame row , col = things [ 'P' ] . position self . _teleport ( ( row - 1 , col ) )


def _fly(self, board, layers, things, the_plot):
    """Handles the behaviour of visible bolts flying toward the player."""
    # Disappear if we've hit a bunker.
    if self.character in the_plot['bunker_hitters']:
      return self._teleport((-1, -1))
    # End the game if we've hit the player.
    if self.position == things['P'].position: the_plot.terminate_episode()
    self._south(board, the_plot)

def _fly ( self , board , layers , things , the_plot ) : if self . character in the_plot [ 'bunker_hitters' ] : return self . _teleport ( ( - 1 , - 1 ) ) if self . position == things [ 'P' ] . position : the_plot . terminate_episode ( ) self . _south ( board , the_plot )

Handles the behaviour of visible bolts flying toward the player .


def _fire(self, layers, the_plot):
    """Launches a new bolt from a random Marauder."""
    # We don't fire if another Marauder fired a bolt just now.
    if the_plot.get('last_marauder_shot') == the_plot.frame: return
    the_plot['last_marauder_shot'] = the_plot.frame
    # Which Marauder should fire the laser bolt?
    col = np.random.choice(np.nonzero(layers['X'].sum(axis=0))[0])
    row = np.nonzero(layers['X'][:, col])[0][-1] + 1
    # Move ourselves just below that Marauder.
    self._teleport((row, col))

def _fire ( self , layers , the_plot ) : if the_plot . get ( 'last_marauder_shot' ) == the_plot . frame : return the_plot [ 'last_marauder_shot' ] = the_plot . frame col = np . random . choice ( np . nonzero ( layers [ 'X' ] . sum ( axis = 0 ) ) [ 0 ] ) row = np . nonzero ( layers [ 'X' ] [ : , col ] ) [ 0 ] [ - 1 ] + 1 self . _teleport ( ( row , col ) )

Launches a new bolt from a random Marauder .


def setup_components_and_tf_funcs(self, custom_getter=None):
        """
        Creates and stores Network and Distribution objects.
        Generates and stores all template functions.
        """
        # Create network before super-call, since non-empty internals_spec attribute (for RNN) is required subsequently.
        self.network = Network.from_spec(
            spec=self.network_spec,
            kwargs=dict(summary_labels=self.summary_labels)
        )

        # Now that we have the network component: We can create the internals placeholders.
        assert len(self.internals_spec) == 0
        self.internals_spec = self.network.internals_spec()
        for name in sorted(self.internals_spec):
            internal = self.internals_spec[name]
            self.internals_input[name] = tf.placeholder(
                dtype=util.tf_dtype(internal['type']),
                shape=(None,) + tuple(internal['shape']),
                name=('internal-' + name)
            )
            if internal['initialization'] == 'zeros':
                self.internals_init[name] = np.zeros(shape=internal['shape'])
            else:
                raise TensorForceError("Invalid internal initialization value.")

        # And only then call super.
        custom_getter = super(DistributionModel, self).setup_components_and_tf_funcs(custom_getter)

        # Distributions
        self.distributions = self.create_distributions()

        # KL divergence function
        self.fn_kl_divergence = tf.make_template(
            name_='kl-divergence',
            func_=self.tf_kl_divergence,
            custom_getter_=custom_getter
        )

        return custom_getter

Creates and stores Network and Distribution objects . Generates and stores all template functions .


def create_distributions(self):
        """
        Creates and returns the Distribution objects based on self.distributions_spec.

        Returns: Dict of distributions according to self.distributions_spec.
        """
        distributions = dict()
        for name in sorted(self.actions_spec):
            action = self.actions_spec[name]

            if self.distributions_spec is not None and name in self.distributions_spec:
                kwargs = dict(action)
                kwargs['scope'] = name
                kwargs['summary_labels'] = self.summary_labels
                distributions[name] = Distribution.from_spec(
                    spec=self.distributions_spec[name],
                    kwargs=kwargs
                )

            elif action['type'] == 'bool':
                distributions[name] = Bernoulli(
                    shape=action['shape'],
                    scope=name,
                    summary_labels=self.summary_labels
                )

            elif action['type'] == 'int':
                distributions[name] = Categorical(
                    shape=action['shape'],
                    num_actions=action['num_actions'],
                    scope=name,
                    summary_labels=self.summary_labels
                )

            elif action['type'] == 'float':
                if 'min_value' in action:
                    distributions[name] = Beta(
                        shape=action['shape'],
                        min_value=action['min_value'],
                        max_value=action['max_value'],
                        scope=name,
                        summary_labels=self.summary_labels
                    )

                else:
                    distributions[name] = Gaussian(
                        shape=action['shape'],
                        scope=name,
                        summary_labels=self.summary_labels
                    )

        return distributions

def create_distributions ( self ) : distributions = dict ( ) for name in sorted ( self . actions_spec ) : action = self . actions_spec [ name ] if self . distributions_spec is not None and name in self . distributions_spec : kwargs = dict ( action ) kwargs [ 'scope' ] = name kwargs [ 'summary_labels' ] = self . summary_labels distributions [ name ] = Distribution . from_spec ( spec = self . distributions_spec [ name ] , kwargs = kwargs ) elif action [ 'type' ] == 'bool' : distributions [ name ] = Bernoulli ( shape = action [ 'shape' ] , scope = name , summary_labels = self . summary_labels ) elif action [ 'type' ] == 'int' : distributions [ name ] = Categorical ( shape = action [ 'shape' ] , num_actions = action [ 'num_actions' ] , scope = name , summary_labels = self . summary_labels ) elif action [ 'type' ] == 'float' : if 'min_value' in action : distributions [ name ] = Beta ( shape = action [ 'shape' ] , min_value = action [ 'min_value' ] , max_value = action [ 'max_value' ] , scope = name , summary_labels = self . summary_labels ) else : distributions [ name ] = Gaussian ( shape = action [ 'shape' ] , scope = name , summary_labels = self . summary_labels ) return distributions

Creates and returns the Distribution objects based on self . distributions_spec .


def from_spec(spec):
        """
        Creates an exploration object from a specification dict.
        """
        exploration = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.explorations.explorations
        )
        assert isinstance(exploration, Exploration)
        return exploration

def from_spec ( spec ) : exploration = util . get_object ( obj = spec , predefined_objects = tensorforce . core . explorations . explorations ) assert isinstance ( exploration , Exploration ) return exploration

Creates an exploration object from a specification dict .


def from_spec(spec, kwargs=None):
        """
        Creates a memory from a specification dict.
        """
        memory = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.memories.memories,
            kwargs=kwargs
        )
        assert isinstance(memory, Memory)
        return memory

def from_spec ( spec , kwargs = None ) : memory = util . get_object ( obj = spec , predefined_objects = tensorforce . core . memories . memories , kwargs = kwargs ) assert isinstance ( memory , Memory ) return memory

Creates a memory from a specification dict .


def tf_retrieve_indices(self, indices):
        """
        Fetches experiences for given indices.

        Args:
            indices: Index tensor

        Returns: Batch of experiences
        """
        states = dict()
        for name in sorted(self.states_memory):
            states[name] = tf.gather(params=self.states_memory[name], indices=indices)

        internals = dict()
        for name in sorted(self.internals_memory):
            internals[name] = tf.gather(params=self.internals_memory[name], indices=indices)

        actions = dict()
        for name in sorted(self.actions_memory):
            actions[name] = tf.gather(params=self.actions_memory[name], indices=indices)

        terminal = tf.gather(params=self.terminal_memory, indices=indices)
        reward = tf.gather(params=self.reward_memory, indices=indices)

        if self.include_next_states:
            assert util.rank(indices) == 1
            next_indices = (indices + 1) % self.capacity

            next_states = dict()
            for name in sorted(self.states_memory):
                next_states[name] = tf.gather(params=self.states_memory[name], indices=next_indices)

            next_internals = dict()
            for name in sorted(self.internals_memory):
                next_internals[name] = tf.gather(params=self.internals_memory[name], indices=next_indices)

            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward,
                next_states=next_states,
                next_internals=next_internals
            )
        else:
            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )

def tf_retrieve_indices ( self , indices ) : states = dict ( ) for name in sorted ( self . states_memory ) : states [ name ] = tf . gather ( params = self . states_memory [ name ] , indices = indices ) internals = dict ( ) for name in sorted ( self . internals_memory ) : internals [ name ] = tf . gather ( params = self . internals_memory [ name ] , indices = indices ) actions = dict ( ) for name in sorted ( self . actions_memory ) : actions [ name ] = tf . gather ( params = self . actions_memory [ name ] , indices = indices ) terminal = tf . gather ( params = self . terminal_memory , indices = indices ) reward = tf . gather ( params = self . reward_memory , indices = indices ) if self . include_next_states : assert util . rank ( indices ) == 1 next_indices = ( indices + 1 ) % self . capacity next_states = dict ( ) for name in sorted ( self . states_memory ) : next_states [ name ] = tf . gather ( params = self . states_memory [ name ] , indices = next_indices ) next_internals = dict ( ) for name in sorted ( self . internals_memory ) : next_internals [ name ] = tf . gather ( params = self . internals_memory [ name ] , indices = next_indices ) return dict ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward , next_states = next_states , next_internals = next_internals ) else : return dict ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward )


def tf_initialize(self, x_init, base_value, target_value, estimated_improvement):
        """
        Initialization step preparing the arguments for the first iteration of the loop body.

        Args:
            x_init: Initial solution guess $x_0$.
            base_value: Value $f(x')$ at $x = x'$.
            target_value: Value $f(x_0)$ at $x = x_0$.
            estimated_improvement: Estimated value at $x = x_0$, $f(x')$ if None.

        Returns:
            Initial arguments for tf_step.
        """
        self.base_value = base_value

        if estimated_improvement is None:  # TODO: Is this a good alternative?
            estimated_improvement = tf.abs(x=base_value)

        first_step = super(LineSearch, self).tf_initialize(x_init)

        improvement = tf.divide(
            x=(target_value - self.base_value),
            y=tf.maximum(x=estimated_improvement, y=util.epsilon)
        )

        last_improvement = improvement - 1.0

        if self.mode == 'linear':
            deltas = [-t * self.parameter for t in x_init]
            self.estimated_incr = -estimated_improvement * self.parameter

        elif self.mode == 'exponential':
            deltas = [-t * self.parameter for t in x_init]

        return first_step + (deltas, improvement, last_improvement, estimated_improvement)

def tf_initialize ( self , x_init , base_value , target_value , estimated_improvement ) : self . base_value = base_value if estimated_improvement is None : estimated_improvement = tf . abs ( x = base_value ) first_step = super ( LineSearch , self ) . tf_initialize ( x_init ) improvement = tf . divide ( x = ( target_value - self . base_value ) , y = tf . maximum ( x = estimated_improvement , y = util . epsilon ) ) last_improvement = improvement - 1.0 if self . mode == 'linear' : deltas = [ - t * self . parameter for t in x_init ] self . estimated_incr = - estimated_improvement * self . parameter elif self . mode == 'exponential' : deltas = [ - t * self . parameter for t in x_init ] return first_step + ( deltas , improvement , last_improvement , estimated_improvement )

Initialization step preparing the arguments for the first iteration of the loop body .


def tf_step(self, x, iteration, deltas, improvement, last_improvement, estimated_improvement):
        """
        Iteration loop body of the line search algorithm.

        Args:
            x: Current solution estimate $x_t$.
            iteration: Current iteration counter $t$.
            deltas: Current difference $x_t - x'$.
            improvement: Current improvement $(f(x_t) - f(x')) / v'$.
            last_improvement: Last improvement $(f(x_{t-1}) - f(x')) / v'$.
            estimated_improvement: Current estimated value $v'$.

        Returns:
            Updated arguments for next iteration.
        """
        x, next_iteration, deltas, improvement, last_improvement, estimated_improvement = super(LineSearch, self).tf_step(
            x, iteration, deltas, improvement, last_improvement, estimated_improvement
        )

        next_x = [t + delta for t, delta in zip(x, deltas)]

        if self.mode == 'linear':
            next_deltas = deltas
            next_estimated_improvement = estimated_improvement + self.estimated_incr

        elif self.mode == 'exponential':
            next_deltas = [delta * self.parameter for delta in deltas]
            next_estimated_improvement = estimated_improvement * self.parameter

        target_value = self.fn_x(next_deltas)

        next_improvement = tf.divide(
            x=(target_value - self.base_value),
            y=tf.maximum(x=next_estimated_improvement, y=util.epsilon)
        )

        return next_x, next_iteration, next_deltas, next_improvement, improvement, next_estimated_improvement

def tf_step ( self , x , iteration , deltas , improvement , last_improvement , estimated_improvement ) : x , next_iteration , deltas , improvement , last_improvement , estimated_improvement = super ( LineSearch , self ) . tf_step ( x , iteration , deltas , improvement , last_improvement , estimated_improvement ) next_x = [ t + delta for t , delta in zip ( x , deltas ) ] if self . mode == 'linear' : next_deltas = deltas next_estimated_improvement = estimated_improvement + self . estimated_incr elif self . mode == 'exponential' : next_deltas = [ delta * self . parameter for delta in deltas ] next_estimated_improvement = estimated_improvement * self . parameter target_value = self . fn_x ( next_deltas ) next_improvement = tf . divide ( x = ( target_value - self . base_value ) , y = tf . maximum ( x = next_estimated_improvement , y = util . epsilon ) ) return next_x , next_iteration , next_deltas , next_improvement , improvement , next_estimated_improvement

Iteration loop body of the line search algorithm .


def markdown(text, escape=True, **kwargs):
    """Render markdown formatted text to html.

    :param text: markdown formatted text content.
    :param escape: if set to False, all html tags will not be escaped.
    :param use_xhtml: output with xhtml tags.
    :param hard_wrap: if set to True, it will use the GFM line breaks feature.
    :param parse_block_html: parse text only in block level html.
    :param parse_inline_html: parse text only in inline level html.
    """
    return Markdown(escape=escape, **kwargs)(text)

def markdown ( text , escape = True , ** kwargs ) : return Markdown ( escape = escape , ** kwargs ) ( text )

Render markdown formatted text to html .


def parse_lheading(self, m):
        """Parse setext heading."""
        self.tokens.append({
            'type': 'heading',
            'level': 1 if m.group(2) == '=' else 2,
            'text': m.group(1),
        })

def parse_lheading ( self , m ) : self . tokens . append ( { 'type' : 'heading' , 'level' : 1 if m . group ( 2 ) == '=' else 2 , 'text' : m . group ( 1 ) , } )


def hard_wrap(self):
        """Grammar for hard wrap linebreak. You don't need to add two
        spaces at the end of a line.
        """
        self.linebreak = re.compile(r'^ *\n(?!\s*$)')
        self.text = re.compile(
            r'^[\s\S]+?(?=[\\<!\[_*`~]|https?://| *\n|$)'
        )

Grammar for hard wrap linebreak . You don t need to add two spaces at the end of a line .


def block_code(self, code, lang=None):
        """Rendering block level code. ``pre > code``.

        :param code: text content of the code block.
        :param lang: language of the given code.
        """
        code = code.rstrip('\n')
        if not lang:
            code = escape(code, smart_amp=False)
            return '
%s\n
\n' % code
        code = escape(code, quote=True, smart_amp=False)
        return '
%s\n
\n' % (lang, code)

Rendering block level code . pre > code .


def block_html(self, html):
        """Rendering block level pure html content.

        :param html: text content of the html snippet.
        """
        if self.options.get('skip_style') and \
           html.lower().startswith('<style'):
            return ''
        if self.options.get('escape'):
            return escape(html)
        return html

def block_html ( self , html ) : if self . options . get ( 'skip_style' ) and html . lower ( ) . startswith ( '<style' ) : return '' if self . options . get ( 'escape' ) : return escape ( html ) return html

Rendering block level pure html content .


def autolink(self, link, is_email=False):
        """Rendering a given link or email address.

        :param link: link content or email address.
        :param is_email: whether this is an email or not.
        """
        text = link = escape(link)
        if is_email:
            link = 'mailto:%s' % link
        return '%s' % (link, text)

def
autolink
(
self
,
link
,
is_email
=
False
)
:
text
=
link
=
escape
(
link
)
if
is_email
:
link
=
'mailto:%s'
%
link
return
'%s'
%
(
link
,
text
)

Rendering a given link or email address .


def footnote_ref(self, key, index):
        """Rendering the ref anchor of a footnote.

        :param key: identity key for the footnote.
        :param index: the index count of current footnote.
        """
        html = (
            ''
            '%d'
        ) % (escape(key), escape(key), index)
        return html

Rendering the ref anchor of a footnote .


def footnote_item(self, key, text):
        """Rendering a footnote item.

        :param key: identity key for the footnote.
        :param text: text content of the footnote.
        """
        back = (
            '↩'
        ) % escape(key)
        text = text.rstrip()
        if text.endswith('
'):
            text = re.sub(r'<\/p>$', r'%s
' % back, text)
        else:
            text = '%s
%s
' % (text, back)
        html = '
%s
\n' % (escape(key), text)
        return html


def build_metagraph_list(self):
        """
        Convert MetaParams into TF Summary Format and create summary_op.

        Returns:
            Merged TF Op for TEXT summary elements, should only be executed once to reduce data duplication.

        """
        ops = []

        self.ignore_unknown_dtypes = True
        for key in sorted(self.meta_params):
            value = self.convert_data_to_string(self.meta_params[key])

            if len(value) == 0:
                continue
            if isinstance(value, str):
                ops.append(tf.contrib.summary.generic(name=key, tensor=tf.convert_to_tensor(str(value))))
            else:
                ops.append(tf.contrib.summary.generic(name=key, tensor=tf.as_string(tf.convert_to_tensor(value))))

        return ops

def build_metagraph_list ( self ) : ops = [ ] self . ignore_unknown_dtypes = True for key in sorted ( self . meta_params ) : value = self . convert_data_to_string ( self . meta_params [ key ] ) if len ( value ) == 0 : continue if isinstance ( value , str ) : ops . append ( tf . contrib . summary . generic ( name = key , tensor = tf . convert_to_tensor ( str ( value ) ) ) ) else : ops . append ( tf . contrib . summary . generic ( name = key , tensor = tf . as_string ( tf . convert_to_tensor ( value ) ) ) ) return ops

Convert MetaParams into TF Summary Format and create summary_op .


def process_docstring(app, what, name, obj, options, lines):
    """Enable markdown syntax in docstrings"""

    markdown = "\n".join(lines)

    # ast = cm_parser.parse(markdown)
    # html = cm_renderer.render(ast)
    rest = m2r(markdown)

    rest.replace("\r\n", "\n")
    del lines[:]
    lines.extend(rest.split("\n"))

def process_docstring ( app , what , name , obj , options , lines ) : markdown = "\n" . join ( lines ) rest = m2r ( markdown ) rest . replace ( "\r\n" , "\n" ) del lines [ : ] lines . extend ( rest . split ( "\n" ) )


def tf_baseline_loss(self, states, internals, reward, update, reference=None):
        """
        Creates the TensorFlow operations for calculating the baseline loss of a batch.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            reward: Reward tensor.
            update: Boolean tensor indicating whether this call happens during an update.
            reference: Optional reference tensor(s), in case of a comparative loss.

        Returns:
            Loss tensor.
        """
        if self.baseline_mode == 'states':
            loss = self.baseline.loss(
                states=states,
                internals=internals,
                reward=reward,
                update=update,
                reference=reference
            )

        elif self.baseline_mode == 'network':
            loss = self.baseline.loss(
                states=self.network.apply(x=states, internals=internals, update=update),
                internals=internals,
                reward=reward,
                update=update,
                reference=reference
            )

        regularization_loss = self.baseline.regularization_loss()
        if regularization_loss is not None:
            loss += regularization_loss

        return loss

def tf_baseline_loss ( self , states , internals , reward , update , reference = None ) : if self . baseline_mode == 'states' : loss = self . baseline . loss ( states = states , internals = internals , reward = reward , update = update , reference = reference ) elif self . baseline_mode == 'network' : loss = self . baseline . loss ( states = self . network . apply ( x = states , internals = internals , update = update ) , internals = internals , reward = reward , update = update , reference = reference ) regularization_loss = self . baseline . regularization_loss ( ) if regularization_loss is not None : loss += regularization_loss return loss

Creates the TensorFlow operations for calculating the baseline loss of a batch .


def baseline_optimizer_arguments(self, states, internals, reward):
        """
        Returns the baseline optimizer arguments including the time, the list of variables to
        optimize, and various functions which the optimizer might require to perform an update
        step.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            reward: Reward tensor.

        Returns:
            Baseline optimizer arguments as dict.
        """
        arguments = dict(
            time=self.global_timestep,
            variables=self.baseline.get_variables(),
            arguments=dict(
                states=states,
                internals=internals,
                reward=reward,
                update=tf.constant(value=True),
            ),
            fn_reference=self.baseline.reference,
            fn_loss=self.fn_baseline_loss,
            # source_variables=self.network.get_variables()
        )
        if self.global_model is not None:
            arguments['global_variables'] = self.global_model.baseline.get_variables()
        return arguments

def baseline_optimizer_arguments ( self , states , internals , reward ) : arguments = dict ( time = self . global_timestep , variables = self . baseline . get_variables ( ) , arguments = dict ( states = states , internals = internals , reward = reward , update = tf . constant ( value = True ) , ) , fn_reference = self . baseline . reference , fn_loss = self . fn_baseline_loss , ) if self . global_model is not None : arguments [ 'global_variables' ] = self . global_model . baseline . get_variables ( ) return arguments

Returns the baseline optimizer arguments including the time the list of variables to optimize and various functions which the optimizer might require to perform an update step .


def tf_step(self, time, variables, **kwargs):
        """
        Creates the TensorFlow operations for performing an optimization step on the given variables, including
        actually changing the values of the variables.

        Args:
            time: Time tensor. Not used for this optimizer.
            variables: List of variables to optimize.
            **kwargs:
                fn_loss : loss function tensor to differentiate.

        Returns:
            List of delta tensors corresponding to the updates for each optimized variable.
        """
        fn_loss = kwargs["fn_loss"]
        if variables is None:
            variables = tf.trainable_variables
        return tf.gradients(fn_loss, variables)

def tf_step ( self , time , variables , ** kwargs ) : fn_loss = kwargs [ "fn_loss" ] if variables is None : variables = tf . trainable_variables return tf . gradients ( fn_loss , variables )

Creates the TensorFlow operations for performing an optimization step on the given variables including actually changing the values of the variables .


def setup_components_and_tf_funcs(self, custom_getter=None):
        """
        Constructs the extra Replay memory.
        """
        custom_getter = super(QDemoModel, self).setup_components_and_tf_funcs(custom_getter)

        self.demo_memory = Replay(
            states=self.states_spec,
            internals=self.internals_spec,
            actions=self.actions_spec,
            include_next_states=True,
            capacity=self.demo_memory_capacity,
            scope='demo-replay',
            summary_labels=self.summary_labels
        )

        # Import demonstration optimization.
        self.fn_import_demo_experience = tf.make_template(
            name_='import-demo-experience',
            func_=self.tf_import_demo_experience,
            custom_getter_=custom_getter
        )

        # Demonstration loss.
        self.fn_demo_loss = tf.make_template(
            name_='demo-loss',
            func_=self.tf_demo_loss,
            custom_getter_=custom_getter
        )

        # Combined loss.
        self.fn_combined_loss = tf.make_template(
            name_='combined-loss',
            func_=self.tf_combined_loss,
            custom_getter_=custom_getter
        )

        # Demonstration optimization.
        self.fn_demo_optimization = tf.make_template(
            name_='demo-optimization',
            func_=self.tf_demo_optimization,
            custom_getter_=custom_getter
        )

        return custom_getter

def setup_components_and_tf_funcs ( self , custom_getter = None ) : custom_getter = super ( QDemoModel , self ) . setup_components_and_tf_funcs ( custom_getter ) self . demo_memory = Replay ( states = self . states_spec , internals = self . internals_spec , actions = self . actions_spec , include_next_states = True , capacity = self . demo_memory_capacity , scope = 'demo-replay' , summary_labels = self . summary_labels ) self . fn_import_demo_experience = tf . make_template ( name_ = 'import-demo-experience' , func_ = self . tf_import_demo_experience , custom_getter_ = custom_getter ) self . fn_demo_loss = tf . make_template ( name_ = 'demo-loss' , func_ = self . tf_demo_loss , custom_getter_ = custom_getter ) self . fn_combined_loss = tf . make_template ( name_ = 'combined-loss' , func_ = self . tf_combined_loss , custom_getter_ = custom_getter ) self . fn_demo_optimization = tf . make_template ( name_ = 'demo-optimization' , func_ = self . tf_demo_optimization , custom_getter_ = custom_getter ) return custom_getter


def tf_import_demo_experience(self, states, internals, actions, terminal, reward):
        """
        Imports a single experience to memory.
        """
        return self.demo_memory.store(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )

def tf_import_demo_experience ( self , states , internals , actions , terminal , reward ) : return self . demo_memory . store ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward )


def tf_demo_loss(self, states, actions, terminal, reward, internals, update, reference=None):
        """
        Extends the q-model loss via the dqfd large-margin loss.
        """
        embedding = self.network.apply(x=states, internals=internals, update=update)
        deltas = list()

        for name in sorted(actions):
            action = actions[name]
            distr_params = self.distributions[name].parameterize(x=embedding)
            state_action_value = self.distributions[name].state_action_value(distr_params=distr_params, action=action)

            # Create the supervised margin loss
            # Zero for the action taken, one for all other actions, now multiply by expert margin
            if self.actions_spec[name]['type'] == 'bool':
                num_actions = 2
                action = tf.cast(x=action, dtype=util.tf_dtype('int'))
            else:
                num_actions = self.actions_spec[name]['num_actions']

            one_hot = tf.one_hot(indices=action, depth=num_actions)
            ones = tf.ones_like(tensor=one_hot, dtype=tf.float32)
            inverted_one_hot = ones - one_hot

            # max_a([Q(s,a) + l(s,a_E,a)], l(s,a_E, a) is 0 for expert action and margin value for others
            state_action_values = self.distributions[name].state_action_value(distr_params=distr_params)
            state_action_values = state_action_values + inverted_one_hot * self.expert_margin
            supervised_selector = tf.reduce_max(input_tensor=state_action_values, axis=-1)

            # J_E(Q) = max_a([Q(s,a) + l(s,a_E,a)] - Q(s,a_E)
            delta = supervised_selector - state_action_value

            action_size = util.prod(self.actions_spec[name]['shape'])
            delta = tf.reshape(tensor=delta, shape=(-1, action_size))
            deltas.append(delta)

        loss_per_instance = tf.reduce_mean(input_tensor=tf.concat(values=deltas, axis=1), axis=1)
        loss_per_instance = tf.square(x=loss_per_instance)

        return tf.reduce_mean(input_tensor=loss_per_instance, axis=0)

def tf_demo_loss ( self , states , actions , terminal , reward , internals , update , reference = None ) : embedding = self . network . apply ( x = states , internals = internals , update = update ) deltas = list ( ) for name in sorted ( actions ) : action = actions [ name ] distr_params = self . distributions [ name ] . parameterize ( x = embedding ) state_action_value = self . distributions [ name ] . state_action_value ( distr_params = distr_params , action = action ) if self . actions_spec [ name ] [ 'type' ] == 'bool' : num_actions = 2 action = tf . cast ( x = action , dtype = util . tf_dtype ( 'int' ) ) else : num_actions = self . actions_spec [ name ] [ 'num_actions' ] one_hot = tf . one_hot ( indices = action , depth = num_actions ) ones = tf . ones_like ( tensor = one_hot , dtype = tf . float32 ) inverted_one_hot = ones - one_hot state_action_values = self . distributions [ name ] . state_action_value ( distr_params = distr_params ) state_action_values = state_action_values + inverted_one_hot * self . expert_margin supervised_selector = tf . reduce_max ( input_tensor = state_action_values , axis = - 1 ) delta = supervised_selector - state_action_value action_size = util . prod ( self . actions_spec [ name ] [ 'shape' ] ) delta = tf . reshape ( tensor = delta , shape = ( - 1 , action_size ) ) deltas . append ( delta ) loss_per_instance = tf . reduce_mean ( input_tensor = tf . concat ( values = deltas , axis = 1 ) , axis = 1 ) loss_per_instance = tf . square ( x = loss_per_instance ) return tf . reduce_mean ( input_tensor = loss_per_instance , axis = 0 )

Extends the q - model loss via the dqfd large - margin loss .


def tf_combined_loss(self, states, internals, actions, terminal, reward, next_states, next_internals, update, reference=None):
        """
        Combines Q-loss and demo loss.
        """
        q_model_loss = self.fn_loss(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward,
            next_states=next_states,
            next_internals=next_internals,
            update=update,
            reference=reference
        )

        demo_loss = self.fn_demo_loss(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward,
            update=update,
            reference=reference
        )

        return q_model_loss + self.supervised_weight * demo_loss

def tf_combined_loss ( self , states , internals , actions , terminal , reward , next_states , next_internals , update , reference = None ) : q_model_loss = self . fn_loss ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward , next_states = next_states , next_internals = next_internals , update = update , reference = reference ) demo_loss = self . fn_demo_loss ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward , update = update , reference = reference ) return q_model_loss + self . supervised_weight * demo_loss