def _keep_alive(x, memo):
    """Keeps a reference to the object x in the memo.

    Because we remember objects by their id, we have
    to assure that possibly temporary objects are kept
    alive by referencing them.
    We store a reference at the id of the memo, which should
    normally not be used unless someone tries to deepcopy
    the memo itself...
    """
    try:
        memo[id(memo)].append(x)
    except KeyError:
        # aha, this is the first one :-)
        memo[id(memo)]=[x]

def _keep_alive ( x , memo ) : try : memo [ id ( memo ) ] . append ( x ) except KeyError : memo [ id ( memo ) ] = [ x ]

Keeps a reference to the object x in the memo .


def warnpy3k(message, category=None, stacklevel=1):
    """Issue a deprecation warning for Python 3.x related changes.

    Warnings are omitted unless Python is started with the -3 option.
    """
    if sys.py3kwarning:
        if category is None:
            category = DeprecationWarning
        warn(message, category, stacklevel+1)

def warnpy3k ( message , category = None , stacklevel = 1 ) : if sys . py3kwarning : if category is None : category = DeprecationWarning warn ( message , category , stacklevel + 1 )

Issue a deprecation warning for Python 3 . x related changes .


def _show_warning(message, category, filename, lineno, file=None, line=None):
    """Hook to write a warning to a file; replace if you like."""
    if file is None:
        file = sys.stderr
        if file is None:
            # sys.stderr is None - warnings get lost
            return
    try:
        file.write(formatwarning(message, category, filename, lineno, line))
    except (IOError, UnicodeError):
        pass # the file (probably stderr) is invalid - this warning gets lost.

def _show_warning ( message , category , filename , lineno , file = None , line = None ) : if file is None : file = sys . stderr if file is None : return try : file . write ( formatwarning ( message , category , filename , lineno , line ) ) except ( IOError , UnicodeError ) : pass

Hook to write a warning to a file ; replace if you like .


def formatwarning(message, category, filename, lineno, line=None):
    """Function to format a warning the standard way."""
    try:
        unicodetype = unicode
    except NameError:
        unicodetype = ()
    try:
        message = str(message)
    except UnicodeEncodeError:
        pass
    s =  "%s: %s: %s\n" % (lineno, category.__name__, message)
    line = linecache.getline(filename, lineno) if line is None else line
    if line:
        line = line.strip()
        if isinstance(s, unicodetype) and isinstance(line, str):
            line = unicode(line, 'latin1')
        s += "  %s\n" % line
    if isinstance(s, unicodetype) and isinstance(filename, str):
        enc = sys.getfilesystemencoding()
        if enc:
            try:
                filename = unicode(filename, enc)
            except UnicodeDecodeError:
                pass
    s = "%s:%s" % (filename, s)
    return s

def
formatwarning
(
message
,
category
,
filename
,
lineno
,
line
=
)
:
try
:
unicodetype
=
unicode
except
NameError
:
unicodetype
=
(
)
try
:
message
=
str
(
message
)
except
UnicodeEncodeError
:
pass
s
=
"%s: %s: %s\n"
%
(
lineno
,
category
.
__name__
,
message
)
line
=
linecache
.
getline
(
filename
,
lineno
)
if
line
is
else
line
if
line
:
line
=
line
.
strip
(
)
if
isinstance
(
s
,
unicodetype
)
and
isinstance
(
line
,
str
)
:
line
=
unicode
(
line
,
'latin1'
)
s
+=
"  %s\n"
%
line
if
isinstance
(
s
,
unicodetype
)
and
isinstance
(
filename
,
str
)
:
enc
=
sys
.
getfilesystemencoding
(
)
if
enc
:
try
:
filename
=
unicode
(
filename
,
enc
)
except
UnicodeDecodeError
:
pass
s
=
"%s:%s"
%
(
filename
,
s
)
return
s

Function to format a warning the standard way .


def warn(message, category=None, stacklevel=1):
    """Issue a warning, or maybe ignore it or raise an exception."""
    # Check if message is already a Warning object
    if isinstance(message, Warning):
        category = message.__class__
    # Check category argument
    if category is None:
        category = UserWarning
    assert issubclass(category, Warning)
    # Get context information
    try:
        caller = sys._getframe(stacklevel)
    except ValueError:
        globals = sys.__dict__
        lineno = 1
    else:
        globals = caller.f_globals
        lineno = caller.f_lineno
    if '__name__' in globals:
        module = globals['__name__']
    else:
        module = "<string>"
    filename = globals.get('__file__')
    if filename:
        fnl = filename.lower()
        if fnl.endswith((".pyc", ".pyo")):
            filename = filename[:-1]
    else:
        if module == "__main__":
            try:
                filename = sys.argv[0]
            except AttributeError:
                # embedded interpreters don't have sys.argv, see bug #839151
                filename = '__main__'
        if not filename:
            filename = module
    registry = globals.setdefault("__warningregistry__", {})
    warn_explicit(message, category, filename, lineno, module, registry,
                  globals)

def warn ( message , category = None , stacklevel = 1 ) : if isinstance ( message , Warning ) : category = message . __class__ if category is None : category = UserWarning assert issubclass ( category , Warning ) try : caller = sys . _getframe ( stacklevel ) except ValueError : globals = sys . __dict__ lineno = 1 else : globals = caller . f_globals lineno = caller . f_lineno if '__name__' in globals : module = globals [ '__name__' ] else : module = "<string>" filename = globals . get ( '__file__' ) if filename : fnl = filename . lower ( ) if fnl . endswith ( ( ".pyc" , ".pyo" ) ) : filename = filename [ : - 1 ] else : if module == "__main__" : try : filename = sys . argv [ 0 ] except AttributeError : filename = '__main__' if not filename : filename = module registry = globals . setdefault ( "__warningregistry__" , { } ) warn_explicit ( message , category , filename , lineno , module , registry , globals )

Issue a warning or maybe ignore it or raise an exception .


def _hash(self):
        """Compute the hash value of a set.

        Note that we don't define __hash__: not all sets are hashable.
        But if you define a hashable set type, its __hash__ should
        call this function.

        This must be compatible __eq__.

        All sets ought to compare equal if they contain the same
        elements, regardless of how they are implemented, and
        regardless of the order of the elements; so there's not much
        freedom for __eq__ or __hash__.  We match the algorithm used
        by the built-in frozenset type.
        """
        MAX = sys.maxint
        MASK = 2 * MAX + 1
        n = len(self)
        h = 1927868237 * (n + 1)
        h &= MASK
        for x in self:
            hx = hash(x)
            h ^= (hx ^ (hx << 16) ^ 89869747)  * 3644798167
            h &= MASK
        h = h * 69069 + 907133923
        h &= MASK
        if h > MAX:
            h -= MASK + 1
        if h == -1:
            h = 590923713
        return h


def remove(self, value):
        """Remove an element. If not a member, raise a KeyError."""
        if value not in self:
            raise KeyError(value)
        self.discard(value)

def remove ( self , value ) : if value not in self : raise KeyError ( value ) self . discard ( value )

Remove an element . If not a member raise a KeyError .


def pop(self):
        """Return the popped value.  Raise KeyError if empty."""
        it = iter(self)
        try:
            value = next(it)
        except StopIteration:
            raise KeyError
        self.discard(value)
        return value

def pop ( self ) : it = iter ( self ) try : value = next ( it ) except StopIteration : raise KeyError self . discard ( value ) return value

Return the popped value . Raise KeyError if empty .


def go_str(value):
  """Returns value as a valid Go string literal."""
  io = StringIO.StringIO()
  io.write('"')
  for c in value:
    if c in _ESCAPES:
      io.write(_ESCAPES[c])
    elif c in _SIMPLE_CHARS:
      io.write(c)
    else:
      io.write(r'\x{:02x}'.format(ord(c)))
  io.write('"')
  return io.getvalue()

def go_str ( value ) : io = StringIO . StringIO ( ) io . write ( '"' ) for c in value : if c in _ESCAPES : io . write ( _ESCAPES [ c ] ) elif c in _SIMPLE_CHARS : io . write ( c ) else : io . write ( r'\x{:02x}' . format ( ord ( c ) ) ) io . write ( '"' ) return io . getvalue ( )

Returns value as a valid Go string literal .


def acquire(self, blocking=1):
        """Acquire a lock, blocking or non-blocking.

        When invoked without arguments: if this thread already owns the lock,
        increment the recursion level by one, and return immediately. Otherwise,
        if another thread owns the lock, block until the lock is unlocked. Once
        the lock is unlocked (not owned by any thread), then grab ownership, set
        the recursion level to one, and return. If more than one thread is
        blocked waiting until the lock is unlocked, only one at a time will be
        able to grab ownership of the lock. There is no return value in this
        case.

        When invoked with the blocking argument set to true, do the same thing
        as when called without arguments, and return true.

        When invoked with the blocking argument set to false, do not block. If a
        call without an argument would block, return false immediately;
        otherwise, do the same thing as when called without arguments, and
        return true.

        """
        me = _get_ident()
        if self.__owner == me:
            self.__count = self.__count + 1
            if __debug__:
                self._note("%s.acquire(%s): recursive success", self, blocking)
            return 1
        rc = self.__block.acquire(blocking)
        if rc:
            self.__owner = me
            self.__count = 1
            if __debug__:
                self._note("%s.acquire(%s): initial success", self, blocking)
        else:
            if __debug__:
                self._note("%s.acquire(%s): failure", self, blocking)
        return rc

def
acquire
(
self
,
blocking
=
1
)
:
me
=
_get_ident
(
)
if
self
.
__owner
==
me
:
self
.
__count
=
self
.
__count
+
1
if
__debug__
:
self
.
_note
(
"%s.acquire(%s): recursive success"
,
self
,
blocking
)
return
1
rc
=
self
.
__block
.
acquire
(
blocking
)
if
rc
:
self
.
__owner
=
me
self
.
__count
=
1
if
__debug__
:
self
.
_note
(
"%s.acquire(%s): initial success"
,
self
,
blocking
)
else
:
if
__debug__
:
self
.
_note
(
"%s.acquire(%s): failure"
,
self
,
blocking
)
return
rc

Acquire a lock blocking or non - blocking .


def release(self):
        """Release a lock, decrementing the recursion level.

        If after the decrement it is zero, reset the lock to unlocked (not owned
        by any thread), and if any other threads are blocked waiting for the
        lock to become unlocked, allow exactly one of them to proceed. If after
        the decrement the recursion level is still nonzero, the lock remains
        locked and owned by the calling thread.

        Only call this method when the calling thread owns the lock. A
        RuntimeError is raised if this method is called when the lock is
        unlocked.

        There is no return value.

        """
        if self.__owner != _get_ident():
            raise RuntimeError("cannot release un-acquired lock")
        self.__count = count = self.__count - 1
        if not count:
            self.__owner = None
            self.__block.release()
            if __debug__:
                self._note("%s.release(): final release", self)
        else:
            if __debug__:
                self._note("%s.release(): non-final release", self)

def
release
(
self
)
:
if
self
.
__owner
!=
_get_ident
(
)
:
raise
RuntimeError
(
"cannot release un-acquired lock"
)
self
.
__count
=
count
=
self
.
__count
1
if
not
count
:
self
.
__owner
=
self
.
__block
.
release
(
)
if
__debug__
:
self
.
_note
(
"%s.release(): final release"
,
self
)
else
:
if
__debug__
:
self
.
_note
(
"%s.release(): non-final release"
,
self
)

Release a lock decrementing the recursion level .


def wait(self, timeout=None):
        """Wait until notified or until a timeout occurs.

        If the calling thread has not acquired the lock when this method is
        called, a RuntimeError is raised.

        This method releases the underlying lock, and then blocks until it is
        awakened by a notify() or notifyAll() call for the same condition
        variable in another thread, or until the optional timeout occurs. Once
        awakened or timed out, it re-acquires the lock and returns.

        When the timeout argument is present and not None, it should be a
        floating point number specifying a timeout for the operation in seconds
        (or fractions thereof).

        When the underlying lock is an RLock, it is not released using its
        release() method, since this may not actually unlock the lock when it
        was acquired multiple times recursively. Instead, an internal interface
        of the RLock class is used, which really unlocks it even when it has
        been recursively acquired several times. Another internal interface is
        then used to restore the recursion level when the lock is reacquired.

        """
        if not self._is_owned():
            raise RuntimeError("cannot wait on un-acquired lock")
        waiter = _allocate_lock()
        waiter.acquire()
        self.__waiters.append(waiter)
        saved_state = self._release_save()
        try:    # restore state no matter what (e.g., KeyboardInterrupt)
            if timeout is None:
                waiter.acquire()
                if __debug__:
                    self._note("%s.wait(): got it", self)
            else:
                # Balancing act:  We can't afford a pure busy loop, so we
                # have to sleep; but if we sleep the whole timeout time,
                # we'll be unresponsive.  The scheme here sleeps very
                # little at first, longer as time goes on, but never longer
                # than 20 times per second (or the timeout time remaining).
                endtime = _time() + timeout
                delay = 0.0005 # 500 us -> initial delay of 1 ms
                while True:
                    gotit = waiter.acquire(0)
                    if gotit:
                        break
                    remaining = endtime - _time()
                    if remaining <= 0:
                        break
                    delay = min(delay * 2, remaining, .05)
                    _sleep(delay)
                if not gotit:
                    if __debug__:
                        self._note("%s.wait(%s): timed out", self, timeout)
                    try:
                        self.__waiters.remove(waiter)
                    except ValueError:
                        pass
                else:
                    if __debug__:
                        self._note("%s.wait(%s): got it", self, timeout)
        finally:
            self._acquire_restore(saved_state)

def
wait
(
self
,
timeout
=
)
:
if
not
self
.
_is_owned
(
)
:
raise
RuntimeError
(
"cannot wait on un-acquired lock"
)
waiter
=
_allocate_lock
(
)
waiter
.
acquire
(
)
self
.
__waiters
.
append
(
waiter
)
saved_state
=
self
.
_release_save
(
)
try
:
if
timeout
is
:
waiter
.
acquire
(
)
if
__debug__
:
self
.
_note
(
"%s.wait(): got it"
,
self
)
else
:
endtime
=
_time
(
)
+
timeout
delay
=
0.0005
while
True
:
gotit
=
waiter
.
acquire
(
0
)
if
gotit
:
break
remaining
=
endtime
_time
(
)
if
remaining
<=
0
:
break
delay
=
min
(
delay
*
2
,
remaining
,
.05
)
_sleep
(
delay
)
if
not
gotit
:
if
__debug__
:
self
.
_note
(
"%s.wait(%s): timed out"
,
self
,
timeout
)
try
:
self
.
__waiters
.
remove
(
waiter
)
except
ValueError
:
pass
else
:
if
__debug__
:
self
.
_note
(
"%s.wait(%s): got it"
,
self
,
timeout
)
finally
:
self
.
_acquire_restore
(
saved_state
)

Wait until notified or until a timeout occurs .


def notify(self, n=1):
        """Wake up one or more threads waiting on this condition, if any.

        If the calling thread has not acquired the lock when this method is
        called, a RuntimeError is raised.

        This method wakes up at most n of the threads waiting for the condition
        variable; it is a no-op if no threads are waiting.

        """
        if not self._is_owned():
            raise RuntimeError("cannot notify on un-acquired lock")
        __waiters = self.__waiters
        waiters = __waiters[:n]
        if not waiters:
            if __debug__:
                self._note("%s.notify(): no waiters", self)
            return
        self._note("%s.notify(): notifying %d waiter%s", self, n,
                   n!=1 and "s" or "")
        for waiter in waiters:
            waiter.release()
            try:
                __waiters.remove(waiter)
            except ValueError:
                pass

def
notify
(
self
,
n
=
1
)
:
if
not
self
.
_is_owned
(
)
:
raise
RuntimeError
(
"cannot notify on un-acquired lock"
)
__waiters
=
self
.
__waiters
waiters
=
__waiters
[
:
n
]
if
not
waiters
:
if
__debug__
:
self
.
_note
(
"%s.notify(): no waiters"
,
self
)
return
self
.
_note
(
"%s.notify(): notifying %d waiter%s"
,
self
,
n
,
n
!=
1
and
"s"
or
""
)
for
waiter
in
waiters
:
waiter
.
release
(
)
try
:
__waiters
.
remove
(
waiter
)
except
ValueError
:
pass

Wake up one or more threads waiting on this condition if any .


def acquire(self, blocking=1):
        """Acquire a semaphore, decrementing the internal counter by one.

        When invoked without arguments: if the internal counter is larger than
        zero on entry, decrement it by one and return immediately. If it is zero
        on entry, block, waiting until some other thread has called release() to
        make it larger than zero. This is done with proper interlocking so that
        if multiple acquire() calls are blocked, release() will wake exactly one
        of them up. The implementation may pick one at random, so the order in
        which blocked threads are awakened should not be relied on. There is no
        return value in this case.

        When invoked with blocking set to true, do the same thing as when called
        without arguments, and return true.

        When invoked with blocking set to false, do not block. If a call without
        an argument would block, return false immediately; otherwise, do the
        same thing as when called without arguments, and return true.

        """
        rc = False
        with self.__cond:
            while self.__value == 0:
                if not blocking:
                    break
                if __debug__:
                    self._note("%s.acquire(%s): blocked waiting, value=%s",
                            self, blocking, self.__value)
                self.__cond.wait()
            else:
                self.__value = self.__value - 1
                if __debug__:
                    self._note("%s.acquire: success, value=%s",
                            self, self.__value)
                rc = True
        return rc

Acquire a semaphore decrementing the internal counter by one .


def set(self):
        """Set the internal flag to true.

        All threads waiting for the flag to become true are awakened. Threads
        that call wait() once the flag is true will not block at all.

        """
        with self.__cond:
            self.__flag = True
            self.__cond.notify_all()

def set ( self ) : with self . __cond : self . __flag = True self . __cond . notify_all ( )


def wait(self, timeout=None):
        """Block until the internal flag is true.

        If the internal flag is true on entry, return immediately. Otherwise,
        block until another thread calls set() to set the flag to true, or until
        the optional timeout occurs.

        When the timeout argument is present and not None, it should be a
        floating point number specifying a timeout for the operation in seconds
        (or fractions thereof).

        This method returns the internal flag on exit, so it will always return
        True except if a timeout is given and the operation times out.

        """
        with self.__cond:
            if not self.__flag:
                self.__cond.wait(timeout)
            return self.__flag

def wait ( self , timeout = None ) : with self . __cond : if not self . __flag : self . __cond . wait ( timeout ) return self . __flag


def start(self):
        """Start the thread's activity.

        It must be called at most once per thread object. It arranges for the
        object's run() method to be invoked in a separate thread of control.

        This method will raise a RuntimeError if called more than once on the
        same thread object.

        """
        if not self.__initialized:
            raise RuntimeError("thread.__init__() not called")
        if self.__started.is_set():
            raise RuntimeError("threads can only be started once")
        if __debug__:
            self._note("%s.start(): starting thread", self)
        with _active_limbo_lock:
            _limbo[self] = self
        try:
            _start_new_thread(self.__bootstrap, ())
        except Exception:
            with _active_limbo_lock:
                del _limbo[self]
            raise
        self.__started.wait()

def
start
(
self
)
:
if
not
self
.
__initialized
:
raise
RuntimeError
(
"thread.__init__() not called"
)
if
self
.
__started
.
is_set
(
)
:
raise
RuntimeError
(
"threads can only be started once"
)
if
__debug__
:
self
.
_note
(
"%s.start(): starting thread"
,
self
)
with
_active_limbo_lock
:
_limbo
[
self
]
=
self
try
:
_start_new_thread
(
self
.
__bootstrap
,
(
)
)
except
Exception
:
with
_active_limbo_lock
:
del
_limbo
[
self
]
raise
self
.
__started
.
wait
(
)


def run(self):
        """Method representing the thread's activity.

        You may override this method in a subclass. The standard run() method
        invokes the callable object passed to the object's constructor as the
        target argument, if any, with sequential and keyword arguments taken
        from the args and kwargs arguments, respectively.

        """
        try:
            if self.__target:
                self.__target(*self.__args, **self.__kwargs)
        finally:
            # Avoid a refcycle if the thread is running a function with
            # an argument that has a member that points to the thread.
            del self.__target, self.__args, self.__kwargs

def run ( self ) : try : if self . __target : self . __target ( * self . __args , ** self . __kwargs ) finally : del self . __target , self . __args , self . __kwargs

Method representing the thread s activity .


def join(self, timeout=None):
        """Wait until the thread terminates.

        This blocks the calling thread until the thread whose join() method is
        called terminates -- either normally or through an unhandled exception
        or until the optional timeout occurs.

        When the timeout argument is present and not None, it should be a
        floating point number specifying a timeout for the operation in seconds
        (or fractions thereof). As join() always returns None, you must call
        isAlive() after join() to decide whether a timeout happened -- if the
        thread is still alive, the join() call timed out.

        When the timeout argument is not present or None, the operation will
        block until the thread terminates.

        A thread can be join()ed many times.

        join() raises a RuntimeError if an attempt is made to join the current
        thread as that would cause a deadlock. It is also an error to join() a
        thread before it has been started and attempts to do so raises the same
        exception.

        """
        if not self.__initialized:
            raise RuntimeError("Thread.__init__() not called")
        if not self.__started.is_set():
            raise RuntimeError("cannot join thread before it is started")
        if self is current_thread():
            raise RuntimeError("cannot join current thread")

        if __debug__:
            if not self.__stopped:
                self._note("%s.join(): waiting until thread stops", self)
        self.__block.acquire()
        try:
            if timeout is None:
                while not self.__stopped:
                    self.__block.wait()
                if __debug__:
                    self._note("%s.join(): thread stopped", self)
            else:
                deadline = _time() + timeout
                while not self.__stopped:
                    delay = deadline - _time()
                    if delay <= 0:
                        if __debug__:
                            self._note("%s.join(): timed out", self)
                        break
                    self.__block.wait(delay)
                else:
                    if __debug__:
                        self._note("%s.join(): thread stopped", self)
        finally:
            self.__block.release()


def _dump_registry(cls, file=None):
        """Debug helper to print the ABC registry."""
        print >> file, "Class: %s.%s" % (cls.__module__, cls.__name__)
        print >> file, "Inv.counter: %s" % ABCMeta._abc_invalidation_counter
        for name in sorted(cls.__dict__.keys()):
            if name.startswith("_abc_"):
                value = getattr(cls, name)
                print >> file, "%s: %r" % (name, value)

def
_dump_registry
(
cls
,
file
=
)
:
print
>>
file
,
"Class: %s.%s"
%
(
cls
.
__module__
,
cls
.
__name__
)
print
>>
file
,
"Inv.counter: %s"
%
ABCMeta
.
_abc_invalidation_counter
for
name
in
sorted
(
cls
.
__dict__
.
keys
(
)
)
:
if
name
.
startswith
(
"_abc_"
)
:
value
=
getattr
(
cls
,
name
)
print
>>
file
,
"%s: %r"
%
(
name
,
value
)

Debug helper to print the ABC registry .


def b2a_qp(data, quotetabs=False, istext=True, header=False):
    """quotetabs=True means that tab and space characters are always
       quoted.
       istext=False means that \r and \n are treated as regular characters
       header=True encodes space characters with '_' and requires
       real '_' characters to be quoted.
    """
    MAXLINESIZE = 76

    # See if this string is using CRLF line ends
    lf = data.find('\n')
    crlf = lf > 0 and data[lf-1] == '\r'

    inp = 0
    linelen = 0
    odata = []
    while inp < len(data):
        c = data[inp]
        if (c > '~' or
            c == '=' or
            (header and c == '_') or
            (c == '.' and linelen == 0 and (inp+1 == len(data) or
                                            data[inp+1] == '\n' or
                                            data[inp+1] == '\r')) or
            (not istext and (c == '\r' or c == '\n')) or
            ((c == '\t' or c == ' ') and (inp + 1 == len(data))) or
            (c <= ' ' and c != '\r' and c != '\n' and
             (quotetabs or (not quotetabs and (c != '\t' and c != ' '))))):
            linelen += 3
            if linelen >= MAXLINESIZE:
                odata.append('=')
                if crlf: odata.append('\r')
                odata.append('\n')
                linelen = 3
            odata.append('=' + two_hex_digits(ord(c)))
            inp += 1
        else:
            if (istext and
                (c == '\n' or (inp+1 < len(data) and c == '\r' and
                               data[inp+1] == '\n'))):
                linelen = 0
                # Protect against whitespace on end of line
                if (len(odata) > 0 and
                    (odata[-1] == ' ' or odata[-1] == '\t')):
                    ch = ord(odata[-1])
                    odata[-1] = '='
                    odata.append(two_hex_digits(ch))

                if crlf: odata.append('\r')
                odata.append('\n')
                if c == '\r':
                    inp += 2
                else:
                    inp += 1
            else:
                if (inp + 1 < len(data) and
                    data[inp+1] != '\n' and
                    (linelen + 1) >= MAXLINESIZE):
                    odata.append('=')
                    if crlf: odata.append('\r')
                    odata.append('\n')
                    linelen = 0

                linelen += 1
                if header and c == ' ':
                    c = '_'
                odata.append(c)
                inp += 1
    return ''.join(odata)

def
b2a_qp
(
data
,
quotetabs
=
False
,
istext
=
True
,
header
=
False
)
:
MAXLINESIZE
=
76
lf
=
data
.
find
(
'\n'
)
crlf
=
lf
>
0
and
data
[
lf
1
]
==
'\r'
inp
=
0
linelen
=
0
odata
=
[
]
while
inp
<
len
(
data
)
:
c
=
data
[
inp
]
if
(
c
>
'~'
or
c
==
'='
or
(
header
and
c
==
'_'
)
or
(
c
==
'.'
and
linelen
==
0
and
(
inp
+
1
==
len
(
data
)
or
data
[
inp
+
1
]
==
'\n'
or
data
[
inp
+
1
]
==
'\r'
)
)
or
(
not
istext
and
(
c
==
'\r'
or
c
==
'\n'
)
)
or
(
(
c
==
'\t'
or
c
==
' '
)
and
(
inp
+
1
==
len
(
data
)
)
)
or
(
c
<=
' '
and
c
!=
'\r'
and
c
!=
'\n'
and
(
quotetabs
or
(
not
quotetabs
and
(
c
!=
'\t'
and
c
!=
' '
)
)
)
)
)
:
linelen
+=
3
if
linelen
>=
MAXLINESIZE
:
odata
.
append
(
'='
)
if
crlf
:
odata
.
append
(
'\r'
)
odata
.
append
(
'\n'
)
linelen
=
3
odata
.
append
(
'='
+
two_hex_digits
(
ord
(
c
)
)
)
inp
+=
1
else
:
if
(
istext
and
(
c
==
'\n'
or
(
inp
+
1
<
len
(
data
)
and
c
==
'\r'
and
data
[
inp
+
1
]
==
'\n'
)
)
)
:
linelen
=
0
if
(
len
(
odata
)
>
0
and
(
odata
[
1
]
==
' '
or
odata
[
1
]
==
'\t'
)
)
:
ch
=
ord
(
odata
[
1
]
)
odata
[
1
]
=
'='
odata
.
append
(
two_hex_digits
(
ch
)
)
if
crlf
:
odata
.
append
(
'\r'
)
odata
.
append
(
'\n'
)
if
c
==
'\r'
:
inp
+=
2
else
:
inp
+=
1
else
:
if
(
inp
+
1
<
len
(
data
)
and
data
[
inp
+
1
]
!=
'\n'
and
(
linelen
+
1
)
>=
MAXLINESIZE
)
:
odata
.
append
(
'='
)
if
crlf
:
odata
.
append
(
'\r'
)
odata
.
append
(
'\n'
)
linelen
=
0
linelen
+=
1
if
header
and
c
==
' '
:
c
=
'_'
odata
.
append
(
c
)
inp
+=
1
return
''
.
join
(
odata
)

quotetabs = True means that tab and space characters are always quoted . istext = False means that \ r and \ n are treated as regular characters header = True encodes space characters with _ and requires real _ characters to be quoted .


def rlecode_hqx(s):
    """
    Run length encoding for binhex4.
    The CPython implementation does not do run length encoding
    of \x90 characters. This implementation does.
    """
    if not s:
        return ''
    result = []
    prev = s[0]
    count = 1
    # Add a dummy character to get the loop to go one extra round.
    # The dummy must be different from the last character of s.
    # In the same step we remove the first character, which has
    # already been stored in prev.
    if s[-1] == '!':
        s = s[1:] + '?'
    else:
        s = s[1:] + '!'

    for c in s:
        if c == prev and count < 255:
            count += 1
        else:
            if count == 1:
                if prev != '\x90':
                    result.append(prev)
                else:
                    result += ['\x90', '\x00']
            elif count < 4:
                if prev != '\x90':
                    result += [prev] * count
                else:
                    result += ['\x90', '\x00'] * count
            else:
                if prev != '\x90':
                    result += [prev, '\x90', chr(count)]
                else:
                    result += ['\x90', '\x00', '\x90', chr(count)]
            count = 1
            prev = c

    return ''.join(result)

def rlecode_hqx ( s ) : if not s : return '' result = [ ] prev = s [ 0 ] count = 1 if s [ - 1 ] == '!' : s = s [ 1 : ] + '?' else : s = s [ 1 : ] + '!' for c in s : if c == prev and count < 255 : count += 1 else : if count == 1 : if prev != '\x90' : result . append ( prev ) else : result += [ '\x90' , '\x00' ] elif count < 4 : if prev != '\x90' : result += [ prev ] * count else : result += [ '\x90' , '\x00' ] * count else : if prev != '\x90' : result += [ prev , '\x90' , chr ( count ) ] else : result += [ '\x90' , '\x00' , '\x90' , chr ( count ) ] count = 1 prev = c return '' . join ( result )

Run length encoding for binhex4 . The CPython implementation does not do run length encoding of \ x90 characters . This implementation does .


def _format_text(self, text):
        """
        Format a paragraph of free-form text for inclusion in the
        help output at the current indentation level.
        """
        text_width = max(self.width - self.current_indent, 11)
        indent = " "*self.current_indent
        return textwrap.fill(text,
                             text_width,
                             initial_indent=indent,
                             subsequent_indent=indent)

def
_format_text
(
self
,
text
)
:
text_width
=
max
(
self
.
width
self
.
current_indent
,
11
)
indent
=
" "
*
self
.
current_indent
return
textwrap
.
fill
(
text
,
text_width
,
initial_indent
=
indent
,
subsequent_indent
=
indent
)

Format a paragraph of free - form text for inclusion in the help output at the current indentation level .


def format_option_strings(self, option):
        """Return a comma-separated list of option strings & metavariables."""
        if option.takes_value():
            metavar = option.metavar or option.dest.upper()
            short_opts = [self._short_opt_fmt % (sopt, metavar)
                          for sopt in option._short_opts]
            long_opts = [self._long_opt_fmt % (lopt, metavar)
                         for lopt in option._long_opts]
        else:
            short_opts = option._short_opts
            long_opts = option._long_opts

        if self.short_first:
            opts = short_opts + long_opts
        else:
            opts = long_opts + short_opts

        return ", ".join(opts)

def
format_option_strings
(
self
,
option
)
:
if
option
.
takes_value
(
)
:
metavar
=
option
.
metavar
or
option
.
dest
.
upper
(
)
short_opts
=
[
self
.
_short_opt_fmt
%
(
sopt
,
metavar
)
for
sopt
in
option
.
_short_opts
]
long_opts
=
[
self
.
_long_opt_fmt
%
(
lopt
,
metavar
)
for
lopt
in
option
.
_long_opts
]
else
:
short_opts
=
option
.
_short_opts
long_opts
=
option
.
_long_opts
if
self
.
short_first
:
opts
=
short_opts
+
long_opts
else
:
opts
=
long_opts
+
short_opts
return
", "
.
join
(
opts
)

Return a comma - separated list of option strings & metavariables .


def _update_careful(self, dict):
        """
        Update the option values from an arbitrary dictionary, but only
        use keys from dict that already have a corresponding attribute
        in self.  Any keys in dict without a corresponding attribute
        are silently ignored.
        """
        for attr in dir(self):
            if attr in dict:
                dval = dict[attr]
                if dval is not None:
                    setattr(self, attr, dval)

def _update_careful ( self , dict ) : for attr in dir ( self ) : if attr in dict : dval = dict [ attr ] if dval is not None : setattr ( self , attr , dval )

Update the option values from an arbitrary dictionary but only use keys from dict that already have a corresponding attribute in self . Any keys in dict without a corresponding attribute are silently ignored .


def insort_right(a, x, lo=0, hi=None):
    """Insert item x in list a, and keep it sorted assuming a is sorted.

    If x is already in a, insert it to the right of the rightmost x.

    Optional args lo (default 0) and hi (default len(a)) bound the
    slice of a to be searched.
    """

    if lo < 0:
        raise ValueError('lo must be non-negative')
    if hi is None:
        hi = len(a)
    while lo < hi:
        mid = (lo+hi)//2
        if x < a[mid]: hi = mid
        else: lo = mid+1
    a.insert(lo, x)

def
insort_right
(
a
,
x
,
lo
=
0
,
hi
=
)
:
if
lo
<
0
:
raise
ValueError
(
'lo must be non-negative'
)
if
hi
is
:
hi
=
len
(
a
)
while
lo
<
hi
:
mid
=
(
lo
+
hi
)
//
2
if
x
<
a
[
mid
]
:
hi
=
mid
else
:
lo
=
mid
+
1
a
.
insert
(
lo
,
x
)

Insert item x in list a and keep it sorted assuming a is sorted .


def lock(self, function, argument):
        """Lock a mutex, call the function with supplied argument
        when it is acquired.  If the mutex is already locked, place
        function and argument in the queue."""
        if self.testandset():
            function(argument)
        else:
            self.queue.append((function, argument))

def lock ( self , function , argument ) : if self . testandset ( ) : function ( argument ) else : self . queue . append ( ( function , argument ) )

Lock a mutex call the function with supplied argument when it is acquired . If the mutex is already locked place function and argument in the queue .


def unlock(self):
        """Unlock a mutex.  If the queue is not empty, call the next
        function with its argument."""
        if self.queue:
            function, argument = self.queue.popleft()
            function(argument)
        else:
            self.locked = False

def unlock ( self ) : if self . queue : function , argument = self . queue . popleft ( ) function ( argument ) else : self . locked = False

Unlock a mutex . If the queue is not empty call the next function with its argument .


def copy(self):
    """Return a clone object.

    Return a copy ('clone') of the md5 object. This can be used
    to efficiently compute the digests of strings that share
    a common initial substring.
    """
    if 0:  # set this to 1 to make the flow space crash
      return copy.deepcopy(self)
    clone = self.__class__()
    clone.length = self.length
    clone.count = [] + self.count[:]
    clone.input = [] + self.input
    clone.A = self.A
    clone.B = self.B
    clone.C = self.C
    clone.D = self.D
    return clone

def copy ( self ) : if 0 : return copy . deepcopy ( self ) clone = self . __class__ ( ) clone . length = self . length clone . count = [ ] + self . count [ : ] clone . input = [ ] + self . input clone . A = self . A clone . B = self . B clone . C = self . C clone . D = self . D return clone


def search(self, string, pos=0, endpos=sys.maxint):
        """Scan through string looking for a location where this regular
        expression produces a match, and return a corresponding MatchObject
        instance. Return None if no position in the string matches the
        pattern."""
        state = _State(string, pos, endpos, self.flags)
        if state.search(self._code):
            return SRE_Match(self, state)
        else:
            return None

def search ( self , string , pos = 0 , endpos = sys . maxint ) : state = _State ( string , pos , endpos , self . flags ) if state . search ( self . _code ) : return SRE_Match ( self , state ) else : return None

Scan through string looking for a location where this regular expression produces a match and return a corresponding MatchObject instance . Return None if no position in the string matches the pattern .


def sub(self, repl, string, count=0):
        """Return the string obtained by replacing the leftmost non-overlapping
        occurrences of pattern in string by the replacement repl."""
        return self._subx(repl, string, count, False)

def sub ( self , repl , string , count = 0 ) : return self . _subx ( repl , string , count , False )

Return the string obtained by replacing the leftmost non - overlapping occurrences of pattern in string by the replacement repl .


def split(self, string, maxsplit=0):
        """Split string by the occurrences of pattern."""
        splitlist = []
        state = _State(string, 0, sys.maxint, self.flags)
        n = 0
        last = state.start
        while not maxsplit or n < maxsplit:
            state.reset()
            state.string_position = state.start
            if not state.search(self._code):
                break
            if state.start == state.string_position: # zero-width match
                if last == state.end:                # or end of string
                    break
                state.start += 1
                continue
            splitlist.append(string[last:state.start])
            # add groups (if any)
            if self.groups:
                match = SRE_Match(self, state)
                # TODO: Use .extend once it is implemented.
                # splitlist.extend(list(match.groups(None)))
                splitlist += (list(match.groups(None)))
            n += 1
            last = state.start = state.string_position
        splitlist.append(string[last:state.end])
        return splitlist

def split ( self , string , maxsplit = 0 ) : splitlist = [ ] state = _State ( string , 0 , sys . maxint , self . flags ) n = 0 last = state . start while not maxsplit or n < maxsplit : state . reset ( ) state . string_position = state . start if not state . search ( self . _code ) : break if state . start == state . string_position : if last == state . end : break state . start += 1 continue splitlist . append ( string [ last : state . start ] ) if self . groups : match = SRE_Match ( self , state ) splitlist += ( list ( match . groups ( None ) ) ) n += 1 last = state . start = state . string_position splitlist . append ( string [ last : state . end ] ) return splitlist

Split string by the occurrences of pattern .


def _create_regs(self, state):
        """Creates a tuple of index pairs representing matched groups."""
        regs = [(state.start, state.string_position)]
        for group in range(self.re.groups):
            mark_index = 2 * group
            if mark_index + 1 < len(state.marks) \
                                    and state.marks[mark_index] is not None \
                                    and state.marks[mark_index + 1] is not None:
                regs.append((state.marks[mark_index], state.marks[mark_index + 1]))
            else:
                regs.append((-1, -1))
        return tuple(regs)

def _create_regs ( self , state ) : regs = [ ( state . start , state . string_position ) ] for group in range ( self . re . groups ) : mark_index = 2 * group if mark_index + 1 < len ( state . marks ) and state . marks [ mark_index ] is not None and state . marks [ mark_index + 1 ] is not None : regs . append ( ( state . marks [ mark_index ] , state . marks [ mark_index + 1 ] ) ) else : regs . append ( ( - 1 , - 1 ) ) return tuple ( regs )

Creates a tuple of index pairs representing matched groups .


def group(self, *args):
        """Returns one or more subgroups of the match. Each argument is either a
        group index or a group name."""
        if len(args) == 0:
            args = (0,)
        grouplist = []
        for group in args:
            grouplist.append(self._get_slice(self._get_index(group), None))
        if len(grouplist) == 1:
            return grouplist[0]
        else:
            return tuple(grouplist)

def group ( self , * args ) : if len ( args ) == 0 : args = ( 0 , ) grouplist = [ ] for group in args : grouplist . append ( self . _get_slice ( self . _get_index ( group ) , None ) ) if len ( grouplist ) == 1 : return grouplist [ 0 ] else : return tuple ( grouplist )

Returns one or more subgroups of the match . Each argument is either a group index or a group name .


def fast_search(self, pattern_codes):
        """Skips forward in a string as fast as possible using information from
        an optimization info block."""
        # pattern starts with a known prefix
        # <5=length> <6=skip> <7=prefix data> <overlap data>
        flags = pattern_codes[2]
        prefix_len = pattern_codes[5]
        prefix_skip = pattern_codes[6] # don't really know what this is good for
        prefix = pattern_codes[7:7 + prefix_len]
        overlap = pattern_codes[7 + prefix_len - 1:pattern_codes[1] + 1]
        pattern_codes = pattern_codes[pattern_codes[1] + 1:]
        i = 0
        string_position = self.string_position
        while string_position < self.end:
            while True:
                if ord(self.string[string_position]) != prefix[i]:
                    if i == 0:
                        break
                    else:
                        i = overlap[i]
                else:
                    i += 1
                    if i == prefix_len:
                        # found a potential match
                        self.start = string_position + 1 - prefix_len
                        self.string_position = string_position + 1 \
                                                     - prefix_len + prefix_skip
                        if flags & SRE_INFO_LITERAL:
                            return True # matched all of pure literal pattern
                        if self.match(pattern_codes[2 * prefix_skip:]):
                            return True
                        i = overlap[i]
                    break
            string_position += 1
        return False

def fast_search ( self , pattern_codes ) : flags = pattern_codes [ 2 ] prefix_len = pattern_codes [ 5 ] prefix_skip = pattern_codes [ 6 ] prefix = pattern_codes [ 7 : 7 + prefix_len ] overlap = pattern_codes [ 7 + prefix_len - 1 : pattern_codes [ 1 ] + 1 ] pattern_codes = pattern_codes [ pattern_codes [ 1 ] + 1 : ] i = 0 string_position = self . string_position while string_position < self . end : while True : if ord ( self . string [ string_position ] ) != prefix [ i ] : if i == 0 : break else : i = overlap [ i ] else : i += 1 if i == prefix_len : self . start = string_position + 1 - prefix_len self . string_position = string_position + 1 - prefix_len + prefix_skip if flags & SRE_INFO_LITERAL : return True if self . match ( pattern_codes [ 2 * prefix_skip : ] ) : return True i = overlap [ i ] break string_position += 1 return False

Skips forward in a string as fast as possible using information from an optimization info block .


def push_new_context(self, pattern_offset):
        """Creates a new child context of this context and pushes it on the
        stack. pattern_offset is the offset off the current code position to
        start interpreting from."""
        child_context = _MatchContext(self.state,
            self.pattern_codes[self.code_position + pattern_offset:])
        self.state.context_stack.append(child_context)
        return child_context

def push_new_context ( self , pattern_offset ) : child_context = _MatchContext ( self . state , self . pattern_codes [ self . code_position + pattern_offset : ] ) self . state . context_stack . append ( child_context ) return child_context

Creates a new child context of this context and pushes it on the stack . pattern_offset is the offset off the current code position to start interpreting from .


def match(self, context):
        """Returns True if the current context matches, False if it doesn't and
        None if matching is not finished, ie must be resumed after child
        contexts have been matched."""
        while context.remaining_codes() > 0 and context.has_matched is None:
            opcode = context.peek_code()
            if not self.dispatch(opcode, context):
                return None
        if context.has_matched is None:
            context.has_matched = False
        return context.has_matched

def match ( self , context ) : while context . remaining_codes ( ) > 0 and context . has_matched is None : opcode = context . peek_code ( ) if not self . dispatch ( opcode , context ) : return None if context . has_matched is None : context . has_matched = False return context . has_matched

Returns True if the current context matches False if it doesn t and None if matching is not finished ie must be resumed after child contexts have been matched .


def dispatch(self, opcode, context):
        """Dispatches a context on a given opcode. Returns True if the context
        is done matching, False if it must be resumed when next encountered."""
        if id(context) in self.executing_contexts:
            generator = self.executing_contexts[id(context)]
            del self.executing_contexts[id(context)]
            has_finished = generator.next()
        else:
            method = self.DISPATCH_TABLE.get(opcode, _OpcodeDispatcher.unknown)
            has_finished = method(self, context)
            if hasattr(has_finished, "next"): # avoid using the types module
                generator = has_finished
                has_finished = generator.next()
        if not has_finished:
            self.executing_contexts[id(context)] = generator
        return has_finished

def dispatch ( self , opcode , context ) : if id ( context ) in self . executing_contexts : generator = self . executing_contexts [ id ( context ) ] del self . executing_contexts [ id ( context ) ] has_finished = generator . next ( ) else : method = self . DISPATCH_TABLE . get ( opcode , _OpcodeDispatcher . unknown ) has_finished = method ( self , context ) if hasattr ( has_finished , "next" ) : generator = has_finished has_finished = generator . next ( ) if not has_finished : self . executing_contexts [ id ( context ) ] = generator return has_finished

Dispatches a context on a given opcode . Returns True if the context is done matching False if it must be resumed when next encountered .


def check_charset(self, ctx, char):
        """Checks whether a character matches set of arbitrary length. Assumes
        the code pointer is at the first member of the set."""
        self.set_dispatcher.reset(char)
        save_position = ctx.code_position
        result = None
        while result is None:
            result = self.set_dispatcher.dispatch(ctx.peek_code(), ctx)
        ctx.code_position = save_position
        return result

def check_charset ( self , ctx , char ) : self . set_dispatcher . reset ( char ) save_position = ctx . code_position result = None while result is None : result = self . set_dispatcher . dispatch ( ctx . peek_code ( ) , ctx ) ctx . code_position = save_position return result

Checks whether a character matches set of arbitrary length . Assumes the code pointer is at the first member of the set .


def unexpo(intpart, fraction, expo):
    """Remove the exponent by changing intpart and fraction."""
    if expo > 0: # Move the point left
        f = len(fraction)
        intpart, fraction = intpart + fraction[:expo], fraction[expo:]
        if expo > f:
            intpart = intpart + '0'*(expo-f)
    elif expo < 0: # Move the point right
        i = len(intpart)
        intpart, fraction = intpart[:expo], intpart[expo:] + fraction
        if expo < -i:
            fraction = '0'*(-expo-i) + fraction
    return intpart, fraction

def unexpo ( intpart , fraction , expo ) : if expo > 0 : f = len ( fraction ) intpart , fraction = intpart + fraction [ : expo ] , fraction [ expo : ] if expo > f : intpart = intpart + '0' * ( expo - f ) elif expo < 0 : i = len ( intpart ) intpart , fraction = intpart [ : expo ] , intpart [ expo : ] + fraction if expo < - i : fraction = '0' * ( - expo - i ) + fraction return intpart , fraction

Remove the exponent by changing intpart and fraction .


def roundfrac(intpart, fraction, digs):
    """Round or extend the fraction to size digs."""
    f = len(fraction)
    if f <= digs:
        return intpart, fraction + '0'*(digs-f)
    i = len(intpart)
    if i+digs < 0:
        return '0'*-digs, ''
    total = intpart + fraction
    nextdigit = total[i+digs]
    if nextdigit >= '5': # Hard case: increment last digit, may have carry!
        n = i + digs - 1
        while n >= 0:
            if total[n] != '9': break
            n = n-1
        else:
            total = '0' + total
            i = i+1
            n = 0
        total = total[:n] + chr(ord(total[n]) + 1) + '0'*(len(total)-n-1)
        intpart, fraction = total[:i], total[i:]
    if digs >= 0:
        return intpart, fraction[:digs]
    else:
        return intpart[:digs] + '0'*-digs, ''

def roundfrac ( intpart , fraction , digs ) : f = len ( fraction ) if f <= digs : return intpart , fraction + '0' * ( digs - f ) i = len ( intpart ) if i + digs < 0 : return '0' * - digs , '' total = intpart + fraction nextdigit = total [ i + digs ] if nextdigit >= '5' : n = i + digs - 1 while n >= 0 : if total [ n ] != '9' : break n = n - 1 else : total = '0' + total i = i + 1 n = 0 total = total [ : n ] + chr ( ord ( total [ n ] ) + 1 ) + '0' * ( len ( total ) - n - 1 ) intpart , fraction = total [ : i ] , total [ i : ] if digs >= 0 : return intpart , fraction [ : digs ] else : return intpart [ : digs ] + '0' * - digs , ''

Round or extend the fraction to size digs .


def _randbelow(self, n):
    """Return a random int in the range [0,n)."""
    # TODO
    # change once int.bit_length is implemented.
    # k = n.bit_length()
    k = _int_bit_length(n)
    r = self.getrandbits(k)
    while r >= n:
      r = self.getrandbits(k)
    return r

def _randbelow ( self , n ) : k = _int_bit_length ( n ) r = self . getrandbits ( k ) while r >= n : r = self . getrandbits ( k ) return r

Return a random int in the range [ 0 n ) .


def filter(names, pat):
    """Return the subset of the list NAMES that match PAT"""
    import os
    # import posixpath
    result=[]
    # pat=os.path.normcase(pat)
    try:
        re_pat = _cache[pat]
    except KeyError:
        res = translate(pat)
        if len(_cache) >= _MAXCACHE:
            # _cache.clear()
            globals()['_cache'] = {}
        _cache[pat] = re_pat = re.compile(res)
    match = re_pat.match
    # if os.path is posixpath:
    if 1:
        # normcase on posix is NOP. Optimize it away from the loop.
        for name in names:
            if match(name):
                result.append(name)
    else:
        for name in names:
            if match(os.path.normcase(name)):
                result.append(name)
    return result

def filter ( names , pat ) : import os result = [ ] try : re_pat = _cache [ pat ] except KeyError : res = translate ( pat ) if len ( _cache ) >= _MAXCACHE : globals ( ) [ '_cache' ] = { } _cache [ pat ] = re_pat = re . compile ( res ) match = re_pat . match if 1 : for name in names : if match ( name ) : result . append ( name ) else : for name in names : if match ( os . path . normcase ( name ) ) : result . append ( name ) return result


def fnmatchcase(name, pat):
    """Test whether FILENAME matches PATTERN, including case.

    This is a version of fnmatch() which doesn't case-normalize
    its arguments.
    """

    try:
        re_pat = _cache[pat]
    except KeyError:
        res = translate(pat)
        if len(_cache) >= _MAXCACHE:
            # _cache.clear()
            globals()['_cache'] = {}
        _cache[pat] = re_pat = re.compile(res)
    return re_pat.match(name) is not None

def fnmatchcase ( name , pat ) : try : re_pat = _cache [ pat ] except KeyError : res = translate ( pat ) if len ( _cache ) >= _MAXCACHE : globals ( ) [ '_cache' ] = { } _cache [ pat ] = re_pat = re . compile ( res ) return re_pat . match ( name ) is not None

Test whether FILENAME matches PATTERN including case .


def translate(pat):
    """Translate a shell PATTERN to a regular expression.

    There is no way to quote meta-characters.
    """

    i, n = 0, len(pat)
    res = ''
    while i < n:
        c = pat[i]
        i = i+1
        if c == '*':
            res = res + '.*'
        elif c == '?':
            res = res + '.'
        elif c == '[':
            j = i
            if j < n and pat[j] == '!':
                j = j+1
            if j < n and pat[j] == ']':
                j = j+1
            while j < n and pat[j] != ']':
                j = j+1
            if j >= n:
                res = res + '\\['
            else:
                stuff = pat[i:j].replace('\\','\\\\')
                i = j+1
                if stuff[0] == '!':
                    stuff = '^' + stuff[1:]
                elif stuff[0] == '^':
                    stuff = '\\' + stuff
                res = '%s[%s]' % (res, stuff)
        else:
            res = res + re.escape(c)
    return res + '\Z(?ms)'

def translate ( pat ) : i , n = 0 , len ( pat ) res = '' while i < n : c = pat [ i ] i = i + 1 if c == '*' : res = res + '.*' elif c == '?' : res = res + '.' elif c == '[' : j = i if j < n and pat [ j ] == '!' : j = j + 1 if j < n and pat [ j ] == ']' : j = j + 1 while j < n and pat [ j ] != ']' : j = j + 1 if j >= n : res = res + '\\[' else : stuff = pat [ i : j ] . replace ( '\\' , '\\\\' ) i = j + 1 if stuff [ 0 ] == '!' : stuff = '^' + stuff [ 1 : ] elif stuff [ 0 ] == '^' : stuff = '\\' + stuff res = '%s[%s]' % ( res , stuff ) else : res = res + re . escape ( c ) return res + '\Z(?ms)'

Translate a shell PATTERN to a regular expression .


def task_done(self):
        """Indicate that a formerly enqueued task is complete.

        Used by Queue consumer threads.  For each get() used to fetch a task,
        a subsequent call to task_done() tells the queue that the processing
        on the task is complete.

        If a join() is currently blocking, it will resume when all items
        have been processed (meaning that a task_done() call was received
        for every item that had been put() into the queue).

        Raises a ValueError if called more times than there were items
        placed in the queue.
        """
        self.all_tasks_done.acquire()
        try:
            unfinished = self.unfinished_tasks - 1
            if unfinished <= 0:
                if unfinished < 0:
                    raise ValueError('task_done() called too many times')
                self.all_tasks_done.notify_all()
            self.unfinished_tasks = unfinished
        finally:
            self.all_tasks_done.release()

def
task_done
(
self
)
:
self
.
all_tasks_done
.
acquire
(
)
try
:
unfinished
=
self
.
unfinished_tasks
1
if
unfinished
<=
0
:
if
unfinished
<
0
:
raise
ValueError
(
'task_done() called too many times'
)
self
.
all_tasks_done
.
notify_all
(
)
self
.
unfinished_tasks
=
unfinished
finally
:
self
.
all_tasks_done
.
release
(
)

Indicate that a formerly enqueued task is complete .


def put(self, item, block=True, timeout=None):
        """Put an item into the queue.

        If optional args 'block' is true and 'timeout' is None (the default),
        block if necessary until a free slot is available. If 'timeout' is
        a non-negative number, it blocks at most 'timeout' seconds and raises
        the Full exception if no free slot was available within that time.
        Otherwise ('block' is false), put an item on the queue if a free slot
        is immediately available, else raise the Full exception ('timeout'
        is ignored in that case).
        """
        self.not_full.acquire()
        try:
            if self.maxsize > 0:
                if not block:
                    if self._qsize() == self.maxsize:
                        raise Full
                elif timeout is None:
                    while self._qsize() == self.maxsize:
                        self.not_full.wait()
                elif timeout < 0:
                    raise ValueError("'timeout' must be a non-negative number")
                else:
                    endtime = _time() + timeout
                    while self._qsize() == self.maxsize:
                        remaining = endtime - _time()
                        if remaining <= 0.0:
                            raise Full
                        self.not_full.wait(remaining)
            self._put(item)
            self.unfinished_tasks += 1
            self.not_empty.notify()
        finally:
            self.not_full.release()


def calculate_transitive_deps(modname, script, gopath):
  """Determines all modules that script transitively depends upon."""
  deps = set()
  def calc(modname, script):
    if modname in deps:
      return
    deps.add(modname)
    for imp in collect_imports(modname, script, gopath):
      if imp.is_native:
        deps.add(imp.name)
        continue
      parts = imp.name.split('.')
      calc(imp.name, imp.script)
      if len(parts) == 1:
        continue
      # For submodules, the parent packages are also deps.
      package_dir, filename = os.path.split(imp.script)
      if filename == '__init__.py':
        package_dir = os.path.dirname(package_dir)
      for i in xrange(len(parts) - 1, 0, -1):
        modname = '.'.join(parts[:i])
        script = os.path.join(package_dir, '__init__.py')
        calc(modname, script)
        package_dir = os.path.dirname(package_dir)
  calc(modname, script)
  deps.remove(modname)
  return deps

def calculate_transitive_deps ( modname , script , gopath ) : deps = set ( ) def calc ( modname , script ) : if modname in deps : return deps . add ( modname ) for imp in collect_imports ( modname , script , gopath ) : if imp . is_native : deps . add ( imp . name ) continue parts = imp . name . split ( '.' ) calc ( imp . name , imp . script ) if len ( parts ) == 1 : continue package_dir , filename = os . path . split ( imp . script ) if filename == '__init__.py' : package_dir = os . path . dirname ( package_dir ) for i in xrange ( len ( parts ) - 1 , 0 , - 1 ) : modname = '.' . join ( parts [ : i ] ) script = os . path . join ( package_dir , '__init__.py' ) calc ( modname , script ) package_dir = os . path . dirname ( package_dir ) calc ( modname , script ) deps . remove ( modname ) return deps

Determines all modules that script transitively depends upon .


def _make_future_features(node):
  """Processes a future import statement, returning set of flags it defines."""
  assert isinstance(node, ast.ImportFrom)
  assert node.module == '__future__'
  features = FutureFeatures()
  for alias in node.names:
    name = alias.name
    if name in _FUTURE_FEATURES:
      if name not in _IMPLEMENTED_FUTURE_FEATURES:
        msg = 'future feature {} not yet implemented by grumpy'.format(name)
        raise util.ParseError(node, msg)
      setattr(features, name, True)
    elif name == 'braces':
      raise util.ParseError(node, 'not a chance')
    elif name not in _REDUNDANT_FUTURE_FEATURES:
      msg = 'future feature {} is not defined'.format(name)
      raise util.ParseError(node, msg)
  return features

Processes a future import statement returning set of flags it defines .


def nested(*managers):
    """Combine multiple context managers into a single nested context manager.

   This function has been deprecated in favour of the multiple manager form
   of the with statement.

   The one advantage of this function over the multiple manager form of the
   with statement is that argument unpacking allows it to be
   used with a variable number of context managers as follows:

      with nested(*managers):
          do_something()

    """
    warn("With-statements now directly support multiple context managers",
         DeprecationWarning, 3)
    exits = []
    vars = []
    exc = (None, None, None)
    try:
        for mgr in managers:
            exit = mgr.__exit__
            enter = mgr.__enter__
            vars.append(enter())
            exits.append(exit)
        yield vars
    except:
        exc = sys.exc_info()
    finally:
        while exits:
            exit = exits.pop()
            try:
                if exit(*exc):
                    exc = (None, None, None)
            except:
                exc = sys.exc_info()
        if exc != (None, None, None):
            # Don't rely on sys.exc_info() still containing
            # the right information. Another exception may
            # have been raised and caught by an exit method
            raise exc[0], exc[1], exc[2]

Combine multiple context managers into a single nested context manager .


def tf_loss(self, states, internals, reward, update, reference=None):
        """
        Creates the TensorFlow operations for calculating the L2 loss between predicted
        state values and actual rewards.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            reward: Reward tensor.
            update: Boolean tensor indicating whether this call happens during an update.
            reference: Optional reference tensor(s), in case of a comparative loss.

        Returns:
            Loss tensor
        """
        prediction = self.predict(states=states, internals=internals, update=update)
        return tf.nn.l2_loss(t=(prediction - reward))

def tf_loss ( self , states , internals , reward , update , reference = None ) : prediction = self . predict ( states = states , internals = internals , update = update ) return tf . nn . l2_loss ( t = ( prediction - reward ) )

Creates the TensorFlow operations for calculating the L2 loss between predicted state values and actual rewards .


def get_variables(self, include_nontrainable=False):
        """
        Returns the TensorFlow variables used by the baseline.

        Returns:
            List of variables
        """
        if include_nontrainable:
            return [self.all_variables[key] for key in sorted(self.all_variables)]
        else:
            return [self.variables[key] for key in sorted(self.variables)]

def get_variables ( self , include_nontrainable = False ) : if include_nontrainable : return [ self . all_variables [ key ] for key in sorted ( self . all_variables ) ] else : return [ self . variables [ key ] for key in sorted ( self . variables ) ]

Returns the TensorFlow variables used by the baseline .


def from_spec(spec, kwargs=None):
        """
        Creates a baseline from a specification dict.
        """
        baseline = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.baselines.baselines,
            kwargs=kwargs
        )
        assert isinstance(baseline, Baseline)
        return baseline

def from_spec ( spec , kwargs = None ) : baseline = util . get_object ( obj = spec , predefined_objects = tensorforce . core . baselines . baselines , kwargs = kwargs ) assert isinstance ( baseline , Baseline ) return baseline

Creates a baseline from a specification dict .


def reset(self):
        """
        Resets the environment to its initialization state. This method needs to be called to start a
        new episode after the last episode ended.

        :return: initial state
        """
        self.level.reset()  # optional: episode=-1, seed=None
        return self.level.observations()[self.state_attribute]

def reset ( self ) : self . level . reset ( ) return self . level . observations ( ) [ self . state_attribute ]

Resets the environment to its initialization state . This method needs to be called to start a new episode after the last episode ended .


def execute(self, action):
        """
        Pass action to universe environment, return reward, next step, terminal state and
        additional info.

        :param action: action to execute as numpy array, should have dtype np.intc and should adhere to
            the specification given in DeepMindLabEnvironment.action_spec(level_id)
        :return: dict containing the next state, the reward, and a boolean indicating if the
            next state is a terminal state
        """
        adjusted_action = list()
        for action_spec in self.level.action_spec():
            if action_spec['min'] == -1 and action_spec['max'] == 1:
                adjusted_action.append(action[action_spec['name']] - 1)
            else:
                adjusted_action.append(action[action_spec['name']])  # clip?
        action = np.array(adjusted_action, dtype=np.intc)

        reward = self.level.step(action=action, num_steps=self.repeat_action)
        state = self.level.observations()['RGB_INTERLACED']
        terminal = not self.level.is_running()
        return state, terminal, reward

def execute ( self , action ) : adjusted_action = list ( ) for action_spec in self . level . action_spec ( ) : if action_spec [ 'min' ] == - 1 and action_spec [ 'max' ] == 1 : adjusted_action . append ( action [ action_spec [ 'name' ] ] - 1 ) else : adjusted_action . append ( action [ action_spec [ 'name' ] ] ) action = np . array ( adjusted_action , dtype = np . intc ) reward = self . level . step ( action = action , num_steps = self . repeat_action ) state = self . level . observations ( ) [ 'RGB_INTERLACED' ] terminal = not self . level . is_running ( ) return state , terminal , reward

Pass action to universe environment return reward next step terminal state and additional info .


def tf_step(self, x, iteration, conjugate, residual, squared_residual):
        """
        Iteration loop body of the conjugate gradient algorithm.

        Args:
            x: Current solution estimate $x_t$.
            iteration: Current iteration counter $t$.
            conjugate: Current conjugate $c_t$.
            residual: Current residual $r_t$.
            squared_residual: Current squared residual $r_t^2$.

        Returns:
            Updated arguments for next iteration.
        """
        x, next_iteration, conjugate, residual, squared_residual = super(ConjugateGradient, self).tf_step(
            x, iteration, conjugate, residual, squared_residual
        )

        # Ac := A * c_t
        A_conjugate = self.fn_x(conjugate)

        # TODO: reference?
        if self.damping > 0.0:
            A_conjugate = [A_conj + self.damping * conj for A_conj, conj in zip(A_conjugate, conjugate)]

        # cAc := c_t^T * Ac
        conjugate_A_conjugate = tf.add_n(
            inputs=[tf.reduce_sum(input_tensor=(conj * A_conj)) for conj, A_conj in zip(conjugate, A_conjugate)]
        )

        # \alpha := r_t^2 / cAc
        alpha = squared_residual / tf.maximum(x=conjugate_A_conjugate, y=util.epsilon)

        # x_{t+1} := x_t + \alpha * c_t
        next_x = [t + alpha * conj for t, conj in zip(x, conjugate)]

        # r_{t+1} := r_t - \alpha * Ac
        next_residual = [res - alpha * A_conj for res, A_conj in zip(residual, A_conjugate)]

        # r_{t+1}^2 := r_{t+1}^T * r_{t+1}
        next_squared_residual = tf.add_n(inputs=[tf.reduce_sum(input_tensor=(res * res)) for res in next_residual])

        # \beta = r_{t+1}^2 / r_t^2
        beta = next_squared_residual / tf.maximum(x=squared_residual, y=util.epsilon)

        # c_{t+1} := r_{t+1} + \beta * c_t
        next_conjugate = [res + beta * conj for res, conj in zip(next_residual, conjugate)]

        return next_x, next_iteration, next_conjugate, next_residual, next_squared_residual

def tf_step ( self , x , iteration , conjugate , residual , squared_residual ) : x , next_iteration , conjugate , residual , squared_residual = super ( ConjugateGradient , self ) . tf_step ( x , iteration , conjugate , residual , squared_residual ) A_conjugate = self . fn_x ( conjugate ) if self . damping > 0.0 : A_conjugate = [ A_conj + self . damping * conj for A_conj , conj in zip ( A_conjugate , conjugate ) ] conjugate_A_conjugate = tf . add_n ( inputs = [ tf . reduce_sum ( input_tensor = ( conj * A_conj ) ) for conj , A_conj in zip ( conjugate , A_conjugate ) ] ) alpha = squared_residual / tf . maximum ( x = conjugate_A_conjugate , y = util . epsilon ) next_x = [ t + alpha * conj for t , conj in zip ( x , conjugate ) ] next_residual = [ res - alpha * A_conj for res , A_conj in zip ( residual , A_conjugate ) ] next_squared_residual = tf . add_n ( inputs = [ tf . reduce_sum ( input_tensor = ( res * res ) ) for res in next_residual ] ) beta = next_squared_residual / tf . maximum ( x = squared_residual , y = util . epsilon ) next_conjugate = [ res + beta * conj for res , conj in zip ( next_residual , conjugate ) ] return next_x , next_iteration , next_conjugate , next_residual , next_squared_residual

Iteration loop body of the conjugate gradient algorithm .


def from_spec(spec, kwargs=None):
        """
        Creates a layer from a specification dict.
        """
        layer = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.networks.layers,
            kwargs=kwargs
        )
        assert isinstance(layer, Layer)
        return layer

def from_spec ( spec , kwargs = None ) : layer = util . get_object ( obj = spec , predefined_objects = tensorforce . core . networks . layers , kwargs = kwargs ) assert isinstance ( layer , Layer ) return layer

Creates a layer from a specification dict .


def target_optimizer_arguments(self):
        """
        Returns the target optimizer arguments including the time, the list of variables to
        optimize, and various functions which the optimizer might require to perform an update
        step.

        Returns:
            Target optimizer arguments as dict.
        """
        variables = self.target_network.get_variables() + [
            variable for name in sorted(self.target_distributions)
            for variable in self.target_distributions[name].get_variables()
        ]
        source_variables = self.network.get_variables() + [
            variable for name in sorted(self.distributions)
            for variable in self.distributions[name].get_variables()
        ]
        arguments = dict(
            time=self.global_timestep,
            variables=variables,
            source_variables=source_variables
        )
        if self.global_model is not None:
            arguments['global_variables'] = self.global_model.target_network.get_variables() + [
                variable for name in sorted(self.global_model.target_distributions)
                for variable in self.global_model.target_distributions[name].get_variables()
            ]
        return arguments

def target_optimizer_arguments ( self ) : variables = self . target_network . get_variables ( ) + [ variable for name in sorted ( self . target_distributions ) for variable in self . target_distributions [ name ] . get_variables ( ) ] source_variables = self . network . get_variables ( ) + [ variable for name in sorted ( self . distributions ) for variable in self . distributions [ name ] . get_variables ( ) ] arguments = dict ( time = self . global_timestep , variables = variables , source_variables = source_variables ) if self . global_model is not None : arguments [ 'global_variables' ] = self . global_model . target_network . get_variables ( ) + [ variable for name in sorted ( self . global_model . target_distributions ) for variable in self . global_model . target_distributions [ name ] . get_variables ( ) ] return arguments

Returns the target optimizer arguments including the time the list of variables to optimize and various functions which the optimizer might require to perform an update step .


def from_spec(spec, kwargs):
        """
        Creates an environment from a specification dict.
        """
        env = tensorforce.util.get_object(
            obj=spec,
            predefined_objects=tensorforce.environments.environments,
            kwargs=kwargs
        )
        assert isinstance(env, Environment)
        return env

def from_spec ( spec , kwargs ) : env = tensorforce . util . get_object ( obj = spec , predefined_objects = tensorforce . environments . environments , kwargs = kwargs ) assert isinstance ( env , Environment ) return env

Creates an environment from a specification dict .


def setup(app):
    """When used for spinx extension."""
    global _is_sphinx
    _is_sphinx = True
    app.add_config_value('no_underscore_emphasis', False, 'env')
    app.add_source_parser('.md', M2RParser)
    app.add_directive('mdinclude', MdInclude)

def setup ( app ) : global _is_sphinx _is_sphinx = True app . add_config_value ( 'no_underscore_emphasis' , False , 'env' ) app . add_source_parser ( '.md' , M2RParser ) app . add_directive ( 'mdinclude' , MdInclude )


def output_image_link(self, m):
        """Pass through rest role."""
        return self.renderer.image_link(
            m.group('url'), m.group('target'), m.group('alt'))

def output_image_link ( self , m ) : return self . renderer . image_link ( m . group ( 'url' ) , m . group ( 'target' ) , m . group ( 'alt' ) )


def output_eol_literal_marker(self, m):
        """Pass through rest link."""
        marker = ':' if m.group(1) is None else ''
        return self.renderer.eol_literal_marker(marker)

def output_eol_literal_marker ( self , m ) : marker = ':' if m . group ( 1 ) is None else '' return self . renderer . eol_literal_marker ( marker )


def table(self, header, body):
        """Rendering table element. Wrap header and body in it.

        :param header: header part of the table.
        :param body: body part of the table.
        """
        table = '\n.. list-table::\n'
        if header and not header.isspace():
            table = (table + self.indent + ':header-rows: 1\n\n' +
                     self._indent_block(header) + '\n')
        else:
            table = table + '\n'
        table = table + self._indent_block(body) + '\n\n'
        return table

def
table
(
self
,
header
,
body
)
:
table
=
'\n.. list-table::\n'
if
header
and
not
header
.
isspace
(
)
:
table
=
(
table
+
self
.
indent
+
':header-rows: 1\n\n'
+
self
.
_indent_block
(
header
)
+
'\n'
)
else
:
table
=
table
+
'\n'
table
=
table
+
self
.
_indent_block
(
body
)
+
'\n\n'
return
table

Rendering table element . Wrap header and body in it .


def run(self):
        """Most of this method is from ``docutils.parser.rst.Directive``.

        docutils version: 0.12
        """
        if not self.state.document.settings.file_insertion_enabled:
            raise self.warning('"%s" directive disabled.' % self.name)
        source = self.state_machine.input_lines.source(
            self.lineno - self.state_machine.input_offset - 1)
        source_dir = os.path.dirname(os.path.abspath(source))
        path = rst.directives.path(self.arguments[0])
        path = os.path.normpath(os.path.join(source_dir, path))
        path = utils.relative_path(None, path)
        path = nodes.reprunicode(path)

        # get options (currently not use directive-specific options)
        encoding = self.options.get(
            'encoding', self.state.document.settings.input_encoding)
        e_handler = self.state.document.settings.input_encoding_error_handler
        tab_width = self.options.get(
            'tab-width', self.state.document.settings.tab_width)

        # open the inclding file
        try:
            self.state.document.settings.record_dependencies.add(path)
            include_file = io.FileInput(source_path=path,
                                        encoding=encoding,
                                        error_handler=e_handler)
        except UnicodeEncodeError as error:
            raise self.severe('Problems with "%s" directive path:\n'
                              'Cannot encode input file path "%s" '
                              '(wrong locale?).' %
                              (self.name, SafeString(path)))
        except IOError as error:
            raise self.severe('Problems with "%s" directive path:\n%s.' %
                              (self.name, ErrorString(error)))

        # read from the file
        try:
            rawtext = include_file.read()
        except UnicodeError as error:
            raise self.severe('Problem with "%s" directive:\n%s' %
                              (self.name, ErrorString(error)))

        config = self.state.document.settings.env.config
        converter = M2R(no_underscore_emphasis=config.no_underscore_emphasis)
        include_lines = statemachine.string2lines(converter(rawtext),
                                                  tab_width,
                                                  convert_whitespace=True)
        self.state_machine.insert_input(include_lines, path)
        return []

Most of this method is from docutils . parser . rst . Directive .


def WorkerAgentGenerator(agent_class):
    """
    Worker Agent generator, receives an Agent class and creates a Worker Agent class that inherits from that Agent.
    """

    # Support special case where class is given as type-string (AgentsDictionary) or class-name-string.
    if isinstance(agent_class, str):
        agent_class = AgentsDictionary.get(agent_class)
        # Last resort: Class name given as string?
        if not agent_class and agent_class.find('.') != -1:
            module_name, function_name = agent_class.rsplit('.', 1)
            module = importlib.import_module(module_name)
            agent_class = getattr(module, function_name)

    class WorkerAgent(agent_class):
        """
        Worker agent receiving a shared model to avoid creating multiple models.
        """

        def __init__(self, model=None, **kwargs):
            # Set our model externally.
            self.model = model
            # Be robust against `network` coming in from kwargs even though this agent doesn't have one
            if not issubclass(agent_class, LearningAgent):
                kwargs.pop("network")
            # Call super c'tor (which will call initialize_model and assign self.model to the return value).
            super(WorkerAgent, self).__init__(**kwargs)

        def initialize_model(self):
            # Return our model (already given and initialized).
            return self.model

    return WorkerAgent

def WorkerAgentGenerator ( agent_class ) : if isinstance ( agent_class , str ) : agent_class = AgentsDictionary . get ( agent_class ) if not agent_class and agent_class . find ( '.' ) != - 1 : module_name , function_name = agent_class . rsplit ( '.' , 1 ) module = importlib . import_module ( module_name ) agent_class = getattr ( module , function_name ) class WorkerAgent ( agent_class ) : def __init__ ( self , model = None , ** kwargs ) : self . model = model if not issubclass ( agent_class , LearningAgent ) : kwargs . pop ( "network" ) super ( WorkerAgent , self ) . __init__ ( ** kwargs ) def initialize_model ( self ) : return self . model return WorkerAgent

Worker Agent generator receives an Agent class and creates a Worker Agent class that inherits from that Agent .


def _run_single(self, thread_id, agent, environment, deterministic=False,
                    max_episode_timesteps=-1, episode_finished=None, testing=False, sleep=None):
        """
        The target function for a thread, runs an agent and environment until signaled to stop.
        Adds rewards to shared episode rewards list.

        Args:
            thread_id (int): The ID of the thread that's running this target function.
            agent (Agent): The Agent object that this particular thread uses.
            environment (Environment): The Environment object that this particular thread uses.
            max_episode_timesteps (int): Max. number of timesteps per episode. Use -1 or 0 for non-limited episodes.
            episode_finished (callable): Function called after each episode that takes an episode summary spec and
                returns False, if this single run should terminate after this episode.
                Can be used e.g. to set a particular mean reward threshold.
        """

        # figure out whether we are using the deprecated way of "episode_finished" reporting
        old_episode_finished = False
        if episode_finished is not None and len(getargspec(episode_finished).args) == 1:
            old_episode_finished = True

        episode = 0
        # Run this single worker (episode loop) as long as global count thresholds have not been reached.
        while not self.should_stop:
            state = environment.reset()
            agent.reset()
            self.global_timestep, self.global_episode = agent.timestep, agent.episode
            episode_reward = 0

            # Time step (within episode) loop
            time_step = 0
            time_start = time.time()
            while True:
                action, internals, states = agent.act(states=state, deterministic=deterministic, buffered=False)
                reward = 0
                for repeat in xrange(self.repeat_actions):
                    state, terminal, step_reward = environment.execute(action=action)
                    reward += step_reward
                    if terminal:
                        break

                if not testing:
                    # agent.observe(reward=reward, terminal=terminal)
                    # Insert everything at once.
                    agent.atomic_observe(
                        states=state,
                        actions=action,
                        internals=internals,
                        reward=reward,
                        terminal=terminal
                    )

                if sleep is not None:
                    time.sleep(sleep)

                time_step += 1
                episode_reward += reward

                if terminal or time_step == max_episode_timesteps:
                    break

                # Abort the episode (discard its results) when global says so.
                if self.should_stop:
                    return

            self.global_timestep += time_step

            # Avoid race condition where order in episode_rewards won't match order in episode_timesteps.
            self.episode_list_lock.acquire()
            self.episode_rewards.append(episode_reward)
            self.episode_timesteps.append(time_step)
            self.episode_times.append(time.time() - time_start)
            self.episode_list_lock.release()

            if episode_finished is not None:
                # old way of calling episode_finished
                if old_episode_finished:
                    summary_data = {
                        "thread_id": thread_id,
                        "episode": episode,
                        "timestep": time_step,
                        "episode_reward": episode_reward
                    }
                    if not episode_finished(summary_data):
                        return
                # New way with BasicRunner (self) and thread-id.
                elif not episode_finished(self, thread_id):
                    return

            episode += 1

def _run_single ( self , thread_id , agent , environment , deterministic = False , max_episode_timesteps = - 1 , episode_finished = None , testing = False , sleep = None ) : old_episode_finished = False if episode_finished is not None and len ( getargspec ( episode_finished ) . args ) == 1 : old_episode_finished = True episode = 0 while not self . should_stop : state = environment . reset ( ) agent . reset ( ) self . global_timestep , self . global_episode = agent . timestep , agent . episode episode_reward = 0 time_step = 0 time_start = time . time ( ) while True : action , internals , states = agent . act ( states = state , deterministic = deterministic , buffered = False ) reward = 0 for repeat in xrange ( self . repeat_actions ) : state , terminal , step_reward = environment . execute ( action = action ) reward += step_reward if terminal : break if not testing : agent . atomic_observe ( states = state , actions = action , internals = internals , reward = reward , terminal = terminal ) if sleep is not None : time . sleep ( sleep ) time_step += 1 episode_reward += reward if terminal or time_step == max_episode_timesteps : break if self . should_stop : return self . global_timestep += time_step self . episode_list_lock . acquire ( ) self . episode_rewards . append ( episode_reward ) self . episode_timesteps . append ( time_step ) self . episode_times . append ( time . time ( ) - time_start ) self . episode_list_lock . release ( ) if episode_finished is not None : if old_episode_finished : summary_data = { "thread_id" : thread_id , "episode" : episode , "timestep" : time_step , "episode_reward" : episode_reward } if not episode_finished ( summary_data ) : return elif not episode_finished ( self , thread_id ) : return episode += 1

The target function for a thread runs an agent and environment until signaled to stop . Adds rewards to shared episode rewards list .


def _int_to_pos(self, flat_position):
        """Returns x, y from flat_position integer.

        Args:
            flat_position: flattened position integer

        Returns: x, y

        """
        return flat_position % self.env.action_space.screen_shape[0],\
            flat_position % self.env.action_space.screen_shape[1]

def _int_to_pos ( self , flat_position ) : return flat_position % self . env . action_space . screen_shape [ 0 ] , flat_position % self . env . action_space . screen_shape [ 1 ]

Returns x y from flat_position integer .


def _wait_state(self, state, reward, terminal):
        """
        Wait until there is a state.
        """
        while state == [None] or not state:
             state, terminal, reward = self._execute(dict(key=0))

        return state, terminal, reward

def _wait_state ( self , state , reward , terminal ) : while state == [ None ] or not state : state , terminal , reward = self . _execute ( dict ( key = 0 ) ) return state , terminal , reward


def from_spec(spec, kwargs=None):
        """
        Creates an optimizer from a specification dict.
        """
        optimizer = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.optimizers.optimizers,
            kwargs=kwargs
        )
        assert isinstance(optimizer, Optimizer)
        return optimizer

def from_spec ( spec , kwargs = None ) : optimizer = util . get_object ( obj = spec , predefined_objects = tensorforce . core . optimizers . optimizers , kwargs = kwargs ) assert isinstance ( optimizer , Optimizer ) return optimizer

Creates an optimizer from a specification dict .


def register_saver_ops(self):
        """
        Registers the saver operations to the graph in context.
        """

        variables = self.get_savable_variables()
        if variables is None or len(variables) == 0:
            self._saver = None
            return

        base_scope = self._get_base_variable_scope()
        variables_map = {strip_name_scope(v.name, base_scope): v for v in variables}

        self._saver = tf.train.Saver(
            var_list=variables_map,
            reshape=False,
            sharded=False,
            max_to_keep=5,
            keep_checkpoint_every_n_hours=10000.0,
            name=None,
            restore_sequentially=False,
            saver_def=None,
            builder=None,
            defer_build=False,
            allow_empty=True,
            write_version=tf.train.SaverDef.V2,
            pad_step_number=False,
            save_relative_paths=True
        )

def register_saver_ops ( self ) : variables = self . get_savable_variables ( ) if variables is None or len ( variables ) == 0 : self . _saver = None return base_scope = self . _get_base_variable_scope ( ) variables_map = { strip_name_scope ( v . name , base_scope ) : v for v in variables } self . _saver = tf . train . Saver ( var_list = variables_map , reshape = False , sharded = False , max_to_keep = 5 , keep_checkpoint_every_n_hours = 10000.0 , name = None , restore_sequentially = False , saver_def = None , builder = None , defer_build = False , allow_empty = True , write_version = tf . train . SaverDef . V2 , pad_step_number = False , save_relative_paths = True )

Registers the saver operations to the graph in context .


def save(self, sess, save_path, timestep=None):
        """
        Saves this component's managed variables.

        Args:
            sess: The session for which to save the managed variables.
            save_path: The path to save data to.
            timestep: Optional, the timestep to append to the file name.

        Returns:
            Checkpoint path where the model was saved.
        """

        if self._saver is None:
            raise TensorForceError("register_saver_ops should be called before save")
        return self._saver.save(
            sess=sess,
            save_path=save_path,
            global_step=timestep,
            write_meta_graph=False,
            write_state=True,  # Do we need this?
        )

Saves this component s managed variables .


def restore(self, sess, save_path):
        """
        Restores the values of the managed variables from disk location.

        Args:
            sess: The session for which to save the managed variables.
            save_path: The path used to save the data to.
        """

        if self._saver is None:
            raise TensorForceError("register_saver_ops should be called before restore")
        self._saver.restore(sess=sess, save_path=save_path)

Restores the values of the managed variables from disk location .


def reset(self):
        """
        Calls `reset` on all our Preprocessor objects.

        Returns:
            A list of tensors to be fetched.
        """
        fetches = []
        for processor in self.preprocessors:
            fetches.extend(processor.reset() or [])
        return fetches

def reset ( self ) : fetches = [ ] for processor in self . preprocessors : fetches . extend ( processor . reset ( ) or [ ] ) return fetches

Calls reset on all our Preprocessor objects .


def process(self, tensor):
        """
        Process state.

        Args:
            tensor: tensor to process

        Returns: processed state

        """
        for processor in self.preprocessors:
            tensor = processor.process(tensor=tensor)
        return tensor

def process ( self , tensor ) : for processor in self . preprocessors : tensor = processor . process ( tensor = tensor ) return tensor


def processed_shape(self, shape):
        """
        Shape of preprocessed state given original shape.

        Args:
            shape: original state shape

        Returns: processed state shape
        """
        for processor in self.preprocessors:
            shape = processor.processed_shape(shape=shape)
        return shape

def processed_shape ( self , shape ) : for processor in self . preprocessors : shape = processor . processed_shape ( shape = shape ) return shape

Shape of preprocessed state given original shape .


def from_spec(spec, kwargs=None):
        """
        Creates a preprocessing stack from a specification dict.
        """
        if isinstance(spec, dict):
            spec = [spec]

        stack = PreprocessorStack()
        for preprocessor_spec in spec:
            # need to deep copy, otherwise will add first processors spec_ to kwargs to second processor
            preprocessor_kwargs = copy.deepcopy(kwargs)
            preprocessor = util.get_object(
                obj=preprocessor_spec,
                predefined_objects=tensorforce.core.preprocessors.preprocessors,
                kwargs=preprocessor_kwargs
            )
            assert isinstance(preprocessor, Preprocessor)
            stack.preprocessors.append(preprocessor)

        return stack

def from_spec ( spec , kwargs = None ) : if isinstance ( spec , dict ) : spec = [ spec ] stack = PreprocessorStack ( ) for preprocessor_spec in spec : preprocessor_kwargs = copy . deepcopy ( kwargs ) preprocessor = util . get_object ( obj = preprocessor_spec , predefined_objects = tensorforce . core . preprocessors . preprocessors , kwargs = preprocessor_kwargs ) assert isinstance ( preprocessor , Preprocessor ) stack . preprocessors . append ( preprocessor ) return stack

Creates a preprocessing stack from a specification dict .


def as_local_model(self):
        """
        Makes sure our optimizer is wrapped into the global_optimizer meta. This is only relevant for distributed RL.
        """
        super(MemoryModel, self).as_local_model()
        self.optimizer_spec = dict(
            type='global_optimizer',
            optimizer=self.optimizer_spec
        )

def as_local_model ( self ) : super ( MemoryModel , self ) . as_local_model ( ) self . optimizer_spec = dict ( type = 'global_optimizer' , optimizer = self . optimizer_spec )

Makes sure our optimizer is wrapped into the global_optimizer meta . This is only relevant for distributed RL .


def setup_components_and_tf_funcs(self, custom_getter=None):
        """
        Constructs the memory and the optimizer objects.
        Generates and stores all template functions.
        """
        custom_getter = super(MemoryModel, self).setup_components_and_tf_funcs(custom_getter)

        # Memory
        self.memory = Memory.from_spec(
            spec=self.memory_spec,
            kwargs=dict(
                states=self.states_spec,
                internals=self.internals_spec,
                actions=self.actions_spec,
                summary_labels=self.summary_labels
            )
        )

        # Optimizer
        self.optimizer = Optimizer.from_spec(
            spec=self.optimizer_spec,
            kwargs=dict(summary_labels=self.summary_labels)
        )

        # TensorFlow functions
        self.fn_discounted_cumulative_reward = tf.make_template(
            name_='discounted-cumulative-reward',
            func_=self.tf_discounted_cumulative_reward,
            custom_getter_=custom_getter
        )
        self.fn_reference = tf.make_template(
            name_='reference',
            func_=self.tf_reference,
            custom_getter_=custom_getter
        )
        self.fn_loss_per_instance = tf.make_template(
            name_='loss-per-instance',
            func_=self.tf_loss_per_instance,
            custom_getter_=custom_getter
        )
        self.fn_regularization_losses = tf.make_template(
            name_='regularization-losses',
            func_=self.tf_regularization_losses,
            custom_getter_=custom_getter
        )
        self.fn_loss = tf.make_template(
            name_='loss',
            func_=self.tf_loss,
            custom_getter_=custom_getter
        )
        self.fn_optimization = tf.make_template(
            name_='optimization',
            func_=self.tf_optimization,
            custom_getter_=custom_getter
        )
        self.fn_import_experience = tf.make_template(
            name_='import-experience',
            func_=self.tf_import_experience,
            custom_getter_=custom_getter
        )

        return custom_getter

def setup_components_and_tf_funcs ( self , custom_getter = None ) : custom_getter = super ( MemoryModel , self ) . setup_components_and_tf_funcs ( custom_getter ) self . memory = Memory . from_spec ( spec = self . memory_spec , kwargs = dict ( states = self . states_spec , internals = self . internals_spec , actions = self . actions_spec , summary_labels = self . summary_labels ) ) self . optimizer = Optimizer . from_spec ( spec = self . optimizer_spec , kwargs = dict ( summary_labels = self . summary_labels ) ) self . fn_discounted_cumulative_reward = tf . make_template ( name_ = 'discounted-cumulative-reward' , func_ = self . tf_discounted_cumulative_reward , custom_getter_ = custom_getter ) self . fn_reference = tf . make_template ( name_ = 'reference' , func_ = self . tf_reference , custom_getter_ = custom_getter ) self . fn_loss_per_instance = tf . make_template ( name_ = 'loss-per-instance' , func_ = self . tf_loss_per_instance , custom_getter_ = custom_getter ) self . fn_regularization_losses = tf . make_template ( name_ = 'regularization-losses' , func_ = self . tf_regularization_losses , custom_getter_ = custom_getter ) self . fn_loss = tf . make_template ( name_ = 'loss' , func_ = self . tf_loss , custom_getter_ = custom_getter ) self . fn_optimization = tf . make_template ( name_ = 'optimization' , func_ = self . tf_optimization , custom_getter_ = custom_getter ) self . fn_import_experience = tf . make_template ( name_ = 'import-experience' , func_ = self . tf_import_experience , custom_getter_ = custom_getter ) return custom_getter

Constructs the memory and the optimizer objects . Generates and stores all template functions .