def _learnBacktrack(self):
    """
    This "backtracks" our learning state, trying to see if we can lock onto
    the current set of inputs by assuming the sequence started up to N steps
    ago on start cells.

    This will adjust @ref lrnActiveState['t'] if it does manage to lock on to a
    sequence that started earlier.

    :returns:          >0 if we managed to lock on to a sequence that started
                      earlier. The value returned is how many steps in the
                      past we locked on.
                      If 0 is returned, the caller needs to change active
                      state to start on start cells.

    How it works:
    -------------------------------------------------------------------
    This method gets called from updateLearningState when we detect either of
    the following two conditions:

    #. Our PAM counter (@ref pamCounter) expired
    #. We reached the max allowed learned sequence length

    Either of these two conditions indicate that we want to start over on start
    cells.

    Rather than start over on start cells on the current input, we can
    accelerate learning by backtracking a few steps ago and seeing if perhaps
    a sequence we already at least partially know already started.

    This updates/modifies:
        - @ref lrnActiveState['t']

    This trashes:
        - @ref lrnActiveState['t-1']
        - @ref lrnPredictedState['t']
        - @ref lrnPredictedState['t-1']

    """
    # How much input history have we accumulated?
    # The current input is always at the end of self._prevInfPatterns (at
    # index -1), and is not a valid startingOffset to evaluate.
    numPrevPatterns = len(self._prevLrnPatterns) - 1
    if numPrevPatterns <= 0:
      if self.verbosity >= 3:
        print "lrnBacktrack: No available history to backtrack from"
      return False

    # We will record which previous input patterns did not generate predictions
    # up to the current time step and remove all the ones at the head of the
    # input history queue so that we don't waste time evaluating them again at
    # a later time step.
    badPatterns = []

    # Let's go back in time and replay the recent inputs from start cells and
    # see if we can lock onto this current set of inputs that way.
    #
    # Start the farthest back and work our way forward. For each starting point,
    # See if firing on start cells at that point would predict the current
    # input.
    #
    # We want to pick the point farthest in the past that has continuity
    # up to the current time step
    inSequence = False
    for startOffset in range(0, numPrevPatterns):
      # Can we backtrack from startOffset?
      inSequence = self._learnBacktrackFrom(startOffset, readOnly=True)

      # Done playing through the sequence from starting point startOffset
      # Break out as soon as we find a good path
      if inSequence:
        break

      # Take this bad starting point out of our input history so we don't
      # try it again later.
      badPatterns.append(startOffset)

    # If we failed to lock on at any starting point, return failure. The caller
    # will start over again on start cells
    if not inSequence:
      if self.verbosity >= 3:
        print ("Failed to lock on. Falling back to start cells on current "
               "time step.")
      # Nothing in our input history was a valid starting point, so get rid
      #  of it so we don't try any of them again at a later iteration
      self._prevLrnPatterns = []
      return False

    # We did find a valid starting point in the past. Now, we need to
    # re-enforce all segments that became active when following this path.
    if self.verbosity >= 3:
      print ("Discovered path to current input by using start cells from %d "
             "steps ago:" % (numPrevPatterns - startOffset),
             self._prevLrnPatterns[startOffset])

    self._learnBacktrackFrom(startOffset, readOnly=False)

    # Remove any useless patterns at the head of the input pattern history
    # queue.
    for i in range(numPrevPatterns):
      if i in badPatterns or i <= startOffset:
        if self.verbosity >= 3:
          print ("Removing useless pattern from history:",
                 self._prevLrnPatterns[0])
        self._prevLrnPatterns.pop(0)
      else:
        break

    return numPrevPatterns - startOffset

This backtracks our learning state trying to see if we can lock onto the current set of inputs by assuming the sequence started up to N steps ago on start cells .


def _learnPhase1(self, activeColumns, readOnly=False):
    """
    Compute the learning active state given the predicted state and
    the bottom-up input.

    :param activeColumns list of active bottom-ups
    :param readOnly      True if being called from backtracking logic.
                         This tells us not to increment any segment
                         duty cycles or queue up any updates.
    :returns: True if the current input was sufficiently predicted, OR
             if we started over on startCells. False indicates that the current
             input was NOT predicted, well enough to consider it as "inSequence"

    This looks at:
        - @ref lrnActiveState['t-1']
        - @ref lrnPredictedState['t-1']

    This modifies:
        - @ref lrnActiveState['t']
        - @ref lrnActiveState['t-1']
    """
    # Save previous active state and start out on a clean slate
    self.lrnActiveState['t'].fill(0)

    # For each column, turn on the predicted cell. There will always be at most
    # one predicted cell per column
    numUnpredictedColumns = 0
    for c in activeColumns:
      predictingCells = numpy.where(self.lrnPredictedState['t-1'][c] == 1)[0]
      numPredictedCells = len(predictingCells)
      assert numPredictedCells <= 1

      # If we have a predicted cell, turn it on. The segment's posActivation
      # count will have already been incremented by processSegmentUpdates
      if numPredictedCells == 1:
        i = predictingCells[0]
        self.lrnActiveState['t'][c, i] = 1
        continue

      numUnpredictedColumns += 1
      if readOnly:
        continue

      # If no predicted cell, pick the closest matching one to reinforce, or
      # if none exists, create a new segment on a cell in that column
      i, s, numActive = self._getBestMatchingCell(
          c, self.lrnActiveState['t-1'], self.minThreshold)
      if s is not None and s.isSequenceSegment():
        if self.verbosity >= 4:
          print "Learn branch 0, found segment match. Learning on col=", c
        self.lrnActiveState['t'][c, i] = 1
        segUpdate = self._getSegmentActiveSynapses(
            c, i, s, self.lrnActiveState['t-1'], newSynapses = True)
        s.totalActivations += 1
        # This will update the permanences, posActivationsCount, and the
        # lastActiveIteration (age).
        trimSegment = self._adaptSegment(segUpdate)
        if trimSegment:
          self._trimSegmentsInCell(c, i, [s], minPermanence = 0.00001,
                                   minNumSyns = 0)

      # If no close match exists, create a new one
      else:
        # Choose a cell in this column to add a new segment to
        i = self._getCellForNewSegment(c)
        if (self.verbosity >= 4):
          print "Learn branch 1, no match. Learning on col=", c,
          print ", newCellIdxInCol=", i
        self.lrnActiveState['t'][c, i] = 1
        segUpdate = self._getSegmentActiveSynapses(
            c, i, None, self.lrnActiveState['t-1'], newSynapses=True)
        segUpdate.sequenceSegment = True # Make it a sequence segment
        self._adaptSegment(segUpdate)  # No need to check whether perm reached 0

    # Determine if we are out of sequence or not and reset our PAM counter
    # if we are in sequence
    numBottomUpColumns = len(activeColumns)
    if numUnpredictedColumns < numBottomUpColumns / 2:
      return True   # in sequence
    else:
      return False

Compute the learning active state given the predicted state and the bottom - up input .


def _learnPhase2(self, readOnly=False):
    """
    Compute the predicted segments given the current set of active cells.

    :param readOnly       True if being called from backtracking logic.
                          This tells us not to increment any segment
                          duty cycles or queue up any updates.

    This computes the lrnPredictedState['t'] and queues up any segments that
    became active (and the list of active synapses for each segment) into
    the segmentUpdates queue

    This looks at:
        - @ref lrnActiveState['t']

    This modifies:
        - @ref lrnPredictedState['t']
        - @ref segmentUpdates
    """
    # Clear out predicted state to start with
    self.lrnPredictedState['t'].fill(0)

    # Compute new predicted state. When computing predictions for
    # phase 2, we predict at  most one cell per column (the one with the best
    # matching segment).
    for c in xrange(self.numberOfCols):

      # Is there a cell predicted to turn on in this column?
      i, s, numActive = self._getBestMatchingCell(
          c, self.lrnActiveState['t'], minThreshold = self.activationThreshold)
      if i is None:
        continue

      # Turn on the predicted state for the best matching cell and queue
      #  the pertinent segment up for an update, which will get processed if
      #  the cell receives bottom up in the future.
      self.lrnPredictedState['t'][c, i] = 1
      if readOnly:
        continue

      # Queue up this segment for updating
      segUpdate = self._getSegmentActiveSynapses(
          c, i, s, activeState=self.lrnActiveState['t'],
          newSynapses=(numActive < self.newSynapseCount))

      s.totalActivations += 1    # increment totalActivations
      self._addToSegmentUpdates(c, i, segUpdate)

      if self.doPooling:
        # creates a new pooling segment if no best matching segment found
        # sum(all synapses) >= minThreshold, "weak" activation
        predSegment = self._getBestMatchingSegment(c, i,
                                                   self.lrnActiveState['t-1'])
        segUpdate = self._getSegmentActiveSynapses(c, i, predSegment,
                                                   self.lrnActiveState['t-1'], newSynapses=True)
        self._addToSegmentUpdates(c, i, segUpdate)

def _learnPhase2 ( self , readOnly = False ) : self . lrnPredictedState [ 't' ] . fill ( 0 ) for c in xrange ( self . numberOfCols ) : i , s , numActive = self . _getBestMatchingCell ( c , self . lrnActiveState [ 't' ] , minThreshold = self . activationThreshold ) if i is None : continue self . lrnPredictedState [ 't' ] [ c , i ] = 1 if readOnly : continue segUpdate = self . _getSegmentActiveSynapses ( c , i , s , activeState = self . lrnActiveState [ 't' ] , newSynapses = ( numActive < self . newSynapseCount ) ) s . totalActivations += 1 self . _addToSegmentUpdates ( c , i , segUpdate ) if self . doPooling : predSegment = self . _getBestMatchingSegment ( c , i , self . lrnActiveState [ 't-1' ] ) segUpdate = self . _getSegmentActiveSynapses ( c , i , predSegment , self . lrnActiveState [ 't-1' ] , newSynapses = True ) self . _addToSegmentUpdates ( c , i , segUpdate )

Compute the predicted segments given the current set of active cells .


def compute(self, bottomUpInput, enableLearn, enableInference=None):
    """
    Handle one compute, possibly learning.

    .. note::  It is an error to have both ``enableLearn`` and
               ``enableInference`` set to False

    .. note:: By default, we don't compute the inference output when learning
              because it slows things down, but you can override this by passing
              in True for ``enableInference``.

    :param bottomUpInput: The bottom-up input as numpy list, typically from a
           spatial pooler.
    :param enableLearn: (bool) If true, perform learning
    :param enableInference: (bool) If None, default behavior is to disable the
           inference output when ``enableLearn`` is on. If true, compute the
           inference output. If false, do not compute the inference output.

    :returns: TODO: document

    """
    # As a speed optimization for now (until we need online learning), skip
    # computing the inference output while learning
    if enableInference is None:
      if enableLearn:
        enableInference = False
      else:
        enableInference = True

    assert (enableLearn or enableInference)

    # Get the list of columns that have bottom-up
    activeColumns = bottomUpInput.nonzero()[0]
    if enableLearn:
      self.lrnIterationIdx += 1
    self.iterationIdx +=  1

    if self.verbosity >= 3:
      print "\n==== PY Iteration: %d =====" % (self.iterationIdx)
      print "Active cols:", activeColumns

    # Update segment duty cycles if we are crossing a "tier"
    # We determine if it's time to update the segment duty cycles. Since the
    # duty cycle calculation is a moving average based on a tiered alpha, it is
    # important that we update all segments on each tier boundary
    if enableLearn:
      if self.lrnIterationIdx in Segment.dutyCycleTiers:
        for c, i in itertools.product(xrange(self.numberOfCols),
                                      xrange(self.cellsPerColumn)):
          for segment in self.cells[c][i]:
            segment.dutyCycle()

    # Update the average input density
    if self.avgInputDensity is None:
      self.avgInputDensity = len(activeColumns)
    else:
      self.avgInputDensity = (0.99 * self.avgInputDensity +
                              0.01 * len(activeColumns))

    # First, update the inference state
    # As a speed optimization for now (until we need online learning), skip
    # computing the inference output while learning
    if enableInference:
      self._updateInferenceState(activeColumns)

    # Next, update the learning state
    if enableLearn:
      self._updateLearningState(activeColumns)

      # Apply global decay, and remove synapses and/or segments.
      # Synapses are removed if their permanence value is <= 0.
      # Segments are removed when they don't have synapses anymore.
      # Removal of synapses can trigger removal of whole segments!
      # todo: isolate the synapse/segment retraction logic so that
      # it can be called in adaptSegments, in the case where we
      # do global decay only episodically.
      if self.globalDecay > 0.0 and ((self.lrnIterationIdx % self.maxAge) == 0):
        for c, i in itertools.product(xrange(self.numberOfCols),
                                      xrange(self.cellsPerColumn)):

          segsToDel = [] # collect and remove outside the loop
          for segment in self.cells[c][i]:
            age = self.lrnIterationIdx - segment.lastActiveIteration
            if age <= self.maxAge:
              continue

            synsToDel = [] # collect and remove outside the loop
            for synapse in segment.syns:

              synapse[2] = synapse[2] - self.globalDecay # decrease permanence

              if synapse[2] <= 0:
                synsToDel.append(synapse) # add to list to delete

            # 1 for sequenceSegment flag
            if len(synsToDel) == segment.getNumSynapses():
              segsToDel.append(segment) # will remove the whole segment
            elif len(synsToDel) > 0:
              for syn in synsToDel: # remove some synapses on segment
                segment.syns.remove(syn)

          for seg in segsToDel: # remove some segments of this cell
            self._cleanUpdatesList(c, i, seg)
            self.cells[c][i].remove(seg)

    # Update the prediction score stats
    # Learning always includes inference
    if self.collectStats:
      if enableInference:
        predictedState = self.infPredictedState['t-1']
      else:
        predictedState = self.lrnPredictedState['t-1']
      self._updateStatsInferEnd(self._internalStats,
                                activeColumns,
                                predictedState,
                                self.colConfidence['t-1'])

    # Finally return the TM output
    output = self._computeOutput()

    # Print diagnostic information based on the current verbosity level
    self.printComputeEnd(output, learn=enableLearn)

    self.resetCalled = False
    return output

def
compute
(
self
,
bottomUpInput
,
enableLearn
,
enableInference
=
)
:
if
enableInference
is
:
if
enableLearn
:
enableInference
=
False
else
:
enableInference
=
True
assert
(
enableLearn
or
enableInference
)
activeColumns
=
bottomUpInput
.
nonzero
(
)
[
0
]
if
enableLearn
:
self
.
lrnIterationIdx
+=
1
self
.
iterationIdx
+=
1
if
self
.
verbosity
>=
3
:
print
"\n==== PY Iteration: %d ====="
%
(
self
.
iterationIdx
)
print
"Active cols:"
,
activeColumns
if
enableLearn
:
if
self
.
lrnIterationIdx
in
Segment
.
dutyCycleTiers
:
for
c
,
i
in
itertools
.
product
(
xrange
(
self
.
numberOfCols
)
,
xrange
(
self
.
cellsPerColumn
)
)
:
for
segment
in
self
.
cells
[
c
]
[
i
]
:
segment
.
dutyCycle
(
)
if
self
.
avgInputDensity
is
:
self
.
avgInputDensity
=
len
(
activeColumns
)
else
:
self
.
avgInputDensity
=
(
0.99
*
self
.
avgInputDensity
+
0.01
*
len
(
activeColumns
)
)
if
enableInference
:
self
.
_updateInferenceState
(
activeColumns
)
if
enableLearn
:
self
.
_updateLearningState
(
activeColumns
)
if
self
.
globalDecay
>
0.0
and
(
(
self
.
lrnIterationIdx
%
self
.
maxAge
)
==
0
)
:
for
c
,
i
in
itertools
.
product
(
xrange
(
self
.
numberOfCols
)
,
xrange
(
self
.
cellsPerColumn
)
)
:
segsToDel
=
[
]
for
segment
in
self
.
cells
[
c
]
[
i
]
:
age
=
self
.
lrnIterationIdx
segment
.
lastActiveIteration
if
age
<=
self
.
maxAge
:
continue
synsToDel
=
[
]
for
synapse
in
segment
.
syns
:
synapse
[
2
]
=
synapse
[
2
]
self
.
globalDecay
if
synapse
[
2
]
<=
0
:
synsToDel
.
append
(
synapse
)
if
len
(
synsToDel
)
==
segment
.
getNumSynapses
(
)
:
segsToDel
.
append
(
segment
)
elif
len
(
synsToDel
)
>
0
:
for
syn
in
synsToDel
:
segment
.
syns
.
remove
(
syn
)
for
seg
in
segsToDel
:
self
.
_cleanUpdatesList
(
c
,
i
,
seg
)
self
.
cells
[
c
]
[
i
]
.
remove
(
seg
)
if
self
.
collectStats
:
if
enableInference
:
predictedState
=
self
.
infPredictedState
[
't-1'
]
else
:
predictedState
=
self
.
lrnPredictedState
[
't-1'
]
self
.
_updateStatsInferEnd
(
self
.
_internalStats
,
activeColumns
,
predictedState
,
self
.
colConfidence
[
't-1'
]
)
output
=
self
.
_computeOutput
(
)
self
.
printComputeEnd
(
output
,
learn
=
enableLearn
)
self
.
resetCalled
=
False
return
output


def _trimSegmentsInCell(self, colIdx, cellIdx, segList, minPermanence,
                          minNumSyns):
    """
    This method goes through a list of segments for a given cell and
    deletes all synapses whose permanence is less than minPermanence and deletes
    any segments that have less than minNumSyns synapses remaining.

    :param colIdx        Column index
    :param cellIdx       Cell index within the column
    :param segList       List of segment references
    :param minPermanence Any syn whose permamence is 0 or < minPermanence will
                         be deleted.
    :param minNumSyns    Any segment with less than minNumSyns synapses remaining
                         in it will be deleted.

    :returns: tuple (numSegsRemoved, numSynsRemoved)
    """
    # Fill in defaults
    if minPermanence is None:
      minPermanence = self.connectedPerm
    if minNumSyns is None:
      minNumSyns = self.activationThreshold

    # Loop through all segments
    nSegsRemoved, nSynsRemoved = 0, 0
    segsToDel = [] # collect and remove segments outside the loop
    for segment in segList:

      # List if synapses to delete
      synsToDel = [syn for syn in segment.syns if syn[2] < minPermanence]

      if len(synsToDel) == len(segment.syns):
        segsToDel.append(segment) # will remove the whole segment
      else:
        if len(synsToDel) > 0:
          for syn in synsToDel: # remove some synapses on segment
            segment.syns.remove(syn)
            nSynsRemoved += 1
        if len(segment.syns) < minNumSyns:
          segsToDel.append(segment)

    # Remove segments that don't have enough synapses and also take them
    # out of the segment update list, if they are in there
    nSegsRemoved += len(segsToDel)
    for seg in segsToDel: # remove some segments of this cell
      self._cleanUpdatesList(colIdx, cellIdx, seg)
      self.cells[colIdx][cellIdx].remove(seg)
      nSynsRemoved += len(seg.syns)

    return nSegsRemoved, nSynsRemoved

def _trimSegmentsInCell ( self , colIdx , cellIdx , segList , minPermanence , minNumSyns ) : if minPermanence is None : minPermanence = self . connectedPerm if minNumSyns is None : minNumSyns = self . activationThreshold nSegsRemoved , nSynsRemoved = 0 , 0 segsToDel = [ ] for segment in segList : synsToDel = [ syn for syn in segment . syns if syn [ 2 ] < minPermanence ] if len ( synsToDel ) == len ( segment . syns ) : segsToDel . append ( segment ) else : if len ( synsToDel ) > 0 : for syn in synsToDel : segment . syns . remove ( syn ) nSynsRemoved += 1 if len ( segment . syns ) < minNumSyns : segsToDel . append ( segment ) nSegsRemoved += len ( segsToDel ) for seg in segsToDel : self . _cleanUpdatesList ( colIdx , cellIdx , seg ) self . cells [ colIdx ] [ cellIdx ] . remove ( seg ) nSynsRemoved += len ( seg . syns ) return nSegsRemoved , nSynsRemoved

This method goes through a list of segments for a given cell and deletes all synapses whose permanence is less than minPermanence and deletes any segments that have less than minNumSyns synapses remaining .


def trimSegments(self, minPermanence=None, minNumSyns=None):
    """
    This method deletes all synapses whose permanence is less than
    minPermanence and deletes any segments that have less than
    minNumSyns synapses remaining.

    :param minPermanence: (float) Any syn whose permanence is 0 or <
           ``minPermanence``  will be deleted. If None is passed in, then
           ``self.connectedPerm`` is used.
    :param minNumSyns: (int) Any segment with less than ``minNumSyns`` synapses
           remaining in it will be deleted. If None is passed in, then
           ``self.activationThreshold`` is used.
    :returns: (tuple) ``numSegsRemoved``, ``numSynsRemoved``
    """
    # Fill in defaults
    if minPermanence is None:
      minPermanence = self.connectedPerm
    if minNumSyns is None:
      minNumSyns = self.activationThreshold

    # Loop through all cells
    totalSegsRemoved, totalSynsRemoved = 0, 0
    for c, i in itertools.product(xrange(self.numberOfCols),
                                  xrange(self.cellsPerColumn)):

      (segsRemoved, synsRemoved) = self._trimSegmentsInCell(
          colIdx=c, cellIdx=i, segList=self.cells[c][i],
          minPermanence=minPermanence, minNumSyns=minNumSyns)
      totalSegsRemoved += segsRemoved
      totalSynsRemoved += synsRemoved

    # Print all cells if verbosity says to
    if self.verbosity >= 5:
      print "Cells, all segments:"
      self.printCells(predictedOnly=False)

    return totalSegsRemoved, totalSynsRemoved

def
trimSegments
(
self
,
minPermanence
=
,
minNumSyns
=
)
:
if
minPermanence
is
:
minPermanence
=
self
.
connectedPerm
if
minNumSyns
is
:
minNumSyns
=
self
.
activationThreshold
totalSegsRemoved
,
totalSynsRemoved
=
0
,
0
for
c
,
i
in
itertools
.
product
(
xrange
(
self
.
numberOfCols
)
,
xrange
(
self
.
cellsPerColumn
)
)
:
(
segsRemoved
,
synsRemoved
)
=
self
.
_trimSegmentsInCell
(
colIdx
=
c
,
cellIdx
=
i
,
segList
=
self
.
cells
[
c
]
[
i
]
,
minPermanence
=
minPermanence
,
minNumSyns
=
minNumSyns
)
totalSegsRemoved
+=
segsRemoved
totalSynsRemoved
+=
synsRemoved
if
self
.
verbosity
>=
5
:
print
"Cells, all segments:"
self
.
printCells
(
predictedOnly
=
False
)
return
totalSegsRemoved
,
totalSynsRemoved

This method deletes all synapses whose permanence is less than minPermanence and deletes any segments that have less than minNumSyns synapses remaining .


def _cleanUpdatesList(self, col, cellIdx, seg):
    """
    Removes any update that would be for the given col, cellIdx, segIdx.

    NOTE: logically, we need to do this when we delete segments, so that if
    an update refers to a segment that was just deleted, we also remove
    that update from the update list. However, I haven't seen it trigger
    in any of the unit tests yet, so it might mean that it's not needed
    and that situation doesn't occur, by construction.
    """
    # TODO: check if the situation described in the docstring above actually
    #       occurs.
    for key, updateList in self.segmentUpdates.iteritems():
      c, i = key[0], key[1]
      if c == col and i == cellIdx:
        for update in updateList:
          if update[1].segment == seg:
            self._removeSegmentUpdate(update)

def _cleanUpdatesList ( self , col , cellIdx , seg ) : for key , updateList in self . segmentUpdates . iteritems ( ) : c , i = key [ 0 ] , key [ 1 ] if c == col and i == cellIdx : for update in updateList : if update [ 1 ] . segment == seg : self . _removeSegmentUpdate ( update )

Removes any update that would be for the given col cellIdx segIdx .


def _getBestMatchingCell(self, c, activeState, minThreshold):
    """
    Find weakly activated cell in column with at least minThreshold active
    synapses.

    :param c            which column to look at
    :param activeState  the active cells
    :param minThreshold minimum number of synapses required

    :returns: tuple (cellIdx, segment, numActiveSynapses)
    """
    # Collect all cells in column c that have at least minThreshold in the most
    # activated segment
    bestActivityInCol = minThreshold
    bestSegIdxInCol = -1
    bestCellInCol = -1

    for i in xrange(self.cellsPerColumn):

      maxSegActivity = 0
      maxSegIdx = 0

      for j, s in enumerate(self.cells[c][i]):

        activity = self._getSegmentActivityLevel(s, activeState)

        if activity > maxSegActivity:
          maxSegActivity = activity
          maxSegIdx = j

      if maxSegActivity >= bestActivityInCol:
        bestActivityInCol = maxSegActivity
        bestSegIdxInCol = maxSegIdx
        bestCellInCol = i

    if bestCellInCol == -1:
      return (None, None, None)
    else:
      return (bestCellInCol, self.cells[c][bestCellInCol][bestSegIdxInCol],
                bestActivityInCol)

def _getBestMatchingCell ( self , c , activeState , minThreshold ) : bestActivityInCol = minThreshold bestSegIdxInCol = - 1 bestCellInCol = - 1 for i in xrange ( self . cellsPerColumn ) : maxSegActivity = 0 maxSegIdx = 0 for j , s in enumerate ( self . cells [ c ] [ i ] ) : activity = self . _getSegmentActivityLevel ( s , activeState ) if activity > maxSegActivity : maxSegActivity = activity maxSegIdx = j if maxSegActivity >= bestActivityInCol : bestActivityInCol = maxSegActivity bestSegIdxInCol = maxSegIdx bestCellInCol = i if bestCellInCol == - 1 : return ( None , None , None ) else : return ( bestCellInCol , self . cells [ c ] [ bestCellInCol ] [ bestSegIdxInCol ] , bestActivityInCol )

Find weakly activated cell in column with at least minThreshold active synapses .


def _getBestMatchingSegment(self, c, i, activeState):
    """
    For the given cell, find the segment with the largest number of active
    synapses. This routine is aggressive in finding the best match. The
    permanence value of synapses is allowed to be below connectedPerm. The number
    of active synapses is allowed to be below activationThreshold, but must be
    above minThreshold. The routine returns the segment index. If no segments are
    found, then an index of -1 is returned.

    :param c TODO: document
    :param i TODO: document
    :param activeState TODO: document
    """
    maxActivity, which = self.minThreshold, -1

    for j, s in enumerate(self.cells[c][i]):
      activity = self._getSegmentActivityLevel(s, activeState,
                                               connectedSynapsesOnly=False)

      if activity >= maxActivity:
        maxActivity, which = activity, j

    if which == -1:
      return None
    else:
      return self.cells[c][i][which]

def _getBestMatchingSegment ( self , c , i , activeState ) : maxActivity , which = self . minThreshold , - 1 for j , s in enumerate ( self . cells [ c ] [ i ] ) : activity = self . _getSegmentActivityLevel ( s , activeState , connectedSynapsesOnly = False ) if activity >= maxActivity : maxActivity , which = activity , j if which == - 1 : return None else : return self . cells [ c ] [ i ] [ which ]

For the given cell find the segment with the largest number of active synapses . This routine is aggressive in finding the best match . The permanence value of synapses is allowed to be below connectedPerm . The number of active synapses is allowed to be below activationThreshold but must be above minThreshold . The routine returns the segment index . If no segments are found then an index of - 1 is returned .


def _getCellForNewSegment(self, colIdx):
    """
    Return the index of a cell in this column which is a good candidate
    for adding a new segment.

    When we have fixed size resources in effect, we insure that we pick a
    cell which does not already have the max number of allowed segments. If
    none exists, we choose the least used segment in the column to re-allocate.

    :param colIdx which column to look at
    :returns: cell index
    """
    # Not fixed size CLA, just choose a cell randomly
    if self.maxSegmentsPerCell < 0:
      if self.cellsPerColumn > 1:
        # Don't ever choose the start cell (cell # 0) in each column
        i = self._random.getUInt32(self.cellsPerColumn-1) + 1
      else:
        i = 0
      return i

    # Fixed size CLA, choose from among the cells that are below the maximum
    # number of segments.
    # NOTE: It is important NOT to always pick the cell with the fewest number
    # of segments. The reason is that if we always do that, we are more likely
    # to run into situations where we choose the same set of cell indices to
    # represent an 'A' in both context 1 and context 2. This is because the
    # cell indices we choose in each column of a pattern will advance in
    # lockstep (i.e. we pick cell indices of 1, then cell indices of 2, etc.).
    candidateCellIdxs = []
    if self.cellsPerColumn == 1:
      minIdx = 0
      maxIdx = 0
    else:
      minIdx = 1                      # Don't include startCell in the mix
      maxIdx = self.cellsPerColumn-1
    for i in xrange(minIdx, maxIdx+1):
      numSegs = len(self.cells[colIdx][i])
      if numSegs < self.maxSegmentsPerCell:
        candidateCellIdxs.append(i)

    # If we found one, return with it. Note we need to use _random to maintain
    # correspondence with CPP code.
    if len(candidateCellIdxs) > 0:
      #candidateCellIdx = random.choice(candidateCellIdxs)
      candidateCellIdx = (
          candidateCellIdxs[self._random.getUInt32(len(candidateCellIdxs))])
      if self.verbosity >= 5:
        print "Cell [%d,%d] chosen for new segment, # of segs is %d" % (
            colIdx, candidateCellIdx, len(self.cells[colIdx][candidateCellIdx]))
      return candidateCellIdx

    # All cells in the column are full, find a segment to free up
    candidateSegment = None
    candidateSegmentDC = 1.0
    # For each cell in this column
    for i in xrange(minIdx, maxIdx+1):
      # For each segment in this cell
      for s in self.cells[colIdx][i]:
        dc = s.dutyCycle()
        if dc < candidateSegmentDC:
          candidateCellIdx = i
          candidateSegmentDC = dc
          candidateSegment = s

    # Free up the least used segment
    if self.verbosity >= 5:
      print ("Deleting segment #%d for cell[%d,%d] to make room for new "
             "segment" % (candidateSegment.segID, colIdx, candidateCellIdx))
      candidateSegment.debugPrint()
    self._cleanUpdatesList(colIdx, candidateCellIdx, candidateSegment)
    self.cells[colIdx][candidateCellIdx].remove(candidateSegment)
    return candidateCellIdx

Return the index of a cell in this column which is a good candidate for adding a new segment .


def _chooseCellsToLearnFrom(self, c, i, s, n, activeState):
    """
    Choose n random cells to learn from.

    This function is called several times while learning with timeStep = t-1, so
    we cache the set of candidates for that case. It's also called once with
    timeStep = t, and we cache that set of candidates.

    :returns: tuple (column index, cell index).
    """
    if n <= 0:
      return []

    tmpCandidates = numpy.where(activeState == 1)

    # Candidates can be empty at this point, in which case we return
    # an empty segment list. adaptSegments will do nothing when getting
    # that list.
    if len(tmpCandidates[0]) == 0:
      return []

    if s is None: # new segment
      cands = [syn for syn in zip(tmpCandidates[0], tmpCandidates[1])]
    else:
      # We exclude any synapse that is already in this segment.
      synapsesAlreadyInSegment = set((syn[0], syn[1]) for syn in s.syns)
      cands = [syn for syn in zip(tmpCandidates[0], tmpCandidates[1])
               if (syn[0], syn[1]) not in synapsesAlreadyInSegment]

    # If we have no more candidates than requested, return all of them,
    # no shuffle necessary.
    if len(cands) <= n:
      return cands

    if n == 1: # so that we don't shuffle if only one is needed
      idx = self._random.getUInt32(len(cands))
      return [cands[idx]]  # col and cell idx in col

    # If we need more than one candidate
    indices = numpy.array([j for j in range(len(cands))], dtype='uint32')
    tmp = numpy.zeros(min(n, len(indices)), dtype='uint32')
    self._random.sample(indices, tmp)
    return sorted([cands[j] for j in tmp])

def _chooseCellsToLearnFrom ( self , c , i , s , n , activeState ) : if n <= 0 : return [ ] tmpCandidates = numpy . where ( activeState == 1 ) if len ( tmpCandidates [ 0 ] ) == 0 : return [ ] if s is None : cands = [ syn for syn in zip ( tmpCandidates [ 0 ] , tmpCandidates [ 1 ] ) ] else : synapsesAlreadyInSegment = set ( ( syn [ 0 ] , syn [ 1 ] ) for syn in s . syns ) cands = [ syn for syn in zip ( tmpCandidates [ 0 ] , tmpCandidates [ 1 ] ) if ( syn [ 0 ] , syn [ 1 ] ) not in synapsesAlreadyInSegment ] if len ( cands ) <= n : return cands if n == 1 : idx = self . _random . getUInt32 ( len ( cands ) ) return [ cands [ idx ] ] indices = numpy . array ( [ j for j in range ( len ( cands ) ) ] , dtype = 'uint32' ) tmp = numpy . zeros ( min ( n , len ( indices ) ) , dtype = 'uint32' ) self . _random . sample ( indices , tmp ) return sorted ( [ cands [ j ] for j in tmp ] )


def _adaptSegment(self, segUpdate):
    """
    This function applies segment update information to a segment in a
    cell.

    Synapses on the active list get their permanence counts incremented by
    permanenceInc. All other synapses get their permanence counts decremented
    by permanenceDec.

    We also increment the positiveActivations count of the segment.

    :param segUpdate SegmentUpdate instance
    :returns: True if some synapses were decremented to 0 and the segment is a
             candidate for trimming
    """
    # This will be set to True if detect that any syapses were decremented to
    #  0
    trimSegment = False

    # segUpdate.segment is None when creating a new segment
    c, i, segment = segUpdate.columnIdx, segUpdate.cellIdx, segUpdate.segment

    # update.activeSynapses can be empty.
    # If not, it can contain either or both integers and tuples.
    # The integers are indices of synapses to update.
    # The tuples represent new synapses to create (src col, src cell in col).
    # We pre-process to separate these various element types.
    # synToCreate is not empty only if positiveReinforcement is True.
    # NOTE: the synapse indices start at *1* to skip the segment flags.
    activeSynapses = segUpdate.activeSynapses
    synToUpdate = set([syn for syn in activeSynapses if type(syn) == int])

    # Modify an existing segment
    if segment is not None:

      if self.verbosity >= 4:
        print "Reinforcing segment #%d for cell[%d,%d]" % (segment.segID, c, i)
        print "  before:",
        segment.debugPrint()

      # Mark it as recently useful
      segment.lastActiveIteration = self.lrnIterationIdx

      # Update frequency and positiveActivations
      segment.positiveActivations += 1       # positiveActivations += 1
      segment.dutyCycle(active=True)

      # First, decrement synapses that are not active
      # s is a synapse *index*, with index 0 in the segment being the tuple
      # (segId, sequence segment flag). See below, creation of segments.
      lastSynIndex = len(segment.syns) - 1
      inactiveSynIndices = [s for s in xrange(0, lastSynIndex+1) \
                            if s not in synToUpdate]
      trimSegment = segment.updateSynapses(inactiveSynIndices,
                                           -self.permanenceDec)

      # Now, increment active synapses
      activeSynIndices = [syn for syn in synToUpdate if syn <= lastSynIndex]
      segment.updateSynapses(activeSynIndices, self.permanenceInc)

      # Finally, create new synapses if needed
      # syn is now a tuple (src col, src cell)
      synsToAdd = [syn for syn in activeSynapses if type(syn) != int]
      # If we have fixed resources, get rid of some old syns if necessary
      if self.maxSynapsesPerSegment > 0 \
          and len(synsToAdd) + len(segment.syns) > self.maxSynapsesPerSegment:
        numToFree = (len(segment.syns) + len(synsToAdd) -
                     self.maxSynapsesPerSegment)
        segment.freeNSynapses(numToFree, inactiveSynIndices, self.verbosity)
      for newSyn in synsToAdd:
        segment.addSynapse(newSyn[0], newSyn[1], self.initialPerm)

      if self.verbosity >= 4:
        print "   after:",
        segment.debugPrint()

    # Create a new segment
    else:

      # (segID, sequenceSegment flag, frequency, positiveActivations,
      #          totalActivations, lastActiveIteration)
      newSegment = Segment(tm=self, isSequenceSeg=segUpdate.sequenceSegment)

      # numpy.float32 important so that we can match with C++
      for synapse in activeSynapses:
        newSegment.addSynapse(synapse[0], synapse[1], self.initialPerm)

      if self.verbosity >= 3:
        print "New segment #%d for cell[%d,%d]" % (self.segID-1, c, i),
        newSegment.debugPrint()

      self.cells[c][i].append(newSegment)

    return trimSegment

This function applies segment update information to a segment in a cell .


def addSynapse(self, srcCellCol, srcCellIdx, perm):
    """Add a new synapse

    :param srcCellCol source cell column
    :param srcCellIdx source cell index within the column
    :param perm       initial permanence
    """
    self.syns.append([int(srcCellCol), int(srcCellIdx), numpy.float32(perm)])

def addSynapse ( self , srcCellCol , srcCellIdx , perm ) : self . syns . append ( [ int ( srcCellCol ) , int ( srcCellIdx ) , numpy . float32 ( perm ) ] )


def getSimplePatterns(numOnes, numPatterns, patternOverlap=0):
  """Very simple patterns. Each pattern has numOnes consecutive
  bits on. The amount of overlap between consecutive patterns is
  configurable, via the patternOverlap parameter.

  Parameters:
  -----------------------------------------------------------------------
  numOnes:        Number of bits ON in each pattern
  numPatterns:    Number of unique patterns to generate
  patternOverlap: Number of bits of overlap between each successive pattern
  retval:         patterns
  """

  assert (patternOverlap < numOnes)

  # How many new bits are introduced in each successive pattern?
  numNewBitsInEachPattern = numOnes - patternOverlap
  numCols = numNewBitsInEachPattern * numPatterns + patternOverlap

  p = []
  for i in xrange(numPatterns):
    x = numpy.zeros(numCols, dtype='float32')

    startBit = i*numNewBitsInEachPattern
    nextStartBit = startBit + numOnes
    x[startBit:nextStartBit] = 1

    p.append(x)

  return p

def getSimplePatterns ( numOnes , numPatterns , patternOverlap = 0 ) : assert ( patternOverlap < numOnes ) numNewBitsInEachPattern = numOnes - patternOverlap numCols = numNewBitsInEachPattern * numPatterns + patternOverlap p = [ ] for i in xrange ( numPatterns ) : x = numpy . zeros ( numCols , dtype = 'float32' ) startBit = i * numNewBitsInEachPattern nextStartBit = startBit + numOnes x [ startBit : nextStartBit ] = 1 p . append ( x ) return p

Very simple patterns . Each pattern has numOnes consecutive bits on . The amount of overlap between consecutive patterns is configurable via the patternOverlap parameter .


def buildOverlappedSequences( numSequences = 2,
                              seqLen = 5,
                              sharedElements = [3,4],
                              numOnBitsPerPattern = 3,
                              patternOverlap = 0,
                              seqOverlap = 0,
                              **kwargs
                              ):
  """ Create training sequences that share some elements in the middle.

  Parameters:
  -----------------------------------------------------
  numSequences:         Number of unique training sequences to generate
  seqLen:               Overall length of each sequence
  sharedElements:       Which element indices of each sequence are shared. These
                          will be in the range between 0 and seqLen-1
  numOnBitsPerPattern:  Number of ON bits in each TM input pattern
  patternOverlap:       Max number of bits of overlap between any 2 patterns
  retval:               (numCols, trainingSequences)
                          numCols - width of the patterns
                          trainingSequences - a list of training sequences

  """

  # Total number of patterns used to build the sequences
  numSharedElements = len(sharedElements)
  numUniqueElements = seqLen - numSharedElements
  numPatterns = numSharedElements + numUniqueElements * numSequences

  # Create the table of patterns
  patterns = getSimplePatterns(numOnBitsPerPattern, numPatterns, patternOverlap)

  # Total number of columns required
  numCols = len(patterns[0])

  # -----------------------------------------------------------------------
  # Create the training sequences
  trainingSequences = []

  uniquePatternIndices = range(numSharedElements, numPatterns)
  for _ in xrange(numSequences):
    sequence = []

    # pattern indices [0 ... numSharedElements-1] are reserved for the shared
    #  middle
    sharedPatternIndices = range(numSharedElements)

    # Build up the sequence
    for j in xrange(seqLen):
      if j in sharedElements:
        patIdx = sharedPatternIndices.pop(0)
      else:
        patIdx = uniquePatternIndices.pop(0)
      sequence.append(patterns[patIdx])

    trainingSequences.append(sequence)

  if VERBOSITY >= 3:
    print "\nTraining sequences"
    printAllTrainingSequences(trainingSequences)

  return (numCols, trainingSequences)

def
buildOverlappedSequences
(
numSequences
=
2
,
seqLen
=
5
,
sharedElements
=
[
3
,
4
]
,
numOnBitsPerPattern
=
3
,
patternOverlap
=
0
,
seqOverlap
=
0
,
**
kwargs
)
:
numSharedElements
=
len
(
sharedElements
)
numUniqueElements
=
seqLen
numSharedElements
numPatterns
=
numSharedElements
+
numUniqueElements
*
numSequences
patterns
=
getSimplePatterns
(
numOnBitsPerPattern
,
numPatterns
,
patternOverlap
)
numCols
=
len
(
patterns
[
0
]
)
trainingSequences
=
[
]
uniquePatternIndices
=
range
(
numSharedElements
,
numPatterns
)
for
_
in
xrange
(
numSequences
)
:
sequence
=
[
]
sharedPatternIndices
=
range
(
numSharedElements
)
for
j
in
xrange
(
seqLen
)
:
if
j
in
sharedElements
:
patIdx
=
sharedPatternIndices
.
pop
(
0
)
else
:
patIdx
=
uniquePatternIndices
.
pop
(
0
)
sequence
.
append
(
patterns
[
patIdx
]
)
trainingSequences
.
append
(
sequence
)
if
VERBOSITY
>=
3
:
print
"\nTraining sequences"
printAllTrainingSequences
(
trainingSequences
)
return
(
numCols
,
trainingSequences
)

Create training sequences that share some elements in the middle .


def buildSequencePool(numSequences = 10,
                      seqLen = [2,3,4],
                      numPatterns = 5,
                      numOnBitsPerPattern = 3,
                      patternOverlap = 0,
                      **kwargs
                      ):
  """ Create a bunch of sequences of various lengths, all built from
  a fixed set of patterns.

  Parameters:
  -----------------------------------------------------
  numSequences:         Number of training sequences to generate
  seqLen:               List of possible sequence lengths
  numPatterns:          How many possible patterns there are to use within
                          sequences
  numOnBitsPerPattern:  Number of ON bits in each TM input pattern
  patternOverlap:       Max number of bits of overlap between any 2 patterns
  retval:               (numCols, trainingSequences)
                          numCols - width of the patterns
                          trainingSequences - a list of training sequences

  """

  # Create the table of patterns
  patterns = getSimplePatterns(numOnBitsPerPattern, numPatterns, patternOverlap)

  # Total number of columns required
  numCols = len(patterns[0])

  # -----------------------------------------------------------------------
  # Create the training sequences
  trainingSequences = []
  for _ in xrange(numSequences):

    # Build it up from patterns
    sequence = []
    length = random.choice(seqLen)
    for _ in xrange(length):
      patIdx = random.choice(xrange(numPatterns))
      sequence.append(patterns[patIdx])

    # Put it in
    trainingSequences.append(sequence)

  if VERBOSITY >= 3:
    print "\nTraining sequences"
    printAllTrainingSequences(trainingSequences)

  return (numCols, trainingSequences)

def
buildSequencePool
(
numSequences
=
10
,
seqLen
=
[
2
,
3
,
4
]
,
numPatterns
=
5
,
numOnBitsPerPattern
=
3
,
patternOverlap
=
0
,
**
kwargs
)
:
patterns
=
getSimplePatterns
(
numOnBitsPerPattern
,
numPatterns
,
patternOverlap
)
numCols
=
len
(
patterns
[
0
]
)
trainingSequences
=
[
]
for
_
in
xrange
(
numSequences
)
:
sequence
=
[
]
length
=
random
.
choice
(
seqLen
)
for
_
in
xrange
(
length
)
:
patIdx
=
random
.
choice
(
xrange
(
numPatterns
)
)
sequence
.
append
(
patterns
[
patIdx
]
)
trainingSequences
.
append
(
sequence
)
if
VERBOSITY
>=
3
:
print
"\nTraining sequences"
printAllTrainingSequences
(
trainingSequences
)
return
(
numCols
,
trainingSequences
)

Create a bunch of sequences of various lengths all built from a fixed set of patterns .


def createTMs(includeCPP = True,
              includePy = True,
              numCols = 100,
              cellsPerCol = 4,
              activationThreshold = 3,
              minThreshold = 3,
              newSynapseCount = 3,
              initialPerm = 0.6,
              permanenceInc = 0.1,
              permanenceDec = 0.0,
              globalDecay = 0.0,
              pamLength = 0,
              checkSynapseConsistency = True,
              maxInfBacktrack = 0,
              maxLrnBacktrack = 0,
              **kwargs
              ):

  """Create one or more TM instances, placing each into a dict keyed by
  name.

  Parameters:
  ------------------------------------------------------------------
  retval:   tms - dict of TM instances
  """

  # Keep these fixed:
  connectedPerm = 0.5

  tms = dict()

  if includeCPP:
    if VERBOSITY >= 2:
      print "Creating BacktrackingTMCPP instance"

    cpp_tm = BacktrackingTMCPP(numberOfCols = numCols, cellsPerColumn = cellsPerCol,
                               initialPerm = initialPerm, connectedPerm = connectedPerm,
                               minThreshold = minThreshold, newSynapseCount = newSynapseCount,
                               permanenceInc = permanenceInc, permanenceDec = permanenceDec,
                               activationThreshold = activationThreshold,
                               globalDecay = globalDecay, burnIn = 1,
                               seed=SEED, verbosity=VERBOSITY,
                               checkSynapseConsistency = checkSynapseConsistency,
                               collectStats = True,
                               pamLength = pamLength,
                               maxInfBacktrack = maxInfBacktrack,
                               maxLrnBacktrack = maxLrnBacktrack,
                               )

    # Ensure we are copying over learning states for TMDiff
    cpp_tm.retrieveLearningStates = True

    tms['CPP'] = cpp_tm

  if includePy:
    if VERBOSITY >= 2:
      print "Creating PY TM instance"

    py_tm = BacktrackingTM(numberOfCols = numCols,
                           cellsPerColumn = cellsPerCol,
                           initialPerm = initialPerm,
                           connectedPerm = connectedPerm,
                           minThreshold = minThreshold,
                           newSynapseCount = newSynapseCount,
                           permanenceInc = permanenceInc,
                           permanenceDec = permanenceDec,
                           activationThreshold = activationThreshold,
                           globalDecay = globalDecay, burnIn = 1,
                           seed=SEED, verbosity=VERBOSITY,
                           collectStats = True,
                           pamLength = pamLength,
                           maxInfBacktrack = maxInfBacktrack,
                           maxLrnBacktrack = maxLrnBacktrack,
                           )

    tms['PY '] = py_tm

  return tms

def
createTMs
(
includeCPP
=
True
,
includePy
=
True
,
numCols
=
100
,
cellsPerCol
=
4
,
activationThreshold
=
3
,
minThreshold
=
3
,
newSynapseCount
=
3
,
initialPerm
=
0.6
,
permanenceInc
=
0.1
,
permanenceDec
=
0.0
,
globalDecay
=
0.0
,
pamLength
=
0
,
checkSynapseConsistency
=
True
,
maxInfBacktrack
=
0
,
maxLrnBacktrack
=
0
,
**
kwargs
)
:
connectedPerm
=
0.5
tms
=
dict
(
)
if
includeCPP
:
if
VERBOSITY
>=
2
:
print
"Creating BacktrackingTMCPP instance"
cpp_tm
=
BacktrackingTMCPP
(
numberOfCols
=
numCols
,
cellsPerColumn
=
cellsPerCol
,
initialPerm
=
initialPerm
,
connectedPerm
=
connectedPerm
,
minThreshold
=
minThreshold
,
newSynapseCount
=
newSynapseCount
,
permanenceInc
=
permanenceInc
,
permanenceDec
=
permanenceDec
,
activationThreshold
=
activationThreshold
,
globalDecay
=
globalDecay
,
burnIn
=
1
,
seed
=
SEED
,
verbosity
=
VERBOSITY
,
checkSynapseConsistency
=
checkSynapseConsistency
,
collectStats
=
True
,
pamLength
=
pamLength
,
maxInfBacktrack
=
maxInfBacktrack
,
maxLrnBacktrack
=
maxLrnBacktrack
,
)
cpp_tm
.
retrieveLearningStates
=
True
tms
[
'CPP'
]
=
cpp_tm
if
includePy
:
if
VERBOSITY
>=
2
:
print
"Creating PY TM instance"
py_tm
=
BacktrackingTM
(
numberOfCols
=
numCols
,
cellsPerColumn
=
cellsPerCol
,
initialPerm
=
initialPerm
,
connectedPerm
=
connectedPerm
,
minThreshold
=
minThreshold
,
newSynapseCount
=
newSynapseCount
,
permanenceInc
=
permanenceInc
,
permanenceDec
=
permanenceDec
,
activationThreshold
=
activationThreshold
,
globalDecay
=
globalDecay
,
burnIn
=
1
,
seed
=
SEED
,
verbosity
=
VERBOSITY
,
collectStats
=
True
,
pamLength
=
pamLength
,
maxInfBacktrack
=
maxInfBacktrack
,
maxLrnBacktrack
=
maxLrnBacktrack
,
)
tms
[
'PY '
]
=
py_tm
return
tms

Create one or more TM instances placing each into a dict keyed by name .


def assertNoTMDiffs(tms):
  """
  Check for diffs among the TM instances in the passed in tms dict and
  raise an assert if any are detected

  Parameters:
  ---------------------------------------------------------------------
  tms:                  dict of TM instances
  """

  if len(tms) == 1:
    return
  if len(tms) > 2:
    raise "Not implemented for more than 2 TMs"

  same = fdrutils.tmDiff2(tms.values(), verbosity=VERBOSITY)
  assert(same)
  return

def
assertNoTMDiffs
(
tms
)
:
if
len
(
tms
)
==
1
:
return
if
len
(
tms
)
>
2
:
raise
"Not implemented for more than 2 TMs"
same
=
fdrutils
.
tmDiff2
(
tms
.
values
(
)
,
verbosity
=
VERBOSITY
)
assert
(
same
)
return


def needsquoting(c, quotetabs, header):
    """Decide whether a particular character needs to be quoted.

    The 'quotetabs' flag indicates whether embedded tabs and spaces should be
    quoted.  Note that line-ending tabs and spaces are always encoded, as per
    RFC 1521.
    """
    if c in ' \t':
        return quotetabs
    # if header, we have to escape _ because _ is used to escape space
    if c == '_':
        return header
    return c == ESCAPE or not (' ' <= c <= '~')

def
needsquoting
(
c
,
quotetabs
,
header
)
:
if
c
in
' \t'
:
return
quotetabs
if
c
==
'_'
:
return
header
return
c
==
ESCAPE
or
not
(
' '
<=
c
<=
'~'
)

Decide whether a particular character needs to be quoted .


def quote(c):
    """Quote a single character."""
    i = ord(c)
    return ESCAPE + HEX[i//16] + HEX[i%16]

def quote ( c ) : i = ord ( c ) return ESCAPE + HEX [ i // 16 ] + HEX [ i % 16 ]


def encode(input, output, quotetabs, header = 0):
    """Read 'input', apply quoted-printable encoding, and write to 'output'.

    'input' and 'output' are files with readline() and write() methods.
    The 'quotetabs' flag indicates whether embedded tabs and spaces should be
    quoted.  Note that line-ending tabs and spaces are always encoded, as per
    RFC 1521.
    The 'header' flag indicates whether we are encoding spaces as _ as per
    RFC 1522.
    """

    if b2a_qp is not None:
        data = input.read()
        odata = b2a_qp(data, quotetabs = quotetabs, header = header)
        output.write(odata)
        return

    def write(s, output=output, lineEnd='\n'):
        # RFC 1521 requires that the line ending in a space or tab must have
        # that trailing character encoded.
        if s and s[-1:] in ' \t':
            output.write(s[:-1] + quote(s[-1]) + lineEnd)
        elif s == '.':
            output.write(quote(s) + lineEnd)
        else:
            output.write(s + lineEnd)

    prevline = None
    while 1:
        line = input.readline()
        if not line:
            break
        outline = []
        # Strip off any readline induced trailing newline
        stripped = ''
        if line[-1:] == '\n':
            line = line[:-1]
            stripped = '\n'
        # Calculate the un-length-limited encoded line
        for c in line:
            if needsquoting(c, quotetabs, header):
                c = quote(c)
            if header and c == ' ':
                outline.append('_')
            else:
                outline.append(c)
        # First, write out the previous line
        if prevline is not None:
            write(prevline)
        # Now see if we need any soft line breaks because of RFC-imposed
        # length limitations.  Then do the thisline->prevline dance.
        thisline = EMPTYSTRING.join(outline)
        while len(thisline) > MAXLINESIZE:
            # Don't forget to include the soft line break `=' sign in the
            # length calculation!
            write(thisline[:MAXLINESIZE-1], lineEnd='=\n')
            thisline = thisline[MAXLINESIZE-1:]
        # Write out the current line
        prevline = thisline
    # Write out the last line, without a trailing newline
    if prevline is not None:
        write(prevline, lineEnd=stripped)

def
encode
(
input
,
output
,
quotetabs
,
header
=
0
)
:
if
b2a_qp
is
not
:
data
=
input
.
read
(
)
odata
=
b2a_qp
(
data
,
quotetabs
=
quotetabs
,
header
=
header
)
output
.
write
(
odata
)
return
def
write
(
s
,
output
=
output
,
lineEnd
=
'\n'
)
:
if
s
and
s
[
1
:
]
in
' \t'
:
output
.
write
(
s
[
:
1
]
+
quote
(
s
[
1
]
)
+
lineEnd
)
elif
s
==
'.'
:
output
.
write
(
quote
(
s
)
+
lineEnd
)
else
:
output
.
write
(
s
+
lineEnd
)
prevline
=
while
1
:
line
=
input
.
readline
(
)
if
not
line
:
break
outline
=
[
]
stripped
=
''
if
line
[
1
:
]
==
'\n'
:
line
=
line
[
:
1
]
stripped
=
'\n'
for
c
in
line
:
if
needsquoting
(
c
,
quotetabs
,
header
)
:
c
=
quote
(
c
)
if
header
and
c
==
' '
:
outline
.
append
(
'_'
)
else
:
outline
.
append
(
c
)
if
prevline
is
not
:
write
(
prevline
)
thisline
=
EMPTYSTRING
.
join
(
outline
)
while
len
(
thisline
)
>
MAXLINESIZE
:
write
(
thisline
[
:
MAXLINESIZE
1
]
,
lineEnd
=
'=\n'
)
thisline
=
thisline
[
MAXLINESIZE
1
:
]
prevline
=
thisline
if
prevline
is
not
:
write
(
prevline
,
lineEnd
=
stripped
)

Read input apply quoted - printable encoding and write to output .


def unhex(s):
    """Get the integer value of a hexadecimal number."""
    bits = 0
    for c in s:
        if '0' <= c <= '9':
            i = ord('0')
        elif 'a' <= c <= 'f':
            i = ord('a')-10
        elif 'A' <= c <= 'F':
            i = ord('A')-10
        else:
            break
        bits = bits*16 + (ord(c) - i)
    return bits

def unhex ( s ) : bits = 0 for c in s : if '0' <= c <= '9' : i = ord ( '0' ) elif 'a' <= c <= 'f' : i = ord ( 'a' ) - 10 elif 'A' <= c <= 'F' : i = ord ( 'A' ) - 10 else : break bits = bits * 16 + ( ord ( c ) - i ) return bits

Get the integer value of a hexadecimal number .


def b64encode(s, altchars=None):
    """Encode a string using Base64.

    s is the string to encode.  Optional altchars must be a string of at least
    length 2 (additional characters are ignored) which specifies an
    alternative alphabet for the '+' and '/' characters.  This allows an
    application to e.g. generate url or filesystem safe Base64 strings.

    The encoded string is returned.
    """
    # Strip off the trailing newline
    encoded = binascii.b2a_base64(s)[:-1]
    if altchars is not None:
        return encoded.translate(string.maketrans(b'+/', altchars[:2]))
    return encoded

def b64encode ( s , altchars = None ) : encoded = binascii . b2a_base64 ( s ) [ : - 1 ] if altchars is not None : return encoded . translate ( string . maketrans ( b'+/' , altchars [ : 2 ] ) ) return encoded


def b64decode(s, altchars=None):
    """Decode a Base64 encoded string.

    s is the string to decode.  Optional altchars must be a string of at least
    length 2 (additional characters are ignored) which specifies the
    alternative alphabet used instead of the '+' and '/' characters.

    The decoded string is returned.  A TypeError is raised if s is
    incorrectly padded.  Characters that are neither in the normal base-64
    alphabet nor the alternative alphabet are discarded prior to the padding
    check.
    """
    if altchars is not None:
        s = s.translate(string.maketrans(altchars[:2], '+/'))
    try:
        return binascii.a2b_base64(s)
    except binascii.Error, msg:
        # Transform this exception for consistency
        raise TypeError(msg)

def b64decode ( s , altchars = None ) : if altchars is not None : s = s . translate ( string . maketrans ( altchars [ : 2 ] , '+/' ) ) try : return binascii . a2b_base64 ( s ) except binascii . Error , msg : raise TypeError ( msg )


def b32encode(s):
    """Encode a string using Base32.

    s is the string to encode.  The encoded string is returned.
    """
    parts = []
    quanta, leftover = divmod(len(s), 5)
    # Pad the last quantum with zero bits if necessary
    if leftover:
        s += ('\0' * (5 - leftover))
        quanta += 1
    for i in range(quanta):
        # c1 and c2 are 16 bits wide, c3 is 8 bits wide.  The intent of this
        # code is to process the 40 bits in units of 5 bits.  So we take the 1
        # leftover bit of c1 and tack it onto c2.  Then we take the 2 leftover
        # bits of c2 and tack them onto c3.  The shifts and masks are intended
        # to give us values of exactly 5 bits in width.
        c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5])
        c2 += (c1 & 1) << 16 # 17 bits wide
        c3 += (c2 & 3) << 8  # 10 bits wide
        parts.extend([_b32tab[c1 >> 11],         # bits 1 - 5
                      _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10
                      _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15
                      _b32tab[c2 >> 12],         # bits 16 - 20 (1 - 5)
                      _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10)
                      _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15)
                      _b32tab[c3 >> 5],          # bits 31 - 35 (1 - 5)
                      _b32tab[c3 & 0x1f],        # bits 36 - 40 (1 - 5)
                      ])
    encoded = EMPTYSTRING.join(parts)
    # Adjust for any leftover partial quanta
    if leftover == 1:
        return encoded[:-6] + '======'
    elif leftover == 2:
        return encoded[:-4] + '===='
    elif leftover == 3:
        return encoded[:-3] + '==='
    elif leftover == 4:
        return encoded[:-1] + '='
    return encoded

def b32encode ( s ) : parts = [ ] quanta , leftover = divmod ( len ( s ) , 5 ) if leftover : s += ( '\0' * ( 5 - leftover ) ) quanta += 1 for i in range ( quanta ) : c1 , c2 , c3 = struct . unpack ( '!HHB' , s [ i * 5 : ( i + 1 ) * 5 ] ) c2 += ( c1 & 1 ) << 16 c3 += ( c2 & 3 ) << 8 parts . extend ( [ _b32tab [ c1 >> 11 ] , _b32tab [ ( c1 >> 6 ) & 0x1f ] , _b32tab [ ( c1 >> 1 ) & 0x1f ] , _b32tab [ c2 >> 12 ] , _b32tab [ ( c2 >> 7 ) & 0x1f ] , _b32tab [ ( c2 >> 2 ) & 0x1f ] , _b32tab [ c3 >> 5 ] , _b32tab [ c3 & 0x1f ] , ] ) encoded = EMPTYSTRING . join ( parts ) if leftover == 1 : return encoded [ : - 6 ] + '======' elif leftover == 2 : return encoded [ : - 4 ] + '====' elif leftover == 3 : return encoded [ : - 3 ] + '===' elif leftover == 4 : return encoded [ : - 1 ] + '=' return encoded


def b32decode(s, casefold=False, map01=None):
    """Decode a Base32 encoded string.

    s is the string to decode.  Optional casefold is a flag specifying whether
    a lowercase alphabet is acceptable as input.  For security purposes, the
    default is False.

    RFC 3548 allows for optional mapping of the digit 0 (zero) to the letter O
    (oh), and for optional mapping of the digit 1 (one) to either the letter I
    (eye) or letter L (el).  The optional argument map01 when not None,
    specifies which letter the digit 1 should be mapped to (when map01 is not
    None, the digit 0 is always mapped to the letter O).  For security
    purposes the default is None, so that 0 and 1 are not allowed in the
    input.

    The decoded string is returned.  A TypeError is raised if s were
    incorrectly padded or if there are non-alphabet characters present in the
    string.
    """
    quanta, leftover = divmod(len(s), 8)
    if leftover:
        raise TypeError('Incorrect padding')
    # Handle section 2.4 zero and one mapping.  The flag map01 will be either
    # False, or the character to map the digit 1 (one) to.  It should be
    # either L (el) or I (eye).
    if map01:
        s = s.translate(string.maketrans(b'01', b'O' + map01))
    if casefold:
        s = s.upper()
    # Strip off pad characters from the right.  We need to count the pad
    # characters because this will tell us how many null bytes to remove from
    # the end of the decoded string.
    padchars = 0
    mo = re.search('(?P<pad>[=]*)$', s)
    if mo:
        padchars = len(mo.group('pad'))
        if padchars > 0:
            s = s[:-padchars]
    # Now decode the full quanta
    parts = []
    acc = 0
    shift = 35
    for c in s:
        val = _b32rev.get(c)
        if val is None:
            raise TypeError('Non-base32 digit found')
        acc += _b32rev[c] << shift
        shift -= 5
        if shift < 0:
            parts.append(binascii.unhexlify('%010x' % acc))
            acc = 0
            shift = 35
    # Process the last, partial quanta
    last = binascii.unhexlify('%010x' % acc)
    if padchars == 0:
        last = ''                       # No characters
    elif padchars == 1:
        last = last[:-1]
    elif padchars == 3:
        last = last[:-2]
    elif padchars == 4:
        last = last[:-3]
    elif padchars == 6:
        last = last[:-4]
    else:
        raise TypeError('Incorrect padding')
    parts.append(last)
    return EMPTYSTRING.join(parts)

def
b32decode
(
s
,
casefold
=
False
,
map01
=
)
:
quanta
,
leftover
=
divmod
(
len
(
s
)
,
8
)
if
leftover
:
raise
TypeError
(
'Incorrect padding'
)
if
map01
:
s
=
s
.
translate
(
string
.
maketrans
(
b'01'
,
b'O'
+
map01
)
)
if
casefold
:
s
=
s
.
upper
(
)
padchars
=
0
mo
=
re
.
search
(
'(?P<pad>[=]*)$'
,
s
)
if
mo
:
padchars
=
len
(
mo
.
group
(
'pad'
)
)
if
padchars
>
0
:
s
=
s
[
:
padchars
]
parts
=
[
]
acc
=
0
shift
=
35
for
c
in
s
:
val
=
_b32rev
.
get
(
c
)
if
val
is
:
raise
TypeError
(
'Non-base32 digit found'
)
acc
+=
_b32rev
[
c
]
<<
shift
shift
-=
5
if
shift
<
0
:
parts
.
append
(
binascii
.
unhexlify
(
'%010x'
%
acc
)
)
acc
=
0
shift
=
35
last
=
binascii
.
unhexlify
(
'%010x'
%
acc
)
if
padchars
==
0
:
last
=
''
elif
padchars
==
1
:
last
=
last
[
:
1
]
elif
padchars
==
3
:
last
=
last
[
:
2
]
elif
padchars
==
4
:
last
=
last
[
:
3
]
elif
padchars
==
6
:
last
=
last
[
:
4
]
else
:
raise
TypeError
(
'Incorrect padding'
)
parts
.
append
(
last
)
return
EMPTYSTRING
.
join
(
parts
)


def b16decode(s, casefold=False):
    """Decode a Base16 encoded string.

    s is the string to decode.  Optional casefold is a flag specifying whether
    a lowercase alphabet is acceptable as input.  For security purposes, the
    default is False.

    The decoded string is returned.  A TypeError is raised if s is
    incorrectly padded or if there are non-alphabet characters present in the
    string.
    """
    if casefold:
        s = s.upper()
    if re.search('[^0-9A-F]', s):
        raise TypeError('Non-base16 digit found')
    return binascii.unhexlify(s)

def
b16decode
(
s
,
casefold
=
False
)
:
if
casefold
:
s
=
s
.
upper
(
)
if
re
.
search
(
'[^0-9A-F]'
,
s
)
:
raise
TypeError
(
'Non-base16 digit found'
)
return
binascii
.
unhexlify
(
s
)


def encode(input, output):
    """Encode a file."""
    while True:
        s = input.read(MAXBINSIZE)
        if not s:
            break
        while len(s) < MAXBINSIZE:
            ns = input.read(MAXBINSIZE-len(s))
            if not ns:
                break
            s += ns
        line = binascii.b2a_base64(s)
        output.write(line)

def encode ( input , output ) : while True : s = input . read ( MAXBINSIZE ) if not s : break while len ( s ) < MAXBINSIZE : ns = input . read ( MAXBINSIZE - len ( s ) ) if not ns : break s += ns line = binascii . b2a_base64 ( s ) output . write ( line )


def decode(input, output):
    """Decode a file."""
    while True:
        line = input.readline()
        if not line:
            break
        s = binascii.a2b_base64(line)
        output.write(s)

def decode ( input , output ) : while True : line = input . readline ( ) if not line : break s = binascii . a2b_base64 ( line ) output . write ( s )


def encodestring(s):
    """Encode a string into multiple lines of base-64 data."""
    pieces = []
    for i in range(0, len(s), MAXBINSIZE):
        chunk = s[i : i + MAXBINSIZE]
        pieces.append(binascii.b2a_base64(chunk))
    return "".join(pieces)

def encodestring ( s ) : pieces = [ ] for i in range ( 0 , len ( s ) , MAXBINSIZE ) : chunk = s [ i : i + MAXBINSIZE ] pieces . append ( binascii . b2a_base64 ( chunk ) ) return "" . join ( pieces )

Encode a string into multiple lines of base - 64 data .


def chain(self, expanded_from):
        """
        Returns a range identical to this one, but indicating that
        it was expanded from the range `expanded_from`.
        """
        return Range(self.source_buffer, self.begin_pos, self.begin_pos,
                     expanded_from=expanded_from)

def chain ( self , expanded_from ) : return Range ( self . source_buffer , self . begin_pos , self . begin_pos , expanded_from = expanded_from )

Returns a range identical to this one but indicating that it was expanded from the range expanded_from .


def begin(self):
        """
        Returns a zero-length range located just before the beginning of this range.
        """
        return Range(self.source_buffer, self.begin_pos, self.begin_pos,
                     expanded_from=self.expanded_from)

def begin ( self ) : return Range ( self . source_buffer , self . begin_pos , self . begin_pos , expanded_from = self . expanded_from )

Returns a zero - length range located just before the beginning of this range .


def end(self):
        """
        Returns a zero-length range located just after the end of this range.
        """
        return Range(self.source_buffer, self.end_pos, self.end_pos,
                     expanded_from=self.expanded_from)

def end ( self ) : return Range ( self . source_buffer , self . end_pos , self . end_pos , expanded_from = self . expanded_from )

Returns a zero - length range located just after the end of this range .


def column(self):
        """
        Returns a zero-based column number of the beginning of this range.
        """
        line, column = self.source_buffer.decompose_position(self.begin_pos)
        return column

def column ( self ) : line , column = self . source_buffer . decompose_position ( self . begin_pos ) return column

Returns a zero - based column number of the beginning of this range .


def line(self):
        """
        Returns the line number of the beginning of this range.
        """
        line, column = self.source_buffer.decompose_position(self.begin_pos)
        return line

def line ( self ) : line , column = self . source_buffer . decompose_position ( self . begin_pos ) return line

Returns the line number of the beginning of this range .


def source_lines(self):
        """
        Returns the lines of source code containing the entirety of this range.
        """
        return [self.source_buffer.source_line(line)
                for line in range(self.line(), self.end().line() + 1)]

def source_lines ( self ) : return [ self . source_buffer . source_line ( line ) for line in range ( self . line ( ) , self . end ( ) . line ( ) + 1 ) ]

Returns the lines of source code containing the entirety of this range .


def compare(left, right, compare_locs=False):
    """
    An AST comparison function. Returns ``True`` if all fields in
    ``left`` are equal to fields in ``right``; if ``compare_locs`` is
    true, all locations should match as well.
    """
    if type(left) != type(right):
        return False

    if isinstance(left, ast.AST):
        for field in left._fields:
            if not compare(getattr(left, field), getattr(right, field)):
                return False

        if compare_locs:
            for loc in left._locs:
                if getattr(left, loc) != getattr(right, loc):
                    return False

        return True
    elif isinstance(left, list):
        if len(left) != len(right):
            return False

        for left_elt, right_elt in zip(left, right):
            if not compare(left_elt, right_elt):
                return False

        return True
    else:
        return left == right

def compare ( left , right , compare_locs = False ) : if type ( left ) != type ( right ) : return False if isinstance ( left , ast . AST ) : for field in left . _fields : if not compare ( getattr ( left , field ) , getattr ( right , field ) ) : return False if compare_locs : for loc in left . _locs : if getattr ( left , loc ) != getattr ( right , loc ) : return False return True elif isinstance ( left , list ) : if len ( left ) != len ( right ) : return False for left_elt , right_elt in zip ( left , right ) : if not compare ( left_elt , right_elt ) : return False return True else : return left == right

An AST comparison function . Returns True if all fields in left are equal to fields in right ; if compare_locs is true all locations should match as well .


def generic_visit(self, node):
        """Called if no explicit visitor function exists for a node."""
        for field_name in node._fields:
            setattr(node, field_name, self.visit(getattr(node, field_name)))
        return node

def generic_visit ( self , node ) : for field_name in node . _fields : setattr ( node , field_name , self . visit ( getattr ( node , field_name ) ) ) return node

Called if no explicit visitor function exists for a node .


def float_unpack(Q, size, le):
  """Convert a 32-bit or 64-bit integer created
  by float_pack into a Python float."""

  if size == 8:
    MIN_EXP = -1021  # = sys.float_info.min_exp
    MAX_EXP = 1024   # = sys.float_info.max_exp
    MANT_DIG = 53    # = sys.float_info.mant_dig
    BITS = 64
  elif size == 4:
    MIN_EXP = -125   # C's FLT_MIN_EXP
    MAX_EXP = 128    # FLT_MAX_EXP
    MANT_DIG = 24    # FLT_MANT_DIG
    BITS = 32
  else:
    raise ValueError("invalid size value")

  if Q >> BITS:
    raise ValueError("input out of range")

  # extract pieces
  sign = Q >> BITS - 1
  exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1
  mant = Q & ((1 << MANT_DIG - 1) - 1)

  if exp == MAX_EXP - MIN_EXP + 2:
    # nan or infinity
    result = float('nan') if mant else float('inf')
  elif exp == 0:
    # subnormal or zero
    result = math.ldexp(float(mant), MIN_EXP - MANT_DIG)
  else:
    # normal
    mant += 1 << MANT_DIG - 1
    result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1)
  return -result if sign else result

def
float_unpack
(
Q
,
size
,
le
)
:
if
size
==
8
:
MIN_EXP
=
1021
MAX_EXP
=
1024
MANT_DIG
=
53
BITS
=
64
elif
size
==
4
:
MIN_EXP
=
125
MAX_EXP
=
128
MANT_DIG
=
24
BITS
=
32
else
:
raise
ValueError
(
"invalid size value"
)
if
Q
>>
BITS
:
raise
ValueError
(
"input out of range"
)
sign
=
Q
>>
BITS
1
exp
=
(
Q
&
(
(
1
<<
BITS
1
)
(
1
<<
MANT_DIG
1
)
)
)
>>
MANT_DIG
1
mant
=
Q
&
(
(
1
<<
MANT_DIG
1
)
1
)
if
exp
==
MAX_EXP
MIN_EXP
+
2
:
result
=
float
(
'nan'
)
if
mant
else
float
(
'inf'
)
elif
exp
==
0
:
result
=
math
.
ldexp
(
float
(
mant
)
,
MIN_EXP
MANT_DIG
)
else
:
mant
+=
1
<<
MANT_DIG
1
result
=
math
.
ldexp
(
float
(
mant
)
,
exp
+
MIN_EXP
MANT_DIG
1
)
return
result
if
sign
else
result

Convert a 32 - bit or 64 - bit integer created by float_pack into a Python float .


def float_pack(x, size):
  """Convert a Python float x into a 64-bit unsigned integer
  with the same byte representation."""

  if size == 8:
    MIN_EXP = -1021  # = sys.float_info.min_exp
    MAX_EXP = 1024   # = sys.float_info.max_exp
    MANT_DIG = 53    # = sys.float_info.mant_dig
    BITS = 64
  elif size == 4:
    MIN_EXP = -125   # C's FLT_MIN_EXP
    MAX_EXP = 128    # FLT_MAX_EXP
    MANT_DIG = 24    # FLT_MANT_DIG
    BITS = 32
  else:
    raise ValueError("invalid size value")

  sign = math.copysign(1.0, x) < 0.0
  if math.isinf(x):
    mant = 0
    exp = MAX_EXP - MIN_EXP + 2
  elif math.isnan(x):
    mant = 1 << (MANT_DIG - 2)  # other values possible
    exp = MAX_EXP - MIN_EXP + 2
  elif x == 0.0:
    mant = 0
    exp = 0
  else:
    m, e = math.frexp(abs(x))  # abs(x) == m * 2**e
    exp = e - (MIN_EXP - 1)
    if exp > 0:
      # Normal case.
      mant = round_to_nearest(m * (1 << MANT_DIG))
      mant -= 1 << MANT_DIG - 1
    else:
      # Subnormal case.
      if exp + MANT_DIG - 1 >= 0:
        mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1))
      else:
        mant = 0
      exp = 0

    # Special case: rounding produced a MANT_DIG-bit mantissa.
    assert 0 <= mant <= 1 << MANT_DIG - 1
    if mant == 1 << MANT_DIG - 1:
      mant = 0
      exp += 1

    # Raise on overflow (in some circumstances, may want to return
    # infinity instead).
    if exp >= MAX_EXP - MIN_EXP + 2:
      raise OverflowError("float too large to pack in this format")

  # check constraints
  assert 0 <= mant < 1 << MANT_DIG - 1
  assert 0 <= exp <= MAX_EXP - MIN_EXP + 2
  assert 0 <= sign <= 1
  return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant

Convert a Python float x into a 64 - bit unsigned integer with the same byte representation .


def context(self, *notes):
        """
        A context manager that appends ``note`` to every diagnostic processed by
        this engine.
        """
        self._appended_notes += notes
        yield
        del self._appended_notes[-len(notes):]

def context ( self , * notes ) : self . _appended_notes += notes yield del self . _appended_notes [ - len ( notes ) : ]

A context manager that appends note to every diagnostic processed by this engine .


def format_list(extracted_list):
    """Format a list of traceback entry tuples for printing.

    Given a list of tuples as returned by extract_tb() or
    extract_stack(), return a list of strings ready for printing.
    Each string in the resulting list corresponds to the item with the
    same index in the argument list.  Each string ends in a newline;
    the strings may contain internal newlines as well, for those items
    whose source text line is not None.
    """
    list = []
    for filename, lineno, name, line in extracted_list:
        item = '  File "%s", line %d, in %s\n' % (filename,lineno,name)
        if line:
            item = item + '    %s\n' % line.strip()
        list.append(item)
    return list

def
format_list
(
extracted_list
)
:
list
=
[
]
for
filename
,
lineno
,
name
,
line
in
extracted_list
:
item
=
'  File "%s", line %d, in %s\n'
%
(
filename
,
lineno
,
name
)
if
line
:
item
=
item
+
'    %s\n'
%
line
.
strip
(
)
list
.
append
(
item
)
return
list

Format a list of traceback entry tuples for printing .


def print_tb(tb, limit=None, file=None):
    """Print up to 'limit' stack trace entries from the traceback 'tb'.

    If 'limit' is omitted or None, all entries are printed.  If 'file'
    is omitted or None, the output goes to sys.stderr; otherwise
    'file' should be an open file or file-like object with a write()
    method.
    """
    if file is None:
        file = sys.stderr
    if limit is None:
        if hasattr(sys, 'tracebacklimit'):
            limit = sys.tracebacklimit
    n = 0
    while tb is not None and (limit is None or n < limit):
        f = tb.tb_frame
        lineno = tb.tb_lineno
        co = f.f_code
        filename = co.co_filename
        name = co.co_name
        _print(file,
               '  File "%s", line %d, in %s' % (filename, lineno, name))
        linecache.checkcache(filename)
        line = linecache.getline(filename, lineno, f.f_globals)
        if line: _print(file, '    ' + line.strip())
        tb = tb.tb_next
        n = n+1

def
print_tb
(
tb
,
limit
=
,
file
=
)
:
if
file
is
:
file
=
sys
.
stderr
if
limit
is
:
if
hasattr
(
sys
,
'tracebacklimit'
)
:
limit
=
sys
.
tracebacklimit
n
=
0
while
tb
is
not
and
(
limit
is
or
n
<
limit
)
:
f
=
tb
.
tb_frame
lineno
=
tb
.
tb_lineno
co
=
f
.
f_code
filename
=
co
.
co_filename
name
=
co
.
co_name
_print
(
file
,
'  File "%s", line %d, in %s'
%
(
filename
,
lineno
,
name
)
)
linecache
.
checkcache
(
filename
)
line
=
linecache
.
getline
(
filename
,
lineno
,
f
.
f_globals
)
if
line
:
_print
(
file
,
'    '
+
line
.
strip
(
)
)
tb
=
tb
.
tb_next
n
=
n
+
1

Print up to limit stack trace entries from the traceback tb .


def print_exception(etype, value, tb, limit=None, file=None):
    """Print exception up to 'limit' stack trace entries from 'tb' to 'file'.

    This differs from print_tb() in the following ways: (1) if
    traceback is not None, it prints a header "Traceback (most recent
    call last):"; (2) it prints the exception type and value after the
    stack trace; (3) if type is SyntaxError and value has the
    appropriate format, it prints the line where the syntax error
    occurred with a caret on the next line indicating the approximate
    position of the error.
    """
    if file is None:
        # TODO: Use sys.stderr when that's implemented.
        file = open('/dev/stderr', 'w')
        #file = sys.stderr
    if tb:
        _print(file, 'Traceback (most recent call last):')
        print_tb(tb, limit, file)
    lines = format_exception_only(etype, value)
    for line in lines:
        _print(file, line, '')

def
print_exception
(
etype
,
value
,
tb
,
limit
=
,
file
=
)
:
if
file
is
:
file
=
open
(
'/dev/stderr'
,
'w'
)
if
tb
:
_print
(
file
,
'Traceback (most recent call last):'
)
print_tb
(
tb
,
limit
,
file
)
lines
=
format_exception_only
(
etype
,
value
)
for
line
in
lines
:
_print
(
file
,
line
,
''
)

Print exception up to limit stack trace entries from tb to file .


def format_exception(etype, value, tb, limit = None):
    """Format a stack trace and the exception information.

    The arguments have the same meaning as the corresponding arguments
    to print_exception().  The return value is a list of strings, each
    ending in a newline and some containing internal newlines.  When
    these lines are concatenated and printed, exactly the same text is
    printed as does print_exception().
    """
    if tb:
        list = ['Traceback (most recent call last):\n']
        list = list + format_tb(tb, limit)
    else:
        list = []
    list = list + format_exception_only(etype, value)
    return list

def
format_exception
(
etype
,
value
,
tb
,
limit
=
)
:
if
tb
:
list
=
[
'Traceback (most recent call last):\n'
]
list
=
list
+
format_tb
(
tb
,
limit
)
else
:
list
=
[
]
list
=
list
+
format_exception_only
(
etype
,
value
)
return
list

Format a stack trace and the exception information .


def format_exception_only(etype, value):
    """Format the exception part of a traceback.

    The arguments are the exception type and value such as given by
    sys.last_type and sys.last_value. The return value is a list of
    strings, each ending in a newline.

    Normally, the list contains a single string; however, for
    SyntaxError exceptions, it contains several lines that (when
    printed) display detailed information about where the syntax
    error occurred.

    The message indicating which exception occurred is always the last
    string in the list.

    """

    # An instance should not have a meaningful value parameter, but
    # sometimes does, particularly for string exceptions, such as
    # >>> raise string1, string2  # deprecated
    #
    # Clear these out first because issubtype(string1, SyntaxError)
    # would raise another exception and mask the original problem.
    if (isinstance(etype, BaseException) or
#        isinstance(etype, types.InstanceType) or
        etype is None or type(etype) is str):
        return [_format_final_exc_line(etype, value)]

    stype = etype.__name__

    if not issubclass(etype, SyntaxError):
        return [_format_final_exc_line(stype, value)]

    # It was a syntax error; show exactly where the problem was found.
    lines = []
    try:
        msg, (filename, lineno, offset, badline) = value.args
    except Exception:
        pass
    else:
        filename = filename or "<string>"
        lines.append('  File "%s", line %d\n' % (filename, lineno))
        if badline is not None:
            lines.append('    %s\n' % badline.strip())
            if offset is not None:
                caretspace = badline.rstrip('\n')
                offset = min(len(caretspace), offset) - 1
                caretspace = caretspace[:offset].lstrip()
                # non-space whitespace (likes tabs) must be kept for alignment
                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
                lines.append('    %s^\n' % ''.join(caretspace))
        value = msg

    lines.append(_format_final_exc_line(stype, value))
    return lines

def
format_exception_only
(
etype
,
value
)
:
if
(
isinstance
(
etype
,
BaseException
)
or
etype
is
or
type
(
etype
)
is
str
)
:
return
[
_format_final_exc_line
(
etype
,
value
)
]
stype
=
etype
.
__name__
if
not
issubclass
(
etype
,
SyntaxError
)
:
return
[
_format_final_exc_line
(
stype
,
value
)
]
lines
=
[
]
try
:
msg
,
(
filename
,
lineno
,
offset
,
badline
)
=
value
.
args
except
Exception
:
pass
else
:
filename
=
filename
or
"<string>"
lines
.
append
(
'  File "%s", line %d\n'
%
(
filename
,
lineno
)
)
if
badline
is
not
:
lines
.
append
(
'    %s\n'
%
badline
.
strip
(
)
)
if
offset
is
not
:
caretspace
=
badline
.
rstrip
(
'\n'
)
offset
=
min
(
len
(
caretspace
)
,
offset
)
1
caretspace
=
caretspace
[
:
offset
]
.
lstrip
(
)
caretspace
=
(
(
c
.
isspace
(
)
and
c
or
' '
)
for
c
in
caretspace
)
lines
.
append
(
'    %s^\n'
%
''
.
join
(
caretspace
)
)
value
=
msg
lines
.
append
(
_format_final_exc_line
(
stype
,
value
)
)
return
lines

Format the exception part of a traceback .


def print_stack(f=None, limit=None, file=None):
    """Print a stack trace from its invocation point.

    The optional 'f' argument can be used to specify an alternate
    stack frame at which to start. The optional 'limit' and 'file'
    arguments have the same meaning as for print_exception().
    """
    if f is None:
        try:
            raise ZeroDivisionError
        except ZeroDivisionError:
            f = sys.exc_info()[2].tb_frame.f_back
    print_list(extract_stack(f, limit), file)

def print_stack ( f = None , limit = None , file = None ) : if f is None : try : raise ZeroDivisionError except ZeroDivisionError : f = sys . exc_info ( ) [ 2 ] . tb_frame . f_back print_list ( extract_stack ( f , limit ) , file )

Print a stack trace from its invocation point .


def extract_stack(f=None, limit = None):
    """Extract the raw traceback from the current stack frame.

    The return value has the same format as for extract_tb().  The
    optional 'f' and 'limit' arguments have the same meaning as for
    print_stack().  Each item in the list is a quadruple (filename,
    line number, function name, text), and the entries are in order
    from oldest to newest stack frame.
    """
    if f is None:
        try:
            raise ZeroDivisionError
        except ZeroDivisionError:
            f = sys.exc_info()[2].tb_frame.f_back
    if limit is None:
        if hasattr(sys, 'tracebacklimit'):
            limit = sys.tracebacklimit
    list = []
    n = 0
    while f is not None and (limit is None or n < limit):
        lineno = f.f_lineno
        co = f.f_code
        filename = co.co_filename
        name = co.co_name
        linecache.checkcache(filename)
        line = linecache.getline(filename, lineno, f.f_globals)
        if line: line = line.strip()
        else: line = None
        list.append((filename, lineno, name, line))
        f = f.f_back
        n = n+1
    list.reverse()
    return list

def extract_stack ( f = None , limit = None ) : if f is None : try : raise ZeroDivisionError except ZeroDivisionError : f = sys . exc_info ( ) [ 2 ] . tb_frame . f_back if limit is None : if hasattr ( sys , 'tracebacklimit' ) : limit = sys . tracebacklimit list = [ ] n = 0 while f is not None and ( limit is None or n < limit ) : lineno = f . f_lineno co = f . f_code filename = co . co_filename name = co . co_name linecache . checkcache ( filename ) line = linecache . getline ( filename , lineno , f . f_globals ) if line : line = line . strip ( ) else : line = None list . append ( ( filename , lineno , name , line ) ) f = f . f_back n = n + 1 list . reverse ( ) return list

Extract the raw traceback from the current stack frame .


def seed(self, a=None):
        """Initialize internal state of the random number generator.

        None or no argument seeds from current time or from an operating
        system specific randomness source if available.

        If a is not None or is an int or long, hash(a) is used instead.
        Hash values for some types are nondeterministic when the
        PYTHONHASHSEED environment variable is enabled.
        """

        super(Random, self).seed(a)
        self.gauss_next = None

def seed ( self , a = None ) : super ( Random , self ) . seed ( a ) self . gauss_next = None

Initialize internal state of the random number generator .


def shuffle(self, x, random=None):
        """x, random=random.random -> shuffle list x in place; return None.

        Optional arg random is a 0-argument function returning a random
        float in [0.0, 1.0); by default, the standard random.random.

        """

        if random is None:
            random = self.random
        _int = int
        for i in reversed(xrange(1, len(x))):
            # pick an element in x[:i+1] with which to exchange x[i]
            j = _int(random() * (i+1))
            x[i], x[j] = x[j], x[i]

def shuffle ( self , x , random = None ) : if random is None : random = self . random _int = int for i in reversed ( xrange ( 1 , len ( x ) ) ) : j = _int ( random ( ) * ( i + 1 ) ) x [ i ] , x [ j ] = x [ j ] , x [ i ]

x random = random . random - > shuffle list x in place ; return None .


def _slotnames(cls):
    """Return a list of slot names for a given class.

    This needs to find slots defined by the class and its bases, so we
    can't simply return the __slots__ attribute.  We must walk down
    the Method Resolution Order and concatenate the __slots__ of each
    class found there.  (This assumes classes don't modify their
    __slots__ attribute to misrepresent their slots after the class is
    defined.)
    """

    # Get the value from a cache in the class if possible
    names = cls.__dict__.get("__slotnames__")
    if names is not None:
        return names

    # Not cached -- calculate the value
    names = []
    if not hasattr(cls, "__slots__"):
        # This class has no slots
        pass
    else:
        # Slots found -- gather slot names from all base classes
        for c in cls.__mro__:
            if "__slots__" in c.__dict__:
                slots = c.__dict__['__slots__']
                # if class has a single slot, it can be given as a string
                if isinstance(slots, basestring):
                    slots = (slots,)
                for name in slots:
                    # special descriptors
                    if name in ("__dict__", "__weakref__"):
                        continue
                    # mangled names
                    elif name.startswith('__') and not name.endswith('__'):
                        names.append('_%s%s' % (c.__name__, name))
                    else:
                        names.append(name)

    # Cache the outcome in the class if at all possible
    try:
        cls.__slotnames__ = names
    except:
        pass # But don't die if we can't

    return names

def _slotnames ( cls ) : names = cls . __dict__ . get ( "__slotnames__" ) if names is not None : return names names = [ ] if not hasattr ( cls , "__slots__" ) : pass else : for c in cls . __mro__ : if "__slots__" in c . __dict__ : slots = c . __dict__ [ '__slots__' ] if isinstance ( slots , basestring ) : slots = ( slots , ) for name in slots : if name in ( "__dict__" , "__weakref__" ) : continue elif name . startswith ( '__' ) and not name . endswith ( '__' ) : names . append ( '_%s%s' % ( c . __name__ , name ) ) else : names . append ( name ) try : cls . __slotnames__ = names except : pass return names

Return a list of slot names for a given class .


def add_extension(module, name, code):
    """Register an extension code."""
    code = int(code)
    if not 1 <= code <= 0x7fffffff:
        raise ValueError, "code out of range"
    key = (module, name)
    if (_extension_registry.get(key) == code and
        _inverted_registry.get(code) == key):
        return # Redundant registrations are benign
    if key in _extension_registry:
        raise ValueError("key %s is already registered with code %s" %
                         (key, _extension_registry[key]))
    if code in _inverted_registry:
        raise ValueError("code %s is already in use for key %s" %
                         (code, _inverted_registry[code]))
    _extension_registry[key] = code
    _inverted_registry[code] = key


def remove_extension(module, name, code):
    """Unregister an extension code.  For testing only."""
    key = (module, name)
    if (_extension_registry.get(key) != code or
        _inverted_registry.get(code) != key):
        raise ValueError("key %s is not registered with code %s" %
                         (key, code))
    del _extension_registry[key]
    del _inverted_registry[code]
    if code in _extension_cache:
        del _extension_cache[code]

def
remove_extension
(
module
,
name
,
code
)
:
key
=
(
module
,
name
)
if
(
_extension_registry
.
get
(
key
)
!=
code
or
_inverted_registry
.
get
(
code
)
!=
key
)
:
raise
ValueError
(
"key %s is not registered with code %s"
%
(
key
,
code
)
)
del
_extension_registry
[
key
]
del
_inverted_registry
[
code
]
if
code
in
_extension_cache
:
del
_extension_cache
[
code
]

Unregister an extension code . For testing only .


def update_wrapper(wrapper,
                   wrapped,
                   assigned = WRAPPER_ASSIGNMENTS,
                   updated = WRAPPER_UPDATES):
    """Update a wrapper function to look like the wrapped function

       wrapper is the function to be updated
       wrapped is the original function
       assigned is a tuple naming the attributes assigned directly
       from the wrapped function to the wrapper function (defaults to
       functools.WRAPPER_ASSIGNMENTS)
       updated is a tuple naming the attributes of the wrapper that
       are updated with the corresponding attribute from the wrapped
       function (defaults to functools.WRAPPER_UPDATES)
    """
    for attr in assigned:
        setattr(wrapper, attr, getattr(wrapped, attr))
    for attr in updated:
        getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
    # Return the wrapper so this can be used as a decorator via partial()
    return wrapper

def update_wrapper ( wrapper , wrapped , assigned = WRAPPER_ASSIGNMENTS , updated = WRAPPER_UPDATES ) : for attr in assigned : setattr ( wrapper , attr , getattr ( wrapped , attr ) ) for attr in updated : getattr ( wrapper , attr ) . update ( getattr ( wrapped , attr , { } ) ) return wrapper


def cmp_to_key(mycmp):
    """Convert a cmp= function into a key= function"""
    class K(object):
        __slots__ = ['obj']
        def __init__(self, obj, *args):
            self.obj = obj
        def __lt__(self, other):
            return mycmp(self.obj, other.obj) < 0
        def __gt__(self, other):
            return mycmp(self.obj, other.obj) > 0
        def __eq__(self, other):
            return mycmp(self.obj, other.obj) == 0
        def __le__(self, other):
            return mycmp(self.obj, other.obj) <= 0
        def __ge__(self, other):
            return mycmp(self.obj, other.obj) >= 0
        def __ne__(self, other):
            return mycmp(self.obj, other.obj) != 0
        def __hash__(self):
            raise TypeError('hash not implemented')
    return K

def
cmp_to_key
(
mycmp
)
:
class
K
(
object
)
:
__slots__
=
[
'obj'
]
def
__init__
(
self
,
obj
,
*
args
)
:
self
.
obj
=
obj
def
__lt__
(
self
,
other
)
:
return
mycmp
(
self
.
obj
,
other
.
obj
)
<
0
def
__gt__
(
self
,
other
)
:
return
mycmp
(
self
.
obj
,
other
.
obj
)
>
0
def
__eq__
(
self
,
other
)
:
return
mycmp
(
self
.
obj
,
other
.
obj
)
==
0
def
__le__
(
self
,
other
)
:
return
mycmp
(
self
.
obj
,
other
.
obj
)
<=
0
def
__ge__
(
self
,
other
)
:
return
mycmp
(
self
.
obj
,
other
.
obj
)
>=
0
def
__ne__
(
self
,
other
)
:
return
mycmp
(
self
.
obj
,
other
.
obj
)
!=
0
def
__hash__
(
self
)
:
raise
TypeError
(
'hash not implemented'
)
return
K

Convert a cmp = function into a key = function


def unquote(s):
    """Remove quotes from a string."""
    if len(s) > 1:
        if s.startswith('"') and s.endswith('"'):
            return s[1:-1].replace('\\\\', '\\').replace('\\"', '"')
        if s.startswith('<') and s.endswith('>'):
            return s[1:-1]
    return s

def unquote ( s ) : if len ( s ) > 1 : if s . startswith ( '"' ) and s . endswith ( '"' ) : return s [ 1 : - 1 ] . replace ( '\\\\' , '\\' ) . replace ( '\\"' , '"' ) if s . startswith ( '<' ) and s . endswith ( '>' ) : return s [ 1 : - 1 ] return s


def formatdate(timeval=None):
    """Returns time format preferred for Internet standards.

    Sun, 06 Nov 1994 08:49:37 GMT  ; RFC 822, updated by RFC 1123

    According to RFC 1123, day and month names must always be in
    English.  If not for that, this code could use strftime().  It
    can't because strftime() honors the locale and could generate
    non-English names.
    """
    if timeval is None:
        timeval = time.time()
    timeval = time.gmtime(timeval)
    return "%s, %02d %s %04d %02d:%02d:%02d GMT" % (
            ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")[timeval[6]],
            timeval[2],
            ("Jan", "Feb", "Mar", "Apr", "May", "Jun",
             "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[timeval[1]-1],
                                timeval[0], timeval[3], timeval[4], timeval[5])

def
formatdate
(
timeval
=
)
:
if
timeval
is
:
timeval
=
time
.
time
(
)
timeval
=
time
.
gmtime
(
timeval
)
return
"%s, %02d %s %04d %02d:%02d:%02d GMT"
%
(
(
"Mon"
,
"Tue"
,
"Wed"
,
"Thu"
,
"Fri"
,
"Sat"
,
"Sun"
)
[
timeval
[
6
]
]
,
timeval
[
2
]
,
(
"Jan"
,
"Feb"
,
"Mar"
,
"Apr"
,
"May"
,
"Jun"
,
"Jul"
,
"Aug"
,
"Sep"
,
"Oct"
,
"Nov"
,
"Dec"
)
[
timeval
[
1
]
1
]
,
timeval
[
0
]
,
timeval
[
3
]
,
timeval
[
4
]
,
timeval
[
5
]
)

Returns time format preferred for Internet standards .