def compute(self, activeColumns, learn=True):
    """
    Perform one time step of the Temporal Memory algorithm.

    This method calls :meth:`activateCells`, then calls
    :meth:`activateDendrites`. Using :class:`TemporalMemory` via its
    :meth:`compute` method ensures that you'll always be able to call
    :meth:`getPredictiveCells` to get predictions for the next time step.

    :param activeColumns: (iter) Indices of active columns.

    :param learn: (bool) Whether or not learning is enabled.
    """
    self.activateCells(sorted(activeColumns), learn)
    self.activateDendrites(learn)

def compute ( self , activeColumns , learn = True ) : self . activateCells ( sorted ( activeColumns ) , learn ) self . activateDendrites ( learn )

Perform one time step of the Temporal Memory algorithm .


def activateCells(self, activeColumns, learn=True):
    """
    Calculate the active cells, using the current active columns and dendrite
    segments. Grow and reinforce synapses.

    :param activeColumns: (iter) A sorted list of active column indices.

    :param learn: (bool) If true, reinforce / punish / grow synapses.

      **Pseudocode:**

      ::

        for each column
          if column is active and has active distal dendrite segments
            call activatePredictedColumn
          if column is active and doesn't have active distal dendrite segments
            call burstColumn
          if column is inactive and has matching distal dendrite segments
            call punishPredictedColumn
    """
    prevActiveCells = self.activeCells
    prevWinnerCells = self.winnerCells
    self.activeCells = []
    self.winnerCells = []

    segToCol = lambda segment: int(segment.cell / self.cellsPerColumn)
    identity = lambda x: x

    for columnData in groupby2(activeColumns, identity,
                               self.activeSegments, segToCol,
                               self.matchingSegments, segToCol):
      (column,
       activeColumns,
       columnActiveSegments,
       columnMatchingSegments) = columnData
      if activeColumns is not None:
        if columnActiveSegments is not None:
          cellsToAdd = self.activatePredictedColumn(column,
                                                    columnActiveSegments,
                                                    columnMatchingSegments,
                                                    prevActiveCells,
                                                    prevWinnerCells,
                                                    learn)

          self.activeCells += cellsToAdd
          self.winnerCells += cellsToAdd
        else:
          (cellsToAdd,
           winnerCell) = self.burstColumn(column,
                                          columnMatchingSegments,
                                          prevActiveCells,
                                          prevWinnerCells,
                                          learn)

          self.activeCells += cellsToAdd
          self.winnerCells.append(winnerCell)
      else:
        if learn:
          self.punishPredictedColumn(column,
                                     columnActiveSegments,
                                     columnMatchingSegments,
                                     prevActiveCells,
                                     prevWinnerCells)

def activateCells ( self , activeColumns , learn = True ) : prevActiveCells = self . activeCells prevWinnerCells = self . winnerCells self . activeCells = [ ] self . winnerCells = [ ] segToCol = lambda segment : int ( segment . cell / self . cellsPerColumn ) identity = lambda x : x for columnData in groupby2 ( activeColumns , identity , self . activeSegments , segToCol , self . matchingSegments , segToCol ) : ( column , activeColumns , columnActiveSegments , columnMatchingSegments ) = columnData if activeColumns is not None : if columnActiveSegments is not None : cellsToAdd = self . activatePredictedColumn ( column , columnActiveSegments , columnMatchingSegments , prevActiveCells , prevWinnerCells , learn ) self . activeCells += cellsToAdd self . winnerCells += cellsToAdd else : ( cellsToAdd , winnerCell ) = self . burstColumn ( column , columnMatchingSegments , prevActiveCells , prevWinnerCells , learn ) self . activeCells += cellsToAdd self . winnerCells . append ( winnerCell ) else : if learn : self . punishPredictedColumn ( column , columnActiveSegments , columnMatchingSegments , prevActiveCells , prevWinnerCells )

Calculate the active cells using the current active columns and dendrite segments . Grow and reinforce synapses .


def activateDendrites(self, learn=True):
    """
    Calculate dendrite segment activity, using the current active cells.

    :param learn: (bool) If true, segment activations will be recorded. This
           information is used during segment cleanup.

    **Pseudocode:**

    ::

      for each distal dendrite segment with activity >= activationThreshold
        mark the segment as active
      for each distal dendrite segment with unconnected activity >= minThreshold
        mark the segment as matching
    """
    (numActiveConnected,
     numActivePotential) = self.connections.computeActivity(
       self.activeCells,
       self.connectedPermanence)

    activeSegments = (
      self.connections.segmentForFlatIdx(i)
      for i in xrange(len(numActiveConnected))
      if numActiveConnected[i] >= self.activationThreshold
    )

    matchingSegments = (
      self.connections.segmentForFlatIdx(i)
      for i in xrange(len(numActivePotential))
      if numActivePotential[i] >= self.minThreshold
    )

    self.activeSegments = sorted(activeSegments,
                                 key=self.connections.segmentPositionSortKey)
    self.matchingSegments = sorted(matchingSegments,
                                   key=self.connections.segmentPositionSortKey)
    self.numActiveConnectedSynapsesForSegment = numActiveConnected
    self.numActivePotentialSynapsesForSegment = numActivePotential

    if learn:
      for segment in self.activeSegments:
        self.lastUsedIterationForSegment[segment.flatIdx] = self.iteration
      self.iteration += 1

def activateDendrites ( self , learn = True ) : ( numActiveConnected , numActivePotential ) = self . connections . computeActivity ( self . activeCells , self . connectedPermanence ) activeSegments = ( self . connections . segmentForFlatIdx ( i ) for i in xrange ( len ( numActiveConnected ) ) if numActiveConnected [ i ] >= self . activationThreshold ) matchingSegments = ( self . connections . segmentForFlatIdx ( i ) for i in xrange ( len ( numActivePotential ) ) if numActivePotential [ i ] >= self . minThreshold ) self . activeSegments = sorted ( activeSegments , key = self . connections . segmentPositionSortKey ) self . matchingSegments = sorted ( matchingSegments , key = self . connections . segmentPositionSortKey ) self . numActiveConnectedSynapsesForSegment = numActiveConnected self . numActivePotentialSynapsesForSegment = numActivePotential if learn : for segment in self . activeSegments : self . lastUsedIterationForSegment [ segment . flatIdx ] = self . iteration self . iteration += 1

Calculate dendrite segment activity using the current active cells .


def reset(self):
    """
    Indicates the start of a new sequence. Clears any predictions and makes sure
    synapses don't grow to the currently active cells in the next time step.
    """
    self.activeCells = []
    self.winnerCells = []
    self.activeSegments = []
    self.matchingSegments = []

def reset ( self ) : self . activeCells = [ ] self . winnerCells = [ ] self . activeSegments = [ ] self . matchingSegments = [ ]

Indicates the start of a new sequence . Clears any predictions and makes sure synapses don t grow to the currently active cells in the next time step .


def activatePredictedColumn(self, column, columnActiveSegments,
                              columnMatchingSegments, prevActiveCells,
                              prevWinnerCells, learn):
    """
    Determines which cells in a predicted column should be added to winner cells
    list, and learns on the segments that correctly predicted this column.

    :param column: (int) Index of bursting column.

    :param columnActiveSegments: (iter) Active segments in this column.

    :param columnMatchingSegments: (iter) Matching segments in this column.

    :param prevActiveCells: (list) Active cells in ``t-1``.

    :param prevWinnerCells: (list) Winner cells in ``t-1``.

    :param learn: (bool) If true, grow and reinforce synapses.

    :returns: (list) A list of predicted cells that will be added to
              active cells and winner cells.
    """
    return self._activatePredictedColumn(
      self.connections, self._random,
      columnActiveSegments, prevActiveCells, prevWinnerCells,
      self.numActivePotentialSynapsesForSegment,
      self.maxNewSynapseCount, self.initialPermanence,
      self.permanenceIncrement, self.permanenceDecrement,
      self.maxSynapsesPerSegment, learn)

def activatePredictedColumn ( self , column , columnActiveSegments , columnMatchingSegments , prevActiveCells , prevWinnerCells , learn ) : return self . _activatePredictedColumn ( self . connections , self . _random , columnActiveSegments , prevActiveCells , prevWinnerCells , self . numActivePotentialSynapsesForSegment , self . maxNewSynapseCount , self . initialPermanence , self . permanenceIncrement , self . permanenceDecrement , self . maxSynapsesPerSegment , learn )

Determines which cells in a predicted column should be added to winner cells list and learns on the segments that correctly predicted this column .


def punishPredictedColumn(self, column, columnActiveSegments,
                            columnMatchingSegments, prevActiveCells,
                            prevWinnerCells):
    """
    Punishes the Segments that incorrectly predicted a column to be active.

    :param column: (int) Index of bursting column.

    :param columnActiveSegments: (iter) Active segments for this column, or None
           if there aren't any.

    :param columnMatchingSegments: (iter) Matching segments for this column, or
           None if there aren't any.

    :param prevActiveCells: (list) Active cells in ``t-1``.

    :param prevWinnerCells: (list) Winner cells in ``t-1``.

    """
    self._punishPredictedColumn(
      self.connections, columnMatchingSegments, prevActiveCells,
      self.predictedSegmentDecrement)

def punishPredictedColumn ( self , column , columnActiveSegments , columnMatchingSegments , prevActiveCells , prevWinnerCells ) : self . _punishPredictedColumn ( self . connections , columnMatchingSegments , prevActiveCells , self . predictedSegmentDecrement )

Punishes the Segments that incorrectly predicted a column to be active .


def _createSegment(cls, connections, lastUsedIterationForSegment, cell,
                     iteration, maxSegmentsPerCell):
    """
    Create a segment on the connections, enforcing the maxSegmentsPerCell
    parameter.
    """
    # Enforce maxSegmentsPerCell.
    while connections.numSegments(cell) >= maxSegmentsPerCell:
      leastRecentlyUsedSegment = min(
        connections.segmentsForCell(cell),
        key=lambda segment : lastUsedIterationForSegment[segment.flatIdx])

      connections.destroySegment(leastRecentlyUsedSegment)

    # Create the segment.
    segment = connections.createSegment(cell)

    # Do TM-specific bookkeeping for the segment.
    if segment.flatIdx == len(lastUsedIterationForSegment):
      lastUsedIterationForSegment.append(iteration)
    elif segment.flatIdx < len(lastUsedIterationForSegment):
      # A flatIdx was recycled.
      lastUsedIterationForSegment[segment.flatIdx] = iteration
    else:
      raise AssertionError(
        "All segments should be created with the TM createSegment method.")

    return segment

Create a segment on the connections enforcing the maxSegmentsPerCell parameter .


def _destroyMinPermanenceSynapses(cls, connections, random, segment,
                                    nDestroy, excludeCells):
    """
    Destroy nDestroy synapses on the specified segment, but don't destroy
    synapses to the "excludeCells".
    """

    destroyCandidates = sorted(
      (synapse for synapse in connections.synapsesForSegment(segment)
       if synapse.presynapticCell not in excludeCells),
      key=lambda s: s._ordinal
    )

    for _ in xrange(nDestroy):
      if len(destroyCandidates) == 0:
        break

      minSynapse = None
      minPermanence = float("inf")

      for synapse in destroyCandidates:
        if synapse.permanence < minPermanence - EPSILON:
          minSynapse = synapse
          minPermanence = synapse.permanence

      connections.destroySynapse(minSynapse)
      destroyCandidates.remove(minSynapse)

def _destroyMinPermanenceSynapses ( cls , connections , random , segment , nDestroy , excludeCells ) : destroyCandidates = sorted ( ( synapse for synapse in connections . synapsesForSegment ( segment ) if synapse . presynapticCell not in excludeCells ) , key = lambda s : s . _ordinal ) for _ in xrange ( nDestroy ) : if len ( destroyCandidates ) == 0 : break minSynapse = None minPermanence = float ( "inf" ) for synapse in destroyCandidates : if synapse . permanence < minPermanence - EPSILON : minSynapse = synapse minPermanence = synapse . permanence connections . destroySynapse ( minSynapse ) destroyCandidates . remove ( minSynapse )

Destroy nDestroy synapses on the specified segment but don t destroy synapses to the excludeCells .


def _leastUsedCell(cls, random, cells, connections):
    """
    Gets the cell with the smallest number of segments.
    Break ties randomly.

    :param random: (Object)
    Random number generator. Gets mutated.

    :param cells: (list)
    Indices of cells.

    :param connections: (Object)
    Connections instance for the TM.

    :returns: (int) Cell index.
    """
    leastUsedCells = []
    minNumSegments = float("inf")
    for cell in cells:
      numSegments = connections.numSegments(cell)

      if numSegments < minNumSegments:
        minNumSegments = numSegments
        leastUsedCells = []

      if numSegments == minNumSegments:
        leastUsedCells.append(cell)

    i = random.getUInt32(len(leastUsedCells))
    return leastUsedCells[i]

def _leastUsedCell ( cls , random , cells , connections ) : leastUsedCells = [ ] minNumSegments = float ( "inf" ) for cell in cells : numSegments = connections . numSegments ( cell ) if numSegments < minNumSegments : minNumSegments = numSegments leastUsedCells = [ ] if numSegments == minNumSegments : leastUsedCells . append ( cell ) i = random . getUInt32 ( len ( leastUsedCells ) ) return leastUsedCells [ i ]

Gets the cell with the smallest number of segments . Break ties randomly .


def _growSynapses(cls, connections, random, segment, nDesiredNewSynapes,
                    prevWinnerCells, initialPermanence, maxSynapsesPerSegment):
    """
    Creates nDesiredNewSynapes synapses on the segment passed in if
    possible, choosing random cells from the previous winner cells that are
    not already on the segment.

    :param connections:        (Object) Connections instance for the tm
    :param random:             (Object) TM object used to generate random
                                        numbers
    :param segment:            (int)    Segment to grow synapses on.
    :param nDesiredNewSynapes: (int)    Desired number of synapses to grow
    :param prevWinnerCells:    (list)   Winner cells in `t-1`
    :param initialPermanence:  (float)  Initial permanence of a new synapse.

    """
    candidates = list(prevWinnerCells)

    for synapse in connections.synapsesForSegment(segment):
      i = binSearch(candidates, synapse.presynapticCell)
      if i != -1:
        del candidates[i]

    nActual = min(nDesiredNewSynapes, len(candidates))

    # Check if we're going to surpass the maximum number of synapses.
    overrun = connections.numSynapses(segment) + nActual - maxSynapsesPerSegment
    if overrun > 0:
      cls._destroyMinPermanenceSynapses(connections, random, segment, overrun,
                                        prevWinnerCells)

    # Recalculate in case we weren't able to destroy as many synapses as needed.
    nActual = min(nActual,
                  maxSynapsesPerSegment - connections.numSynapses(segment))

    for _ in range(nActual):
      i = random.getUInt32(len(candidates))
      connections.createSynapse(segment, candidates[i], initialPermanence)
      del candidates[i]

def _growSynapses ( cls , connections , random , segment , nDesiredNewSynapes , prevWinnerCells , initialPermanence , maxSynapsesPerSegment ) : candidates = list ( prevWinnerCells ) for synapse in connections . synapsesForSegment ( segment ) : i = binSearch ( candidates , synapse . presynapticCell ) if i != - 1 : del candidates [ i ] nActual = min ( nDesiredNewSynapes , len ( candidates ) ) overrun = connections . numSynapses ( segment ) + nActual - maxSynapsesPerSegment if overrun > 0 : cls . _destroyMinPermanenceSynapses ( connections , random , segment , overrun , prevWinnerCells ) nActual = min ( nActual , maxSynapsesPerSegment - connections . numSynapses ( segment ) ) for _ in range ( nActual ) : i = random . getUInt32 ( len ( candidates ) ) connections . createSynapse ( segment , candidates [ i ] , initialPermanence ) del candidates [ i ]

Creates nDesiredNewSynapes synapses on the segment passed in if possible choosing random cells from the previous winner cells that are not already on the segment .


def _adaptSegment(cls, connections, segment, prevActiveCells,
                    permanenceIncrement, permanenceDecrement):
    """
    Updates synapses on segment.
    Strengthens active synapses; weakens inactive synapses.

    :param connections:          (Object) Connections instance for the tm
    :param segment:              (int)    Segment to adapt
    :param prevActiveCells:      (list)   Active cells in `t-1`
    :param permanenceIncrement:  (float)  Amount to increment active synapses
    :param permanenceDecrement:  (float)  Amount to decrement inactive synapses
    """

    # Destroying a synapse modifies the set that we're iterating through.
    synapsesToDestroy = []

    for synapse in connections.synapsesForSegment(segment):
      permanence = synapse.permanence

      if binSearch(prevActiveCells, synapse.presynapticCell) != -1:
        permanence += permanenceIncrement
      else:
        permanence -= permanenceDecrement

      # Keep permanence within min/max bounds
      permanence = max(0.0, min(1.0, permanence))

      if permanence < EPSILON:
        synapsesToDestroy.append(synapse)
      else:
        connections.updateSynapsePermanence(synapse, permanence)

    for synapse in synapsesToDestroy:
      connections.destroySynapse(synapse)

    if connections.numSynapses(segment) == 0:
      connections.destroySegment(segment)

def _adaptSegment ( cls , connections , segment , prevActiveCells , permanenceIncrement , permanenceDecrement ) : synapsesToDestroy = [ ] for synapse in connections . synapsesForSegment ( segment ) : permanence = synapse . permanence if binSearch ( prevActiveCells , synapse . presynapticCell ) != - 1 : permanence += permanenceIncrement else : permanence -= permanenceDecrement permanence = max ( 0.0 , min ( 1.0 , permanence ) ) if permanence < EPSILON : synapsesToDestroy . append ( synapse ) else : connections . updateSynapsePermanence ( synapse , permanence ) for synapse in synapsesToDestroy : connections . destroySynapse ( synapse ) if connections . numSynapses ( segment ) == 0 : connections . destroySegment ( segment )

Updates synapses on segment . Strengthens active synapses ; weakens inactive synapses .


def columnForCell(self, cell):
    """
    Returns the index of the column that a cell belongs to.

    :param cell: (int) Cell index

    :returns: (int) Column index
    """
    self._validateCell(cell)

    return int(cell / self.cellsPerColumn)

def columnForCell ( self , cell ) : self . _validateCell ( cell ) return int ( cell / self . cellsPerColumn )

Returns the index of the column that a cell belongs to .


def cellsForColumn(self, column):
    """
    Returns the indices of cells that belong to a column.

    :param column: (int) Column index

    :returns: (list) Cell indices
    """
    self._validateColumn(column)

    start = self.cellsPerColumn * column
    end = start + self.cellsPerColumn
    return range(start, end)

def cellsForColumn ( self , column ) : self . _validateColumn ( column ) start = self . cellsPerColumn * column end = start + self . cellsPerColumn return range ( start , end )

Returns the indices of cells that belong to a column .


def mapCellsToColumns(self, cells):
    """
    Maps cells to the columns they belong to.

    :param cells: (set) Cells

    :returns: (dict) Mapping from columns to their cells in `cells`
    """
    cellsForColumns = defaultdict(set)

    for cell in cells:
      column = self.columnForCell(cell)
      cellsForColumns[column].add(cell)

    return cellsForColumns

def mapCellsToColumns ( self , cells ) : cellsForColumns = defaultdict ( set ) for cell in cells : column = self . columnForCell ( cell ) cellsForColumns [ column ] . add ( cell ) return cellsForColumns

Maps cells to the columns they belong to .


def getPredictiveCells(self):
    """ Returns the indices of the predictive cells.

    :returns: (list) Indices of predictive cells.
    """
    previousCell = None
    predictiveCells = []
    for segment in self.activeSegments:
      if segment.cell != previousCell:
        predictiveCells.append(segment.cell)
        previousCell = segment.cell

    return predictiveCells

def getPredictiveCells ( self ) : previousCell = None predictiveCells = [ ] for segment in self . activeSegments : if segment . cell != previousCell : predictiveCells . append ( segment . cell ) previousCell = segment . cell return predictiveCells

Returns the indices of the predictive cells .


def read(cls, proto):
    """
    Reads deserialized data from proto object.

    :param proto: (DynamicStructBuilder) Proto object

    :returns: (:class:TemporalMemory) TemporalMemory instance
    """
    tm = object.__new__(cls)

    # capnp fails to save a tuple, so proto.columnDimensions was forced to
    # serialize as a list.  We prefer a tuple, however, because columnDimensions
    # should be regarded as immutable.
    tm.columnDimensions = tuple(proto.columnDimensions)
    tm.cellsPerColumn = int(proto.cellsPerColumn)
    tm.activationThreshold = int(proto.activationThreshold)
    tm.initialPermanence = round(proto.initialPermanence, EPSILON_ROUND)
    tm.connectedPermanence = round(proto.connectedPermanence, EPSILON_ROUND)
    tm.minThreshold = int(proto.minThreshold)
    tm.maxNewSynapseCount = int(proto.maxNewSynapseCount)
    tm.permanenceIncrement = round(proto.permanenceIncrement, EPSILON_ROUND)
    tm.permanenceDecrement = round(proto.permanenceDecrement, EPSILON_ROUND)
    tm.predictedSegmentDecrement = round(proto.predictedSegmentDecrement,
                                         EPSILON_ROUND)

    tm.maxSegmentsPerCell = int(proto.maxSegmentsPerCell)
    tm.maxSynapsesPerSegment = int(proto.maxSynapsesPerSegment)

    tm.connections = Connections.read(proto.connections)
    #pylint: disable=W0212
    tm._random = Random()
    tm._random.read(proto.random)
    #pylint: enable=W0212

    tm.activeCells = [int(x) for x in proto.activeCells]
    tm.winnerCells = [int(x) for x in proto.winnerCells]

    flatListLength = tm.connections.segmentFlatListLength()
    tm.numActiveConnectedSynapsesForSegment = [0] * flatListLength
    tm.numActivePotentialSynapsesForSegment = [0] * flatListLength
    tm.lastUsedIterationForSegment = [0] * flatListLength

    tm.activeSegments = []
    tm.matchingSegments = []

    for protoSegment in proto.activeSegments:
      tm.activeSegments.append(
        tm.connections.getSegment(protoSegment.cell,
                                  protoSegment.idxOnCell))

    for protoSegment in proto.matchingSegments:
      tm.matchingSegments.append(
        tm.connections.getSegment(protoSegment.cell,
                                  protoSegment.idxOnCell))

    for protoSegment in proto.numActivePotentialSynapsesForSegment:
      segment = tm.connections.getSegment(protoSegment.cell,
                                          protoSegment.idxOnCell)

      tm.numActivePotentialSynapsesForSegment[segment.flatIdx] = (
        int(protoSegment.number))

    tm.iteration = long(proto.iteration)

    for protoSegment in proto.lastUsedIterationForSegment:
      segment = tm.connections.getSegment(protoSegment.cell,
                                          protoSegment.idxOnCell)

      tm.lastUsedIterationForSegment[segment.flatIdx] = (
        long(protoSegment.number))

    return tm

def read ( cls , proto ) : tm = object . __new__ ( cls ) tm . columnDimensions = tuple ( proto . columnDimensions ) tm . cellsPerColumn = int ( proto . cellsPerColumn ) tm . activationThreshold = int ( proto . activationThreshold ) tm . initialPermanence = round ( proto . initialPermanence , EPSILON_ROUND ) tm . connectedPermanence = round ( proto . connectedPermanence , EPSILON_ROUND ) tm . minThreshold = int ( proto . minThreshold ) tm . maxNewSynapseCount = int ( proto . maxNewSynapseCount ) tm . permanenceIncrement = round ( proto . permanenceIncrement , EPSILON_ROUND ) tm . permanenceDecrement = round ( proto . permanenceDecrement , EPSILON_ROUND ) tm . predictedSegmentDecrement = round ( proto . predictedSegmentDecrement , EPSILON_ROUND ) tm . maxSegmentsPerCell = int ( proto . maxSegmentsPerCell ) tm . maxSynapsesPerSegment = int ( proto . maxSynapsesPerSegment ) tm . connections = Connections . read ( proto . connections ) tm . _random = Random ( ) tm . _random . read ( proto . random ) tm . activeCells = [ int ( x ) for x in proto . activeCells ] tm . winnerCells = [ int ( x ) for x in proto . winnerCells ] flatListLength = tm . connections . segmentFlatListLength ( ) tm . numActiveConnectedSynapsesForSegment = [ 0 ] * flatListLength tm . numActivePotentialSynapsesForSegment = [ 0 ] * flatListLength tm . lastUsedIterationForSegment = [ 0 ] * flatListLength tm . activeSegments = [ ] tm . matchingSegments = [ ] for protoSegment in proto . activeSegments : tm . activeSegments . append ( tm . connections . getSegment ( protoSegment . cell , protoSegment . idxOnCell ) ) for protoSegment in proto . matchingSegments : tm . matchingSegments . append ( tm . connections . getSegment ( protoSegment . cell , protoSegment . idxOnCell ) ) for protoSegment in proto . numActivePotentialSynapsesForSegment : segment = tm . connections . getSegment ( protoSegment . cell , protoSegment . idxOnCell ) tm . numActivePotentialSynapsesForSegment [ segment . flatIdx ] = ( int ( protoSegment . number ) ) tm . iteration = long ( proto . iteration ) for protoSegment in proto . lastUsedIterationForSegment : segment = tm . connections . getSegment ( protoSegment . cell , protoSegment . idxOnCell ) tm . lastUsedIterationForSegment [ segment . flatIdx ] = ( long ( protoSegment . number ) ) return tm

Reads deserialized data from proto object .


def generateFromNumbers(self, numbers):
    """
    Generate a sequence from a list of numbers.

    Note: Any `None` in the list of numbers is considered a reset.

    @param numbers (list) List of numbers

    @return (list) Generated sequence
    """
    sequence = []

    for number in numbers:
      if number == None:
        sequence.append(number)
      else:
        pattern = self.patternMachine.get(number)
        sequence.append(pattern)

    return sequence

def generateFromNumbers ( self , numbers ) : sequence = [ ] for number in numbers : if number == None : sequence . append ( number ) else : pattern = self . patternMachine . get ( number ) sequence . append ( pattern ) return sequence

Generate a sequence from a list of numbers .


def addSpatialNoise(self, sequence, amount):
    """
    Add spatial noise to each pattern in the sequence.

    @param sequence (list)  Sequence
    @param amount   (float) Amount of spatial noise

    @return (list) Sequence with spatial noise
    """
    newSequence = []

    for pattern in sequence:
      if pattern is not None:
        pattern = self.patternMachine.addNoise(pattern, amount)
      newSequence.append(pattern)

    return newSequence

def addSpatialNoise ( self , sequence , amount ) : newSequence = [ ] for pattern in sequence : if pattern is not None : pattern = self . patternMachine . addNoise ( pattern , amount ) newSequence . append ( pattern ) return newSequence

Add spatial noise to each pattern in the sequence .


def prettyPrintSequence(self, sequence, verbosity=1):
    """
    Pretty print a sequence.

    @param sequence  (list) Sequence
    @param verbosity (int)  Verbosity level

    @return (string) Pretty-printed text
    """
    text = ""

    for i in xrange(len(sequence)):
      pattern = sequence[i]

      if pattern == None:
        text += "<reset>"
        if i < len(sequence) - 1:
          text += "\n"
      else:
        text += self.patternMachine.prettyPrintPattern(pattern,
                                                       verbosity=verbosity)

    return text

def prettyPrintSequence ( self , sequence , verbosity = 1 ) : text = "" for i in xrange ( len ( sequence ) ) : pattern = sequence [ i ] if pattern == None : text += "<reset>" if i < len ( sequence ) - 1 : text += "\n" else : text += self . patternMachine . prettyPrintPattern ( pattern , verbosity = verbosity ) return text


def mmPrettyPrintTraces(traces, breakOnResets=None):
    """
    Returns pretty-printed table of traces.

    @param traces (list) Traces to print in table
    @param breakOnResets (BoolsTrace) Trace of resets to break table on

    @return (string) Pretty-printed table of traces.
    """
    assert len(traces) > 0, "No traces found"
    table = PrettyTable(["#"] + [trace.prettyPrintTitle() for trace in traces])

    for i in xrange(len(traces[0].data)):
      if breakOnResets and breakOnResets.data[i]:
        table.add_row(["<reset>"] * (len(traces) + 1))
      table.add_row([i] +
        [trace.prettyPrintDatum(trace.data[i]) for trace in traces])

    return table.get_string().encode("utf-8")

def
mmPrettyPrintTraces
(
traces
,
breakOnResets
=
)
:
assert
len
(
traces
)
>
0
,
"No traces found"
table
=
PrettyTable
(
[
"#"
]
+
[
trace
.
prettyPrintTitle
(
)
for
trace
in
traces
]
)
for
i
in
xrange
(
len
(
traces
[
0
]
.
data
)
)
:
if
breakOnResets
and
breakOnResets
.
data
[
i
]
:
table
.
add_row
(
[
"<reset>"
]
*
(
len
(
traces
)
+
1
)
)
table
.
add_row
(
[
i
]
+
[
trace
.
prettyPrintDatum
(
trace
.
data
[
i
]
)
for
trace
in
traces
]
)
return
table
.
get_string
(
)
.
encode
(
"utf-8"
)

Returns pretty - printed table of traces .


def mmPrettyPrintMetrics(metrics, sigFigs=5):
    """
    Returns pretty-printed table of metrics.

    @param metrics (list) Traces to print in table
    @param sigFigs (int)  Number of significant figures to print

    @return (string) Pretty-printed table of metrics.
    """
    assert len(metrics) > 0, "No metrics found"
    table = PrettyTable(["Metric", "mean", "standard deviation",
                         "min", "max", "sum", ])

    for metric in metrics:
      table.add_row([metric.prettyPrintTitle()] + metric.getStats())

    return table.get_string().encode("utf-8")

def
mmPrettyPrintMetrics
(
metrics
,
sigFigs
=
5
)
:
assert
len
(
metrics
)
>
0
,
"No metrics found"
table
=
PrettyTable
(
[
"Metric"
,
"mean"
,
"standard deviation"
,
"min"
,
"max"
,
"sum"
,
]
)
for
metric
in
metrics
:
table
.
add_row
(
[
metric
.
prettyPrintTitle
(
)
]
+
metric
.
getStats
(
)
)
return
table
.
get_string
(
)
.
encode
(
"utf-8"
)

Returns pretty - printed table of metrics .


def updateAnomalyLikelihoods(anomalyScores,
                             params,
                             verbosity=0):
  """
  Compute updated probabilities for anomalyScores using the given params.

  :param anomalyScores: a list of records. Each record is a list with the
                        following three elements: [timestamp, value, score]

                        Example::

                            [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]

  :param params: the JSON dict returned by estimateAnomalyLikelihoods
  :param verbosity: integer controlling extent of printouts for debugging
  :type verbosity: int

  :returns: 3-tuple consisting of:

            - likelihoods

              numpy array of likelihoods, one for each aggregated point

            - avgRecordList

              list of averaged input records

            - params

              an updated JSON object containing the state of this metric.

  """
  if verbosity > 3:
    print("In updateAnomalyLikelihoods.")
    print("Number of anomaly scores:", len(anomalyScores))
    print("First 20:", anomalyScores[0:min(20, len(anomalyScores))])
    print("Params:", params)

  if len(anomalyScores) == 0:
    raise ValueError("Must have at least one anomalyScore")

  if not isValidEstimatorParams(params):
    raise ValueError("'params' is not a valid params structure")

  # For backward compatibility.
  if "historicalLikelihoods" not in params:
    params["historicalLikelihoods"] = [1.0]

  # Compute moving averages of these new scores using the previous values
  # as well as likelihood for these scores using the old estimator
  historicalValues  = params["movingAverage"]["historicalValues"]
  total             = params["movingAverage"]["total"]
  windowSize        = params["movingAverage"]["windowSize"]

  aggRecordList = numpy.zeros(len(anomalyScores), dtype=float)
  likelihoods = numpy.zeros(len(anomalyScores), dtype=float)
  for i, v in enumerate(anomalyScores):
    newAverage, historicalValues, total = (
      MovingAverage.compute(historicalValues, total, v[2], windowSize)
    )
    aggRecordList[i] = newAverage
    likelihoods[i]   = tailProbability(newAverage, params["distribution"])

  # Filter the likelihood values. First we prepend the historical likelihoods
  # to the current set. Then we filter the values.  We peel off the likelihoods
  # to return and the last windowSize values to store for later.
  likelihoods2 = params["historicalLikelihoods"] + list(likelihoods)
  filteredLikelihoods = _filterLikelihoods(likelihoods2)
  likelihoods[:] = filteredLikelihoods[-len(likelihoods):]
  historicalLikelihoods = likelihoods2[-min(windowSize, len(likelihoods2)):]

  # Update the estimator
  newParams = {
    "distribution": params["distribution"],
    "movingAverage": {
      "historicalValues": historicalValues,
      "total": total,
      "windowSize": windowSize,
    },
    "historicalLikelihoods": historicalLikelihoods,
  }

  assert len(newParams["historicalLikelihoods"]) <= windowSize

  if verbosity > 3:
    print("Number of likelihoods:", len(likelihoods))
    print("First 20 likelihoods:", likelihoods[0:min(20, len(likelihoods))])
    print("Leaving updateAnomalyLikelihoods.")

  return (likelihoods, aggRecordList, newParams)

Compute updated probabilities for anomalyScores using the given params .


def _calcSkipRecords(numIngested, windowSize, learningPeriod):
    """Return the value of skipRecords for passing to estimateAnomalyLikelihoods

    If `windowSize` is very large (bigger than the amount of data) then this
    could just return `learningPeriod`. But when some values have fallen out of
    the historical sliding window of anomaly records, then we have to take those
    into account as well so we return the `learningPeriod` minus the number
    shifted out.

    :param numIngested - (int) number of data points that have been added to the
      sliding window of historical data points.
    :param windowSize - (int) size of sliding window of historical data points.
    :param learningPeriod - (int) the number of iterations required for the
      algorithm to learn the basic patterns in the dataset and for the anomaly
      score to 'settle down'.
    """
    numShiftedOut = max(0, numIngested - windowSize)
    return min(numIngested, max(0, learningPeriod - numShiftedOut))

def _calcSkipRecords ( numIngested , windowSize , learningPeriod ) : numShiftedOut = max ( 0 , numIngested - windowSize ) return min ( numIngested , max ( 0 , learningPeriod - numShiftedOut ) )


def read(cls, proto):
    """ capnp deserialization method for the anomaly likelihood object

    :param proto: (Object) capnp proto object specified in
                          nupic.regions.anomaly_likelihood.capnp

    :returns: (Object) the deserialized AnomalyLikelihood object
    """
    # pylint: disable=W0212
    anomalyLikelihood = object.__new__(cls)
    anomalyLikelihood._iteration = proto.iteration

    anomalyLikelihood._historicalScores = collections.deque(
      maxlen=proto.historicWindowSize)
    for i, score in enumerate(proto.historicalScores):
      anomalyLikelihood._historicalScores.append((i, score.value,
                                                  score.anomalyScore))
    if proto.distribution.name: # is "" when there is no distribution.
      anomalyLikelihood._distribution = dict()
      anomalyLikelihood._distribution['distribution'] = dict()
      anomalyLikelihood._distribution['distribution']["name"] = proto.distribution.name
      anomalyLikelihood._distribution['distribution']["mean"] = proto.distribution.mean
      anomalyLikelihood._distribution['distribution']["variance"] = proto.distribution.variance
      anomalyLikelihood._distribution['distribution']["stdev"] = proto.distribution.stdev

      anomalyLikelihood._distribution["movingAverage"] = {}
      anomalyLikelihood._distribution["movingAverage"]["windowSize"] = proto.distribution.movingAverage.windowSize
      anomalyLikelihood._distribution["movingAverage"]["historicalValues"] = []
      for value in proto.distribution.movingAverage.historicalValues:
        anomalyLikelihood._distribution["movingAverage"]["historicalValues"].append(value)
      anomalyLikelihood._distribution["movingAverage"]["total"] = proto.distribution.movingAverage.total

      anomalyLikelihood._distribution["historicalLikelihoods"] = []
      for likelihood in proto.distribution.historicalLikelihoods:
        anomalyLikelihood._distribution["historicalLikelihoods"].append(likelihood)
    else:
      anomalyLikelihood._distribution = None

    anomalyLikelihood._probationaryPeriod = proto.probationaryPeriod
    anomalyLikelihood._learningPeriod = proto.learningPeriod
    anomalyLikelihood._reestimationPeriod = proto.reestimationPeriod
    # pylint: enable=W0212

    return anomalyLikelihood

def read ( cls , proto ) : anomalyLikelihood = object . __new__ ( cls ) anomalyLikelihood . _iteration = proto . iteration anomalyLikelihood . _historicalScores = collections . deque ( maxlen = proto . historicWindowSize ) for i , score in enumerate ( proto . historicalScores ) : anomalyLikelihood . _historicalScores . append ( ( i , score . value , score . anomalyScore ) ) if proto . distribution . name : anomalyLikelihood . _distribution = dict ( ) anomalyLikelihood . _distribution [ 'distribution' ] = dict ( ) anomalyLikelihood . _distribution [ 'distribution' ] [ "name" ] = proto . distribution . name anomalyLikelihood . _distribution [ 'distribution' ] [ "mean" ] = proto . distribution . mean anomalyLikelihood . _distribution [ 'distribution' ] [ "variance" ] = proto . distribution . variance anomalyLikelihood . _distribution [ 'distribution' ] [ "stdev" ] = proto . distribution . stdev anomalyLikelihood . _distribution [ "movingAverage" ] = { } anomalyLikelihood . _distribution [ "movingAverage" ] [ "windowSize" ] = proto . distribution . movingAverage . windowSize anomalyLikelihood . _distribution [ "movingAverage" ] [ "historicalValues" ] = [ ] for value in proto . distribution . movingAverage . historicalValues : anomalyLikelihood . _distribution [ "movingAverage" ] [ "historicalValues" ] . append ( value ) anomalyLikelihood . _distribution [ "movingAverage" ] [ "total" ] = proto . distribution . movingAverage . total anomalyLikelihood . _distribution [ "historicalLikelihoods" ] = [ ] for likelihood in proto . distribution . historicalLikelihoods : anomalyLikelihood . _distribution [ "historicalLikelihoods" ] . append ( likelihood ) else : anomalyLikelihood . _distribution = None anomalyLikelihood . _probationaryPeriod = proto . probationaryPeriod anomalyLikelihood . _learningPeriod = proto . learningPeriod anomalyLikelihood . _reestimationPeriod = proto . reestimationPeriod return anomalyLikelihood


def write(self, proto):
    """ capnp serialization method for the anomaly likelihood object

    :param proto: (Object) capnp proto object specified in
                          nupic.regions.anomaly_likelihood.capnp
    """

    proto.iteration = self._iteration

    pHistScores = proto.init('historicalScores', len(self._historicalScores))
    for i, score in enumerate(list(self._historicalScores)):
      _, value, anomalyScore = score
      record = pHistScores[i]
      record.value = float(value)
      record.anomalyScore = float(anomalyScore)

    if self._distribution:
      proto.distribution.name = self._distribution["distribution"]["name"]
      proto.distribution.mean = float(self._distribution["distribution"]["mean"])
      proto.distribution.variance = float(self._distribution["distribution"]["variance"])
      proto.distribution.stdev = float(self._distribution["distribution"]["stdev"])

      proto.distribution.movingAverage.windowSize = float(self._distribution["movingAverage"]["windowSize"])

      historicalValues = self._distribution["movingAverage"]["historicalValues"]
      pHistValues = proto.distribution.movingAverage.init(
        "historicalValues", len(historicalValues))
      for i, value in enumerate(historicalValues):
        pHistValues[i] = float(value)

      #proto.distribution.movingAverage.historicalValues = self._distribution["movingAverage"]["historicalValues"]
      proto.distribution.movingAverage.total = float(self._distribution["movingAverage"]["total"])

      historicalLikelihoods = self._distribution["historicalLikelihoods"]
      pHistLikelihoods = proto.distribution.init("historicalLikelihoods",
                                                 len(historicalLikelihoods))
      for i, likelihood in enumerate(historicalLikelihoods):
        pHistLikelihoods[i] = float(likelihood)

    proto.probationaryPeriod = self._probationaryPeriod
    proto.learningPeriod = self._learningPeriod
    proto.reestimationPeriod = self._reestimationPeriod
    proto.historicWindowSize = self._historicalScores.maxlen

def write ( self , proto ) : proto . iteration = self . _iteration pHistScores = proto . init ( 'historicalScores' , len ( self . _historicalScores ) ) for i , score in enumerate ( list ( self . _historicalScores ) ) : _ , value , anomalyScore = score record = pHistScores [ i ] record . value = float ( value ) record . anomalyScore = float ( anomalyScore ) if self . _distribution : proto . distribution . name = self . _distribution [ "distribution" ] [ "name" ] proto . distribution . mean = float ( self . _distribution [ "distribution" ] [ "mean" ] ) proto . distribution . variance = float ( self . _distribution [ "distribution" ] [ "variance" ] ) proto . distribution . stdev = float ( self . _distribution [ "distribution" ] [ "stdev" ] ) proto . distribution . movingAverage . windowSize = float ( self . _distribution [ "movingAverage" ] [ "windowSize" ] ) historicalValues = self . _distribution [ "movingAverage" ] [ "historicalValues" ] pHistValues = proto . distribution . movingAverage . init ( "historicalValues" , len ( historicalValues ) ) for i , value in enumerate ( historicalValues ) : pHistValues [ i ] = float ( value ) proto . distribution . movingAverage . total = float ( self . _distribution [ "movingAverage" ] [ "total" ] ) historicalLikelihoods = self . _distribution [ "historicalLikelihoods" ] pHistLikelihoods = proto . distribution . init ( "historicalLikelihoods" , len ( historicalLikelihoods ) ) for i , likelihood in enumerate ( historicalLikelihoods ) : pHistLikelihoods [ i ] = float ( likelihood ) proto . probationaryPeriod = self . _probationaryPeriod proto . learningPeriod = self . _learningPeriod proto . reestimationPeriod = self . _reestimationPeriod proto . historicWindowSize = self . _historicalScores . maxlen


def anomalyProbability(self, value, anomalyScore, timestamp=None):
    """
    Compute the probability that the current value plus anomaly score represents
    an anomaly given the historical distribution of anomaly scores. The closer
    the number is to 1, the higher the chance it is an anomaly.

    :param value: the current metric ("raw") input value, eg. "orange", or
                   '21.2' (deg. Celsius), ...
    :param anomalyScore: the current anomaly score
    :param timestamp: [optional] timestamp of the ocurrence,
                       default (None) results in using iteration step.
    :returns: the anomalyLikelihood for this record.
    """
    if timestamp is None:
      timestamp = self._iteration

    dataPoint = (timestamp, value, anomalyScore)
    # We ignore the first probationaryPeriod data points
    if self._iteration < self._probationaryPeriod:
      likelihood = 0.5
    else:
      # On a rolling basis we re-estimate the distribution
      if ( (self._distribution is None) or
           (self._iteration % self._reestimationPeriod == 0) ):

        numSkipRecords = self._calcSkipRecords(
          numIngested=self._iteration,
          windowSize=self._historicalScores.maxlen,
          learningPeriod=self._learningPeriod)

        _, _, self._distribution = estimateAnomalyLikelihoods(
          self._historicalScores,
          skipRecords=numSkipRecords)

      likelihoods, _, self._distribution = updateAnomalyLikelihoods(
        [dataPoint],
        self._distribution)

      likelihood = 1.0 - likelihoods[0]

    # Before we exit update historical scores and iteration
    self._historicalScores.append(dataPoint)
    self._iteration += 1

    return likelihood

def anomalyProbability ( self , value , anomalyScore , timestamp = None ) : if timestamp is None : timestamp = self . _iteration dataPoint = ( timestamp , value , anomalyScore ) if self . _iteration < self . _probationaryPeriod : likelihood = 0.5 else : if ( ( self . _distribution is None ) or ( self . _iteration % self . _reestimationPeriod == 0 ) ) : numSkipRecords = self . _calcSkipRecords ( numIngested = self . _iteration , windowSize = self . _historicalScores . maxlen , learningPeriod = self . _learningPeriod ) _ , _ , self . _distribution = estimateAnomalyLikelihoods ( self . _historicalScores , skipRecords = numSkipRecords ) likelihoods , _ , self . _distribution = updateAnomalyLikelihoods ( [ dataPoint ] , self . _distribution ) likelihood = 1.0 - likelihoods [ 0 ] self . _historicalScores . append ( dataPoint ) self . _iteration += 1 return likelihood

Compute the probability that the current value plus anomaly score represents an anomaly given the historical distribution of anomaly scores . The closer the number is to 1 the higher the chance it is an anomaly .


def replaceIterationCycle(self, phaseSpecs):
    """ Replaces the Iteration Cycle phases

    :param phaseSpecs: Iteration cycle description consisting of a sequence of
                  IterationPhaseSpecXXXXX elements that are performed in the
                  given order
    """

    # -----------------------------------------------------------------------
    # Replace our phase manager
    #
    self.__phaseManager = _PhaseManager(
      model=self.__model,
      phaseSpecs=phaseSpecs)

    return

def replaceIterationCycle ( self , phaseSpecs ) : self . __phaseManager = _PhaseManager ( model = self . __model , phaseSpecs = phaseSpecs ) return


def handleInputRecord(self, inputRecord):
    """
    Processes the given record according to the current iteration cycle phase

    :param inputRecord: (object) record expected to be returned from
           :meth:`nupic.data.record_stream.RecordStreamIface.getNextRecord`.

    :returns: :class:`nupic.frameworks.opf.opf_utils.ModelResult`
    """
    assert inputRecord, "Invalid inputRecord: %r" % inputRecord

    results = self.__phaseManager.handleInputRecord(inputRecord)
    metrics = self.__metricsMgr.update(results)

    # Execute task-postIter callbacks
    for cb in self.__userCallbacks['postIter']:
      cb(self.__model)

    results.metrics = metrics

    # Return the input and predictions for this record
    return results

def
handleInputRecord
(
self
,
inputRecord
)
:
assert
inputRecord
,
"Invalid inputRecord: %r"
%
inputRecord
results
=
self
.
__phaseManager
.
handleInputRecord
(
inputRecord
)
metrics
=
self
.
__metricsMgr
.
update
(
results
)
for
cb
in
self
.
__userCallbacks
[
'postIter'
]
:
cb
(
self
.
__model
)
results
.
metrics
=
metrics
return
results


def __advancePhase(self):
    """ Advance to the next iteration cycle phase
    """
    self.__currentPhase = self.__phaseCycler.next()
    self.__currentPhase.enterPhase()

    return

def __advancePhase ( self ) : self . __currentPhase = self . __phaseCycler . next ( ) self . __currentPhase . enterPhase ( ) return


def handleInputRecord(self, inputRecord):
    """ Processes the given record according to the current phase

    inputRecord:  record object formatted according to
                  nupic.data.FileSource.getNext() result format.

    Returns:      An opf_utils.ModelResult object with the inputs and inferences
                  after the current record is processed by the model
    """

    results = self.__model.run(inputRecord)

    shouldContinue = self.__currentPhase.advance()
    if not shouldContinue:
      self.__advancePhase()

    return results

def handleInputRecord ( self , inputRecord ) : results = self . __model . run ( inputRecord ) shouldContinue = self . __currentPhase . advance ( ) if not shouldContinue : self . __advancePhase ( ) return results


def advance(self):
    """ Advances the iteration;

    Returns:      True if more iterations remain; False if this is the final
                  iteration.
    """
    hasMore = True
    try:
      self.__iter.next()
    except StopIteration:
      self.__iter = None
      hasMore = False

    return hasMore

def advance ( self ) : hasMore = True try : self . __iter . next ( ) except StopIteration : self . __iter = None hasMore = False return hasMore


def write(self, proto):
    """ Serialize via capnp

    :param proto: capnp PreviousValueModelProto message builder
    """
    super(PreviousValueModel, self).writeBaseToProto(proto.modelBase)

    proto.fieldNames = self._fieldNames
    proto.fieldTypes = self._fieldTypes
    if self._predictedField:
      proto.predictedField = self._predictedField
    proto.predictionSteps = self._predictionSteps

def write ( self , proto ) : super ( PreviousValueModel , self ) . writeBaseToProto ( proto . modelBase ) proto . fieldNames = self . _fieldNames proto . fieldTypes = self . _fieldTypes if self . _predictedField : proto . predictedField = self . _predictedField proto . predictionSteps = self . _predictionSteps


def read(cls, proto):
    """Deserialize via capnp

    :param proto: capnp PreviousValueModelProto message reader

    :returns: new instance of PreviousValueModel deserialized from the given
              proto
    """
    instance = object.__new__(cls)
    super(PreviousValueModel, instance).__init__(proto=proto.modelBase)

    instance._logger = opf_utils.initLogger(instance)
    if len(proto.predictedField):
      instance._predictedField = proto.predictedField
    else:
      instance._predictedField = None
    instance._fieldNames = list(proto.fieldNames)
    instance._fieldTypes = list(proto.fieldTypes)
    instance._predictionSteps = list(proto.predictionSteps)

    return instance

def read ( cls , proto ) : instance = object . __new__ ( cls ) super ( PreviousValueModel , instance ) . __init__ ( proto = proto . modelBase ) instance . _logger = opf_utils . initLogger ( instance ) if len ( proto . predictedField ) : instance . _predictedField = proto . predictedField else : instance . _predictedField = None instance . _fieldNames = list ( proto . fieldNames ) instance . _fieldTypes = list ( proto . fieldTypes ) instance . _predictionSteps = list ( proto . predictionSteps ) return instance


def lscsum(lx, epsilon=None):
  """
  Accepts log-values as input, exponentiates them, computes the sum,
  then converts the sum back to log-space and returns the result.
  Handles underflow by rescaling so that the largest values is exactly 1.0.
  """
  lx = numpy.asarray(lx)
  base = lx.max()

  # If the input is the log of 0's, catch this condition before we generate
  #  an exception, and return the log(0)
  if numpy.isinf(base):
    return base

  # If the user specified an epsilon and we are below it, return epsilon
  if (epsilon is not None) and (base < epsilon):
    return epsilon

  x = numpy.exp(lx - base)
  ssum = x.sum()

  result = numpy.log(ssum) + base
  # try:
  #   conventional = numpy.log(numpy.exp(lx).sum())
  #   if not similar(result, conventional):
  #     if numpy.isinf(conventional).any() and not numpy.isinf(result).any():
  #       # print "Scaled log sum avoided underflow or overflow."
  #       pass
  #     else:
  #       import sys
  #       print >>sys.stderr, "Warning: scaled log sum did not match."
  #       print >>sys.stderr, "Scaled log result:"
  #       print >>sys.stderr, result
  #       print >>sys.stderr, "Conventional result:"
  #       print >>sys.stderr, conventional
  # except FloatingPointError, e:
  #   # print "Scaled log sum avoided underflow or overflow."
  #   pass

  return result

def lscsum ( lx , epsilon = None ) : lx = numpy . asarray ( lx ) base = lx . max ( ) if numpy . isinf ( base ) : return base if ( epsilon is not None ) and ( base < epsilon ) : return epsilon x = numpy . exp ( lx - base ) ssum = x . sum ( ) result = numpy . log ( ssum ) + base return result

Accepts log - values as input exponentiates them computes the sum then converts the sum back to log - space and returns the result . Handles underflow by rescaling so that the largest values is exactly 1 . 0 .


def normalize(lx):
  """
  Accepts log-values as input, exponentiates them,
  normalizes and returns the result.
  Handles underflow by rescaling so that the largest values is exactly 1.0.
  """
  lx = numpy.asarray(lx)
  base = lx.max()
  x = numpy.exp(lx - base)
  result = x / x.sum()

  conventional = (numpy.exp(lx) / numpy.exp(lx).sum())
  assert similar(result, conventional)

  return result

def normalize ( lx ) : lx = numpy . asarray ( lx ) base = lx . max ( ) x = numpy . exp ( lx - base ) result = x / x . sum ( ) conventional = ( numpy . exp ( lx ) / numpy . exp ( lx ) . sum ( ) ) assert similar ( result , conventional ) return result

Accepts log - values as input exponentiates them normalizes and returns the result . Handles underflow by rescaling so that the largest values is exactly 1 . 0 .


def debug(self, msg, *args, **kwargs):
    """
    Log 'msg % args' with severity 'DEBUG'.

    To pass exception information, use the keyword argument exc_info with
    a true value, e.g.

    logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)
    """
    self._baseLogger.debug(self, self.getExtendedMsg(msg), *args, **kwargs)

def debug ( self , msg , * args , ** kwargs ) : self . _baseLogger . debug ( self , self . getExtendedMsg ( msg ) , * args , ** kwargs )


def info(self, msg, *args, **kwargs):
    """
    Log 'msg % args' with severity 'INFO'.

    To pass exception information, use the keyword argument exc_info with
    a true value, e.g.

    logger.info("Houston, we have a %s", "interesting problem", exc_info=1)
    """
    self._baseLogger.info(self, self.getExtendedMsg(msg), *args, **kwargs)

def info ( self , msg , * args , ** kwargs ) : self . _baseLogger . info ( self , self . getExtendedMsg ( msg ) , * args , ** kwargs )


def warning(self, msg, *args, **kwargs):
    """
    Log 'msg % args' with severity 'WARNING'.

    To pass exception information, use the keyword argument exc_info with
    a true value, e.g.

    logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1)
    """
    self._baseLogger.warning(self, self.getExtendedMsg(msg), *args, **kwargs)

def warning ( self , msg , * args , ** kwargs ) : self . _baseLogger . warning ( self , self . getExtendedMsg ( msg ) , * args , ** kwargs )


def error(self, msg, *args, **kwargs):
    """
    Log 'msg % args' with severity 'ERROR'.

    To pass exception information, use the keyword argument exc_info with
    a true value, e.g.

    logger.error("Houston, we have a %s", "major problem", exc_info=1)
    """
    self._baseLogger.error(self, self.getExtendedMsg(msg), *args, **kwargs)

def error ( self , msg , * args , ** kwargs ) : self . _baseLogger . error ( self , self . getExtendedMsg ( msg ) , * args , ** kwargs )


def critical(self, msg, *args, **kwargs):
    """
    Log 'msg % args' with severity 'CRITICAL'.

    To pass exception information, use the keyword argument exc_info with
    a true value, e.g.

    logger.critical("Houston, we have a %s", "major disaster", exc_info=1)
    """
    self._baseLogger.critical(self, self.getExtendedMsg(msg), *args, **kwargs)

def critical ( self , msg , * args , ** kwargs ) : self . _baseLogger . critical ( self , self . getExtendedMsg ( msg ) , * args , ** kwargs )


def log(self, level, msg, *args, **kwargs):
      """
      Log 'msg % args' with the integer severity 'level'.

      To pass exception information, use the keyword argument exc_info with
      a true value, e.g.

      logger.log(level, "We have a %s", "mysterious problem", exc_info=1)
      """
      self._baseLogger.log(self, level, self.getExtendedMsg(msg), *args,
                           **kwargs)

def log ( self , level , msg , * args , ** kwargs ) : self . _baseLogger . log ( self , level , self . getExtendedMsg ( msg ) , * args , ** kwargs )

Log msg % args with the integer severity level .


def _filterRecord(filterList, record):
  """ Takes a record and returns true if record meets filter criteria,
  false otherwise
  """

  for (fieldIdx, fp, params) in filterList:
    x = dict()
    x['value'] = record[fieldIdx]
    x['acceptValues'] = params['acceptValues']
    x['min'] = params['min']
    x['max'] = params['max']
    if not fp(x):
      return False

  # None of the field filters triggered, accept the record as a good one
  return True

def _filterRecord ( filterList , record ) : for ( fieldIdx , fp , params ) in filterList : x = dict ( ) x [ 'value' ] = record [ fieldIdx ] x [ 'acceptValues' ] = params [ 'acceptValues' ] x [ 'min' ] = params [ 'min' ] x [ 'max' ] = params [ 'max' ] if not fp ( x ) : return False return True

Takes a record and returns true if record meets filter criteria false otherwise


def _aggr_sum(inList):
  """ Returns sum of the elements in the list. Missing items are replaced with
  the mean value
  """
  aggrMean = _aggr_mean(inList)
  if aggrMean == None:
    return None

  aggrSum = 0
  for elem in inList:
    if elem != SENTINEL_VALUE_FOR_MISSING_DATA:
      aggrSum += elem
    else:
      aggrSum += aggrMean

  return aggrSum

def _aggr_sum ( inList ) : aggrMean = _aggr_mean ( inList ) if aggrMean == None : return None aggrSum = 0 for elem in inList : if elem != SENTINEL_VALUE_FOR_MISSING_DATA : aggrSum += elem else : aggrSum += aggrMean return aggrSum

Returns sum of the elements in the list . Missing items are replaced with the mean value


def _aggr_mean(inList):
  """ Returns mean of non-None elements of the list
  """
  aggrSum = 0
  nonNone = 0
  for elem in inList:
    if elem != SENTINEL_VALUE_FOR_MISSING_DATA:
      aggrSum += elem
      nonNone += 1
  if nonNone != 0:
    return aggrSum / nonNone
  else:
    return None

def _aggr_mean ( inList ) : aggrSum = 0 nonNone = 0 for elem in inList : if elem != SENTINEL_VALUE_FOR_MISSING_DATA : aggrSum += elem nonNone += 1 if nonNone != 0 : return aggrSum / nonNone else : return None

Returns mean of non - None elements of the list


def _aggr_mode(inList):
  """ Returns most common value seen in the non-None elements of the list
  """

  valueCounts = dict()
  nonNone = 0

  for elem in inList:
    if elem == SENTINEL_VALUE_FOR_MISSING_DATA:
      continue

    nonNone += 1
    if elem in valueCounts:
      valueCounts[elem] += 1
    else:
      valueCounts[elem] = 1

  # Get the most common one
  if nonNone == 0:
    return None

  # Sort by counts
  sortedCounts = valueCounts.items()
  sortedCounts.sort(cmp=lambda x,y: x[1] - y[1], reverse=True)
  return sortedCounts[0][0]

def _aggr_mode ( inList ) : valueCounts = dict ( ) nonNone = 0 for elem in inList : if elem == SENTINEL_VALUE_FOR_MISSING_DATA : continue nonNone += 1 if elem in valueCounts : valueCounts [ elem ] += 1 else : valueCounts [ elem ] = 1 if nonNone == 0 : return None sortedCounts = valueCounts . items ( ) sortedCounts . sort ( cmp = lambda x , y : x [ 1 ] - y [ 1 ] , reverse = True ) return sortedCounts [ 0 ] [ 0 ]

Returns most common value seen in the non - None elements of the list


def generateDataset(aggregationInfo, inputFilename, outputFilename=None):
  """Generate a dataset of aggregated values

  Parameters:
  ----------------------------------------------------------------------------
  aggregationInfo: a dictionary that contains the following entries
    - fields: a list of pairs. Each pair is a field name and an
      aggregation function (e.g. sum). The function will be used to aggregate
      multiple values during the aggregation period.

  aggregation period: 0 or more of unit=value fields; allowed units are:
        [years months] |
        [weeks days hours minutes seconds milliseconds microseconds]
        NOTE: years and months are mutually-exclusive with the other units.
              See getEndTime() and _aggregate() for more details.
        Example1: years=1, months=6,
        Example2: hours=1, minutes=30,
        If none of the period fields are specified or if all that are specified
        have values of 0, then aggregation will be suppressed, and the given
        inputFile parameter value will be returned.

  inputFilename: filename of the input dataset within examples/prediction/data

  outputFilename: name for the output file. If not given, a name will be
        generated based on the input filename and the aggregation params

  retval: Name of the generated output file. This will be the same as the input
      file name if no aggregation needed to be performed

  If the input file contained a time field, sequence id field or reset field
  that were not specified in aggregationInfo fields, those fields will be
  added automatically with the following rules:

  1. The order will be R, S, T, rest of the fields
  2. The aggregation function for all will be to pick the first: lambda x: x[0]

    Returns: the path of the aggregated data file if aggregation was performed
      (in the same directory as the given input file); if aggregation did not
      need to be performed, then the given inputFile argument value is returned.
  """

  # Create the input stream
  inputFullPath = resource_filename("nupic.datafiles", inputFilename)
  inputObj = FileRecordStream(inputFullPath)

  # Instantiate the aggregator
  aggregator = Aggregator(aggregationInfo=aggregationInfo,
                          inputFields=inputObj.getFields())

  # Is it a null aggregation? If so, just return the input file unmodified
  if aggregator.isNullAggregation():
    return inputFullPath

  # ------------------------------------------------------------------------
  # If we were not given an output filename, create one based on the
  #  aggregation settings
  if outputFilename is None:
    outputFilename = 'agg_%s' % \
                        os.path.splitext(os.path.basename(inputFullPath))[0]
    timePeriods = 'years months weeks days '\
                  'hours minutes seconds milliseconds microseconds'
    for k in timePeriods.split():
      if aggregationInfo.get(k, 0) > 0:
        outputFilename += '_%s_%d' % (k, aggregationInfo[k])

    outputFilename += '.csv'
    outputFilename = os.path.join(os.path.dirname(inputFullPath), outputFilename)

  # ------------------------------------------------------------------------
  # If some other process already started creating this file, simply
  #   wait for it to finish and return without doing anything
  lockFilePath = outputFilename + '.please_wait'
  if os.path.isfile(outputFilename) or \
     os.path.isfile(lockFilePath):
    while os.path.isfile(lockFilePath):
      print 'Waiting for %s to be fully written by another process' % \
            lockFilePath
      time.sleep(1)
    return outputFilename

  # Create the lock file
  lockFD = open(lockFilePath, 'w')

  # -------------------------------------------------------------------------
  # Create the output stream
  outputObj = FileRecordStream(streamID=outputFilename, write=True,
                               fields=inputObj.getFields())

  # -------------------------------------------------------------------------
  # Write all aggregated records to the output
  while True:
    inRecord = inputObj.getNextRecord()

    (aggRecord, aggBookmark) = aggregator.next(inRecord, None)

    if aggRecord is None and inRecord is None:
      break

    if aggRecord is not None:
      outputObj.appendRecord(aggRecord)

  return outputFilename


def getFilename(aggregationInfo, inputFile):
  """Generate the filename for aggregated dataset

  The filename is based on the input filename and the
  aggregation period.

  Returns the inputFile if no aggregation required (aggregation
  info has all 0's)
  """

  # Find the actual file, with an absolute path
  inputFile = resource_filename("nupic.datafiles", inputFile)

  a = defaultdict(lambda: 0, aggregationInfo)
  outputDir = os.path.dirname(inputFile)
  outputFile = 'agg_%s' % os.path.splitext(os.path.basename(inputFile))[0]
  noAggregation = True
  timePeriods = 'years months weeks days '\
                'hours minutes seconds milliseconds microseconds'
  for k in timePeriods.split():
    if a[k] > 0:
      noAggregation = False
      outputFile += '_%s_%d' % (k, a[k])

  if noAggregation:
    return inputFile
  outputFile += '.csv'
  outputFile = os.path.join(outputDir, outputFile)

  return outputFile


def _getEndTime(self, t):
    """Add the aggregation period to the input time t and return a datetime object

    Years and months are handled as aspecial case due to leap years
    and months with different number of dates. They can't be converted
    to a strict timedelta because a period of 3 months will have different
    durations actually. The solution is to just add the years and months
    fields directly to the current time.

    Other periods are converted to timedelta and just added to current time.
    """

    assert isinstance(t, datetime.datetime)
    if self._aggTimeDelta:
      return t + self._aggTimeDelta
    else:
      year = t.year + self._aggYears + (t.month - 1 + self._aggMonths) / 12
      month = (t.month - 1 + self._aggMonths) % 12 + 1
      return t.replace(year=year, month=month)

def _getEndTime ( self , t ) : assert isinstance ( t , datetime . datetime ) if self . _aggTimeDelta : return t + self . _aggTimeDelta else : year = t . year + self . _aggYears + ( t . month - 1 + self . _aggMonths ) / 12 month = ( t . month - 1 + self . _aggMonths ) % 12 + 1 return t . replace ( year = year , month = month )


def _getFuncPtrAndParams(self, funcName):
    """ Given the name of an aggregation function, returns the function pointer
    and param.

    Parameters:
    ------------------------------------------------------------------------
    funcName:  a string (name of function) or funcPtr
    retval:   (funcPtr, param)
    """

    params = None
    if isinstance(funcName, basestring):
      if funcName == 'sum':
        fp = _aggr_sum
      elif funcName == 'first':
        fp = _aggr_first
      elif funcName == 'last':
        fp = _aggr_last
      elif funcName == 'mean':
        fp = _aggr_mean
      elif funcName == 'max':
        fp = max
      elif funcName == 'min':
        fp = min
      elif funcName == 'mode':
        fp = _aggr_mode
      elif funcName.startswith('wmean:'):
        fp = _aggr_weighted_mean
        paramsName = funcName[6:]
        params = [f[0] for f in self._inputFields].index(paramsName)
    else:
      fp = funcName

    return (fp, params)

def _getFuncPtrAndParams ( self , funcName ) : params = None if isinstance ( funcName , basestring ) : if funcName == 'sum' : fp = _aggr_sum elif funcName == 'first' : fp = _aggr_first elif funcName == 'last' : fp = _aggr_last elif funcName == 'mean' : fp = _aggr_mean elif funcName == 'max' : fp = max elif funcName == 'min' : fp = min elif funcName == 'mode' : fp = _aggr_mode elif funcName . startswith ( 'wmean:' ) : fp = _aggr_weighted_mean paramsName = funcName [ 6 : ] params = [ f [ 0 ] for f in self . _inputFields ] . index ( paramsName ) else : fp = funcName return ( fp , params )

Given the name of an aggregation function returns the function pointer and param .


def _createAggregateRecord(self):
    """ Generate the aggregated output record

    Parameters:
    ------------------------------------------------------------------------
    retval: outputRecord

    """

    record = []

    for i, (fieldIdx, aggFP, paramIdx) in enumerate(self._fields):
      if aggFP is None: # this field is not supposed to be aggregated.
        continue

      values = self._slice[i]
      refIndex = None
      if paramIdx is not None:
        record.append(aggFP(values, self._slice[paramIdx]))
      else:
        record.append(aggFP(values))

    return record

def _createAggregateRecord ( self ) : record = [ ] for i , ( fieldIdx , aggFP , paramIdx ) in enumerate ( self . _fields ) : if aggFP is None : continue values = self . _slice [ i ] refIndex = None if paramIdx is not None : record . append ( aggFP ( values , self . _slice [ paramIdx ] ) ) else : record . append ( aggFP ( values ) ) return record


def next(self, record, curInputBookmark):
    """ Return the next aggregated record, if any

    Parameters:
    ------------------------------------------------------------------------
    record:         The input record (values only) from the input source, or
                    None if the input has reached EOF (this will cause this
                    method to force completion of and return any partially
                    aggregated time period)
    curInputBookmark: The bookmark to the next input record
    retval:
      (outputRecord, inputBookmark)

      outputRecord: the aggregated record
      inputBookmark: a bookmark to the last position from the input that
                      contributed to this aggregated record.

      If we don't have any aggregated records yet, returns (None, None)

    The caller should generally do a loop like this:
      while True:
        inRecord = reader.getNextRecord()
        bookmark = reader.getBookmark()

        (aggRecord, aggBookmark) = aggregator.next(inRecord, bookmark)

        # reached EOF?
        if inRecord is None and aggRecord is None:
          break

        if aggRecord is not None:
          proessRecord(aggRecord, aggBookmark)

    This method makes use of the self._slice member variable to build up
    the values we need to aggregate. This is a dict of lists. The keys are
    the field indices and the elements of each list are the values for that
    field. For example:

      self._siice = { 0: [42, 53], 1: [4.0, 5.1] }

    """

    # This will hold the aggregated record we return
    outRecord = None

    # This will hold the bookmark of the last input used within the
    #  aggregated record we return.
    retInputBookmark = None

    if record is not None:

      # Increment input count
      self._inIdx += 1

      #print self._inIdx, record

      # Apply the filter, ignore the record if any field is unacceptable
      if self._filter != None and not self._filter[0](self._filter[1], record):
        return (None, None)

      # If no aggregation info just return as-is
      if self._nullAggregation:
        return (record, curInputBookmark)

      # ----------------------------------------------------------------------
      # Do aggregation

      #
      # Remember the very first record time stamp - it will be used as
      # the timestamp for all first records in all sequences to align
      # times for the aggregation/join of sequences.
      #
      # For a set of aggregated records, it will use the beginning of the time
      # window as a timestamp for the set
      #
      t = record[self._timeFieldIdx]

      if self._firstSequenceStartTime == None:
        self._firstSequenceStartTime = t

      # Create initial startTime and endTime if needed
      if self._startTime is None:
        self._startTime = t
      if self._endTime is None:
        self._endTime = self._getEndTime(t)
        assert self._endTime > t

      #print 'Processing line:', i, t, endTime
      #from dbgp.client import brk; brk(port=9011)

      # ----------------------------------------------------------------------
      # Does this record have a reset signal or sequence Id associated with it?
      # If so, see if we've reached a sequence boundary
      if self._resetFieldIdx is not None:
        resetSignal = record[self._resetFieldIdx]
      else:
        resetSignal = None

      if self._sequenceIdFieldIdx is not None:
        currSequenceId = record[self._sequenceIdFieldIdx]
      else:
        currSequenceId = None

      newSequence = (resetSignal == 1 and self._inIdx > 0) \
                      or self._sequenceId != currSequenceId \
                      or self._inIdx == 0

      if newSequence:
        self._sequenceId = currSequenceId

      # --------------------------------------------------------------------
      # We end the aggregation chunk if we go past the end time
      # -OR- we get an out of order record (t < startTime)
      sliceEnded = (t >= self._endTime or t < self._startTime)

      # -------------------------------------------------------------------
      # Time to generate a new output record?
      if (newSequence or sliceEnded) and len(self._slice) > 0:
        # Create aggregated record
        # print 'Creating aggregate record...'

        # Make first record timestamp as the beginning of the time period,
        # in case the first record wasn't falling on the beginning of the period
        for j, f in enumerate(self._fields):
          index = f[0]
          if index == self._timeFieldIdx:
            self._slice[j][0] = self._startTime
            break

        # Generate the aggregated record
        outRecord = self._createAggregateRecord()
        retInputBookmark = self._aggrInputBookmark

        # Reset the slice
        self._slice = defaultdict(list)

      # --------------------------------------------------------------------
      # Add current record to slice (Note keeping slices in memory). Each
      # field in the slice is a list of field values from all the sliced
      # records
      for j, f in enumerate(self._fields):
        index = f[0]
        # append the parsed field value to the proper aggregated slice field.
        self._slice[j].append(record[index])
        self._aggrInputBookmark = curInputBookmark

      # --------------------------------------------------------------------
      # If we've encountered a new sequence, start aggregation over again
      if newSequence:
        # TODO: May use self._firstSequenceStartTime as a start for the new
        # sequence (to align all sequences)
        self._startTime = t
        self._endTime = self._getEndTime(t)

      # --------------------------------------------------------------------
      # If a slice just ended, re-compute the start and end time for the
      #  next aggregated record
      if sliceEnded:
        # Did we receive an out of order record? If so, go back and iterate
        #   till we get to the next end time boundary.
        if t < self._startTime:
          self._endTime = self._firstSequenceStartTime
        while t >= self._endTime:
          self._startTime = self._endTime
          self._endTime = self._getEndTime(self._endTime)

      # If we have a record to return, do it now
      if outRecord is not None:
        return (outRecord, retInputBookmark)

    # ---------------------------------------------------------------------
    # Input reached EOF
    # Aggregate one last time in the end if necessary
    elif self._slice:

      # Make first record timestamp as the beginning of the time period,
      # in case the first record wasn't falling on the beginning of the period
      for j, f in enumerate(self._fields):
        index = f[0]
        if index == self._timeFieldIdx:
          self._slice[j][0] = self._startTime
          break

      outRecord = self._createAggregateRecord()
      retInputBookmark = self._aggrInputBookmark

      self._slice = defaultdict(list)

    # Return aggregated record
    return (outRecord, retInputBookmark)

def next ( self , record , curInputBookmark ) : outRecord = None retInputBookmark = None if record is not None : self . _inIdx += 1 if self . _filter != None and not self . _filter [ 0 ] ( self . _filter [ 1 ] , record ) : return ( None , None ) if self . _nullAggregation : return ( record , curInputBookmark ) t = record [ self . _timeFieldIdx ] if self . _firstSequenceStartTime == None : self . _firstSequenceStartTime = t if self . _startTime is None : self . _startTime = t if self . _endTime is None : self . _endTime = self . _getEndTime ( t ) assert self . _endTime > t if self . _resetFieldIdx is not None : resetSignal = record [ self . _resetFieldIdx ] else : resetSignal = None if self . _sequenceIdFieldIdx is not None : currSequenceId = record [ self . _sequenceIdFieldIdx ] else : currSequenceId = None newSequence = ( resetSignal == 1 and self . _inIdx > 0 ) or self . _sequenceId != currSequenceId or self . _inIdx == 0 if newSequence : self . _sequenceId = currSequenceId sliceEnded = ( t >= self . _endTime or t < self . _startTime ) if ( newSequence or sliceEnded ) and len ( self . _slice ) > 0 : for j , f in enumerate ( self . _fields ) : index = f [ 0 ] if index == self . _timeFieldIdx : self . _slice [ j ] [ 0 ] = self . _startTime break outRecord = self . _createAggregateRecord ( ) retInputBookmark = self . _aggrInputBookmark self . _slice = defaultdict ( list ) for j , f in enumerate ( self . _fields ) : index = f [ 0 ] self . _slice [ j ] . append ( record [ index ] ) self . _aggrInputBookmark = curInputBookmark if newSequence : self . _startTime = t self . _endTime = self . _getEndTime ( t ) if sliceEnded : if t < self . _startTime : self . _endTime = self . _firstSequenceStartTime while t >= self . _endTime : self . _startTime = self . _endTime self . _endTime = self . _getEndTime ( self . _endTime ) if outRecord is not None : return ( outRecord , retInputBookmark ) elif self . _slice : for j , f in enumerate ( self . _fields ) : index = f [ 0 ] if index == self . _timeFieldIdx : self . _slice [ j ] [ 0 ] = self . _startTime break outRecord = self . _createAggregateRecord ( ) retInputBookmark = self . _aggrInputBookmark self . _slice = defaultdict ( list ) return ( outRecord , retInputBookmark )

Return the next aggregated record if any


def run(self, inputRecord):
    """
    Run one iteration of this model.

    :param inputRecord: (object)
           A record object formatted according to
           :meth:`~nupic.data.record_stream.RecordStreamIface.getNextRecord` or
           :meth:`~nupic.data.record_stream.RecordStreamIface.getNextRecordDict`
           result format.
    :returns: (:class:`~nupic.frameworks.opf.opf_utils.ModelResult`)
             An ModelResult namedtuple. The contents of ModelResult.inferences
             depends on the the specific inference type of this model, which
             can be queried by :meth:`.getInferenceType`.
    """
    # 0-based prediction index for ModelResult
    predictionNumber = self._numPredictions
    self._numPredictions += 1
    result = opf_utils.ModelResult(predictionNumber=predictionNumber,
                                   rawInput=inputRecord)
    return result

def run ( self , inputRecord ) : predictionNumber = self . _numPredictions self . _numPredictions += 1 result = opf_utils . ModelResult ( predictionNumber = predictionNumber , rawInput = inputRecord ) return result


def _getModelCheckpointFilePath(checkpointDir):
    """ Return the absolute path of the model's checkpoint file.

    :param checkpointDir: (string)
           Directory of where the experiment is to be or was saved
    :returns: (string) An absolute path.
    """
    path = os.path.join(checkpointDir, "model.data")
    path = os.path.abspath(path)
    return path

def _getModelCheckpointFilePath ( checkpointDir ) : path = os . path . join ( checkpointDir , "model.data" ) path = os . path . abspath ( path ) return path

Return the absolute path of the model s checkpoint file .


def writeToCheckpoint(self, checkpointDir):
    """Serializes model using capnproto and writes data to ``checkpointDir``"""
    proto = self.getSchema().new_message()

    self.write(proto)

    checkpointPath = self._getModelCheckpointFilePath(checkpointDir)

    # Clean up old saved state, if any
    if os.path.exists(checkpointDir):
      if not os.path.isdir(checkpointDir):
        raise Exception(("Existing filesystem entry <%s> is not a model"
                         " checkpoint -- refusing to delete (not a directory)") \
                          % checkpointDir)
      if not os.path.isfile(checkpointPath):
        raise Exception(("Existing filesystem entry <%s> is not a model"
                         " checkpoint -- refusing to delete"\
                         " (%s missing or not a file)") % \
                          (checkpointDir, checkpointPath))

      shutil.rmtree(checkpointDir)

    # Create a new directory for saving state
    self.__makeDirectoryFromAbsolutePath(checkpointDir)

    with open(checkpointPath, 'wb') as f:
      proto.write(f)

Serializes model using capnproto and writes data to checkpointDir


def readFromCheckpoint(cls, checkpointDir):
    """Deserializes model from checkpointDir using capnproto"""
    checkpointPath = cls._getModelCheckpointFilePath(checkpointDir)

    with open(checkpointPath, 'r') as f:
      proto = cls.getSchema().read(f,
                                   traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)

    model = cls.read(proto)
    return model

def readFromCheckpoint ( cls , checkpointDir ) : checkpointPath = cls . _getModelCheckpointFilePath ( checkpointDir ) with open ( checkpointPath , 'r' ) as f : proto = cls . getSchema ( ) . read ( f , traversal_limit_in_words = _TRAVERSAL_LIMIT_IN_WORDS ) model = cls . read ( proto ) return model


def writeBaseToProto(self, proto):
    """Save the state maintained by the Model base class

    :param proto: capnp ModelProto message builder
    """
    inferenceType = self.getInferenceType()
    # lower-case first letter to be compatible with capnproto enum naming
    inferenceType = inferenceType[:1].lower() + inferenceType[1:]
    proto.inferenceType = inferenceType

    proto.numPredictions = self._numPredictions

    proto.learningEnabled = self.__learningEnabled
    proto.inferenceEnabled = self.__inferenceEnabled
    proto.inferenceArgs = json.dumps(self.__inferenceArgs)

def writeBaseToProto ( self , proto ) : inferenceType = self . getInferenceType ( ) inferenceType = inferenceType [ : 1 ] . lower ( ) + inferenceType [ 1 : ] proto . inferenceType = inferenceType proto . numPredictions = self . _numPredictions proto . learningEnabled = self . __learningEnabled proto . inferenceEnabled = self . __inferenceEnabled proto . inferenceArgs = json . dumps ( self . __inferenceArgs )


def save(self, saveModelDir):
    """ Save the model in the given directory.

    :param saveModelDir: (string)
         Absolute directory path for saving the model. This directory should
         only be used to store a saved model. If the directory does not exist,
         it will be created automatically and populated with model data. A
         pre-existing directory will only be accepted if it contains previously
         saved model data. If such a directory is given, the full contents of
         the directory will be deleted and replaced with current model data.
    """
    logger = self._getLogger()
    logger.debug("(%s) Creating local checkpoint in %r...",
                       self, saveModelDir)

    modelPickleFilePath = self._getModelPickleFilePath(saveModelDir)

    # Clean up old saved state, if any
    if os.path.exists(saveModelDir):
      if not os.path.isdir(saveModelDir):
        raise Exception(("Existing filesystem entry <%s> is not a model"
                         " checkpoint -- refusing to delete (not a directory)") \
                          % saveModelDir)
      if not os.path.isfile(modelPickleFilePath):
        raise Exception(("Existing filesystem entry <%s> is not a model"
                         " checkpoint -- refusing to delete"\
                         " (%s missing or not a file)") % \
                          (saveModelDir, modelPickleFilePath))

      shutil.rmtree(saveModelDir)

    # Create a new directory for saving state
    self.__makeDirectoryFromAbsolutePath(saveModelDir)

    with open(modelPickleFilePath, 'wb') as modelPickleFile:
      logger.debug("(%s) Pickling Model instance...", self)

      pickle.dump(self, modelPickleFile, protocol=pickle.HIGHEST_PROTOCOL)

      logger.debug("(%s) Finished pickling Model instance", self)

    # Tell the model to save extra data, if any, that's too big for pickling
    self._serializeExtraData(extraDataDir=self._getModelExtraDataDir(saveModelDir))

    logger.debug("(%s) Finished creating local checkpoint", self)

    return


def _getModelPickleFilePath(saveModelDir):
    """ Return the absolute path of the model's pickle file.

    :param saveModelDir: (string)
           Directory of where the experiment is to be or was saved
    :returns: (string) An absolute path.
    """
    path = os.path.join(saveModelDir, "model.pkl")
    path = os.path.abspath(path)
    return path

def _getModelPickleFilePath ( saveModelDir ) : path = os . path . join ( saveModelDir , "model.pkl" ) path = os . path . abspath ( path ) return path

Return the absolute path of the model s pickle file .


def runExperiment(args, model=None):
  """
  Run a single OPF experiment.

  .. note:: The caller is responsible for initializing python logging before
     calling this function (e.g., import :mod:`nupic.support`;
     :meth:`nupic.support.initLogging`)

  See also: :meth:`.initExperimentPrng`.

  :param args: (string) Experiment command-line args list. Too see all options,
      run with ``--help``:

      .. code-block:: text

        Options:
          -h, --help           show this help message and exit
          -c <CHECKPOINT>      Create a model and save it under the given <CHECKPOINT>
                               name, but don't run it
          --listCheckpoints    List all available checkpoints
          --listTasks          List all task labels in description.py
          --load=<CHECKPOINT>  Load a model from the given <CHECKPOINT> and run it.
                               Run with --listCheckpoints flag for more details.
          --newSerialization   Use new capnproto serialization
          --tasks              Run the tasks with the given TASK LABELS in the order
                               they are given.  Either end of arg-list, or a
                               standalone dot ('.') arg or the next short or long
                               option name (-a or --blah) terminates the list. NOTE:
                               FAILS TO RECOGNIZE task label names with one or more
                               leading dashes. [default: run all of the tasks in
                               description.py]
          --testMode           Reduce iteration count for testing
          --noCheckpoint       Don't checkpoint the model after running each task.

  :param model: (:class:`~nupic.frameworks.opf.model.Model`) For testing, may
      pass in an existing OPF Model to use instead of creating a new one.

  :returns: (:class:`~nupic.frameworks.opf.model.Model`)
    reference to OPF Model instance that was constructed (this
    is provided to aid with debugging) or None, if none was
    created.
  """
  # Parse command-line options
  opt = _parseCommandLineOptions(args)

  #print "runExperiment: Parsed Command Options: ", opt

  model = _runExperimentImpl(opt, model)

  return model

def runExperiment ( args , model = None ) : opt = _parseCommandLineOptions ( args ) model = _runExperimentImpl ( opt , model ) return model


def reapVarArgsCallback(option, optStr, value, parser):
  """Used as optparse callback for reaping a variable number of option args.
  The option may be specified multiple times, and all the args associated with
  that option name will be accumulated in the order that they are encountered
  """
  newValues = []

  # Reap the args, taking care to stop before the next option or '.'
  gotDot = False
  for arg in parser.rargs:
    # Stop on --longname options
    if arg.startswith("--") and len(arg) > 2:
      break

    # Stop on -b options
    if arg.startswith("-") and len(arg) > 1:
      break

    if arg == ".":
      gotDot = True
      break

    newValues.append(arg)

  if not newValues:
    raise optparse.OptionValueError(
      ("Empty arg list for option %r expecting one or more args "
       "(remaining tokens: %r)") % (optStr, parser.rargs))

  del parser.rargs[:len(newValues) + int(gotDot)]

  # Retrieve the existing arg accumulator, if any
  value = getattr(parser.values, option.dest, [])
  #print "Previous value: %r" % value
  if value is None:
    value = []

  # Append the new args to the existing ones and save to the parser
  value.extend(newValues)
  setattr(parser.values, option.dest, value)

Used as optparse callback for reaping a variable number of option args . The option may be specified multiple times and all the args associated with that option name will be accumulated in the order that they are encountered


def _reportCommandLineUsageErrorAndExit(parser, message):
  """Report usage error and exit program with error indication."""
  print parser.get_usage()
  print message
  sys.exit(1)

def _reportCommandLineUsageErrorAndExit ( parser , message ) : print parser . get_usage ( ) print message sys . exit ( 1 )

Report usage error and exit program with error indication .


def _runExperimentImpl(options, model=None):
  """Creates and runs the experiment

  Args:
    options: namedtuple ParseCommandLineOptionsResult
    model: For testing: may pass in an existing OPF Model instance
        to use instead of creating a new one.

  Returns: reference to OPFExperiment instance that was constructed (this
      is provided to aid with debugging) or None, if none was
      created.
  """
  json_helpers.validate(options.privateOptions,
                        schemaDict=g_parsedPrivateCommandLineOptionsSchema)

  # Load the experiment's description.py module
  experimentDir = options.experimentDir
  descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir(
      experimentDir)
  expIface = helpers.getExperimentDescriptionInterfaceFromModule(
      descriptionPyModule)

  # Handle "list checkpoints" request
  if options.privateOptions['listAvailableCheckpoints']:
    _printAvailableCheckpoints(experimentDir)
    return None

  # Load experiment tasks
  experimentTasks = expIface.getModelControl().get('tasks', [])

  # If the tasks list is empty, and this is a nupic environment description
  # file being run from the OPF, convert it to a simple OPF description file.
  if (len(experimentTasks) == 0 and
      expIface.getModelControl()['environment'] == OpfEnvironment.Nupic):
    expIface.convertNupicEnvToOPF()
    experimentTasks = expIface.getModelControl().get('tasks', [])

  # Ensures all the source locations are either absolute paths or relative to
  # the nupic.datafiles package_data location.
  expIface.normalizeStreamSources()

  # Extract option
  newSerialization = options.privateOptions['newSerialization']

  # Handle listTasks
  if options.privateOptions['listTasks']:
    print "Available tasks:"

    for label in [t['taskLabel'] for t in experimentTasks]:
      print "\t", label

    return None

  # Construct the experiment instance
  if options.privateOptions['runCheckpointName']:

    assert model is None

    checkpointName = options.privateOptions['runCheckpointName']

    model = ModelFactory.loadFromCheckpoint(
          savedModelDir=_getModelCheckpointDir(experimentDir, checkpointName),
          newSerialization=newSerialization)

  elif model is not None:
    print "Skipping creation of OPFExperiment instance: caller provided his own"
  else:
    modelDescription = expIface.getModelDescription()
    model = ModelFactory.create(modelDescription)

  # Handle "create model" request
  if options.privateOptions['createCheckpointName']:
    checkpointName = options.privateOptions['createCheckpointName']
    _saveModel(model=model,
               experimentDir=experimentDir,
               checkpointLabel=checkpointName,
               newSerialization=newSerialization)

    return model

  # Build the task list

  # Default task execution index list is in the natural list order of the tasks
  taskIndexList = range(len(experimentTasks))

  customTaskExecutionLabelsList = options.privateOptions['taskLabels']
  if customTaskExecutionLabelsList:
    taskLabelsList = [t['taskLabel'] for t in experimentTasks]
    taskLabelsSet = set(taskLabelsList)

    customTaskExecutionLabelsSet = set(customTaskExecutionLabelsList)

    assert customTaskExecutionLabelsSet.issubset(taskLabelsSet), \
           ("Some custom-provided task execution labels don't correspond "
            "to actual task labels: mismatched labels: %r; actual task "
            "labels: %r.") % (customTaskExecutionLabelsSet - taskLabelsSet,
                              customTaskExecutionLabelsList)

    taskIndexList = [taskLabelsList.index(label) for label in
                     customTaskExecutionLabelsList]

    print "#### Executing custom task list: %r" % [taskLabelsList[i] for
                                                   i in taskIndexList]

  # Run all experiment tasks
  for taskIndex in taskIndexList:

    task = experimentTasks[taskIndex]

    # Create a task runner and run it!
    taskRunner = _TaskRunner(model=model,
                             task=task,
                             cmdOptions=options)
    taskRunner.run()
    del taskRunner

    if options.privateOptions['checkpointModel']:
      _saveModel(model=model,
                 experimentDir=experimentDir,
                 checkpointLabel=task['taskLabel'],
                 newSerialization=newSerialization)

  return model


def _getModelCheckpointDir(experimentDir, checkpointLabel):
  """Creates directory for serialization of the model

  checkpointLabel:
      Checkpoint label (string)

  Returns:
    absolute path to the serialization directory
  """
  checkpointDir = os.path.join(getCheckpointParentDir(experimentDir),
                               checkpointLabel + g_defaultCheckpointExtension)
  checkpointDir = os.path.abspath(checkpointDir)

  return checkpointDir

def _getModelCheckpointDir ( experimentDir , checkpointLabel ) : checkpointDir = os . path . join ( getCheckpointParentDir ( experimentDir ) , checkpointLabel + g_defaultCheckpointExtension ) checkpointDir = os . path . abspath ( checkpointDir ) return checkpointDir


def getCheckpointParentDir(experimentDir):
  """Get checkpoint parent dir.

  Returns: absolute path to the base serialization directory within which
      model checkpoints for this experiment are created
  """
  baseDir = os.path.join(experimentDir, "savedmodels")
  baseDir = os.path.abspath(baseDir)

  return baseDir

def getCheckpointParentDir ( experimentDir ) : baseDir = os . path . join ( experimentDir , "savedmodels" ) baseDir = os . path . abspath ( baseDir ) return baseDir


def _checkpointLabelFromCheckpointDir(checkpointDir):
  """Returns a checkpoint label string for the given model checkpoint directory

  checkpointDir: relative or absolute model checkpoint directory path
  """
  assert checkpointDir.endswith(g_defaultCheckpointExtension)

  lastSegment = os.path.split(checkpointDir)[1]

  checkpointLabel = lastSegment[0:-len(g_defaultCheckpointExtension)]

  return checkpointLabel

def _checkpointLabelFromCheckpointDir ( checkpointDir ) : assert checkpointDir . endswith ( g_defaultCheckpointExtension ) lastSegment = os . path . split ( checkpointDir ) [ 1 ] checkpointLabel = lastSegment [ 0 : - len ( g_defaultCheckpointExtension ) ] return checkpointLabel


def _isCheckpointDir(checkpointDir):
  """Return true iff checkpointDir appears to be a checkpoint directory."""
  lastSegment = os.path.split(checkpointDir)[1]
  if lastSegment[0] == '.':
    return False

  if not checkpointDir.endswith(g_defaultCheckpointExtension):
    return False

  if not os.path.isdir(checkpointDir):
    return False

  return True

def _isCheckpointDir ( checkpointDir ) : lastSegment = os . path . split ( checkpointDir ) [ 1 ] if lastSegment [ 0 ] == '.' : return False if not checkpointDir . endswith ( g_defaultCheckpointExtension ) : return False if not os . path . isdir ( checkpointDir ) : return False return True

Return true iff checkpointDir appears to be a checkpoint directory .


def _printAvailableCheckpoints(experimentDir):
  """List available checkpoints for the specified experiment."""
  checkpointParentDir = getCheckpointParentDir(experimentDir)

  if not os.path.exists(checkpointParentDir):
    print "No available checkpoints."
    return

  checkpointDirs = [x for x in os.listdir(checkpointParentDir)
                    if _isCheckpointDir(os.path.join(checkpointParentDir, x))]
  if not checkpointDirs:
    print "No available checkpoints."
    return

  print "Available checkpoints:"
  checkpointList = [_checkpointLabelFromCheckpointDir(x)
                    for x in checkpointDirs]

  for checkpoint in sorted(checkpointList):
    print "\t", checkpoint

  print
  print "To start from a checkpoint:"
  print "  python run_opf_experiment.py experiment --load <CHECKPOINT>"
  print "For example, to start from the checkpoint \"MyCheckpoint\":"
  print "  python run_opf_experiment.py experiment --load MyCheckpoint"

List available checkpoints for the specified experiment .