def enableTap(self, tapPath):
    """
    Begin writing output tap files.

    :param tapPath: (string) base name of the output tap files to write.
    """

    self._tapFileIn = open(tapPath + '.in', 'w')
    self._tapFileOut = open(tapPath + '.out', 'w')

def enableTap ( self , tapPath ) : self . _tapFileIn = open ( tapPath + '.in' , 'w' ) self . _tapFileOut = open ( tapPath + '.out' , 'w' )


def disableTap(self):
    """
    Disable writing of output tap files.
    """

    if self._tapFileIn is not None:
      self._tapFileIn.close()
      self._tapFileIn = None
    if self._tapFileOut is not None:
      self._tapFileOut.close()
      self._tapFileOut = None

def disableTap ( self ) : if self . _tapFileIn is not None : self . _tapFileIn . close ( ) self . _tapFileIn = None if self . _tapFileOut is not None : self . _tapFileOut . close ( ) self . _tapFileOut = None


def handleLogOutput(self, output):
    """
    Write outputs to output tap file.

    :param outputs: (iter) some outputs.
    """
    #raise Exception('MULTI-LINE DUMMY\nMULTI-LINE DUMMY')
    if self._tapFileOut is not None:
      for k in range(len(output)):
        print >> self._tapFileOut, output[k],
      print >> self._tapFileOut

def handleLogOutput ( self , output ) : if self . _tapFileOut is not None : for k in range ( len ( output ) ) : print >> self . _tapFileOut , output [ k ] , print >> self . _tapFileOut


def _storeSample(self, inputVector, trueCatIndex, partition=0):
    """
    Store a training sample and associated category label
    """

    # If this is the first sample, then allocate a numpy array
    # of the appropriate size in which to store all samples.
    if self._samples is None:
      self._samples = numpy.zeros((0, len(inputVector)), dtype=RealNumpyDType)
      assert self._labels is None
      self._labels = []

    # Add the sample vector and category lable
    self._samples = numpy.concatenate((self._samples, numpy.atleast_2d(inputVector)), axis=0)
    self._labels += [trueCatIndex]

    # Add the partition ID
    if self._partitions is None:
      self._partitions = []
    if partition is None:
      partition = 0
    self._partitions += [partition]

def _storeSample ( self , inputVector , trueCatIndex , partition = 0 ) : if self . _samples is None : self . _samples = numpy . zeros ( ( 0 , len ( inputVector ) ) , dtype = RealNumpyDType ) assert self . _labels is None self . _labels = [ ] self . _samples = numpy . concatenate ( ( self . _samples , numpy . atleast_2d ( inputVector ) ) , axis = 0 ) self . _labels += [ trueCatIndex ] if self . _partitions is None : self . _partitions = [ ] if partition is None : partition = 0 self . _partitions += [ partition ]


def _finishLearning(self):
    """Does nothing. Kept here for API compatibility """
    if self._doSphering:
      self._finishSphering()

    self._knn.finishLearning()

    # Compute leave-one-out validation accuracy if
    # we actually received non-trivial partition info
    self._accuracy = None

def _finishLearning ( self ) : if self . _doSphering : self . _finishSphering ( ) self . _knn . finishLearning ( ) self . _accuracy = None

Does nothing . Kept here for API compatibility


def generateStats(filename, statsInfo, maxSamples = None, filters=[], cache=True):
  """Generate requested statistics for a dataset and cache to a file.
  If filename is None, then don't cache to a file"""

  # Sanity checking
  if not isinstance(statsInfo, dict):
    raise RuntimeError("statsInfo must be a dict -- "
                       "found '%s' instead" % type(statsInfo))

  filename = resource_filename("nupic.datafiles", filename)

  if cache:
    statsFilename = getStatsFilename(filename, statsInfo, filters)
    # Use cached stats if found AND if it has the right data
    if os.path.exists(statsFilename):
      try:
        r = pickle.load(open(statsFilename, "rb"))
      except:
        # Ok to ignore errors -- we will just re-generate the file
        print "Warning: unable to load stats for %s -- " \
              "will regenerate" % filename
        r = dict()
      requestedKeys = set([s for s in statsInfo])
      availableKeys = set(r.keys())
      unavailableKeys = requestedKeys.difference(availableKeys)
      if len(unavailableKeys ) == 0:
        return r
      else:
        print "generateStats: re-generating stats file %s because " \
              "keys %s are not available" %  \
              (filename, str(unavailableKeys))
        os.remove(filename)

  print "Generating statistics for file '%s' with filters '%s'" % (filename, filters)
  sensor = RecordSensor()
  sensor.dataSource = FileRecordStream(filename)
  sensor.preEncodingFilters = filters

  # Convert collector description to collector object
  stats = []
  for field in statsInfo:
    # field = key from statsInfo
    if statsInfo[field] == "number":
      # This wants a field name e.g. consumption and the field type as the value
      statsInfo[field] = NumberStatsCollector()
    elif statsInfo[field] == "category":
      statsInfo[field] = CategoryStatsCollector()
    else:
      raise RuntimeError("Unknown stats type '%s' for field '%s'" % (statsInfo[field], field))

  # Now collect the stats
  if maxSamples is None:
    maxSamples = 500000
  for i in xrange(maxSamples):
    try:
      record = sensor.getNextRecord()
    except StopIteration:
      break
    for (name, collector) in statsInfo.items():
      collector.add(record[name])

  del sensor

  # Assemble the results and return
  r = dict()
  for (field, collector) in statsInfo.items():
    stats = collector.getStats()
    if field not in r:
      r[field] = stats
    else:
      r[field].update(stats)

  if cache:
    f = open(statsFilename, "wb")
    pickle.dump(r, f)
    f.close()
    # caller may need to know name of cached file
    r["_filename"] = statsFilename

  return r

Generate requested statistics for a dataset and cache to a file . If filename is None then don t cache to a file


def _fixupRandomEncoderParams(params, minVal, maxVal, minResolution):
  """
  Given model params, figure out the correct parameters for the
  RandomDistributed encoder. Modifies params in place.
  """
  encodersDict = (
    params["modelConfig"]["modelParams"]["sensorParams"]["encoders"]
  )

  for encoder in encodersDict.itervalues():
    if encoder is not None:
      if encoder["type"] == "RandomDistributedScalarEncoder":
        resolution = max(minResolution,
                         (maxVal - minVal) / encoder.pop("numBuckets")
                        )
        encodersDict["c1"]["resolution"] = resolution

def _fixupRandomEncoderParams ( params , minVal , maxVal , minResolution ) : encodersDict = ( params [ "modelConfig" ] [ "modelParams" ] [ "sensorParams" ] [ "encoders" ] ) for encoder in encodersDict . itervalues ( ) : if encoder is not None : if encoder [ "type" ] == "RandomDistributedScalarEncoder" : resolution = max ( minResolution , ( maxVal - minVal ) / encoder . pop ( "numBuckets" ) ) encodersDict [ "c1" ] [ "resolution" ] = resolution

Given model params figure out the correct parameters for the RandomDistributed encoder . Modifies params in place .


def read(cls, proto):
    """
    Intercepts TemporalMemory deserialization request in order to initialize
    `TemporalMemoryMonitorMixin` state

    @param proto (DynamicStructBuilder) Proto object

    @return (TemporalMemory) TemporalMemory shim instance
    """
    tm = super(TemporalMemoryMonitorMixin, cls).read(proto)

    # initialize `TemporalMemoryMonitorMixin` attributes
    tm.mmName = None
    tm._mmTraces = None
    tm._mmData = None
    tm.mmClearHistory()
    tm._mmResetActive = True
    return tm

def read ( cls , proto ) : tm = super ( TemporalMemoryMonitorMixin , cls ) . read ( proto ) tm . mmName = None tm . _mmTraces = None tm . _mmData = None tm . mmClearHistory ( ) tm . _mmResetActive = True return tm

Intercepts TemporalMemory deserialization request in order to initialize TemporalMemoryMonitorMixin state


def pickByDistribution(distribution, r=None):
  """
  Pick a value according to the provided distribution.

  Example:

  ::

    pickByDistribution([.2, .1])

  Returns 0 two thirds of the time and 1 one third of the time.

  :param distribution: Probability distribution. Need not be normalized.
  :param r: Instance of random.Random. Uses the system instance if one is
         not provided.
  """

  if r is None:
    r = random

  x = r.uniform(0, sum(distribution))
  for i, d in enumerate(distribution):
    if x <= d:
      return i
    x -= d

def pickByDistribution ( distribution , r = None ) : if r is None : r = random x = r . uniform ( 0 , sum ( distribution ) ) for i , d in enumerate ( distribution ) : if x <= d : return i x -= d

Pick a value according to the provided distribution .


def Indicator(pos, size, dtype):
  """
  Returns an array of length size and type dtype that is everywhere 0,
  except in the index in pos.

  :param pos: (int) specifies the position of the one entry that will be set.
  :param size: (int) The total size of the array to be returned.
  :param dtype: The element type (compatible with NumPy array())
         of the array to be returned.
  :returns: (list) of length ``size`` and element type ``dtype``.
  """
  x = numpy.zeros(size, dtype=dtype)
  x[pos] = 1
  return x

def Indicator ( pos , size , dtype ) : x = numpy . zeros ( size , dtype = dtype ) x [ pos ] = 1 return x

Returns an array of length size and type dtype that is everywhere 0 except in the index in pos .


def MultiIndicator(pos, size, dtype):
  """
  Returns an array of length size and type dtype that is everywhere 0,
  except in the indices listed in sequence pos.

  :param pos:   A single integer or sequence of integers that specify
         the position of ones to be set.
  :param size:  The total size of the array to be returned.
  :param dtype: The element type (compatible with NumPy array())
         of the array to be returned.
  :returns: An array of length size and element type dtype.
  """
  x = numpy.zeros(size, dtype=dtype)
  if hasattr(pos, '__iter__'):
    for i in pos: x[i] = 1
  else: x[pos] = 1
  return x

def MultiIndicator ( pos , size , dtype ) : x = numpy . zeros ( size , dtype = dtype ) if hasattr ( pos , '__iter__' ) : for i in pos : x [ i ] = 1 else : x [ pos ] = 1 return x

Returns an array of length size and type dtype that is everywhere 0 except in the indices listed in sequence pos .


def Distribution(pos, size, counts, dtype):
  """
  Returns an array of length size and type dtype that is everywhere 0,
  except in the indices listed in sequence pos.  The non-zero indices
  contain a normalized distribution based on the counts.

  :param pos:    A single integer or sequence of integers that specify
          the position of ones to be set.
  :param size:   The total size of the array to be returned.
  :param counts: The number of times we have observed each index.
  :param dtype:  The element type (compatible with NumPy array())
          of the array to be returned.
  :returns: An array of length size and element type dtype.
  """
  x = numpy.zeros(size, dtype=dtype)
  if hasattr(pos, '__iter__'):
    # calculate normalization constant
    total = 0
    for i in pos:
      total += counts[i]
    total = float(total)
    # set included positions to normalized probability
    for i in pos:
      x[i] = counts[i]/total
  # If we don't have a set of positions, assume there's only one position
  else: x[pos] = 1
  return x

def Distribution ( pos , size , counts , dtype ) : x = numpy . zeros ( size , dtype = dtype ) if hasattr ( pos , '__iter__' ) : total = 0 for i in pos : total += counts [ i ] total = float ( total ) for i in pos : x [ i ] = counts [ i ] / total else : x [ pos ] = 1 return x

Returns an array of length size and type dtype that is everywhere 0 except in the indices listed in sequence pos . The non - zero indices contain a normalized distribution based on the counts .


def grow(self, rows, cols):
    """
    Grows the histogram to have rows rows and cols columns.
    Must not have been initialized before, or already have the same
    number of columns.
    If rows is smaller than the current number of rows,
    does not shrink.
    Also updates the sizes of the row and column sums.

    :param rows: Integer number of rows.
    :param cols: Integer number of columns.
    """
    if not self.hist_:
      self.hist_ = SparseMatrix(rows, cols)
      self.rowSums_ = numpy.zeros(rows, dtype=dtype)
      self.colSums_ = numpy.zeros(cols, dtype=dtype)
      self.hack_ = None
    else:
      oldRows = self.hist_.nRows()
      oldCols = self.hist_.nCols()
      nextRows = max(oldRows, rows)
      nextCols = max(oldCols, cols)
      if (oldRows < nextRows) or (oldCols < nextCols):
        self.hist_.resize(nextRows, nextCols)
        if oldRows < nextRows:
          oldSums = self.rowSums_
          self.rowSums_ = numpy.zeros(nextRows, dtype=dtype)
          self.rowSums_[0:len(oldSums)] = oldSums
          self.hack_ = None
        if oldCols < nextCols:
          oldSums = self.colSums_
          self.colSums_ = numpy.zeros(nextCols, dtype=dtype)
          self.colSums_[0:len(oldSums)] = oldSums
          self.hack_ = None

def grow ( self , rows , cols ) : if not self . hist_ : self . hist_ = SparseMatrix ( rows , cols ) self . rowSums_ = numpy . zeros ( rows , dtype = dtype ) self . colSums_ = numpy . zeros ( cols , dtype = dtype ) self . hack_ = None else : oldRows = self . hist_ . nRows ( ) oldCols = self . hist_ . nCols ( ) nextRows = max ( oldRows , rows ) nextCols = max ( oldCols , cols ) if ( oldRows < nextRows ) or ( oldCols < nextCols ) : self . hist_ . resize ( nextRows , nextCols ) if oldRows < nextRows : oldSums = self . rowSums_ self . rowSums_ = numpy . zeros ( nextRows , dtype = dtype ) self . rowSums_ [ 0 : len ( oldSums ) ] = oldSums self . hack_ = None if oldCols < nextCols : oldSums = self . colSums_ self . colSums_ = numpy . zeros ( nextCols , dtype = dtype ) self . colSums_ [ 0 : len ( oldSums ) ] = oldSums self . hack_ = None

Grows the histogram to have rows rows and cols columns . Must not have been initialized before or already have the same number of columns . If rows is smaller than the current number of rows does not shrink . Also updates the sizes of the row and column sums .


def updateRow(self, row, distribution):
    """
    Add distribution to row row.
    Distribution should be an array of probabilities or counts.

    :param row:   Integer index of the row to add to.
                  May be larger than the current number of rows, in which case
                  the histogram grows.
    :param distribution: Array of length equal to the number of columns.
    """
    self.grow(row+1, len(distribution))
    self.hist_.axby(row, 1, 1, distribution)
    self.rowSums_[row] += distribution.sum()
    self.colSums_ += distribution
    self.hack_ = None

def updateRow ( self , row , distribution ) : self . grow ( row + 1 , len ( distribution ) ) self . hist_ . axby ( row , 1 , 1 , distribution ) self . rowSums_ [ row ] += distribution . sum ( ) self . colSums_ += distribution self . hack_ = None

Add distribution to row row . Distribution should be an array of probabilities or counts .


def importAndRunFunction(
    path,
    moduleName,
    funcName,
    **keywords
  ):
  """
  Run a named function specified by a filesystem path, module name
  and function name.

  Returns the value returned by the imported function.

  Use this when access is needed to code that has
  not been added to a package accessible from the ordinary Python
  path. Encapsulates the multiple lines usually needed to
  safely manipulate and restore the Python path.

  Parameters
  ----------
  path: filesystem path
  Path to the directory where the desired module is stored.
  This will be used to temporarily augment the Python path.

  moduleName: basestring
  Name of the module, without trailing extension, where the desired
  function is stored. This module should be in the directory specified
  with path.

  funcName: basestring
  Name of the function to import and call.

  keywords:
  Keyword arguments to be passed to the imported function.
  """
  import sys
  originalPath = sys.path
  try:
    augmentedPath = [path] + sys.path
    sys.path = augmentedPath
    func = getattr(__import__(moduleName, fromlist=[funcName]), funcName)
    sys.path = originalPath
  except:
    # Restore the original path in case of an exception.
    sys.path = originalPath
    raise
  return func(**keywords)

def importAndRunFunction ( path , moduleName , funcName , ** keywords ) : import sys originalPath = sys . path try : augmentedPath = [ path ] + sys . path sys . path = augmentedPath func = getattr ( __import__ ( moduleName , fromlist = [ funcName ] ) , funcName ) sys . path = originalPath except : sys . path = originalPath raise return func ( ** keywords )

Run a named function specified by a filesystem path module name and function name .


def compute(slidingWindow, total, newVal, windowSize):
    """Routine for computing a moving average.

    @param slidingWindow a list of previous values to use in computation that
        will be modified and returned
    @param total the sum of the values in slidingWindow to be used in the
        calculation of the moving average
    @param newVal a new number compute the new windowed average
    @param windowSize how many values to use in the moving window

    @returns an updated windowed average, the modified input slidingWindow list,
        and the new total sum of the sliding window
    """
    if len(slidingWindow) == windowSize:
      total -= slidingWindow.pop(0)

    slidingWindow.append(newVal)
    total += newVal
    return float(total) / len(slidingWindow), slidingWindow, total

def compute ( slidingWindow , total , newVal , windowSize ) : if len ( slidingWindow ) == windowSize : total -= slidingWindow . pop ( 0 ) slidingWindow . append ( newVal ) total += newVal return float ( total ) / len ( slidingWindow ) , slidingWindow , total

Routine for computing a moving average .


def next(self, newValue):
    """Instance method wrapper around compute."""
    newAverage, self.slidingWindow, self.total = self.compute(
        self.slidingWindow, self.total, newValue, self.windowSize)
    return newAverage

def next ( self , newValue ) : newAverage , self . slidingWindow , self . total = self . compute ( self . slidingWindow , self . total , newValue , self . windowSize ) return newAverage

Instance method wrapper around compute .


def addInstance(self, groundTruth, prediction, record = None, result = None):
    """Compute and store metric value"""
    self.value = self.avg(prediction)

def addInstance ( self , groundTruth , prediction , record = None , result = None ) : self . value = self . avg ( prediction )


def mostLikely(self, pred):
    """ Helper function to return a scalar value representing the most
        likely outcome given a probability distribution
    """
    if len(pred) == 1:
      return pred.keys()[0]

    mostLikelyOutcome = None
    maxProbability = 0

    for prediction, probability in pred.items():
      if probability > maxProbability:
        mostLikelyOutcome = prediction
        maxProbability = probability

    return mostLikelyOutcome

def mostLikely ( self , pred ) : if len ( pred ) == 1 : return pred . keys ( ) [ 0 ] mostLikelyOutcome = None maxProbability = 0 for prediction , probability in pred . items ( ) : if probability > maxProbability : mostLikelyOutcome = prediction maxProbability = probability return mostLikelyOutcome


def expValue(self, pred):
    """ Helper function to return a scalar value representing the expected
        value of a probability distribution
    """
    if len(pred) == 1:
      return pred.keys()[0]

    return sum([x*p for x,p in pred.items()])

def expValue ( self , pred ) : if len ( pred ) == 1 : return pred . keys ( ) [ 0 ] return sum ( [ x * p for x , p in pred . items ( ) ] )


def getScalarNames(self, parentFieldName=''):
    """
    Return the field names for each of the scalar values returned by
    getScalars.

    :param parentFieldName: The name of the encoder which is our parent. This
        name is prefixed to each of the field names within this encoder to
        form the keys of the dict() in the retval.

    :return: array of field names
    """
    names = []

    if self.encoders is not None:
      for (name, encoder, offset) in self.encoders:
        subNames = encoder.getScalarNames(parentFieldName=name)
        if parentFieldName != '':
          subNames = ['%s.%s' % (parentFieldName, name) for name in subNames]
        names.extend(subNames)
    else:
      if parentFieldName != '':
        names.append(parentFieldName)
      else:
        names.append(self.name)

    return names

def getScalarNames ( self , parentFieldName = '' ) : names = [ ] if self . encoders is not None : for ( name , encoder , offset ) in self . encoders : subNames = encoder . getScalarNames ( parentFieldName = name ) if parentFieldName != '' : subNames = [ '%s.%s' % ( parentFieldName , name ) for name in subNames ] names . extend ( subNames ) else : if parentFieldName != '' : names . append ( parentFieldName ) else : names . append ( self . name ) return names

Return the field names for each of the scalar values returned by getScalars .


def _getInputValue(self, obj, fieldName):
    """
    Gets the value of a given field from the input record
    """
    if isinstance(obj, dict):
      if not fieldName in obj:
        knownFields = ", ".join(
          key for key in obj.keys() if not key.startswith("_")
        )
        raise ValueError(
          "Unknown field name '%s' in input record. Known fields are '%s'.\n"
          "This could be because input headers are mislabeled, or because "
          "input data rows do not contain a value for '%s'." % (
            fieldName, knownFields, fieldName
          )
        )
      return obj[fieldName]
    else:
      return getattr(obj, fieldName)


def getFieldDescription(self, fieldName):
    """
    Return the offset and length of a given field within the encoded output.

    :param fieldName: Name of the field
    :return: tuple(``offset``, ``width``) of the field within the encoded output
    """

    # Find which field it's in
    description = self.getDescription() + [("end", self.getWidth())]
    for i in xrange(len(description)):
      (name, offset) = description[i]
      if (name == fieldName):
        break

    if i >= len(description)-1:
      raise RuntimeError("Field name %s not found in this encoder" % fieldName)

    # Return the offset and width
    return (offset, description[i+1][1] - offset)

Return the offset and length of a given field within the encoded output .


def encodedBitDescription(self, bitOffset, formatted=False):
    """
    Return a description of the given bit in the encoded output.
    This will include the field name and the offset within the field.

    :param bitOffset:      Offset of the bit to get the description of
    :param formatted:      If True, the bitOffset is w.r.t. formatted output,
                          which includes separators
    :return:             tuple(``fieldName``, ``offsetWithinField``)
    """

    # Find which field it's in
    (prevFieldName, prevFieldOffset) = (None, None)
    description = self.getDescription()
    for i in xrange(len(description)):
      (name, offset) = description[i]
      if formatted:
        offset = offset + i
        if bitOffset == offset-1:
          prevFieldName = "separator"
          prevFieldOffset = bitOffset
          break

      if bitOffset < offset:
        break
      (prevFieldName, prevFieldOffset) = (name, offset)

    # Return the field name and offset within the field
    # return (fieldName, bitOffset - fieldOffset)
    width = self.getDisplayWidth() if formatted else self.getWidth()

    if prevFieldOffset is None or bitOffset > self.getWidth():
      raise IndexError("Bit is outside of allowable range: [0 - %d]" % width)

    return (prevFieldName, bitOffset - prevFieldOffset)

Return a description of the given bit in the encoded output . This will include the field name and the offset within the field .


def pprint(self, output, prefix=""):
    """
    Pretty-print the encoded output using ascii art.

    :param output: to print
    :param prefix: printed before the header if specified
    """
    print prefix,
    description = self.getDescription() + [("end", self.getWidth())]
    for i in xrange(len(description) - 1):
      offset = description[i][1]
      nextoffset = description[i+1][1]
      print "%s |" % bitsToString(output[offset:nextoffset]),
    print

def
pprint
(
self
,
output
,
prefix
=
""
)
:
print
prefix
,
description
=
self
.
getDescription
(
)
+
[
(
"end"
,
self
.
getWidth
(
)
)
]
for
i
in
xrange
(
len
(
description
)
1
)
:
offset
=
description
[
i
]
[
1
]
nextoffset
=
description
[
i
+
1
]
[
1
]
print
"%s |"
%
bitsToString
(
output
[
offset
:
nextoffset
]
)
,
print

Pretty - print the encoded output using ascii art .


def decode(self, encoded, parentFieldName=''):
    """
    Takes an encoded output and does its best to work backwards and generate
    the input that would have generated it.

    In cases where the encoded output contains more ON bits than an input
    would have generated, this routine will return one or more ranges of inputs
    which, if their encoded outputs were ORed together, would produce the
    target output. This behavior makes this method suitable for doing things
    like generating a description of a learned coincidence in the SP, which
    in many cases might be a union of one or more inputs.

    If instead, you want to figure the *most likely* single input scalar value
    that would have generated a specific encoded output, use the
    :meth:`.topDownCompute` method.

    If you want to pretty print the return value from this method, use the
    :meth:`.decodedToStr` method.

    :param encoded:      The encoded output that you want decode
    :param parentFieldName: The name of the encoder which is our parent. This name
           is prefixed to each of the field names within this encoder to form the
           keys of the dict() in the retval.

    :return: tuple(``fieldsDict``, ``fieldOrder``)

              ``fieldsDict`` is a dict() where the keys represent field names
              (only 1 if this is a simple encoder, > 1 if this is a multi
              or date encoder) and the values are the result of decoding each
              field. If there are  no bits in encoded that would have been
              generated by a field, it won't be present in the dict. The
              key of each entry in the dict is formed by joining the passed in
              parentFieldName with the child encoder name using a '.'.

              Each 'value' in ``fieldsDict`` consists of (ranges, desc), where
              ranges is a list of one or more (minVal, maxVal) ranges of
              input that would generate bits in the encoded output and 'desc'
              is a pretty print description of the ranges. For encoders like
              the category encoder, the 'desc' will contain the category
              names that correspond to the scalar values included in the
              ranges.

              ``fieldOrder`` is a list of the keys from ``fieldsDict``, in the
              same order as the fields appear in the encoded output.

              TODO: when we switch to Python 2.7 or 3.x, use OrderedDict

    Example retvals for a scalar encoder:

    .. code-block:: python

       {'amount':  ( [[1,3], [7,10]], '1-3, 7-10' )}
       {'amount':  ( [[2.5,2.5]],     '2.5'       )}

    Example retval for a category encoder:

    .. code-block:: python

       {'country': ( [[1,1], [5,6]], 'US, GB, ES' )}

    Example retval for a multi encoder:

    .. code-block:: python

       {'amount':  ( [[2.5,2.5]],     '2.5'       ),
        'country': ( [[1,1], [5,6]],  'US, GB, ES' )}

    """

    fieldsDict = dict()
    fieldsOrder = []

    # What is the effective parent name?
    if parentFieldName == '':
      parentName = self.name
    else:
      parentName = "%s.%s" % (parentFieldName, self.name)

    if self.encoders is not None:
      # Merge decodings of all child encoders together
      for i in xrange(len(self.encoders)):

        # Get the encoder and the encoded output
        (name, encoder, offset) = self.encoders[i]
        if i < len(self.encoders)-1:
          nextOffset = self.encoders[i+1][2]
        else:
          nextOffset = self.width
        fieldOutput = encoded[offset:nextOffset]
        (subFieldsDict, subFieldsOrder) = encoder.decode(fieldOutput,
                                              parentFieldName=parentName)

        fieldsDict.update(subFieldsDict)
        fieldsOrder.extend(subFieldsOrder)

    return (fieldsDict, fieldsOrder)

def decode ( self , encoded , parentFieldName = '' ) : fieldsDict = dict ( ) fieldsOrder = [ ] if parentFieldName == '' : parentName = self . name else : parentName = "%s.%s" % ( parentFieldName , self . name ) if self . encoders is not None : for i in xrange ( len ( self . encoders ) ) : ( name , encoder , offset ) = self . encoders [ i ] if i < len ( self . encoders ) - 1 : nextOffset = self . encoders [ i + 1 ] [ 2 ] else : nextOffset = self . width fieldOutput = encoded [ offset : nextOffset ] ( subFieldsDict , subFieldsOrder ) = encoder . decode ( fieldOutput , parentFieldName = parentName ) fieldsDict . update ( subFieldsDict ) fieldsOrder . extend ( subFieldsOrder ) return ( fieldsDict , fieldsOrder )

Takes an encoded output and does its best to work backwards and generate the input that would have generated it .


def drawFile(dataset, matrix, patterns, cells, w, fnum):
  '''The similarity of two patterns in the bit-encoding space is displayed alongside
  their similarity in the sp-coinc space.'''
  score=0
  count = 0
  assert len(patterns)==len(cells)
  for p in xrange(len(patterns)-1):
    matrix[p+1:,p] = [len(set(patterns[p]).intersection(set(q)))*100/w for q in patterns[p+1:]]
    matrix[p,p+1:] = [len(set(cells[p]).intersection(set(r)))*5/2 for r in cells[p+1:]]

    score += sum(abs(np.array(matrix[p+1:,p])-np.array(matrix[p,p+1:])))
    count += len(matrix[p+1:,p])

  print 'Score', score/count

  fig = pyl.figure(figsize = (10,10), num = fnum)
  pyl.matshow(matrix, fignum = fnum)
  pyl.colorbar()
  pyl.title('Coincidence Space', verticalalignment='top', fontsize=12)
  pyl.xlabel('The Mirror Image Visualization for '+dataset, fontsize=17)
  pyl.ylabel('Encoding space', fontsize=12)

def
drawFile
(
dataset
,
matrix
,
patterns
,
cells
,
w
,
fnum
)
:
score
=
0
count
=
0
assert
len
(
patterns
)
==
len
(
cells
)
for
p
in
xrange
(
len
(
patterns
)
1
)
:
matrix
[
p
+
1
:
,
p
]
=
[
len
(
set
(
patterns
[
p
]
)
.
intersection
(
set
(
q
)
)
)
*
100
/
w
for
q
in
patterns
[
p
+
1
:
]
]
matrix
[
p
,
p
+
1
:
]
=
[
len
(
set
(
cells
[
p
]
)
.
intersection
(
set
(
r
)
)
)
*
5
/
2
for
r
in
cells
[
p
+
1
:
]
]
score
+=
sum
(
abs
(
np
.
array
(
matrix
[
p
+
1
:
,
p
]
)
np
.
array
(
matrix
[
p
,
p
+
1
:
]
)
)
)
count
+=
len
(
matrix
[
p
+
1
:
,
p
]
)
print
'Score'
,
score
/
count
fig
=
pyl
.
figure
(
figsize
=
(
10
,
10
)
,
num
=
fnum
)
pyl
.
matshow
(
matrix
,
fignum
=
fnum
)
pyl
.
colorbar
(
)
pyl
.
title
(
'Coincidence Space'
,
verticalalignment
=
'top'
,
fontsize
=
12
)
pyl
.
xlabel
(
'The Mirror Image Visualization for '
+
dataset
,
fontsize
=
17
)
pyl
.
ylabel
(
'Encoding space'
,
fontsize
=
12
)

The similarity of two patterns in the bit - encoding space is displayed alongside their similarity in the sp - coinc space .


def createInput(self):
    """create a random input vector"""

    print "-" * 70 + "Creating a random input vector" + "-" * 70

    #clear the inputArray to zero before creating a new input vector
    self.inputArray[0:] = 0

    for i in range(self.inputSize):
      #randrange returns 0 or 1
      self.inputArray[i] = random.randrange(2)

def
createInput
(
self
)
:
print
"-"
*
70
+
"Creating a random input vector"
+
"-"
*
70
self
.
inputArray
[
0
:
]
=
0
for
i
in
range
(
self
.
inputSize
)
:
self
.
inputArray
[
i
]
=
random
.
randrange
(
2
)


def run(self):
    """Run the spatial pooler with the input vector"""

    print "-" * 80 + "Computing the SDR" + "-" * 80

    #activeArray[column]=1 if column is active after spatial pooling
    self.sp.compute(self.inputArray, True, self.activeArray)

    print self.activeArray.nonzero()

def
run
(
self
)
:
print
"-"
*
80
+
"Computing the SDR"
+
"-"
*
80
self
.
sp
.
compute
(
self
.
inputArray
,
True
,
self
.
activeArray
)
print
self
.
activeArray
.
nonzero
(
)


def clear(self):
    """Clears the state of the KNNClassifier."""
    self._Memory = None
    self._numPatterns = 0
    self._M = None
    self._categoryList = []
    self._partitionIdList = []
    self._partitionIdMap = {}
    self._finishedLearning = False
    self._iterationIdx = -1

    # Fixed capacity KNN
    if self.maxStoredPatterns > 0:
      assert self.useSparseMemory, ("Fixed capacity KNN is implemented only "
                                    "in the sparse memory mode")
      self.fixedCapacity = True
      self._categoryRecencyList = []
    else:
      self.fixedCapacity = False

    # Cached value of the store prototype sizes
    self._protoSizes = None

    # Used by PCA
    self._s = None
    self._vt = None
    self._nc = None
    self._mean = None

    # Used by Network Builder
    self._specificIndexTraining = False
    self._nextTrainingIndices = None


def _removeRows(self, rowsToRemove):
    """
    A list of row indices to remove. There are two caveats. First, this is
    a potentially slow operation. Second, pattern indices will shift if
    patterns before them are removed.
    """
    # Form a numpy array of row indices to be removed
    removalArray = numpy.array(rowsToRemove)

    # Remove categories
    self._categoryList = numpy.delete(numpy.array(self._categoryList),
                                      removalArray).tolist()

    if self.fixedCapacity:
      self._categoryRecencyList = numpy.delete(
        numpy.array(self._categoryRecencyList), removalArray).tolist()

    # Remove the partition ID, if any for these rows and rebuild the id map.
    for row in reversed(rowsToRemove):  # Go backwards
      # Remove these patterns from partitionList
      self._partitionIdList.pop(row)
    self._rebuildPartitionIdMap(self._partitionIdList)

    # Remove actual patterns
    if self.useSparseMemory:
      # Delete backwards
      for rowIndex in rowsToRemove[::-1]:
        self._Memory.deleteRow(rowIndex)
    else:
      self._M = numpy.delete(self._M, removalArray, 0)

    numRemoved = len(rowsToRemove)

    # Sanity checks
    numRowsExpected = self._numPatterns - numRemoved
    if self.useSparseMemory:
      if self._Memory is not None:
        assert self._Memory.nRows() == numRowsExpected
    else:
      assert self._M.shape[0] == numRowsExpected
    assert len(self._categoryList) == numRowsExpected

    self._numPatterns -= numRemoved
    return numRemoved

def _removeRows ( self , rowsToRemove ) : removalArray = numpy . array ( rowsToRemove ) self . _categoryList = numpy . delete ( numpy . array ( self . _categoryList ) , removalArray ) . tolist ( ) if self . fixedCapacity : self . _categoryRecencyList = numpy . delete ( numpy . array ( self . _categoryRecencyList ) , removalArray ) . tolist ( ) for row in reversed ( rowsToRemove ) : self . _partitionIdList . pop ( row ) self . _rebuildPartitionIdMap ( self . _partitionIdList ) if self . useSparseMemory : for rowIndex in rowsToRemove [ : : - 1 ] : self . _Memory . deleteRow ( rowIndex ) else : self . _M = numpy . delete ( self . _M , removalArray , 0 ) numRemoved = len ( rowsToRemove ) numRowsExpected = self . _numPatterns - numRemoved if self . useSparseMemory : if self . _Memory is not None : assert self . _Memory . nRows ( ) == numRowsExpected else : assert self . _M . shape [ 0 ] == numRowsExpected assert len ( self . _categoryList ) == numRowsExpected self . _numPatterns -= numRemoved return numRemoved

A list of row indices to remove . There are two caveats . First this is a potentially slow operation . Second pattern indices will shift if patterns before them are removed .


def getDistances(self, inputPattern):
    """Return the distances between the input pattern and all other
    stored patterns.

    :param inputPattern: pattern to check distance with

    :returns: (distances, categories) numpy arrays of the same length.
        - overlaps: an integer overlap amount for each category
        - categories: category index for each element of distances
    """
    dist = self._getDistances(inputPattern)
    return (dist, self._categoryList)

def getDistances ( self , inputPattern ) : dist = self . _getDistances ( inputPattern ) return ( dist , self . _categoryList )

Return the distances between the input pattern and all other stored patterns .


def infer(self, inputPattern, computeScores=True, overCategories=True,
            partitionId=None):
    """Finds the category that best matches the input pattern. Returns the
    winning category index as well as a distribution over all categories.

    :param inputPattern: (list or array) The pattern to be classified. This
        must be a dense representation of the array (e.g. [0, 0, 1, 1, 0, 1]).

    :param computeScores: NO EFFECT

    :param overCategories: NO EFFECT

    :param partitionId: (int) If provided, all training vectors with partitionId
        equal to that of the input pattern are ignored.
        For example, this may be used to perform k-fold cross validation
        without repopulating the classifier. First partition all the data into
        k equal partitions numbered 0, 1, 2, ... and then call learn() for each
        vector passing in its partitionId. Then, during inference, by passing
        in the partition ID in the call to infer(), all other vectors with the
        same partitionId are ignored simulating the effect of repopulating the
        classifier while ommitting the training vectors in the same partition.

    :returns: 4-tuple with these keys:

      - ``winner``: The category with the greatest number of nearest neighbors
          within the kth nearest neighbors. If the inferenceResult contains no
          neighbors, the value of winner is None. This can happen, for example,
          in cases of exact matching, if there are no stored vectors, or if
          minSparsity is not met.
      - ``inferenceResult``: A list of length numCategories, each entry contains
          the number of neighbors within the top k neighbors that are in that
          category.
      - ``dist``: A list of length numPrototypes. Each entry is the distance
          from the unknown to that prototype. All distances are between 0.0 and
          1.0.
      - ``categoryDist``: A list of length numCategories. Each entry is the
                        distance from the unknown to the nearest prototype of
                        that category. All distances are between 0 and 1.0.
    """

    # Calculate sparsity. If sparsity is too low, we do not want to run
    # inference with this vector
    sparsity = 0.0
    if self.minSparsity > 0.0:
      sparsity = ( float(len(inputPattern.nonzero()[0])) /
                   len(inputPattern) )

    if len(self._categoryList) == 0 or sparsity < self.minSparsity:
      # No categories learned yet; i.e. first inference w/ online learning or
      # insufficient sparsity
      winner = None
      inferenceResult = numpy.zeros(1)
      dist = numpy.ones(1)
      categoryDist = numpy.ones(1)

    else:
      maxCategoryIdx = max(self._categoryList)
      inferenceResult = numpy.zeros(maxCategoryIdx+1)
      dist = self._getDistances(inputPattern, partitionId=partitionId)
      validVectorCount = len(self._categoryList) - self._categoryList.count(-1)

      # Loop through the indices of the nearest neighbors.
      if self.exact:
        # Is there an exact match in the distances?
        exactMatches = numpy.where(dist<0.00001)[0]
        if len(exactMatches) > 0:
          for i in exactMatches[:min(self.k, validVectorCount)]:
            inferenceResult[self._categoryList[i]] += 1.0
      else:
        sorted = dist.argsort()
        for j in sorted[:min(self.k, validVectorCount)]:
          inferenceResult[self._categoryList[j]] += 1.0

      # Prepare inference results.
      if inferenceResult.any():
        winner = inferenceResult.argmax()
        inferenceResult /= inferenceResult.sum()
      else:
        winner = None
      categoryDist = min_score_per_category(maxCategoryIdx,
                                            self._categoryList, dist)
      categoryDist.clip(0, 1.0, categoryDist)

    if self.verbosity >= 1:
      print "%s infer:" % (g_debugPrefix)
      print "  active inputs:",  _labeledInput(inputPattern,
                                               cellsPerCol=self.cellsPerCol)
      print "  winner category:", winner
      print "  pct neighbors of each category:", inferenceResult
      print "  dist of each prototype:", dist
      print "  dist of each category:", categoryDist

    result = (winner, inferenceResult, dist, categoryDist)
    return result

def
infer
(
self
,
inputPattern
,
computeScores
=
True
,
overCategories
=
True
,
partitionId
=
)
:
sparsity
=
0.0
if
self
.
minSparsity
>
0.0
:
sparsity
=
(
float
(
len
(
inputPattern
.
nonzero
(
)
[
0
]
)
)
/
len
(
inputPattern
)
)
if
len
(
self
.
_categoryList
)
==
0
or
sparsity
<
self
.
minSparsity
:
winner
=
inferenceResult
=
numpy
.
zeros
(
1
)
dist
=
numpy
.
ones
(
1
)
categoryDist
=
numpy
.
ones
(
1
)
else
:
maxCategoryIdx
=
max
(
self
.
_categoryList
)
inferenceResult
=
numpy
.
zeros
(
maxCategoryIdx
+
1
)
dist
=
self
.
_getDistances
(
inputPattern
,
partitionId
=
partitionId
)
validVectorCount
=
len
(
self
.
_categoryList
)
self
.
_categoryList
.
count
(
1
)
if
self
.
exact
:
exactMatches
=
numpy
.
where
(
dist
<
0.00001
)
[
0
]
if
len
(
exactMatches
)
>
0
:
for
i
in
exactMatches
[
:
min
(
self
.
k
,
validVectorCount
)
]
:
inferenceResult
[
self
.
_categoryList
[
i
]
]
+=
1.0
else
:
sorted
=
dist
.
argsort
(
)
for
j
in
sorted
[
:
min
(
self
.
k
,
validVectorCount
)
]
:
inferenceResult
[
self
.
_categoryList
[
j
]
]
+=
1.0
if
inferenceResult
.
any
(
)
:
winner
=
inferenceResult
.
argmax
(
)
inferenceResult
/=
inferenceResult
.
sum
(
)
else
:
winner
=
categoryDist
=
min_score_per_category
(
maxCategoryIdx
,
self
.
_categoryList
,
dist
)
categoryDist
.
clip
(
0
,
1.0
,
categoryDist
)
if
self
.
verbosity
>=
1
:
print
"%s infer:"
%
(
g_debugPrefix
)
print
"  active inputs:"
,
_labeledInput
(
inputPattern
,
cellsPerCol
=
self
.
cellsPerCol
)
print
"  winner category:"
,
winner
print
"  pct neighbors of each category:"
,
inferenceResult
print
"  dist of each prototype:"
,
dist
print
"  dist of each category:"
,
categoryDist
result
=
(
winner
,
inferenceResult
,
dist
,
categoryDist
)
return
result

Finds the category that best matches the input pattern . Returns the winning category index as well as a distribution over all categories .


def getClosest(self, inputPattern, topKCategories=3):
    """Returns the index of the pattern that is closest to inputPattern,
    the distances of all patterns to inputPattern, and the indices of the k
    closest categories.
    """
    inferenceResult = numpy.zeros(max(self._categoryList)+1)
    dist = self._getDistances(inputPattern)

    sorted = dist.argsort()

    validVectorCount = len(self._categoryList) - self._categoryList.count(-1)
    for j in sorted[:min(self.k, validVectorCount)]:
      inferenceResult[self._categoryList[j]] += 1.0

    winner = inferenceResult.argmax()

    topNCats = []
    for i in range(topKCategories):
      topNCats.append((self._categoryList[sorted[i]], dist[sorted[i]] ))

    return winner, dist, topNCats

def getClosest ( self , inputPattern , topKCategories = 3 ) : inferenceResult = numpy . zeros ( max ( self . _categoryList ) + 1 ) dist = self . _getDistances ( inputPattern ) sorted = dist . argsort ( ) validVectorCount = len ( self . _categoryList ) - self . _categoryList . count ( - 1 ) for j in sorted [ : min ( self . k , validVectorCount ) ] : inferenceResult [ self . _categoryList [ j ] ] += 1.0 winner = inferenceResult . argmax ( ) topNCats = [ ] for i in range ( topKCategories ) : topNCats . append ( ( self . _categoryList [ sorted [ i ] ] , dist [ sorted [ i ] ] ) ) return winner , dist , topNCats

Returns the index of the pattern that is closest to inputPattern the distances of all patterns to inputPattern and the indices of the k closest categories .


def closestTrainingPattern(self, inputPattern, cat):
    """Returns the closest training pattern to inputPattern that belongs to
    category "cat".

    :param inputPattern: The pattern whose closest neighbor is sought

    :param cat: The required category of closest neighbor

    :returns: A dense version of the closest training pattern, or None if no
        such patterns exist
    """
    dist = self._getDistances(inputPattern)
    sorted = dist.argsort()

    for patIdx in sorted:
      patternCat = self._categoryList[patIdx]

      # If closest pattern belongs to desired category, return it
      if patternCat == cat:
        if self.useSparseMemory:
          closestPattern = self._Memory.getRow(int(patIdx))
        else:
          closestPattern = self._M[patIdx]

        return closestPattern

    # No patterns were found!
    return None

def closestTrainingPattern ( self , inputPattern , cat ) : dist = self . _getDistances ( inputPattern ) sorted = dist . argsort ( ) for patIdx in sorted : patternCat = self . _categoryList [ patIdx ] if patternCat == cat : if self . useSparseMemory : closestPattern = self . _Memory . getRow ( int ( patIdx ) ) else : closestPattern = self . _M [ patIdx ] return closestPattern return None

Returns the closest training pattern to inputPattern that belongs to category cat .


def getPattern(self, idx, sparseBinaryForm=False, cat=None):
    """Gets a training pattern either by index or category number.

    :param idx: Index of the training pattern

    :param sparseBinaryForm: If true, returns a list of the indices of the
        non-zero bits in the training pattern

    :param cat: If not None, get the first pattern belonging to category cat. If
        this is specified, idx must be None.

    :returns: The training pattern with specified index
    """
    if cat is not None:
      assert idx is None
      idx = self._categoryList.index(cat)

    if not self.useSparseMemory:
      pattern = self._Memory[idx]
      if sparseBinaryForm:
        pattern = pattern.nonzero()[0]

    else:
      (nz, values) = self._Memory.rowNonZeros(idx)
      if not sparseBinaryForm:
        pattern = numpy.zeros(self._Memory.nCols())
        numpy.put(pattern, nz, 1)
      else:
        pattern = nz

    return pattern

def getPattern ( self , idx , sparseBinaryForm = False , cat = None ) : if cat is not None : assert idx is None idx = self . _categoryList . index ( cat ) if not self . useSparseMemory : pattern = self . _Memory [ idx ] if sparseBinaryForm : pattern = pattern . nonzero ( ) [ 0 ] else : ( nz , values ) = self . _Memory . rowNonZeros ( idx ) if not sparseBinaryForm : pattern = numpy . zeros ( self . _Memory . nCols ( ) ) numpy . put ( pattern , nz , 1 ) else : pattern = nz return pattern

Gets a training pattern either by index or category number .


def getPartitionId(self, i):
    """
    Gets the partition id given an index.

    :param i: index of partition
    :returns: the partition id associated with pattern i. Returns None if no id
        is associated with it.
    """
    if (i < 0) or (i >= self._numPatterns):
      raise RuntimeError("index out of bounds")
    partitionId = self._partitionIdList[i]
    if partitionId == numpy.inf:
      return None
    else:
      return partitionId

def
getPartitionId
(
self
,
i
)
:
if
(
i
<
0
)
or
(
i
>=
self
.
_numPatterns
)
:
raise
RuntimeError
(
"index out of bounds"
)
partitionId
=
self
.
_partitionIdList
[
i
]
if
partitionId
==
numpy
.
inf
:
return
else
:
return
partitionId


def _addPartitionId(self, index, partitionId=None):
    """
    Adds partition id for pattern index
    """
    if partitionId is None:
      self._partitionIdList.append(numpy.inf)
    else:
      self._partitionIdList.append(partitionId)
      indices = self._partitionIdMap.get(partitionId, [])
      indices.append(index)
      self._partitionIdMap[partitionId] = indices

def _addPartitionId ( self , index , partitionId = None ) : if partitionId is None : self . _partitionIdList . append ( numpy . inf ) else : self . _partitionIdList . append ( partitionId ) indices = self . _partitionIdMap . get ( partitionId , [ ] ) indices . append ( index ) self . _partitionIdMap [ partitionId ] = indices


def _rebuildPartitionIdMap(self, partitionIdList):
    """
    Rebuilds the partition Id map using the given partitionIdList
    """
    self._partitionIdMap = {}
    for row, partitionId in enumerate(partitionIdList):
      indices = self._partitionIdMap.get(partitionId, [])
      indices.append(row)
      self._partitionIdMap[partitionId] = indices

def _rebuildPartitionIdMap ( self , partitionIdList ) : self . _partitionIdMap = { } for row , partitionId in enumerate ( partitionIdList ) : indices = self . _partitionIdMap . get ( partitionId , [ ] ) indices . append ( row ) self . _partitionIdMap [ partitionId ] = indices


def _calcDistance(self, inputPattern, distanceNorm=None):
    """Calculate the distances from inputPattern to all stored patterns. All
    distances are between 0.0 and 1.0

    :param inputPattern The pattern from which distances to all other patterns
        are calculated

    :param distanceNorm Degree of the distance norm
    """
    if distanceNorm is None:
      distanceNorm = self.distanceNorm

    # Sparse memory
    if self.useSparseMemory:
      if self._protoSizes is None:
        self._protoSizes = self._Memory.rowSums()
      overlapsWithProtos = self._Memory.rightVecSumAtNZ(inputPattern)
      inputPatternSum = inputPattern.sum()

      if self.distanceMethod == "rawOverlap":
        dist = inputPattern.sum() - overlapsWithProtos
      elif self.distanceMethod == "pctOverlapOfInput":
        dist = inputPatternSum - overlapsWithProtos
        if inputPatternSum > 0:
          dist /= inputPatternSum
      elif self.distanceMethod == "pctOverlapOfProto":
        overlapsWithProtos /= self._protoSizes
        dist = 1.0 - overlapsWithProtos
      elif self.distanceMethod == "pctOverlapOfLarger":
        maxVal = numpy.maximum(self._protoSizes, inputPatternSum)
        if maxVal.all() > 0:
          overlapsWithProtos /= maxVal
        dist = 1.0 - overlapsWithProtos
      elif self.distanceMethod == "norm":
        dist = self._Memory.vecLpDist(self.distanceNorm, inputPattern)
        distMax = dist.max()
        if distMax > 0:
          dist /= distMax
      else:
        raise RuntimeError("Unimplemented distance method %s" %
          self.distanceMethod)

    # Dense memory
    else:
      if self.distanceMethod == "norm":
        dist = numpy.power(numpy.abs(self._M - inputPattern), self.distanceNorm)
        dist = dist.sum(1)
        dist = numpy.power(dist, 1.0/self.distanceNorm)
        dist /= dist.max()
      else:
        raise RuntimeError ("Not implemented yet for dense storage....")

    return dist

Calculate the distances from inputPattern to all stored patterns . All distances are between 0 . 0 and 1 . 0


def _getDistances(self, inputPattern, partitionId=None):
    """Return the distances from inputPattern to all stored patterns.

    :param inputPattern The pattern from which distances to all other patterns
        are returned

    :param partitionId If provided, ignore all training vectors with this
        partitionId.
    """
    if not self._finishedLearning:
      self.finishLearning()
      self._finishedLearning = True

    if self._vt is not None and len(self._vt) > 0:
      inputPattern = numpy.dot(self._vt, inputPattern - self._mean)

    sparseInput = self._sparsifyVector(inputPattern)

    # Compute distances
    dist = self._calcDistance(sparseInput)
    # Invalidate results where category is -1
    if self._specificIndexTraining:
      dist[numpy.array(self._categoryList) == -1] = numpy.inf

    # Ignore vectors with this partition id by setting their distances to inf
    if partitionId is not None:
      dist[self._partitionIdMap.get(partitionId, [])] = numpy.inf

    return dist

def _getDistances ( self , inputPattern , partitionId = None ) : if not self . _finishedLearning : self . finishLearning ( ) self . _finishedLearning = True if self . _vt is not None and len ( self . _vt ) > 0 : inputPattern = numpy . dot ( self . _vt , inputPattern - self . _mean ) sparseInput = self . _sparsifyVector ( inputPattern ) dist = self . _calcDistance ( sparseInput ) if self . _specificIndexTraining : dist [ numpy . array ( self . _categoryList ) == - 1 ] = numpy . inf if partitionId is not None : dist [ self . _partitionIdMap . get ( partitionId , [ ] ) ] = numpy . inf return dist

Return the distances from inputPattern to all stored patterns .


def remapCategories(self, mapping):
    """Change the category indices.

    Used by the Network Builder to keep the category indices in sync with the
    ImageSensor categoryInfo when the user renames or removes categories.

    :param mapping: List of new category indices. For example, mapping=[2,0,1]
        would change all vectors of category 0 to be category 2, category 1 to
        0, and category 2 to 1
    """
    categoryArray = numpy.array(self._categoryList)
    newCategoryArray = numpy.zeros(categoryArray.shape[0])
    newCategoryArray.fill(-1)
    for i in xrange(len(mapping)):
      newCategoryArray[categoryArray==i] = mapping[i]
    self._categoryList = list(newCategoryArray)

def remapCategories ( self , mapping ) : categoryArray = numpy . array ( self . _categoryList ) newCategoryArray = numpy . zeros ( categoryArray . shape [ 0 ] ) newCategoryArray . fill ( - 1 ) for i in xrange ( len ( mapping ) ) : newCategoryArray [ categoryArray == i ] = mapping [ i ] self . _categoryList = list ( newCategoryArray )


def _convertNonNumericData(self, spatialOutput, temporalOutput, output):
    """
    Converts all of the non-numeric fields from spatialOutput and temporalOutput
    into their scalar equivalents and records them in the output dictionary.

    :param spatialOutput: The results of topDownCompute() for the spatial input.
    :param temporalOutput: The results of topDownCompute() for the temporal
      input.
    :param output: The main dictionary of outputs passed to compute(). It is
      expected to have keys 'spatialTopDownOut' and 'temporalTopDownOut' that
      are mapped to numpy arrays.
    """
    encoders = self.encoder.getEncoderList()
    types = self.encoder.getDecoderOutputFieldTypes()
    for i, (encoder, type) in enumerate(zip(encoders, types)):
      spatialData = spatialOutput[i]
      temporalData = temporalOutput[i]

      if type != FieldMetaType.integer and type != FieldMetaType.float:
        # TODO: Make sure that this doesn't modify any state
        spatialData = encoder.getScalars(spatialData)[0]
        temporalData = encoder.getScalars(temporalData)[0]

      assert isinstance(spatialData, (float, int))
      assert isinstance(temporalData, (float, int))
      output['spatialTopDownOut'][i] = spatialData
      output['temporalTopDownOut'][i] = temporalData

def _convertNonNumericData ( self , spatialOutput , temporalOutput , output ) : encoders = self . encoder . getEncoderList ( ) types = self . encoder . getDecoderOutputFieldTypes ( ) for i , ( encoder , type ) in enumerate ( zip ( encoders , types ) ) : spatialData = spatialOutput [ i ] temporalData = temporalOutput [ i ] if type != FieldMetaType . integer and type != FieldMetaType . float : spatialData = encoder . getScalars ( spatialData ) [ 0 ] temporalData = encoder . getScalars ( temporalData ) [ 0 ] assert isinstance ( spatialData , ( float , int ) ) assert isinstance ( temporalData , ( float , int ) ) output [ 'spatialTopDownOut' ] [ i ] = spatialData output [ 'temporalTopDownOut' ] [ i ] = temporalData

Converts all of the non - numeric fields from spatialOutput and temporalOutput into their scalar equivalents and records them in the output dictionary .


def getOutputElementCount(self, name):
    """
    Computes the width of dataOut.

    Overrides
    :meth:`nupic.bindings.regions.PyRegion.PyRegion.getOutputElementCount`.
    """

    if name == "resetOut":
      print ("WARNING: getOutputElementCount should not have been called with "
             "resetOut")
      return 1

    elif name == "sequenceIdOut":
      print ("WARNING: getOutputElementCount should not have been called with "
             "sequenceIdOut")
      return 1

    elif name == "dataOut":
      if self.encoder is None:
        raise Exception("NuPIC requested output element count for 'dataOut' "
                        "on a RecordSensor node, but the encoder has not "
                        "been set")
      return self.encoder.getWidth()

    elif name == "sourceOut":
      if self.encoder is None:
        raise Exception("NuPIC requested output element count for 'sourceOut' "
                        "on a RecordSensor node, "
                        "but the encoder has not been set")
      return len(self.encoder.getDescription())

    elif name == "bucketIdxOut":
      return 1

    elif name == "actValueOut":
      return 1

    elif name == "categoryOut":
      return self.numCategories

    elif name == 'spatialTopDownOut' or name == 'temporalTopDownOut':
      if self.encoder is None:
        raise Exception("NuPIC requested output element count for 'sourceOut' "
                        "on a RecordSensor node, "
                        "but the encoder has not been set")
      return len(self.encoder.getDescription())
    else:
      raise Exception("Unknown output %s" % name)


def setParameter(self, parameterName, index, parameterValue):
    """
      Set the value of a Spec parameter. Most parameters are handled
      automatically by PyRegion's parameter set mechanism. The ones that need
      special treatment are explicitly handled here.
    """
    if parameterName == 'topDownMode':
      self.topDownMode = parameterValue
    elif parameterName == 'predictedField':
      self.predictedField = parameterValue
    else:
      raise Exception('Unknown parameter: ' + parameterName)

def
setParameter
(
self
,
parameterName
,
index
,
parameterValue
)
:
if
parameterName
==
'topDownMode'
:
self
.
topDownMode
=
parameterValue
elif
parameterName
==
'predictedField'
:
self
.
predictedField
=
parameterValue
else
:
raise
Exception
(
'Unknown parameter: '
+
parameterName
)

Set the value of a Spec parameter . Most parameters are handled automatically by PyRegion s parameter set mechanism . The ones that need special treatment are explicitly handled here .


def rewind(self):
    """
    Put us back at the beginning of the file again.
    """

    # Superclass rewind
    super(FileRecordStream, self).rewind()

    self.close()
    self._file = open(self._filename, self._mode)
    self._reader = csv.reader(self._file, dialect="excel")

    # Skip header rows
    self._reader.next()
    self._reader.next()
    self._reader.next()

    # Reset record count, etc.
    self._recordCount = 0

def rewind ( self ) : super ( FileRecordStream , self ) . rewind ( ) self . close ( ) self . _file = open ( self . _filename , self . _mode ) self . _reader = csv . reader ( self . _file , dialect = "excel" ) self . _reader . next ( ) self . _reader . next ( ) self . _reader . next ( ) self . _recordCount = 0

Put us back at the beginning of the file again .


def getNextRecord(self, useCache=True):
    """ Returns next available data record from the file.

    :returns: a data row (a list or tuple) if available; None, if no more
              records in the table (End of Stream - EOS); empty sequence (list
              or tuple) when timing out while waiting for the next record.
    """
    assert self._file is not None
    assert self._mode == self._FILE_READ_MODE

    # Read the line
    try:
      line = self._reader.next()

    except StopIteration:
      if self.rewindAtEOF:
        if self._recordCount == 0:
          raise Exception("The source configured to reset at EOF but "
                          "'%s' appears to be empty" % self._filename)
        self.rewind()
        line = self._reader.next()

      else:
        return None

    # Keep score of how many records were read
    self._recordCount += 1

    # Split the line to text fields and convert each text field to a Python
    # object if value is missing (empty string) encode appropriately for
    # upstream consumers in the case of numeric types, this means replacing
    # missing data with a sentinel value for string type, we can leave the empty
    # string in place
    record = []
    for i, f in enumerate(line):
      #print "DEBUG: Evaluating field @ index %s: %r" % (i, f)
      #sys.stdout.flush()
      if f in self._missingValues:
        record.append(SENTINEL_VALUE_FOR_MISSING_DATA)
      else:
        # either there is valid data, or the field is string type,
        # in which case the adapter does the right thing by default
        record.append(self._adapters[i](f))

    return record

Returns next available data record from the file .


def appendRecord(self, record):
    """
    Saves the record in the underlying csv file.

    :param record: a list of Python objects that will be string-ified
    """

    assert self._file is not None
    assert self._mode == self._FILE_WRITE_MODE
    assert isinstance(record, (list, tuple)), \
      "unexpected record type: " + repr(type(record))

    assert len(record) == self._fieldCount, \
      "len(record): %s, fieldCount: %s" % (len(record), self._fieldCount)

    # Write header if needed
    if self._recordCount == 0:
      # Write the header
      names, types, specials = zip(*self.getFields())
      for line in names, types, specials:
        self._writer.writerow(line)

    # Keep track of sequences, make sure time flows forward
    self._updateSequenceInfo(record)

    line = [self._adapters[i](f) for i, f in enumerate(record)]

    self._writer.writerow(line)
    self._recordCount += 1

def
appendRecord
(
self
,
record
)
:
assert
self
.
_file
is
not
assert
self
.
_mode
==
self
.
_FILE_WRITE_MODE
assert
isinstance
(
record
,
(
list
,
tuple
)
)
,
"unexpected record type: "
+
repr
(
type
(
record
)
)
assert
len
(
record
)
==
self
.
_fieldCount
,
"len(record): %s, fieldCount: %s"
%
(
len
(
record
)
,
self
.
_fieldCount
)
if
self
.
_recordCount
==
0
:
names
,
types
,
specials
=
zip
(
*
self
.
getFields
(
)
)
for
line
in
names
,
types
,
specials
:
self
.
_writer
.
writerow
(
line
)
self
.
_updateSequenceInfo
(
record
)
line
=
[
self
.
_adapters
[
i
]
(
f
)
for
i
,
f
in
enumerate
(
record
)
]
self
.
_writer
.
writerow
(
line
)
self
.
_recordCount
+=
1

Saves the record in the underlying csv file .


def appendRecords(self, records, progressCB=None):
    """
    Saves multiple records in the underlying storage.

    :param records: array of records as in
                    :meth:`~.FileRecordStream.appendRecord`
    :param progressCB: (function) callback to report progress
    """

    for record in records:
      self.appendRecord(record)
      if progressCB is not None:
        progressCB()

def appendRecords ( self , records , progressCB = None ) : for record in records : self . appendRecord ( record ) if progressCB is not None : progressCB ( )

Saves multiple records in the underlying storage .


def getBookmark(self):
    """
    Gets a bookmark or anchor to the current position.

    :returns: an anchor to the current position in the data. Passing this
              anchor to a constructor makes the current position to be the first
              returned record.
    """

    if self._write and self._recordCount==0:
      return None

    rowDict = dict(filepath=os.path.realpath(self._filename),
                   currentRow=self._recordCount)
    return json.dumps(rowDict)

def getBookmark ( self ) : if self . _write and self . _recordCount == 0 : return None rowDict = dict ( filepath = os . path . realpath ( self . _filename ) , currentRow = self . _recordCount ) return json . dumps ( rowDict )

Gets a bookmark or anchor to the current position .


def seekFromEnd(self, numRecords):
    """
    Seeks to ``numRecords`` from the end and returns a bookmark to the new
    position.

    :param numRecords: how far to seek from end of file.
    :return: bookmark to desired location.
    """
    self._file.seek(self._getTotalLineCount() - numRecords)
    return self.getBookmark()

def seekFromEnd ( self , numRecords ) : self . _file . seek ( self . _getTotalLineCount ( ) - numRecords ) return self . getBookmark ( )

Seeks to numRecords from the end and returns a bookmark to the new position .


def _updateSequenceInfo(self, r):
    """Keep track of sequence and make sure time goes forward

    Check if the current record is the beginning of a new sequence
    A new sequence starts in 2 cases:

    1. The sequence id changed (if there is a sequence id field)
    2. The reset field is 1 (if there is a reset field)

    Note that if there is no sequenceId field or resetId field then the entire
    dataset is technically one big sequence. The function will not return True
    for the first record in this case. This is Ok because it is important to
    detect new sequences only when there are multiple sequences in the file.
    """

    # Get current sequence id (if any)
    newSequence = False
    sequenceId = (r[self._sequenceIdIdx]
                  if self._sequenceIdIdx is not None else None)
    if sequenceId != self._currSequence:
      # verify that the new sequence didn't show up before
      if sequenceId in self._sequences:
        raise Exception('Broken sequence: %s, record: %s' % \
                        (sequenceId, r))

      # add the finished sequence to the set of sequence
      self._sequences.add(self._currSequence)
      self._currSequence = sequenceId

      # Verify that the reset is consistent (if there is one)
      if self._resetIdx:
        assert r[self._resetIdx] == 1
      newSequence = True

    else:
      # Check the reset
      reset = False
      if self._resetIdx:
        reset = r[self._resetIdx]
        if reset == 1:
          newSequence = True

    # If it's still the same old sequence make sure the time flows forward
    if not newSequence:
      if self._timeStampIdx and self._currTime is not None:
        t = r[self._timeStampIdx]
        if t < self._currTime:
          raise Exception('No time travel. Early timestamp for record: %s' % r)

    if self._timeStampIdx:
      self._currTime = r[self._timeStampIdx]


def _getStartRow(self, bookmark):
    """ Extracts start row from the bookmark information
    """
    bookMarkDict = json.loads(bookmark)

    realpath = os.path.realpath(self._filename)

    bookMarkFile = bookMarkDict.get('filepath', None)

    if bookMarkFile != realpath:
      print ("Ignoring bookmark due to mismatch between File's "
             "filename realpath vs. bookmark; realpath: %r; bookmark: %r") % (
        realpath, bookMarkDict)
      return 0
    else:
      return bookMarkDict['currentRow']


def isTemporal(inferenceElement):
    """ Returns True if the inference from this timestep is predicted the input
    for the NEXT timestep.

    NOTE: This should only be checked IF THE MODEL'S INFERENCE TYPE IS ALSO
    TEMPORAL. That is, a temporal model CAN have non-temporal inference elements,
    but a non-temporal model CANNOT have temporal inference elements
    """
    if InferenceElement.__temporalInferenceElements is None:
      InferenceElement.__temporalInferenceElements = \
                                set([InferenceElement.prediction])

    return inferenceElement in InferenceElement.__temporalInferenceElements

def isTemporal ( inferenceElement ) : if InferenceElement . __temporalInferenceElements is None : InferenceElement . __temporalInferenceElements = set ( [ InferenceElement . prediction ] ) return inferenceElement in InferenceElement . __temporalInferenceElements

Returns True if the inference from this timestep is predicted the input for the NEXT timestep .


def getTemporalDelay(inferenceElement, key=None):
    """ Returns the number of records that elapse between when an inference is
    made and when the corresponding input record will appear. For example, a
    multistep prediction for 3 timesteps out will have a delay of 3

    Parameters:
    -----------------------------------------------------------------------

    inferenceElement:   The InferenceElement value being delayed
    key:                If the inference is a dictionary type, this specifies
                        key for the sub-inference that is being delayed
    """
    # -----------------------------------------------------------------------
    # For next step prediction, we shift by 1
    if inferenceElement in (InferenceElement.prediction,
                            InferenceElement.encodings):
      return 1
    # -----------------------------------------------------------------------
    # For classification, anomaly scores, the inferences immediately succeed the
    # inputs
    if inferenceElement in (InferenceElement.anomalyScore,
                            InferenceElement.anomalyLabel,
                            InferenceElement.classification,
                            InferenceElement.classConfidences):
      return 0
    # -----------------------------------------------------------------------
    # For multistep prediction, the delay is based on the key in the inference
    # dictionary
    if inferenceElement in (InferenceElement.multiStepPredictions,
                            InferenceElement.multiStepBestPredictions):
      return int(key)

    # -----------------------------------------------------------------------
    # default: return 0
    return 0

def getTemporalDelay ( inferenceElement , key = None ) : if inferenceElement in ( InferenceElement . prediction , InferenceElement . encodings ) : return 1 if inferenceElement in ( InferenceElement . anomalyScore , InferenceElement . anomalyLabel , InferenceElement . classification , InferenceElement . classConfidences ) : return 0 if inferenceElement in ( InferenceElement . multiStepPredictions , InferenceElement . multiStepBestPredictions ) : return int ( key ) return 0

Returns the number of records that elapse between when an inference is made and when the corresponding input record will appear . For example a multistep prediction for 3 timesteps out will have a delay of 3


def getMaxDelay(inferences):
    """
    Returns the maximum delay for the InferenceElements in the inference
    dictionary

    Parameters:
    -----------------------------------------------------------------------
    inferences:   A dictionary where the keys are InferenceElements
    """
    maxDelay = 0
    for inferenceElement, inference in inferences.iteritems():
      if isinstance(inference, dict):
        for key in inference.iterkeys():
          maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement,
                                                            key),
                         maxDelay)
      else:
        maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement),
                       maxDelay)

    return maxDelay

def getMaxDelay ( inferences ) : maxDelay = 0 for inferenceElement , inference in inferences . iteritems ( ) : if isinstance ( inference , dict ) : for key in inference . iterkeys ( ) : maxDelay = max ( InferenceElement . getTemporalDelay ( inferenceElement , key ) , maxDelay ) else : maxDelay = max ( InferenceElement . getTemporalDelay ( inferenceElement ) , maxDelay ) return maxDelay


def isTemporal(inferenceType):
    """ Returns True if the inference type is 'temporal', i.e. requires a
    temporal memory in the network.
    """
    if InferenceType.__temporalInferenceTypes is None:
      InferenceType.__temporalInferenceTypes = \
                                set([InferenceType.TemporalNextStep,
                                     InferenceType.TemporalClassification,
                                     InferenceType.TemporalAnomaly,
                                     InferenceType.TemporalMultiStep,
                                     InferenceType.NontemporalMultiStep])

    return inferenceType in InferenceType.__temporalInferenceTypes

def isTemporal ( inferenceType ) : if InferenceType . __temporalInferenceTypes is None : InferenceType . __temporalInferenceTypes = set ( [ InferenceType . TemporalNextStep , InferenceType . TemporalClassification , InferenceType . TemporalAnomaly , InferenceType . TemporalMultiStep , InferenceType . NontemporalMultiStep ] ) return inferenceType in InferenceType . __temporalInferenceTypes

Returns True if the inference type is temporal i . e . requires a temporal memory in the network .


def Enum(*args, **kwargs):
  """
  Utility function for creating enumerations in python

  Example Usage:
    >> Color = Enum("Red", "Green", "Blue", "Magenta")
    >> print Color.Red
    >> 0
    >> print Color.Green
    >> 1
    >> print Color.Blue
    >> 2
    >> print Color.Magenta
    >> 3
    >> Color.Violet
    >> 'violet'
    >> Color.getLabel(Color.Red)
    >> 'Red'
    >> Color.getLabel(2)
    >> 'Blue'
  """

  def getLabel(cls, val):
    """ Get a string label for the current value of the enum """
    return cls.__labels[val]

  def validate(cls, val):
    """ Returns True if val is a valid value for the enumeration """
    return val in cls.__values

  def getValues(cls):
    """ Returns a list of all the possible values for this enum """
    return list(cls.__values)

  def getLabels(cls):
    """ Returns a list of all possible labels for this enum """
    return list(cls.__labels.values())

  def getValue(cls, label):
    """ Returns value given a label """
    return cls.__labels[label]

  for arg in list(args)+kwargs.keys():
    if type(arg) is not str:
      raise TypeError("Enum arg {0} must be a string".format(arg))

    if not __isidentifier(arg):
      raise ValueError("Invalid enum value '{0}'. "\
                       "'{0}' is not a valid identifier".format(arg))

  #kwargs.update(zip(args, range(len(args))))
  kwargs.update(zip(args, args))
  newType = type("Enum", (object,), kwargs)

  newType.__labels = dict( (v,k) for k,v in kwargs.iteritems())
  newType.__values = set(newType.__labels.keys())
  newType.getLabel = functools.partial(getLabel, newType)
  newType.validate = functools.partial(validate, newType)
  newType.getValues = functools.partial(getValues, newType)
  newType.getLabels = functools.partial(getLabels, newType)
  newType.getValue = functools.partial(getValue, newType)

  return newType

def
Enum
(
*
args
,
**
kwargs
)
:
def
getLabel
(
cls
,
val
)
:
return
cls
.
__labels
[
val
]
def
validate
(
cls
,
val
)
:
return
val
in
cls
.
__values
def
getValues
(
cls
)
:
return
list
(
cls
.
__values
)
def
getLabels
(
cls
)
:
return
list
(
cls
.
__labels
.
values
(
)
)
def
getValue
(
cls
,
label
)
:
return
cls
.
__labels
[
label
]
for
arg
in
list
(
args
)
+
kwargs
.
keys
(
)
:
if
type
(
arg
)
is
not
str
:
raise
TypeError
(
"Enum arg {0} must be a string"
.
format
(
arg
)
)
if
not
__isidentifier
(
arg
)
:
raise
ValueError
(
"Invalid enum value '{0}'. "
"'{0}' is not a valid identifier"
.
format
(
arg
)
)
kwargs
.
update
(
zip
(
args
,
args
)
)
newType
=
type
(
"Enum"
,
(
object
,
)
,
kwargs
)
newType
.
__labels
=
dict
(
(
v
,
k
)
for
k
,
v
in
kwargs
.
iteritems
(
)
)
newType
.
__values
=
set
(
newType
.
__labels
.
keys
(
)
)
newType
.
getLabel
=
functools
.
partial
(
getLabel
,
newType
)
newType
.
validate
=
functools
.
partial
(
validate
,
newType
)
newType
.
getValues
=
functools
.
partial
(
getValues
,
newType
)
newType
.
getLabels
=
functools
.
partial
(
getLabels
,
newType
)
newType
.
getValue
=
functools
.
partial
(
getValue
,
newType
)
return
newType


def makeDirectoryFromAbsolutePath(absDirPath):
  """ Makes directory for the given directory path with default permissions.
  If the directory already exists, it is treated as success.

  absDirPath:   absolute path of the directory to create.

  Returns:      absDirPath arg

  Exceptions:         OSError if directory creation fails
  """

  assert os.path.isabs(absDirPath)

  try:
    os.makedirs(absDirPath)
  except OSError, e:
    if e.errno != os.errno.EEXIST:
      raise

  return absDirPath

def makeDirectoryFromAbsolutePath ( absDirPath ) : assert os . path . isabs ( absDirPath ) try : os . makedirs ( absDirPath ) except OSError , e : if e . errno != os . errno . EEXIST : raise return absDirPath

Makes directory for the given directory path with default permissions . If the directory already exists it is treated as success .


def _readConfigFile(cls, filename, path=None):
    """ Parse the given XML file and return a dict describing the file.

    Parameters:
    ----------------------------------------------------------------
    filename:  name of XML file to parse (no path)
    path:      path of the XML file. If None, then use the standard
                  configuration search path.
    retval:    returns a dict with each property as a key and a dict of all
               the property's attributes as value
    """

    outputProperties = dict()

    # Get the path to the config files.
    if path is None:
      filePath = cls.findConfigFile(filename)
    else:
      filePath = os.path.join(path, filename)

    # ------------------------------------------------------------------
    # Read in the config file
    try:
      if filePath is not None:
        try:
          # Use warn since console log level is set to warning
          _getLoggerBase().debug("Loading config file: %s", filePath)
          with open(filePath, 'r') as inp:
            contents = inp.read()
        except Exception:
          raise RuntimeError("Expected configuration file at %s" % filePath)
      else:
        # If the file was not found in the normal search paths, which includes
        # checking the NTA_CONF_PATH, we'll try loading it from pkg_resources.
        try:
          contents = resource_string("nupic.support", filename)
        except Exception as resourceException:
          # We expect these to be read, and if they don't exist we'll just use
          # an empty configuration string.
          if filename in [USER_CONFIG, CUSTOM_CONFIG]:
            contents = '<configuration/>'
          else:
            raise resourceException

      elements = ElementTree.XML(contents)

      if elements.tag != 'configuration':
        raise RuntimeError("Expected top-level element to be 'configuration' "
                           "but got '%s'" % (elements.tag))

      # ------------------------------------------------------------------
      # Add in each property found
      propertyElements = elements.findall('./property')

      for propertyItem in propertyElements:

        propInfo = dict()

        # Parse this property element
        propertyAttributes = list(propertyItem)
        for propertyAttribute in propertyAttributes:
          propInfo[propertyAttribute.tag] = propertyAttribute.text

        # Get the name
        name = propInfo.get('name', None)

        # value is allowed to be empty string
        if 'value' in propInfo and propInfo['value'] is None:
          value = ''
        else:
          value = propInfo.get('value', None)

          if value is None:
            if 'novalue' in propInfo:
              # Placeholder "novalue" properties are intended to be overridden
              # via dynamic configuration or another configuration layer.
              continue
            else:
              raise RuntimeError("Missing 'value' element within the property "
                                 "element: => %s " % (str(propInfo)))

        # The value is allowed to contain substitution tags of the form
        # ${env.VARNAME}, which should be substituted with the corresponding
        # environment variable values
        restOfValue = value
        value = ''
        while True:
          # Find the beginning of substitution tag
          pos = restOfValue.find('${env.')
          if pos == -1:
            # No more environment variable substitutions
            value += restOfValue
            break

          # Append prefix to value accumulator
          value += restOfValue[0:pos]

          # Find the end of current substitution tag
          varTailPos = restOfValue.find('}', pos)
          if varTailPos == -1:
            raise RuntimeError(
              "Trailing environment variable tag delimiter '}'"
              " not found in %r" % (restOfValue))

          # Extract environment variable name from tag
          varname = restOfValue[pos + 6:varTailPos]
          if varname not in os.environ:
            raise RuntimeError("Attempting to use the value of the environment"
                               " variable %r, which is not defined" % (
                               varname))
          envVarValue = os.environ[varname]

          value += envVarValue

          restOfValue = restOfValue[varTailPos + 1:]

        # Check for errors
        if name is None:
          raise RuntimeError(
            "Missing 'name' element within following property "
            "element:\n => %s " % (str(propInfo)))

        propInfo['value'] = value
        outputProperties[name] = propInfo

      return outputProperties
    except Exception:
      _getLoggerBase().exception("Error while parsing configuration file: %s.",
                             filePath)
      raise

Parse the given XML file and return a dict describing the file .


def setCustomProperties(cls, properties):
    """ Set multiple custom properties and persist them to the custom
    configuration store.

    Parameters:
    ----------------------------------------------------------------
    properties: a dict of property name/value pairs to set
    """
    _getLogger().info("Setting custom configuration properties=%r; caller=%r",
                      properties, traceback.format_stack())

    _CustomConfigurationFileWrapper.edit(properties)

    for propertyName, value in properties.iteritems():
      cls.set(propertyName, value)

Set multiple custom properties and persist them to the custom configuration store .


def clear(cls):
    """ Clear all configuration properties from in-memory cache, but do NOT
    alter the custom configuration file. Used in unit-testing.
    """
    # Clear the in-memory settings cache, forcing reload upon subsequent "get"
    # request.
    super(Configuration, cls).clear()

    # Reset in-memory custom configuration info.
    _CustomConfigurationFileWrapper.clear(persistent=False)

def clear ( cls ) : super ( Configuration , cls ) . clear ( ) _CustomConfigurationFileWrapper . clear ( persistent = False )

Clear all configuration properties from in - memory cache but do NOT alter the custom configuration file . Used in unit - testing .


def resetCustomConfig(cls):
    """ Clear all custom configuration settings and delete the persistent
    custom configuration store.
    """
    _getLogger().info("Resetting all custom configuration properties; "
                      "caller=%r", traceback.format_stack())

    # Clear the in-memory settings cache, forcing reload upon subsequent "get"
    # request.
    super(Configuration, cls).clear()

    # Delete the persistent custom configuration store and reset in-memory
    # custom configuration info
    _CustomConfigurationFileWrapper.clear(persistent=True)

Clear all custom configuration settings and delete the persistent custom configuration store .


def clear(cls, persistent=False):
    """ If persistent is True, delete the temporary file

    Parameters:
    ----------------------------------------------------------------
    persistent: if True, custom configuration file is deleted
    """
    if persistent:
      try:
        os.unlink(cls.getPath())
      except OSError, e:
        if e.errno != errno.ENOENT:
          _getLogger().exception("Error %s while trying to remove dynamic " \
                                 "configuration file: %s", e.errno,
                                 cls.getPath())
          raise
    cls._path = None

If persistent is True delete the temporary file


def getCustomDict(cls):
    """ Returns a dict of all temporary values in custom configuration file

    """
    if not os.path.exists(cls.getPath()):
      return dict()

    properties = Configuration._readConfigFile(os.path.basename(
      cls.getPath()), os.path.dirname(cls.getPath()))

    values = dict()
    for propName in properties:
      if 'value' in properties[propName]:
        values[propName] = properties[propName]['value']

    return values

def getCustomDict ( cls ) : if not os . path . exists ( cls . getPath ( ) ) : return dict ( ) properties = Configuration . _readConfigFile ( os . path . basename ( cls . getPath ( ) ) , os . path . dirname ( cls . getPath ( ) ) ) values = dict ( ) for propName in properties : if 'value' in properties [ propName ] : values [ propName ] = properties [ propName ] [ 'value' ] return values


def edit(cls, properties):
    """ Edits the XML configuration file with the parameters specified by
    properties

    Parameters:
    ----------------------------------------------------------------
    properties: dict of settings to be applied to the custom configuration store
                 (key is property name, value is value)
    """
    copyOfProperties = copy(properties)

    configFilePath = cls.getPath()

    try:
      with open(configFilePath, 'r') as fp:
        contents = fp.read()
    except IOError, e:
      if e.errno != errno.ENOENT:
        _getLogger().exception("Error %s reading custom configuration store "
                               "from %s, while editing properties %s.",
                               e.errno, configFilePath, properties)
        raise
      contents = '<configuration/>'

    try:
      elements = ElementTree.XML(contents)
      ElementTree.tostring(elements)
    except Exception, e:
      # Raising error as RuntimeError with custom message since ElementTree
      # exceptions aren't clear.
      msg = "File contents of custom configuration is corrupt.  File " \
            "location: %s; Contents: '%s'. Original Error (%s): %s." % \
            (configFilePath, contents, type(e), e)
      _getLogger().exception(msg)
      raise RuntimeError(msg), None, sys.exc_info()[2]

    if elements.tag != 'configuration':
      e = "Expected top-level element to be 'configuration' but got '%s'" % \
          (elements.tag)
      _getLogger().error(e)
      raise RuntimeError(e)

    # Apply new properties to matching settings in the custom config store;
    # pop matching properties from our copy of the properties dict
    for propertyItem in elements.findall('./property'):
      propInfo = dict((attr.tag, attr.text) for attr in propertyItem)
      name = propInfo['name']
      if name in copyOfProperties:
        foundValues = propertyItem.findall('./value')
        if len(foundValues) > 0:
          foundValues[0].text = str(copyOfProperties.pop(name))
          if not copyOfProperties:
            break
        else:
          e = "Property %s missing value tag." % (name,)
          _getLogger().error(e)
          raise RuntimeError(e)

    # Add unmatched remaining properties to custom config store
    for propertyName, value in copyOfProperties.iteritems():
      newProp = ElementTree.Element('property')
      nameTag = ElementTree.Element('name')
      nameTag.text = propertyName
      newProp.append(nameTag)

      valueTag = ElementTree.Element('value')
      valueTag.text = str(value)
      newProp.append(valueTag)

      elements.append(newProp)

    try:
      makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))
      with open(configFilePath, 'w') as fp:
        fp.write(ElementTree.tostring(elements))
    except Exception, e:
      _getLogger().exception("Error while saving custom configuration "
                             "properties %s in %s.", properties,
                             configFilePath)
      raise


def _setPath(cls):
    """ Sets the path of the custom configuration file
    """
    cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'],
                             cls.customFileName)

def _setPath ( cls ) : cls . _path = os . path . join ( os . environ [ 'NTA_DYNAMIC_CONF_DIR' ] , cls . customFileName )


def getState(self):
    """Get the particle state as a dict. This is enough information to
    instantiate this particle on another worker."""
    varStates = dict()
    for varName, var in self.permuteVars.iteritems():
      varStates[varName] = var.getState()

    return dict(id=self.particleId,
                genIdx=self.genIdx,
                swarmId=self.swarmId,
                varStates=varStates)

def getState ( self ) : varStates = dict ( ) for varName , var in self . permuteVars . iteritems ( ) : varStates [ varName ] = var . getState ( ) return dict ( id = self . particleId , genIdx = self . genIdx , swarmId = self . swarmId , varStates = varStates )

Get the particle state as a dict . This is enough information to instantiate this particle on another worker .


def initStateFrom(self, particleId, particleState, newBest):
    """Init all of our variable positions, velocities, and optionally the best
    result and best position from the given particle.

    If newBest is true, we get the best result and position for this new
    generation from the resultsDB, This is used when evoloving a particle
    because the bestResult and position as stored in was the best AT THE TIME
    THAT PARTICLE STARTED TO RUN and does not include the best since that
    particle completed.
    """
    # Get the update best position and result?
    if newBest:
      (bestResult, bestPosition) = self._resultsDB.getParticleBest(particleId)
    else:
      bestResult = bestPosition = None

    # Replace with the position and velocity of each variable from
    #  saved state
    varStates = particleState['varStates']
    for varName in varStates.keys():
      varState = copy.deepcopy(varStates[varName])
      if newBest:
        varState['bestResult'] = bestResult
      if bestPosition is not None:
        varState['bestPosition'] = bestPosition[varName]
      self.permuteVars[varName].setState(varState)

def initStateFrom ( self , particleId , particleState , newBest ) : if newBest : ( bestResult , bestPosition ) = self . _resultsDB . getParticleBest ( particleId ) else : bestResult = bestPosition = None varStates = particleState [ 'varStates' ] for varName in varStates . keys ( ) : varState = copy . deepcopy ( varStates [ varName ] ) if newBest : varState [ 'bestResult' ] = bestResult if bestPosition is not None : varState [ 'bestPosition' ] = bestPosition [ varName ] self . permuteVars [ varName ] . setState ( varState )

Init all of our variable positions velocities and optionally the best result and best position from the given particle .


def copyVarStatesFrom(self, particleState, varNames):
    """Copy specific variables from particleState into this particle.

    Parameters:
    --------------------------------------------------------------
    particleState:        dict produced by a particle's getState() method
    varNames:             which variables to copy
    """
    # Set this to false if you don't want the variable to move anymore
    #  after we set the state
    allowedToMove = True

    for varName in particleState['varStates']:
      if varName in varNames:

        # If this particle doesn't include this field, don't copy it
        if varName not in self.permuteVars:
          continue

        # Set the best position to the copied position
        state = copy.deepcopy(particleState['varStates'][varName])
        state['_position'] = state['position']
        state['bestPosition'] = state['position']

        if not allowedToMove:
          state['velocity'] = 0

        # Set the state now
        self.permuteVars[varName].setState(state)

        if allowedToMove:
          # Let the particle move in both directions from the best position
          #  it found previously and set it's initial velocity to a known
          #  fraction of the total distance.
          self.permuteVars[varName].resetVelocity(self._rng)

def copyVarStatesFrom ( self , particleState , varNames ) : allowedToMove = True for varName in particleState [ 'varStates' ] : if varName in varNames : if varName not in self . permuteVars : continue state = copy . deepcopy ( particleState [ 'varStates' ] [ varName ] ) state [ '_position' ] = state [ 'position' ] state [ 'bestPosition' ] = state [ 'position' ] if not allowedToMove : state [ 'velocity' ] = 0 self . permuteVars [ varName ] . setState ( state ) if allowedToMove : self . permuteVars [ varName ] . resetVelocity ( self . _rng )

Copy specific variables from particleState into this particle .


def getPositionFromState(pState):
    """Return the position of a particle given its state dict.

    Parameters:
    --------------------------------------------------------------
    retval:     dict() of particle position, keys are the variable names,
                  values are their positions
    """
    result = dict()
    for (varName, value) in pState['varStates'].iteritems():
      result[varName] = value['position']

    return result

def getPositionFromState ( pState ) : result = dict ( ) for ( varName , value ) in pState [ 'varStates' ] . iteritems ( ) : result [ varName ] = value [ 'position' ] return result

Return the position of a particle given its state dict .


def agitate(self):
    """Agitate this particle so that it is likely to go to a new position.
    Every time agitate is called, the particle is jiggled an even greater
    amount.

    Parameters:
    --------------------------------------------------------------
    retval:               None
    """
    for (varName, var) in self.permuteVars.iteritems():
      var.agitate()

    self.newPosition()

def agitate ( self ) : for ( varName , var ) in self . permuteVars . iteritems ( ) : var . agitate ( ) self . newPosition ( )

Agitate this particle so that it is likely to go to a new position . Every time agitate is called the particle is jiggled an even greater amount .


def newPosition(self, whichVars=None):
    # TODO: incorporate data from choice variables....
    # TODO: make sure we're calling this when appropriate.
    """Choose a new position based on results obtained so far from all other
    particles.

    Parameters:
    --------------------------------------------------------------
    whichVars:       If not None, only move these variables
    retval:               new position
    """
    # Get the global best position for this swarm generation
    globalBestPosition = None
    # If speculative particles are enabled, use the global best considering
    #  even particles in the current generation. This gives better results
    #  but does not provide repeatable results because it depends on
    #  worker timing
    if self._hsObj._speculativeParticles:
      genIdx = self.genIdx
    else:
      genIdx = self.genIdx - 1

    if genIdx >= 0:
      (bestModelId, _) = self._resultsDB.bestModelIdAndErrScore(self.swarmId,
                                                                genIdx)
      if bestModelId is not None:
        (particleState, _, _, _, _) = self._resultsDB.getParticleInfo(
          bestModelId)
        globalBestPosition = Particle.getPositionFromState(particleState)

    # Update each variable
    for (varName, var) in self.permuteVars.iteritems():
      if whichVars is not None and varName not in whichVars:
        continue
      if globalBestPosition is None:
        var.newPosition(None, self._rng)
      else:
        var.newPosition(globalBestPosition[varName], self._rng)

    # get the new position
    position = self.getPosition()

    # Log the new position
    if self.logger.getEffectiveLevel() <= logging.DEBUG:
      msg = StringIO.StringIO()
      print >> msg, "New particle position: \n%s" % (pprint.pformat(position,
                                                                    indent=4))
      print >> msg, "Particle variables:"
      for (varName, var) in self.permuteVars.iteritems():
        print >> msg, "  %s: %s" % (varName, str(var))
      self.logger.debug(msg.getvalue())
      msg.close()

    return position

def
newPosition
(
self
,
whichVars
=
)
:
globalBestPosition
=
if
self
.
_hsObj
.
_speculativeParticles
:
genIdx
=
self
.
genIdx
else
:
genIdx
=
self
.
genIdx
1
if
genIdx
>=
0
:
(
bestModelId
,
_
)
=
self
.
_resultsDB
.
bestModelIdAndErrScore
(
self
.
swarmId
,
genIdx
)
if
bestModelId
is
not
:
(
particleState
,
_
,
_
,
_
,
_
)
=
self
.
_resultsDB
.
getParticleInfo
(
bestModelId
)
globalBestPosition
=
Particle
.
getPositionFromState
(
particleState
)
for
(
varName
,
var
)
in
self
.
permuteVars
.
iteritems
(
)
:
if
whichVars
is
not
and
varName
not
in
whichVars
:
continue
if
globalBestPosition
is
:
var
.
newPosition
(
,
self
.
_rng
)
else
:
var
.
newPosition
(
globalBestPosition
[
varName
]
,
self
.
_rng
)
position
=
self
.
getPosition
(
)
if
self
.
logger
.
getEffectiveLevel
(
)
<=
logging
.
DEBUG
:
msg
=
StringIO
.
StringIO
(
)
print
>>
msg
,
"New particle position: \n%s"
%
(
pprint
.
pformat
(
position
,
indent
=
4
)
)
print
>>
msg
,
"Particle variables:"
for
(
varName
,
var
)
in
self
.
permuteVars
.
iteritems
(
)
:
print
>>
msg
,
"  %s: %s"
%
(
varName
,
str
(
var
)
)
self
.
logger
.
debug
(
msg
.
getvalue
(
)
)
msg
.
close
(
)
return
position

Choose a new position based on results obtained so far from all other particles .


def __getLogger(cls):
    """ Get the logger for this object.

    :returns: (Logger) A Logger object.
    """
    if cls.__logger is None:
      cls.__logger = opf_utils.initLogger(cls)
    return cls.__logger

def __getLogger ( cls ) : if cls . __logger is None : cls . __logger = opf_utils . initLogger ( cls ) return cls . __logger


def create(modelConfig, logLevel=logging.ERROR):
    """ Create a new model instance, given a description dictionary.

    :param modelConfig: (dict)
           A dictionary describing the current model,
           `described here <../../quick-start/example-model-params.html>`_.

    :param logLevel: (int) The level of logging output that should be generated

    :raises Exception: Unsupported model type

    :returns: :class:`nupic.frameworks.opf.model.Model`
    """
    logger = ModelFactory.__getLogger()
    logger.setLevel(logLevel)
    logger.debug("ModelFactory returning Model from dict: %s", modelConfig)

    modelClass = None
    if modelConfig['model'] == "HTMPrediction":
      modelClass = HTMPredictionModel
    elif modelConfig['model'] == "TwoGram":
      modelClass = TwoGramModel
    elif modelConfig['model'] == "PreviousValue":
      modelClass = PreviousValueModel
    else:
      raise Exception("ModelFactory received unsupported Model type: %s" % \
                      modelConfig['model'])

    return modelClass(**modelConfig['modelParams'])

Create a new model instance given a description dictionary .