def _mergeFiles(key, chunkCount, outputFile, fields):
  """Merge sorted chunk files into a sorted output file

  chunkCount - the number of available chunk files
  outputFile the name of the sorted output file

  _mergeFiles()

  """
  title()

  # Open all chun files
  files = [FileRecordStream('chunk_%d.csv' % i) for i in range(chunkCount)]

  # Open output file
  with FileRecordStream(outputFile, write=True, fields=fields) as o:
    # Open all chunk files
    files = [FileRecordStream('chunk_%d.csv' % i) for i in range(chunkCount)]
    records = [f.getNextRecord() for f in files]

    # This loop will run until all files are exhausted
    while not all(r is None for r in records):
      # Cleanup None values (files that were exhausted)
      indices = [i for i,r in enumerate(records) if r is not None]
      records = [records[i] for i in indices]
      files = [files[i] for i in indices]

      # Find the current record
      r = min(records, key=itemgetter(*key))
      # Write it to the file
      o.appendRecord(r)

      # Find the index of file that produced the current record
      index = records.index(r)
      # Read a new record from the file
      records[index] = files[index].getNextRecord()

  # Cleanup chunk files
  for i, f in enumerate(files):
    f.close()
    os.remove('chunk_%d.csv' % i)

def _mergeFiles ( key , chunkCount , outputFile , fields ) : title ( ) files = [ FileRecordStream ( 'chunk_%d.csv' % i ) for i in range ( chunkCount ) ] with FileRecordStream ( outputFile , write = True , fields = fields ) as o : files = [ FileRecordStream ( 'chunk_%d.csv' % i ) for i in range ( chunkCount ) ] records = [ f . getNextRecord ( ) for f in files ] while not all ( r is None for r in records ) : indices = [ i for i , r in enumerate ( records ) if r is not None ] records = [ records [ i ] for i in indices ] files = [ files [ i ] for i in indices ] r = min ( records , key = itemgetter ( * key ) ) o . appendRecord ( r ) index = records . index ( r ) records [ index ] = files [ index ] . getNextRecord ( ) for i , f in enumerate ( files ) : f . close ( ) os . remove ( 'chunk_%d.csv' % i )


def compute(self, activeColumns, learn=True):
    """
    Feeds input record through TM, performing inference and learning.
    Updates member variables with new state.

    @param activeColumns (set) Indices of active columns in `t`
    """
    bottomUpInput = numpy.zeros(self.numberOfCols, dtype=dtype)
    bottomUpInput[list(activeColumns)] = 1
    super(TemporalMemoryShim, self).compute(bottomUpInput,
                                            enableLearn=learn,
                                            enableInference=True)

    predictedState = self.getPredictedState()
    self.predictiveCells = set(numpy.flatnonzero(predictedState))

def compute ( self , activeColumns , learn = True ) : bottomUpInput = numpy . zeros ( self . numberOfCols , dtype = dtype ) bottomUpInput [ list ( activeColumns ) ] = 1 super ( TemporalMemoryShim , self ) . compute ( bottomUpInput , enableLearn = learn , enableInference = True ) predictedState = self . getPredictedState ( ) self . predictiveCells = set ( numpy . flatnonzero ( predictedState ) )

Feeds input record through TM performing inference and learning . Updates member variables with new state .


def cPrint(self, level, message, *args, **kw):
    """Print a message to the console.

    Prints only if level <= self.consolePrinterVerbosity
    Printing with level 0 is equivalent to using a print statement,
    and should normally be avoided.

    :param level: (int) indicating the urgency of the message with
           lower values meaning more urgent (messages at level 0  are the most
           urgent and are always printed)

    :param message: (string) possibly with format specifiers

    :param args: specifies the values for any format specifiers in message

    :param kw: newline is the only keyword argument. True (default) if a newline
           should be printed
    """

    if level > self.consolePrinterVerbosity:
      return

    if len(kw) > 1:
      raise KeyError("Invalid keywords for cPrint: %s" % str(kw.keys()))

    newline = kw.get("newline", True)
    if len(kw) == 1 and 'newline' not in kw:
      raise KeyError("Invalid keyword for cPrint: %s" % kw.keys()[0])

    if len(args) == 0:
      if newline:
        print message
      else:
        print message,
    else:
      if newline:
        print message % args
      else:
        print message % args,

def
cPrint
(
self
,
level
,
message
,
*
args
,
**
kw
)
:
if
level
>
self
.
consolePrinterVerbosity
:
return
if
len
(
kw
)
>
1
:
raise
KeyError
(
"Invalid keywords for cPrint: %s"
%
str
(
kw
.
keys
(
)
)
)
newline
=
kw
.
get
(
"newline"
,
True
)
if
len
(
kw
)
==
1
and
'newline'
not
in
kw
:
raise
KeyError
(
"Invalid keyword for cPrint: %s"
%
kw
.
keys
(
)
[
0
]
)
if
len
(
args
)
==
0
:
if
newline
:
print
message
else
:
print
message
,
else
:
if
newline
:
print
message
%
args
else
:
print
message
%
args
,


def coordinateForPosition(self, longitude, latitude, altitude=None):
    """
    Returns coordinate for given GPS position.

    :param: longitude (float) Longitude of position
    :param: latitude (float) Latitude of position
    :param: altitude (float) Altitude of position
    :returns: (numpy.array) Coordinate that the given GPS position
                          maps to
    """
    coords = PROJ(longitude, latitude)

    if altitude is not None:
      coords = transform(PROJ, geocentric, coords[0], coords[1], altitude)

    coordinate = numpy.array(coords)
    coordinate = coordinate / self.scale
    return coordinate.astype(int)

def coordinateForPosition ( self , longitude , latitude , altitude = None ) : coords = PROJ ( longitude , latitude ) if altitude is not None : coords = transform ( PROJ , geocentric , coords [ 0 ] , coords [ 1 ] , altitude ) coordinate = numpy . array ( coords ) coordinate = coordinate / self . scale return coordinate . astype ( int )

Returns coordinate for given GPS position .


def radiusForSpeed(self, speed):
    """
    Returns radius for given speed.

    Tries to get the encodings of consecutive readings to be
    adjacent with some overlap.

    :param: speed (float) Speed (in meters per second)
    :returns: (int) Radius for given speed
    """
    overlap = 1.5
    coordinatesPerTimestep = speed * self.timestep / self.scale
    radius = int(round(float(coordinatesPerTimestep) / 2 * overlap))
    minRadius = int(math.ceil((math.sqrt(self.w) - 1) / 2))
    return max(radius, minRadius)

def radiusForSpeed ( self , speed ) : overlap = 1.5 coordinatesPerTimestep = speed * self . timestep / self . scale radius = int ( round ( float ( coordinatesPerTimestep ) / 2 * overlap ) ) minRadius = int ( math . ceil ( ( math . sqrt ( self . w ) - 1 ) / 2 ) ) return max ( radius , minRadius )


def readFromFile(cls, f, packed=True):
    """
    Read serialized object from file.

    :param f: input file
    :param packed: If true, will assume content is packed
    :return: first-class instance initialized from proto obj
    """
    # Get capnproto schema from instance
    schema = cls.getSchema()

    # Read from file
    if packed:
      proto = schema.read_packed(f)
    else:
      proto = schema.read(f)

    # Return first-class instance initialized from proto obj
    return cls.read(proto)

def readFromFile ( cls , f , packed = True ) : schema = cls . getSchema ( ) if packed : proto = schema . read_packed ( f ) else : proto = schema . read ( f ) return cls . read ( proto )


def writeToFile(self, f, packed=True):
    """
    Write serialized object to file.

    :param f: output file
    :param packed: If true, will pack contents.
    """
    # Get capnproto schema from instance
    schema = self.getSchema()

    # Construct new message, otherwise refered to as `proto`
    proto = schema.new_message()

    # Populate message w/ `write()` instance method
    self.write(proto)

    # Finally, write to file
    if packed:
      proto.write_packed(f)
    else:
      proto.write(f)

def writeToFile ( self , f , packed = True ) : schema = self . getSchema ( ) proto = schema . new_message ( ) self . write ( proto ) if packed : proto . write_packed ( f ) else : proto . write ( f )


def requireAnomalyModel(func):
  """
  Decorator for functions that require anomaly models.
  """
  @wraps(func)
  def _decorator(self, *args, **kwargs):
    if not self.getInferenceType() == InferenceType.TemporalAnomaly:
      raise RuntimeError("Method required a TemporalAnomaly model.")
    if self._getAnomalyClassifier() is None:
      raise RuntimeError("Model does not support this command. Model must"
          "be an active anomalyDetector model.")
    return func(self, *args, **kwargs)
  return _decorator

Decorator for functions that require anomaly models .


def anomalyRemoveLabels(self, start, end, labelFilter):
    """
    Remove labels from the anomaly classifier within this model. Removes all
    records if ``labelFilter==None``, otherwise only removes the labels equal to
    ``labelFilter``.

    :param start: (int) index to start removing labels
    :param end: (int) index to end removing labels
    :param labelFilter: (string) If specified, only removes records that match
    """
    self._getAnomalyClassifier().getSelf().removeLabels(start, end, labelFilter)

def anomalyRemoveLabels ( self , start , end , labelFilter ) : self . _getAnomalyClassifier ( ) . getSelf ( ) . removeLabels ( start , end , labelFilter )

Remove labels from the anomaly classifier within this model . Removes all records if labelFilter == None otherwise only removes the labels equal to labelFilter .


def anomalyAddLabel(self, start, end, labelName):
    """
    Add labels from the anomaly classifier within this model.

    :param start: (int) index to start label
    :param end: (int) index to end label
    :param labelName: (string) name of label
    """
    self._getAnomalyClassifier().getSelf().addLabel(start, end, labelName)

def anomalyAddLabel ( self , start , end , labelName ) : self . _getAnomalyClassifier ( ) . getSelf ( ) . addLabel ( start , end , labelName )

Add labels from the anomaly classifier within this model .


def anomalyGetLabels(self, start, end):
    """
    Get labels from the anomaly classifier within this model.

    :param start: (int) index to start getting labels
    :param end: (int) index to end getting labels
    """
    return self._getAnomalyClassifier().getSelf().getLabels(start, end)

def anomalyGetLabels ( self , start , end ) : return self . _getAnomalyClassifier ( ) . getSelf ( ) . getLabels ( start , end )

Get labels from the anomaly classifier within this model .


def _anomalyCompute(self):
    """
    Compute Anomaly score, if required
    """
    inferenceType = self.getInferenceType()

    inferences = {}
    sp = self._getSPRegion()
    score = None
    if inferenceType == InferenceType.NontemporalAnomaly:
      score = sp.getOutputData("anomalyScore")[0] #TODO move from SP to Anomaly ?

    elif inferenceType == InferenceType.TemporalAnomaly:
      tm = self._getTPRegion()

      if sp is not None:
        activeColumns = sp.getOutputData("bottomUpOut").nonzero()[0]
      else:
        sensor = self._getSensorRegion()
        activeColumns = sensor.getOutputData('dataOut').nonzero()[0]

      if not self._predictedFieldName in self._input:
        raise ValueError(
          "Expected predicted field '%s' in input row, but was not found!"
          % self._predictedFieldName
        )
      # Calculate the anomaly score using the active columns
      # and previous predicted columns.
      score = tm.getOutputData("anomalyScore")[0]

      # Calculate the classifier's output and use the result as the anomaly
      # label. Stores as string of results.

      # TODO: make labels work with non-SP models
      if sp is not None:
        self._getAnomalyClassifier().setParameter(
            "activeColumnCount", len(activeColumns))
        self._getAnomalyClassifier().prepareInputs()
        self._getAnomalyClassifier().compute()

        labels = self._getAnomalyClassifier().getSelf().getLabelResults()
        inferences[InferenceElement.anomalyLabel] = "%s" % labels

    inferences[InferenceElement.anomalyScore] = score
    return inferences


def _removeUnlikelyPredictions(cls, likelihoodsDict, minLikelihoodThreshold,
                                 maxPredictionsPerStep):
    """Remove entries with 0 likelihood or likelihood less than
    minLikelihoodThreshold, but don't leave an empty dict.
    """
    maxVal = (None, None)
    for (k, v) in likelihoodsDict.items():
      if len(likelihoodsDict) <= 1:
        break
      if maxVal[0] is None or v >= maxVal[1]:
        if maxVal[0] is not None and maxVal[1] < minLikelihoodThreshold:
          del likelihoodsDict[maxVal[0]]
        maxVal = (k, v)
      elif v < minLikelihoodThreshold:
        del likelihoodsDict[k]
    # Limit the number of predictions to include.
    likelihoodsDict = dict(sorted(likelihoodsDict.iteritems(),
                                  key=itemgetter(1),
                                  reverse=True)[:maxPredictionsPerStep])
    return likelihoodsDict

def _removeUnlikelyPredictions ( cls , likelihoodsDict , minLikelihoodThreshold , maxPredictionsPerStep ) : maxVal = ( None , None ) for ( k , v ) in likelihoodsDict . items ( ) : if len ( likelihoodsDict ) <= 1 : break if maxVal [ 0 ] is None or v >= maxVal [ 1 ] : if maxVal [ 0 ] is not None and maxVal [ 1 ] < minLikelihoodThreshold : del likelihoodsDict [ maxVal [ 0 ] ] maxVal = ( k , v ) elif v < minLikelihoodThreshold : del likelihoodsDict [ k ] likelihoodsDict = dict ( sorted ( likelihoodsDict . iteritems ( ) , key = itemgetter ( 1 ) , reverse = True ) [ : maxPredictionsPerStep ] ) return likelihoodsDict

Remove entries with 0 likelihood or likelihood less than minLikelihoodThreshold but don t leave an empty dict .


def _getClassifierRegion(self):
    """
    Returns reference to the network's Classifier region
    """
    if (self._netInfo.net is not None and
        "Classifier" in self._netInfo.net.regions):
      return self._netInfo.net.regions["Classifier"]
    else:
      return None

def _getClassifierRegion ( self ) : if ( self . _netInfo . net is not None and "Classifier" in self . _netInfo . net . regions ) : return self . _netInfo . net . regions [ "Classifier" ] else : return None

Returns reference to the network s Classifier region


def _addAnomalyClassifierRegion(self, network, params, spEnable, tmEnable):
    """
    Attaches an 'AnomalyClassifier' region to the network. Will remove current
    'AnomalyClassifier' region if it exists.

    Parameters
    -----------
    network - network to add the AnomalyClassifier region
    params - parameters to pass to the region
    spEnable - True if network has an SP region
    tmEnable - True if network has a TM region; Currently requires True
    """

    allParams = copy.deepcopy(params)
    knnParams = dict(k=1,
                     distanceMethod='rawOverlap',
                     distanceNorm=1,
                     doBinarization=1,
                     replaceDuplicates=0,
                     maxStoredPatterns=1000)
    allParams.update(knnParams)

    # Set defaults if not set
    if allParams['trainRecords'] is None:
      allParams['trainRecords'] = DEFAULT_ANOMALY_TRAINRECORDS

    if allParams['cacheSize'] is None:
      allParams['cacheSize'] = DEFAULT_ANOMALY_CACHESIZE

    # Remove current instance if already created (used for deserializing)
    if self._netInfo is not None and self._netInfo.net is not None \
              and self._getAnomalyClassifier() is not None:
      self._netInfo.net.removeRegion('AnomalyClassifier')

    network.addRegion("AnomalyClassifier",
                      "py.KNNAnomalyClassifierRegion",
                      json.dumps(allParams))

    # Attach link to SP
    if spEnable:
      network.link("SP", "AnomalyClassifier", "UniformLink", "",
          srcOutput="bottomUpOut", destInput="spBottomUpOut")
    else:
      network.link("sensor", "AnomalyClassifier", "UniformLink", "",
          srcOutput="dataOut", destInput="spBottomUpOut")

    # Attach link to TM
    if tmEnable:
      network.link("TM", "AnomalyClassifier", "UniformLink", "",
              srcOutput="topDownOut", destInput="tpTopDownOut")
      network.link("TM", "AnomalyClassifier", "UniformLink", "",
              srcOutput="lrnActiveStateT", destInput="tpLrnActiveStateT")
    else:
      raise RuntimeError("TemporalAnomaly models require a TM region.")

Attaches an AnomalyClassifier region to the network . Will remove current AnomalyClassifier region if it exists .


def setResultsPerChoice(self, resultsPerChoice):
    """Setup our resultsPerChoice history based on the passed in
    resultsPerChoice.

    For example, if this variable has the following choices:
      ['a', 'b', 'c']

    resultsPerChoice will have up to 3 elements, each element is a tuple
    containing (choiceValue, errors) where errors is the list of errors
    received from models that used the specific choice:
    retval:
      [('a', [0.1, 0.2, 0.3]), ('b', [0.5, 0.1, 0.6]), ('c', [0.2])]
    """
    # Keep track of the results obtained for each choice.
    self._resultsPerChoice = [[]] * len(self.choices)
    for (choiceValue, values) in resultsPerChoice:
      choiceIndex = self.choices.index(choiceValue)
      self._resultsPerChoice[choiceIndex] = list(values)

def setResultsPerChoice ( self , resultsPerChoice ) : self . _resultsPerChoice = [ [ ] ] * len ( self . choices ) for ( choiceValue , values ) in resultsPerChoice : choiceIndex = self . choices . index ( choiceValue ) self . _resultsPerChoice [ choiceIndex ] = list ( values )

Setup our resultsPerChoice history based on the passed in resultsPerChoice .


def _translateMetricsToJSON(self, metrics, label):
    """ Translates the given metrics value to JSON string

    metrics:        A list of dictionaries per OPFTaskDriver.getMetrics():

    Returns:        JSON string representing the given metrics object.
    """

    # Transcode the MetricValueElement values into JSON-compatible
    # structure
    metricsDict = metrics

    # Convert the structure to a display-friendly JSON string
    def _mapNumpyValues(obj):
      """
      """
      import numpy

      if isinstance(obj, numpy.float32):
        return float(obj)

      elif isinstance(obj, numpy.bool_):
        return bool(obj)

      elif isinstance(obj, numpy.ndarray):
        return obj.tolist()

      else:
        raise TypeError("UNEXPECTED OBJ: %s; class=%s" % (obj, obj.__class__))

    jsonString = json.dumps(metricsDict, indent=4, default=_mapNumpyValues)

    return jsonString

def
_translateMetricsToJSON
(
self
,
metrics
,
label
)
:
metricsDict
=
metrics
def
_mapNumpyValues
(
obj
)
:
import
numpy
if
isinstance
(
obj
,
numpy
.
float32
)
:
return
float
(
obj
)
elif
isinstance
(
obj
,
numpy
.
bool_
)
:
return
bool
(
obj
)
elif
isinstance
(
obj
,
numpy
.
ndarray
)
:
return
obj
.
tolist
(
)
else
:
raise
TypeError
(
"UNEXPECTED OBJ: %s; class=%s"
%
(
obj
,
obj
.
__class__
)
)
jsonString
=
json
.
dumps
(
metricsDict
,
indent
=
4
,
default
=
_mapNumpyValues
)
return
jsonString


def setLoggedMetrics(self, metricNames):
    """ Tell the writer which metrics should be written

    Parameters:
    -----------------------------------------------------------------------
    metricsNames: A list of metric lables to be written
    """
    if metricNames is None:
      self.__metricNames = set([])
    else:
      self.__metricNames = set(metricNames)

def setLoggedMetrics ( self , metricNames ) : if metricNames is None : self . __metricNames = set ( [ ] ) else : self . __metricNames = set ( metricNames )


def __getDictMetaInfo(self, inferenceElement, inferenceDict):
    """Get field metadate information for inferences that are of dict type"""
    fieldMetaInfo = []
    inferenceLabel = InferenceElement.getLabel(inferenceElement)

    if InferenceElement.getInputElement(inferenceElement):
      fieldMetaInfo.append(FieldMetaInfo(name=inferenceLabel+".actual",
                                         type=FieldMetaType.string,
                                         special = ''))

    keys = sorted(inferenceDict.keys())
    for key in keys:
      fieldMetaInfo.append(FieldMetaInfo(name=inferenceLabel+"."+str(key),
                                         type=FieldMetaType.string,
                                         special=''))

    return fieldMetaInfo

def __getDictMetaInfo ( self , inferenceElement , inferenceDict ) : fieldMetaInfo = [ ] inferenceLabel = InferenceElement . getLabel ( inferenceElement ) if InferenceElement . getInputElement ( inferenceElement ) : fieldMetaInfo . append ( FieldMetaInfo ( name = inferenceLabel + ".actual" , type = FieldMetaType . string , special = '' ) ) keys = sorted ( inferenceDict . keys ( ) ) for key in keys : fieldMetaInfo . append ( FieldMetaInfo ( name = inferenceLabel + "." + str ( key ) , type = FieldMetaType . string , special = '' ) ) return fieldMetaInfo


def createExperimentInferenceDir(cls, experimentDir):
    """ Creates the inference output directory for the given experiment

    experimentDir:  experiment directory path that contains description.py

    Returns:  path of the inference output directory
    """
    path = cls.getExperimentInferenceDirPath(experimentDir)

    cls.makeDirectory(path)

    return path

def createExperimentInferenceDir ( cls , experimentDir ) : path = cls . getExperimentInferenceDirPath ( experimentDir ) cls . makeDirectory ( path ) return path


def _allow_new_attributes(f):
  """A decorator that maintains the attribute lock state of an object

  It coperates with the LockAttributesMetaclass (see bellow) that replaces
  the __setattr__ method with a custom one that checks the _canAddAttributes
  counter and allows setting new attributes only if _canAddAttributes > 0.

  New attributes can be set only from methods decorated
  with this decorator (should be only __init__ and __setstate__ normally)

  The decorator is reentrant (e.g. if from inside a decorated function another
  decorated function is invoked). Before invoking the target function it
  increments the counter (or sets it to 1). After invoking the target function
  it decrements the counter and if it's 0 it removed the counter.
  """
  def decorated(self, *args, **kw):
    """The decorated function that replaces __init__() or __setstate__()

    """
    # Run the original function
    if not hasattr(self, '_canAddAttributes'):
      self.__dict__['_canAddAttributes'] = 1
    else:
      self._canAddAttributes += 1
    assert self._canAddAttributes >= 1

    # Save add attribute counter
    count = self._canAddAttributes
    f(self, *args, **kw)

    # Restore _CanAddAttributes if deleted from dict (can happen in __setstte__)
    if hasattr(self, '_canAddAttributes'):
      self._canAddAttributes -= 1
    else:
      self._canAddAttributes = count - 1

    assert self._canAddAttributes >= 0
    if self._canAddAttributes == 0:
      del self._canAddAttributes

  decorated.__doc__ = f.__doc__
  decorated.__name__ = f.__name__
  return decorated

def _allow_new_attributes ( f ) : def decorated ( self , * args , ** kw ) : if not hasattr ( self , '_canAddAttributes' ) : self . __dict__ [ '_canAddAttributes' ] = 1 else : self . _canAddAttributes += 1 assert self . _canAddAttributes >= 1 count = self . _canAddAttributes f ( self , * args , ** kw ) if hasattr ( self , '_canAddAttributes' ) : self . _canAddAttributes -= 1 else : self . _canAddAttributes = count - 1 assert self . _canAddAttributes >= 0 if self . _canAddAttributes == 0 : del self . _canAddAttributes decorated . __doc__ = f . __doc__ decorated . __name__ = f . __name__ return decorated


def generateRandomInput(numRecords, elemSize = 400, numSet = 42):
  """ Generates a set of input record

  Params:
          numRecords - how many records to generate
          elemSize - the size of each record (num 0s or 1s)
          numSet - how many 1s in each record

  Returns: a list of inputs
  """

  inputs = []

  for _ in xrange(numRecords):

    input = np.zeros(elemSize, dtype=realDType)
    for _ in range(0,numSet):
      ind = np.random.random_integers(0, elemSize-1, 1)[0]
      input[ind] = 1
    while abs(input.sum() - numSet) > 0.1:
      ind = np.random.random_integers(0, elemSize-1, 1)[0]
      input[ind] = 1

    inputs.append(input)

  return inputs

def generateRandomInput ( numRecords , elemSize = 400 , numSet = 42 ) : inputs = [ ] for _ in xrange ( numRecords ) : input = np . zeros ( elemSize , dtype = realDType ) for _ in range ( 0 , numSet ) : ind = np . random . random_integers ( 0 , elemSize - 1 , 1 ) [ 0 ] input [ ind ] = 1 while abs ( input . sum ( ) - numSet ) > 0.1 : ind = np . random . random_integers ( 0 , elemSize - 1 , 1 ) [ 0 ] input [ ind ] = 1 inputs . append ( input ) return inputs


def appendInputWithSimilarValues(inputs):
  """ Creates an 'one-off' record for each record in the inputs. Appends new
  records to the same inputs list.
  """
  numInputs = len(inputs)
  for i in xrange(numInputs):
    input = inputs[i]
    for j in xrange(len(input)-1):
      if input[j] == 1 and input[j+1] == 0:
        newInput = copy.deepcopy(input)
        newInput[j] = 0
        newInput[j+1] = 1
        inputs.append(newInput)
        break

def appendInputWithSimilarValues ( inputs ) : numInputs = len ( inputs ) for i in xrange ( numInputs ) : input = inputs [ i ] for j in xrange ( len ( input ) - 1 ) : if input [ j ] == 1 and input [ j + 1 ] == 0 : newInput = copy . deepcopy ( input ) newInput [ j ] = 0 newInput [ j + 1 ] = 1 inputs . append ( newInput ) break

Creates an one - off record for each record in the inputs . Appends new records to the same inputs list .


def appendInputWithNSimilarValues(inputs, numNear = 10):
  """ Creates a neighboring record for each record in the inputs and adds
  new records at the end of the inputs list
  """
  numInputs = len(inputs)
  skipOne = False
  for i in xrange(numInputs):
    input = inputs[i]
    numChanged = 0
    newInput = copy.deepcopy(input)
    for j in xrange(len(input)-1):
      if skipOne:
        skipOne = False
        continue
      if input[j] == 1 and input[j+1] == 0:
        newInput[j] = 0
        newInput[j+1] = 1
        inputs.append(newInput)
        newInput = copy.deepcopy(newInput)
        #print input
        #print newInput
        numChanged += 1
        skipOne = True
        if numChanged == numNear:
          break

def appendInputWithNSimilarValues ( inputs , numNear = 10 ) : numInputs = len ( inputs ) skipOne = False for i in xrange ( numInputs ) : input = inputs [ i ] numChanged = 0 newInput = copy . deepcopy ( input ) for j in xrange ( len ( input ) - 1 ) : if skipOne : skipOne = False continue if input [ j ] == 1 and input [ j + 1 ] == 0 : newInput [ j ] = 0 newInput [ j + 1 ] = 1 inputs . append ( newInput ) newInput = copy . deepcopy ( newInput ) numChanged += 1 skipOne = True if numChanged == numNear : break


def modifyBits(inputVal, maxChanges):
  """ Modifies up to maxChanges number of bits in the inputVal
  """
  changes = np.random.random_integers(0, maxChanges, 1)[0]

  if changes == 0:
    return inputVal

  inputWidth = len(inputVal)

  whatToChange = np.random.random_integers(0, 41, changes)

  runningIndex = -1
  numModsDone = 0
  for i in xrange(inputWidth):
    if numModsDone >= changes:
      break
    if inputVal[i] == 1:
      runningIndex += 1
      if runningIndex in whatToChange:
        if i != 0 and inputVal[i-1] == 0:
          inputVal[i-1] = 1
          inputVal[i] = 0

  return inputVal

def modifyBits ( inputVal , maxChanges ) : changes = np . random . random_integers ( 0 , maxChanges , 1 ) [ 0 ] if changes == 0 : return inputVal inputWidth = len ( inputVal ) whatToChange = np . random . random_integers ( 0 , 41 , changes ) runningIndex = - 1 numModsDone = 0 for i in xrange ( inputWidth ) : if numModsDone >= changes : break if inputVal [ i ] == 1 : runningIndex += 1 if runningIndex in whatToChange : if i != 0 and inputVal [ i - 1 ] == 0 : inputVal [ i - 1 ] = 1 inputVal [ i ] = 0 return inputVal


def getRandomWithMods(inputSpace, maxChanges):
  """ Returns a random selection from the inputSpace with randomly modified
  up to maxChanges number of bits.
  """
  size = len(inputSpace)
  ind = np.random.random_integers(0, size-1, 1)[0]

  value = copy.deepcopy(inputSpace[ind])

  if maxChanges == 0:
    return value

  return modifyBits(value, maxChanges)

def getRandomWithMods ( inputSpace , maxChanges ) : size = len ( inputSpace ) ind = np . random . random_integers ( 0 , size - 1 , 1 ) [ 0 ] value = copy . deepcopy ( inputSpace [ ind ] ) if maxChanges == 0 : return value return modifyBits ( value , maxChanges )

Returns a random selection from the inputSpace with randomly modified up to maxChanges number of bits .


def createRecordSensor(network, name, dataSource):
  """
  Creates a RecordSensor region that allows us to specify a file record
  stream as the input source.
  """

  # Specific type of region. Possible options can be found in /nupic/regions/
  regionType = "py.RecordSensor"

  # Creates a json from specified dictionary.
  regionParams = json.dumps({"verbosity": _VERBOSITY})
  network.addRegion(name, regionType, regionParams)

  # getSelf returns the actual region, instead of a region wrapper
  sensorRegion = network.regions[name].getSelf()

  # Specify how RecordSensor encodes input values
  sensorRegion.encoder = createEncoder()

  # Specify which sub-encoder should be used for "actValueOut"
  network.regions[name].setParameter("predictedField", "consumption")

  # Specify the dataSource as a file record stream instance
  sensorRegion.dataSource = dataSource
  return sensorRegion

def createRecordSensor ( network , name , dataSource ) : regionType = "py.RecordSensor" regionParams = json . dumps ( { "verbosity" : _VERBOSITY } ) network . addRegion ( name , regionType , regionParams ) sensorRegion = network . regions [ name ] . getSelf ( ) sensorRegion . encoder = createEncoder ( ) network . regions [ name ] . setParameter ( "predictedField" , "consumption" ) sensorRegion . dataSource = dataSource return sensorRegion

Creates a RecordSensor region that allows us to specify a file record stream as the input source .


def createNetwork(dataSource):
  """Creates and returns a new Network with a sensor region reading data from
  'dataSource'. There are two hierarchical levels, each with one SP and one TM.
  @param dataSource - A RecordStream containing the input data
  @returns a Network ready to run
  """
  network = Network()

  # Create and add a record sensor and a SP region
  sensor = createRecordSensor(network, name=_RECORD_SENSOR,
                              dataSource=dataSource)
  createSpatialPooler(network, name=_L1_SPATIAL_POOLER,
                      inputWidth=sensor.encoder.getWidth())

  # Link the SP region to the sensor input
  linkType = "UniformLink"
  linkParams = ""
  network.link(_RECORD_SENSOR, _L1_SPATIAL_POOLER, linkType, linkParams)

  # Create and add a TM region
  l1temporalMemory = createTemporalMemory(network, _L1_TEMPORAL_MEMORY)

  # Link SP region to TM region in the feedforward direction
  network.link(_L1_SPATIAL_POOLER, _L1_TEMPORAL_MEMORY, linkType, linkParams)

  # Add a classifier
  classifierParams = {  # Learning rate. Higher values make it adapt faster.
                        'alpha': 0.005,

                        # A comma separated list of the number of steps the
                        # classifier predicts in the future. The classifier will
                        # learn predictions of each order specified.
                        'steps': '1',

                        # The specific implementation of the classifier to use
                        # See SDRClassifierFactory#create for options
                        'implementation': 'py',

                        # Diagnostic output verbosity control;
                        # 0: silent; [1..6]: increasing levels of verbosity
                        'verbosity': 0}

  l1Classifier = network.addRegion(_L1_CLASSIFIER, "py.SDRClassifierRegion",
                                   json.dumps(classifierParams))
  l1Classifier.setParameter('inferenceMode', True)
  l1Classifier.setParameter('learningMode', True)
  network.link(_L1_TEMPORAL_MEMORY, _L1_CLASSIFIER, linkType, linkParams,
               srcOutput="bottomUpOut", destInput="bottomUpIn")
  network.link(_RECORD_SENSOR, _L1_CLASSIFIER, linkType, linkParams,
               srcOutput="categoryOut", destInput="categoryIn")
  network.link(_RECORD_SENSOR, _L1_CLASSIFIER, linkType, linkParams,
               srcOutput="bucketIdxOut", destInput="bucketIdxIn")
  network.link(_RECORD_SENSOR, _L1_CLASSIFIER, linkType, linkParams,
               srcOutput="actValueOut", destInput="actValueIn")

  # Second Level
  l2inputWidth = l1temporalMemory.getSelf().getOutputElementCount("bottomUpOut")
  createSpatialPooler(network, name=_L2_SPATIAL_POOLER, inputWidth=l2inputWidth)
  network.link(_L1_TEMPORAL_MEMORY, _L2_SPATIAL_POOLER, linkType, linkParams)

  createTemporalMemory(network, _L2_TEMPORAL_MEMORY)
  network.link(_L2_SPATIAL_POOLER, _L2_TEMPORAL_MEMORY, linkType, linkParams)

  l2Classifier = network.addRegion(_L2_CLASSIFIER, "py.SDRClassifierRegion",
                                   json.dumps(classifierParams))
  l2Classifier.setParameter('inferenceMode', True)
  l2Classifier.setParameter('learningMode', True)
  network.link(_L2_TEMPORAL_MEMORY, _L2_CLASSIFIER, linkType, linkParams,
               srcOutput="bottomUpOut", destInput="bottomUpIn")
  network.link(_RECORD_SENSOR, _L2_CLASSIFIER, linkType, linkParams,
               srcOutput="categoryOut", destInput="categoryIn")
  network.link(_RECORD_SENSOR, _L2_CLASSIFIER, linkType, linkParams,
               srcOutput="bucketIdxOut", destInput="bucketIdxIn")
  network.link(_RECORD_SENSOR, _L2_CLASSIFIER, linkType, linkParams,
               srcOutput="actValueOut", destInput="actValueIn")
  return network

def createNetwork ( dataSource ) : network = Network ( ) sensor = createRecordSensor ( network , name = _RECORD_SENSOR , dataSource = dataSource ) createSpatialPooler ( network , name = _L1_SPATIAL_POOLER , inputWidth = sensor . encoder . getWidth ( ) ) linkType = "UniformLink" linkParams = "" network . link ( _RECORD_SENSOR , _L1_SPATIAL_POOLER , linkType , linkParams ) l1temporalMemory = createTemporalMemory ( network , _L1_TEMPORAL_MEMORY ) network . link ( _L1_SPATIAL_POOLER , _L1_TEMPORAL_MEMORY , linkType , linkParams ) classifierParams = { 'alpha' : 0.005 , 'steps' : '1' , 'implementation' : 'py' , 'verbosity' : 0 } l1Classifier = network . addRegion ( _L1_CLASSIFIER , "py.SDRClassifierRegion" , json . dumps ( classifierParams ) ) l1Classifier . setParameter ( 'inferenceMode' , True ) l1Classifier . setParameter ( 'learningMode' , True ) network . link ( _L1_TEMPORAL_MEMORY , _L1_CLASSIFIER , linkType , linkParams , srcOutput = "bottomUpOut" , destInput = "bottomUpIn" ) network . link ( _RECORD_SENSOR , _L1_CLASSIFIER , linkType , linkParams , srcOutput = "categoryOut" , destInput = "categoryIn" ) network . link ( _RECORD_SENSOR , _L1_CLASSIFIER , linkType , linkParams , srcOutput = "bucketIdxOut" , destInput = "bucketIdxIn" ) network . link ( _RECORD_SENSOR , _L1_CLASSIFIER , linkType , linkParams , srcOutput = "actValueOut" , destInput = "actValueIn" ) l2inputWidth = l1temporalMemory . getSelf ( ) . getOutputElementCount ( "bottomUpOut" ) createSpatialPooler ( network , name = _L2_SPATIAL_POOLER , inputWidth = l2inputWidth ) network . link ( _L1_TEMPORAL_MEMORY , _L2_SPATIAL_POOLER , linkType , linkParams ) createTemporalMemory ( network , _L2_TEMPORAL_MEMORY ) network . link ( _L2_SPATIAL_POOLER , _L2_TEMPORAL_MEMORY , linkType , linkParams ) l2Classifier = network . addRegion ( _L2_CLASSIFIER , "py.SDRClassifierRegion" , json . dumps ( classifierParams ) ) l2Classifier . setParameter ( 'inferenceMode' , True ) l2Classifier . setParameter ( 'learningMode' , True ) network . link ( _L2_TEMPORAL_MEMORY , _L2_CLASSIFIER , linkType , linkParams , srcOutput = "bottomUpOut" , destInput = "bottomUpIn" ) network . link ( _RECORD_SENSOR , _L2_CLASSIFIER , linkType , linkParams , srcOutput = "categoryOut" , destInput = "categoryIn" ) network . link ( _RECORD_SENSOR , _L2_CLASSIFIER , linkType , linkParams , srcOutput = "bucketIdxOut" , destInput = "bucketIdxIn" ) network . link ( _RECORD_SENSOR , _L2_CLASSIFIER , linkType , linkParams , srcOutput = "actValueOut" , destInput = "actValueIn" ) return network

Creates and returns a new Network with a sensor region reading data from dataSource . There are two hierarchical levels each with one SP and one TM .


def runNetwork(network, numRecords, writer):
  """
  Runs specified Network writing the ensuing anomaly
  scores to writer.

  @param network: The Network instance to be run
  @param writer: A csv.writer used to write to output file.
  """
  sensorRegion = network.regions[_RECORD_SENSOR]
  l1SpRegion = network.regions[_L1_SPATIAL_POOLER]
  l1TpRegion = network.regions[_L1_TEMPORAL_MEMORY]
  l1Classifier = network.regions[_L1_CLASSIFIER]

  l2SpRegion = network.regions[_L2_SPATIAL_POOLER]
  l2TpRegion = network.regions[_L2_TEMPORAL_MEMORY]
  l2Classifier = network.regions[_L2_CLASSIFIER]

  l1PreviousPredictedColumns = []
  l2PreviousPredictedColumns = []

  l1PreviousPrediction = None
  l2PreviousPrediction = None
  l1ErrorSum = 0.0
  l2ErrorSum = 0.0
  for record in xrange(numRecords):
    # Run the network for a single iteration
    network.run(1)

    actual = float(sensorRegion.getOutputData("actValueOut")[0])

    l1Predictions = l1Classifier.getOutputData("actualValues")
    l1Probabilities = l1Classifier.getOutputData("probabilities")
    l1Prediction = l1Predictions[l1Probabilities.argmax()]
    if l1PreviousPrediction is not None:
      l1ErrorSum += math.fabs(l1PreviousPrediction - actual)
    l1PreviousPrediction = l1Prediction

    l2Predictions = l2Classifier.getOutputData("actualValues")
    l2Probabilities = l2Classifier.getOutputData("probabilities")
    l2Prediction = l2Predictions[l2Probabilities.argmax()]
    if l2PreviousPrediction is not None:
      l2ErrorSum += math.fabs(l2PreviousPrediction - actual)
    l2PreviousPrediction = l2Prediction

    l1AnomalyScore = l1TpRegion.getOutputData("anomalyScore")[0]
    l2AnomalyScore = l2TpRegion.getOutputData("anomalyScore")[0]

    # Write record number, actualInput, and anomaly scores
    writer.writerow((record, actual, l1PreviousPrediction, l1AnomalyScore, l2PreviousPrediction, l2AnomalyScore))

    # Store the predicted columns for the next timestep
    l1PredictedColumns = l1TpRegion.getOutputData("topDownOut").nonzero()[0]
    l1PreviousPredictedColumns = copy.deepcopy(l1PredictedColumns)
    #
    l2PredictedColumns = l2TpRegion.getOutputData("topDownOut").nonzero()[0]
    l2PreviousPredictedColumns = copy.deepcopy(l2PredictedColumns)

  # Output absolute average error for each level
  if numRecords > 1:
    print "L1 ave abs class. error: %f" % (l1ErrorSum / (numRecords - 1))
    print "L2 ave abs class. error: %f" % (l2ErrorSum / (numRecords - 1))

def
runNetwork
(
network
,
numRecords
,
writer
)
:
sensorRegion
=
network
.
regions
[
_RECORD_SENSOR
]
l1SpRegion
=
network
.
regions
[
_L1_SPATIAL_POOLER
]
l1TpRegion
=
network
.
regions
[
_L1_TEMPORAL_MEMORY
]
l1Classifier
=
network
.
regions
[
_L1_CLASSIFIER
]
l2SpRegion
=
network
.
regions
[
_L2_SPATIAL_POOLER
]
l2TpRegion
=
network
.
regions
[
_L2_TEMPORAL_MEMORY
]
l2Classifier
=
network
.
regions
[
_L2_CLASSIFIER
]
l1PreviousPredictedColumns
=
[
]
l2PreviousPredictedColumns
=
[
]
l1PreviousPrediction
=
l2PreviousPrediction
=
l1ErrorSum
=
0.0
l2ErrorSum
=
0.0
for
record
in
xrange
(
numRecords
)
:
network
.
run
(
1
)
actual
=
float
(
sensorRegion
.
getOutputData
(
"actValueOut"
)
[
0
]
)
l1Predictions
=
l1Classifier
.
getOutputData
(
"actualValues"
)
l1Probabilities
=
l1Classifier
.
getOutputData
(
"probabilities"
)
l1Prediction
=
l1Predictions
[
l1Probabilities
.
argmax
(
)
]
if
l1PreviousPrediction
is
not
:
l1ErrorSum
+=
math
.
fabs
(
l1PreviousPrediction
actual
)
l1PreviousPrediction
=
l1Prediction
l2Predictions
=
l2Classifier
.
getOutputData
(
"actualValues"
)
l2Probabilities
=
l2Classifier
.
getOutputData
(
"probabilities"
)
l2Prediction
=
l2Predictions
[
l2Probabilities
.
argmax
(
)
]
if
l2PreviousPrediction
is
not
:
l2ErrorSum
+=
math
.
fabs
(
l2PreviousPrediction
actual
)
l2PreviousPrediction
=
l2Prediction
l1AnomalyScore
=
l1TpRegion
.
getOutputData
(
"anomalyScore"
)
[
0
]
l2AnomalyScore
=
l2TpRegion
.
getOutputData
(
"anomalyScore"
)
[
0
]
writer
.
writerow
(
(
record
,
actual
,
l1PreviousPrediction
,
l1AnomalyScore
,
l2PreviousPrediction
,
l2AnomalyScore
)
)
l1PredictedColumns
=
l1TpRegion
.
getOutputData
(
"topDownOut"
)
.
nonzero
(
)
[
0
]
l1PreviousPredictedColumns
=
copy
.
deepcopy
(
l1PredictedColumns
)
l2PredictedColumns
=
l2TpRegion
.
getOutputData
(
"topDownOut"
)
.
nonzero
(
)
[
0
]
l2PreviousPredictedColumns
=
copy
.
deepcopy
(
l2PredictedColumns
)
if
numRecords
>
1
:
print
"L1 ave abs class. error: %f"
%
(
l1ErrorSum
/
(
numRecords
1
)
)
print
"L2 ave abs class. error: %f"
%
(
l2ErrorSum
/
(
numRecords
1
)
)

Runs specified Network writing the ensuing anomaly scores to writer .


def clean(s):
  """Removes trailing whitespace on each line."""
  lines = [l.rstrip() for l in s.split('\n')]
  return '\n'.join(lines)

def clean ( s ) : lines = [ l . rstrip ( ) for l in s . split ( '\n' ) ] return '\n' . join ( lines )

Removes trailing whitespace on each line .


def getMetrics(self):
    """
    Gets the current metric values

    :returns: (dict) where each key is the metric-name, and the values are
              it scalar value. Same as the output of
              :meth:`~nupic.frameworks.opf.prediction_metrics_manager.MetricsManager.update`
    """

    result = {}

    for metricObj, label in zip(self.__metrics, self.__metricLabels):
      value = metricObj.getMetric()
      result[label] = value['value']

    return result

def getMetrics ( self ) : result = { } for metricObj , label in zip ( self . __metrics , self . __metricLabels ) : value = metricObj . getMetric ( ) result [ label ] = value [ 'value' ] return result


def getMetricDetails(self, metricLabel):
    """
    Gets detailed info about a given metric, in addition to its value. This
    may including any statistics or auxilary data that are computed for a given
    metric.

    :param metricLabel: (string) label of the given metric (see
           :class:`~nupic.frameworks.opf.metrics.MetricSpec`)

    :returns: (dict) of metric information, as returned by
             :meth:`nupic.frameworks.opf.metrics.MetricsIface.getMetric`.
    """
    try:
      metricIndex = self.__metricLabels.index(metricLabel)
    except IndexError:
      return None

    return self.__metrics[metricIndex].getMetric()

def getMetricDetails ( self , metricLabel ) : try : metricIndex = self . __metricLabels . index ( metricLabel ) except IndexError : return None return self . __metrics [ metricIndex ] . getMetric ( )

Gets detailed info about a given metric in addition to its value . This may including any statistics or auxilary data that are computed for a given metric .


def _addResults(self, results):
    """
    Stores the current model results in the manager's internal store

    Parameters:
    -----------------------------------------------------------------------
    results:  A ModelResults object that contains the current timestep's
              input/inferences
    """
    # -----------------------------------------------------------------------
    # If the model potentially has temporal inferences.
    if self.__isTemporal:
      shiftedInferences = self.__inferenceShifter.shift(results).inferences
      self.__currentResult = copy.deepcopy(results)
      self.__currentResult.inferences = shiftedInferences
      self.__currentInference = shiftedInferences

    # -----------------------------------------------------------------------
    # The current model has no temporal inferences.
    else:
      self.__currentResult = copy.deepcopy(results)
      self.__currentInference = copy.deepcopy(results.inferences)

    # -----------------------------------------------------------------------
    # Save the current ground-truth results
    self.__currentGroundTruth = copy.deepcopy(results)

def _addResults ( self , results ) : if self . __isTemporal : shiftedInferences = self . __inferenceShifter . shift ( results ) . inferences self . __currentResult = copy . deepcopy ( results ) self . __currentResult . inferences = shiftedInferences self . __currentInference = shiftedInferences else : self . __currentResult = copy . deepcopy ( results ) self . __currentInference = copy . deepcopy ( results . inferences ) self . __currentGroundTruth = copy . deepcopy ( results )

Stores the current model results in the manager s internal store


def _getGroundTruth(self, inferenceElement):
    """
    Get the actual value for this field

    Parameters:
    -----------------------------------------------------------------------
    sensorInputElement:       The inference element (part of the inference) that
                            is being used for this metric
    """
    sensorInputElement = InferenceElement.getInputElement(inferenceElement)
    if sensorInputElement is None:
      return None
    return getattr(self.__currentGroundTruth.sensorInput, sensorInputElement)

def _getGroundTruth ( self , inferenceElement ) : sensorInputElement = InferenceElement . getInputElement ( inferenceElement ) if sensorInputElement is None : return None return getattr ( self . __currentGroundTruth . sensorInput , sensorInputElement )


def __constructMetricsModules(self, metricSpecs):
    """
    Creates the required metrics modules

    Parameters:
    -----------------------------------------------------------------------
    metricSpecs:
      A sequence of MetricSpec objects that specify which metric modules to
      instantiate
    """
    if not metricSpecs:
      return

    self.__metricSpecs = metricSpecs
    for spec in metricSpecs:
      if not InferenceElement.validate(spec.inferenceElement):
        raise ValueError("Invalid inference element for metric spec: %r" %spec)

      self.__metrics.append(metrics.getModule(spec))
      self.__metricLabels.append(spec.getLabel())


def shift(self, modelResult):
    """Shift the model result and return the new instance.

    Queues up the T(i+1) prediction value and emits a T(i)
    input/prediction pair, if possible. E.g., if the previous T(i-1)
    iteration was learn-only, then we would not have a T(i) prediction in our
    FIFO and would not be able to emit a meaningful input/prediction pair.

    :param modelResult: A :class:`~.nupic.frameworks.opf.opf_utils.ModelResult`
                        instance to shift.
    :return: A :class:`~.nupic.frameworks.opf.opf_utils.ModelResult` instance that
             has been shifted
    """
    inferencesToWrite = {}

    if self._inferenceBuffer is None:
      maxDelay = InferenceElement.getMaxDelay(modelResult.inferences)
      self._inferenceBuffer = collections.deque(maxlen=maxDelay + 1)

    self._inferenceBuffer.appendleft(copy.deepcopy(modelResult.inferences))

    for inferenceElement, inference in modelResult.inferences.iteritems():
      if isinstance(inference, dict):
        inferencesToWrite[inferenceElement] = {}
        for key, _ in inference.iteritems():
          delay = InferenceElement.getTemporalDelay(inferenceElement, key)
          if len(self._inferenceBuffer) > delay:
            prevInference = self._inferenceBuffer[delay][inferenceElement][key]
            inferencesToWrite[inferenceElement][key] = prevInference
          else:
            inferencesToWrite[inferenceElement][key] = None
      else:
        delay = InferenceElement.getTemporalDelay(inferenceElement)
        if len(self._inferenceBuffer) > delay:
          inferencesToWrite[inferenceElement] = (
              self._inferenceBuffer[delay][inferenceElement])
        else:
          if type(inference) in (list, tuple):
            inferencesToWrite[inferenceElement] = [None] * len(inference)
          else:
            inferencesToWrite[inferenceElement] = None

    shiftedResult = ModelResult(rawInput=modelResult.rawInput,
                                sensorInput=modelResult.sensorInput,
                                inferences=inferencesToWrite,
                                metrics=modelResult.metrics,
                                predictedFieldIdx=modelResult.predictedFieldIdx,
                                predictedFieldName=modelResult.predictedFieldName)
    return shiftedResult

def shift ( self , modelResult ) : inferencesToWrite = { } if self . _inferenceBuffer is None : maxDelay = InferenceElement . getMaxDelay ( modelResult . inferences ) self . _inferenceBuffer = collections . deque ( maxlen = maxDelay + 1 ) self . _inferenceBuffer . appendleft ( copy . deepcopy ( modelResult . inferences ) ) for inferenceElement , inference in modelResult . inferences . iteritems ( ) : if isinstance ( inference , dict ) : inferencesToWrite [ inferenceElement ] = { } for key , _ in inference . iteritems ( ) : delay = InferenceElement . getTemporalDelay ( inferenceElement , key ) if len ( self . _inferenceBuffer ) > delay : prevInference = self . _inferenceBuffer [ delay ] [ inferenceElement ] [ key ] inferencesToWrite [ inferenceElement ] [ key ] = prevInference else : inferencesToWrite [ inferenceElement ] [ key ] = None else : delay = InferenceElement . getTemporalDelay ( inferenceElement ) if len ( self . _inferenceBuffer ) > delay : inferencesToWrite [ inferenceElement ] = ( self . _inferenceBuffer [ delay ] [ inferenceElement ] ) else : if type ( inference ) in ( list , tuple ) : inferencesToWrite [ inferenceElement ] = [ None ] * len ( inference ) else : inferencesToWrite [ inferenceElement ] = None shiftedResult = ModelResult ( rawInput = modelResult . rawInput , sensorInput = modelResult . sensorInput , inferences = inferencesToWrite , metrics = modelResult . metrics , predictedFieldIdx = modelResult . predictedFieldIdx , predictedFieldName = modelResult . predictedFieldName ) return shiftedResult

Shift the model result and return the new instance .


def generateStats(filename, maxSamples = None,):
  """
  Collect statistics for each of the fields in the user input data file and
  return a stats dict object.

  Parameters:
  ------------------------------------------------------------------------------
  filename:             The path and name of the data file.
  maxSamples:           Upper bound on the number of rows to be processed
  retval:               A dictionary of dictionaries. The top level keys are the
                        field names and the corresponding values are the statistics
                        collected for the individual file.
                        Example:
                        {
                          'consumption':{'min':0,'max':90,'mean':50,...},
                          'gym':{'numDistinctCategories':10,...},
                          ...
                         }

  """
  # Mapping from field type to stats collector object
  statsCollectorMapping = {'float':    FloatStatsCollector,
                           'int':      IntStatsCollector,
                           'string':   StringStatsCollector,
                           'datetime': DateTimeStatsCollector,
                           'bool':     BoolStatsCollector,
                           }

  filename = resource_filename("nupic.datafiles", filename)
  print "*"*40
  print "Collecting statistics for file:'%s'" % (filename,)
  dataFile = FileRecordStream(filename)

  # Initialize collector objects
  # statsCollectors list holds statsCollector objects for each field
  statsCollectors = []
  for fieldName, fieldType, fieldSpecial in dataFile.getFields():
    # Find the corresponding stats collector for each field based on field type
    # and intialize an instance
    statsCollector = \
            statsCollectorMapping[fieldType](fieldName, fieldType, fieldSpecial)
    statsCollectors.append(statsCollector)

  # Now collect the stats
  if maxSamples is None:
    maxSamples = 500000
  for i in xrange(maxSamples):
    record = dataFile.getNextRecord()
    if record is None:
      break
    for i, value in enumerate(record):
      statsCollectors[i].addValue(value)

  # stats dict holds the statistics for each field
  stats = {}
  for statsCollector in statsCollectors:
    statsCollector.getStats(stats)

  # We don't want to include reset field in permutations
  # TODO: handle reset field in a clean way
  if dataFile.getResetFieldIdx() is not None:
    resetFieldName,_,_ = dataFile.getFields()[dataFile.reset]
    stats.pop(resetFieldName)

  if VERBOSITY > 0:
    pprint.pprint(stats)

  return stats

def
generateStats
(
filename
,
maxSamples
=
,
)
:
statsCollectorMapping
=
{
'float'
:
FloatStatsCollector
,
'int'
:
IntStatsCollector
,
'string'
:
StringStatsCollector
,
'datetime'
:
DateTimeStatsCollector
,
'bool'
:
BoolStatsCollector
,
}
filename
=
resource_filename
(
"nupic.datafiles"
,
filename
)
print
"*"
*
40
print
"Collecting statistics for file:'%s'"
%
(
filename
,
)
dataFile
=
FileRecordStream
(
filename
)
statsCollectors
=
[
]
for
fieldName
,
fieldType
,
fieldSpecial
in
dataFile
.
getFields
(
)
:
statsCollector
=
statsCollectorMapping
[
fieldType
]
(
fieldName
,
fieldType
,
fieldSpecial
)
statsCollectors
.
append
(
statsCollector
)
if
maxSamples
is
:
maxSamples
=
500000
for
i
in
xrange
(
maxSamples
)
:
record
=
dataFile
.
getNextRecord
(
)
if
record
is
:
break
for
i
,
value
in
enumerate
(
record
)
:
statsCollectors
[
i
]
.
addValue
(
value
)
stats
=
{
}
for
statsCollector
in
statsCollectors
:
statsCollector
.
getStats
(
stats
)
if
dataFile
.
getResetFieldIdx
(
)
is
not
:
resetFieldName
,
_
,
_
=
dataFile
.
getFields
(
)
[
dataFile
.
reset
]
stats
.
pop
(
resetFieldName
)
if
VERBOSITY
>
0
:
pprint
.
pprint
(
stats
)
return
stats

Collect statistics for each of the fields in the user input data file and return a stats dict object .


def main():
  """Run according to options in sys.argv and diff classifiers."""
  initLogging(verbose=True)

  # Initialize PRNGs
  initExperimentPrng()

  # Mock out the creation of the SDRClassifier.
  @staticmethod
  def _mockCreate(*args, **kwargs):
    kwargs.pop('implementation', None)
    return SDRClassifierDiff(*args, **kwargs)
  SDRClassifierFactory.create = _mockCreate

  # Run it!
  runExperiment(sys.argv[1:])

def main ( ) : initLogging ( verbose = True ) initExperimentPrng ( ) @ staticmethod def _mockCreate ( * args , ** kwargs ) : kwargs . pop ( 'implementation' , None ) return SDRClassifierDiff ( * args , ** kwargs ) SDRClassifierFactory . create = _mockCreate runExperiment ( sys . argv [ 1 : ] )

Run according to options in sys . argv and diff classifiers .


def _abbreviate(text, threshold):
  """ Abbreviate the given text to threshold chars and append an ellipsis if its
  length exceeds threshold; used for logging;

  NOTE: the resulting text could be longer than threshold due to the ellipsis
  """
  if text is not None and len(text) > threshold:
    text = text[:threshold] + "..."

  return text

def _abbreviate ( text , threshold ) : if text is not None and len ( text ) > threshold : text = text [ : threshold ] + "..." return text

Abbreviate the given text to threshold chars and append an ellipsis if its length exceeds threshold ; used for logging ;


def __getDBNameForVersion(cls, dbVersion):
    """ Generates the ClientJobs database name for the given version of the
    database

    Parameters:
    ----------------------------------------------------------------
    dbVersion:      ClientJobs database version number

    retval:         the ClientJobs database name for the given DB version
    """

    # DB Name prefix for the given version
    prefix = cls.__getDBNamePrefixForVersion(dbVersion)

    # DB Name suffix
    suffix = Configuration.get('nupic.cluster.database.nameSuffix')

    # Replace dash and dot with underscore (e.g. 'ec2-user' or ec2.user will break SQL)
    suffix = suffix.replace("-", "_")
    suffix = suffix.replace(".", "_")

    # Create the name of the database for the given DB version
    dbName = '%s_%s' % (prefix, suffix)

    return dbName

def __getDBNameForVersion ( cls , dbVersion ) : prefix = cls . __getDBNamePrefixForVersion ( dbVersion ) suffix = Configuration . get ( 'nupic.cluster.database.nameSuffix' ) suffix = suffix . replace ( "-" , "_" ) suffix = suffix . replace ( "." , "_" ) dbName = '%s_%s' % ( prefix , suffix ) return dbName


def connect(self, deleteOldVersions=False, recreate=False):
    """ Locate the current version of the jobs DB or create a new one, and
    optionally delete old versions laying around. If desired, this method
    can be called at any time to re-create the tables from scratch, delete
    old versions of the database, etc.

    Parameters:
    ----------------------------------------------------------------
    deleteOldVersions:   if true, delete any old versions of the DB left
                          on the server
    recreate:            if true, recreate the database from scratch even
                          if it already exists.
    """

    # Initialize tables, if needed
    with ConnectionFactory.get() as conn:
      # Initialize tables
      self._initTables(cursor=conn.cursor, deleteOldVersions=deleteOldVersions,
                       recreate=recreate)

      # Save our connection id
      conn.cursor.execute('SELECT CONNECTION_ID()')
      self._connectionID = conn.cursor.fetchall()[0][0]
      self._logger.info("clientJobsConnectionID=%r", self._connectionID)

    return

def
connect
(
self
,
deleteOldVersions
=
False
,
recreate
=
False
)
:
with
ConnectionFactory
.
get
(
)
as
conn
:
self
.
_initTables
(
cursor
=
conn
.
cursor
,
deleteOldVersions
=
deleteOldVersions
,
recreate
=
recreate
)
conn
.
cursor
.
execute
(
'SELECT CONNECTION_ID()'
)
self
.
_connectionID
=
conn
.
cursor
.
fetchall
(
)
[
0
]
[
0
]
self
.
_logger
.
info
(
"clientJobsConnectionID=%r"
,
self
.
_connectionID
)
return

Locate the current version of the jobs DB or create a new one and optionally delete old versions laying around . If desired this method can be called at any time to re - create the tables from scratch delete old versions of the database etc .


def _getMatchingRowsNoRetries(self, tableInfo, conn, fieldsToMatch,
                                selectFieldNames, maxRows=None):
    """ Return a sequence of matching rows with the requested field values from
    a table or empty sequence if nothing matched.

    tableInfo:       Table information: a ClientJobsDAO._TableInfoBase  instance
    conn:            Owned connection acquired from ConnectionFactory.get()
    fieldsToMatch:   Dictionary of internal fieldName/value mappings that
                     identify the desired rows. If a value is an instance of
                     ClientJobsDAO._SEQUENCE_TYPES (list/set/tuple), then the
                     operator 'IN' will be used in the corresponding SQL
                     predicate; if the value is bool: "IS TRUE/FALSE"; if the
                     value is None: "IS NULL"; '=' will be used for all other
                     cases.
    selectFieldNames:
                     list of fields to return, using internal field names
    maxRows:         maximum number of rows to return; unlimited if maxRows
                      is None

    retval:          A sequence of matching rows, each row consisting of field
                      values in the order of the requested field names.  Empty
                      sequence is returned when not match exists.
    """

    assert fieldsToMatch, repr(fieldsToMatch)
    assert all(k in tableInfo.dbFieldNames
               for k in fieldsToMatch.iterkeys()), repr(fieldsToMatch)

    assert selectFieldNames, repr(selectFieldNames)
    assert all(f in tableInfo.dbFieldNames for f in selectFieldNames), repr(
      selectFieldNames)

    # NOTE: make sure match expressions and values are in the same order
    matchPairs = fieldsToMatch.items()
    matchExpressionGen = (
      p[0] +
      (' IS ' + {True:'TRUE', False:'FALSE'}[p[1]] if isinstance(p[1], bool)
       else ' IS NULL' if p[1] is None
       else ' IN %s' if isinstance(p[1], self._SEQUENCE_TYPES)
       else '=%s')
      for p in matchPairs)
    matchFieldValues = [p[1] for p in matchPairs
                        if (not isinstance(p[1], (bool)) and p[1] is not None)]

    query = 'SELECT %s FROM %s WHERE (%s)' % (
      ','.join(selectFieldNames), tableInfo.tableName,
      ' AND '.join(matchExpressionGen))
    sqlParams = matchFieldValues
    if maxRows is not None:
      query += ' LIMIT %s'
      sqlParams.append(maxRows)

    conn.cursor.execute(query, sqlParams)
    rows = conn.cursor.fetchall()

    if rows:
      assert maxRows is None or len(rows) <= maxRows, "%d !<= %d" % (
        len(rows), maxRows)
      assert len(rows[0]) == len(selectFieldNames), "%d != %d" % (
        len(rows[0]), len(selectFieldNames))
    else:
      rows = tuple()

    return rows

def
_getMatchingRowsNoRetries
(
self
,
tableInfo
,
conn
,
fieldsToMatch
,
selectFieldNames
,
maxRows
=
)
:
assert
fieldsToMatch
,
repr
(
fieldsToMatch
)
assert
all
(
k
in
tableInfo
.
dbFieldNames
for
k
in
fieldsToMatch
.
iterkeys
(
)
)
,
repr
(
fieldsToMatch
)
assert
selectFieldNames
,
repr
(
selectFieldNames
)
assert
all
(
f
in
tableInfo
.
dbFieldNames
for
f
in
selectFieldNames
)
,
repr
(
selectFieldNames
)
matchPairs
=
fieldsToMatch
.
items
(
)
matchExpressionGen
=
(
p
[
0
]
+
(
' IS '
+
{
True
:
'TRUE'
,
False
:
'FALSE'
}
[
p
[
1
]
]
if
isinstance
(
p
[
1
]
,
bool
)
else
' IS NULL'
if
p
[
1
]
is
else
' IN %s'
if
isinstance
(
p
[
1
]
,
self
.
_SEQUENCE_TYPES
)
else
'=%s'
)
for
p
in
matchPairs
)
matchFieldValues
=
[
p
[
1
]
for
p
in
matchPairs
if
(
not
isinstance
(
p
[
1
]
,
(
bool
)
)
and
p
[
1
]
is
not
)
]
query
=
'SELECT %s FROM %s WHERE (%s)'
%
(
','
.
join
(
selectFieldNames
)
,
tableInfo
.
tableName
,
' AND '
.
join
(
matchExpressionGen
)
)
sqlParams
=
matchFieldValues
if
maxRows
is
not
:
query
+=
' LIMIT %s'
sqlParams
.
append
(
maxRows
)
conn
.
cursor
.
execute
(
query
,
sqlParams
)
rows
=
conn
.
cursor
.
fetchall
(
)
if
rows
:
assert
maxRows
is
or
len
(
rows
)
<=
maxRows
,
"%d !<= %d"
%
(
len
(
rows
)
,
maxRows
)
assert
len
(
rows
[
0
]
)
==
len
(
selectFieldNames
)
,
"%d != %d"
%
(
len
(
rows
[
0
]
)
,
len
(
selectFieldNames
)
)
else
:
rows
=
tuple
(
)
return
rows

Return a sequence of matching rows with the requested field values from a table or empty sequence if nothing matched .


def _getOneMatchingRowNoRetries(self, tableInfo, conn, fieldsToMatch,
                                  selectFieldNames):
    """ Return a single matching row with the requested field values from the
    the requested table or None if nothing matched.

    tableInfo:       Table information: a ClientJobsDAO._TableInfoBase  instance
    conn:            Owned connection acquired from ConnectionFactory.get()
    fieldsToMatch:   Dictionary of internal fieldName/value mappings that
                     identify the desired rows. If a value is an instance of
                     ClientJobsDAO._SEQUENCE_TYPES (list/set/tuple), then the
                     operator 'IN' will be used in the corresponding SQL
                     predicate; if the value is bool: "IS TRUE/FALSE"; if the
                     value is None: "IS NULL"; '=' will be used for all other
                     cases.
    selectFieldNames:
                     list of fields to return, using internal field names

    retval:          A sequence of field values of the matching row in the order
                      of the given field names; or None if there was no match.
    """
    rows = self._getMatchingRowsNoRetries(tableInfo, conn, fieldsToMatch,
                                          selectFieldNames, maxRows=1)
    if rows:
      assert len(rows) == 1, repr(len(rows))
      result = rows[0]
    else:
      result = None

    return result

def _getOneMatchingRowNoRetries ( self , tableInfo , conn , fieldsToMatch , selectFieldNames ) : rows = self . _getMatchingRowsNoRetries ( tableInfo , conn , fieldsToMatch , selectFieldNames , maxRows = 1 ) if rows : assert len ( rows ) == 1 , repr ( len ( rows ) ) result = rows [ 0 ] else : result = None return result

Return a single matching row with the requested field values from the the requested table or None if nothing matched .


def jobInsert(self, client, cmdLine, clientInfo='', clientKey='', params='',
                alreadyRunning=False, minimumWorkers=0, maximumWorkers=0,
                jobType='', priority=DEFAULT_JOB_PRIORITY):
    """ Add an entry to the jobs table for a new job request. This is called by
    clients that wish to startup a new job, like a Hypersearch, stream job, or
    specific model evaluation from the engine.

    This puts a new entry into the jobs table. The CJM is always periodically
    sweeping the jobs table and when it finds a new job, will proceed to start it
    up on Hadoop.

    Parameters:
    ----------------------------------------------------------------
    client:          Name of the client submitting the job
    cmdLine:         Command line to use to launch each worker process; must be
                      a non-empty string
    clientInfo:      JSON encoded dict of client specific information.
    clientKey:       Foreign key.
    params:          JSON encoded dict of the parameters for the job. This
                      can be fetched out of the database by the worker processes
                      based on the jobID.
    alreadyRunning:  Used for unit test purposes only. This inserts the job
                      in the running state. It is used when running a worker
                      in standalone mode without hadoop - it gives it a job
                      record to work with.
    minimumWorkers:  minimum number of workers design at a time.
    maximumWorkers:  maximum number of workers desired at a time.
    jobType:         The type of job that this is. This should be one of the
                      JOB_TYPE_XXXX enums. This is needed to allow a standard
                      way of recognizing a job's function and capabilities.
    priority:        Job scheduling priority; 0 is the default priority (
                      ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are
                      higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY),
                      and negative values are lower priority (down to
                      ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will
                      be scheduled to run at the expense of the lower-priority
                      jobs, and higher-priority job tasks will preempt those
                      with lower priority if there is inadequate supply of
                      scheduling slots. Excess lower priority job tasks will
                      starve as long as slot demand exceeds supply. Most jobs
                      should be scheduled with DEFAULT_JOB_PRIORITY. System jobs
                      that must run at all cost, such as Multi-Model-Master,
                      should be scheduled with MAX_JOB_PRIORITY.

    retval:          jobID - unique ID assigned to this job
    """

    jobHash = self._normalizeHash(uuid.uuid1().bytes)

    @g_retrySQL
    def insertWithRetries():
      with ConnectionFactory.get() as conn:
        return self._insertOrGetUniqueJobNoRetries(
          conn, client=client, cmdLine=cmdLine, jobHash=jobHash,
          clientInfo=clientInfo, clientKey=clientKey, params=params,
          minimumWorkers=minimumWorkers, maximumWorkers=maximumWorkers,
          jobType=jobType, priority=priority, alreadyRunning=alreadyRunning)

    try:
      jobID = insertWithRetries()
    except:
      self._logger.exception(
        'jobInsert FAILED: jobType=%r; client=%r; clientInfo=%r; clientKey=%r;'
        'jobHash=%r; cmdLine=%r',
        jobType, client, _abbreviate(clientInfo, 48), clientKey, jobHash,
        cmdLine)
      raise
    else:
      self._logger.info(
        'jobInsert: returning jobID=%s. jobType=%r; client=%r; clientInfo=%r; '
        'clientKey=%r; jobHash=%r; cmdLine=%r',
        jobID, jobType, client, _abbreviate(clientInfo, 48), clientKey,
        jobHash, cmdLine)

    return jobID

Add an entry to the jobs table for a new job request . This is called by clients that wish to startup a new job like a Hypersearch stream job or specific model evaluation from the engine .


def _startJobWithRetries(self, jobID):
    """ Place the given job in STATUS_RUNNING mode; the job is expected to be
    STATUS_NOTSTARTED.

    NOTE: this function was factored out of jobStartNext because it's also
     needed for testing (e.g., test_client_jobs_dao.py)
    """
    with ConnectionFactory.get() as conn:
      query = 'UPDATE %s SET status=%%s, ' \
                '            _eng_cjm_conn_id=%%s, ' \
                '            start_time=UTC_TIMESTAMP(), ' \
                '            _eng_last_update_time=UTC_TIMESTAMP() ' \
                '          WHERE (job_id=%%s AND status=%%s)' \
                % (self.jobsTableName,)
      sqlParams = [self.STATUS_RUNNING, self._connectionID,
                   jobID, self.STATUS_NOTSTARTED]
      numRowsUpdated = conn.cursor.execute(query, sqlParams)
      if numRowsUpdated != 1:
        self._logger.warn('jobStartNext: numRowsUpdated=%r instead of 1; '
                          'likely side-effect of transient connection '
                          'failure', numRowsUpdated)
    return

'            start_time=UTC_TIMESTAMP(), '

'          WHERE (job_id=%%s AND status=%%s)'

Place the given job in STATUS_RUNNING mode ; the job is expected to be STATUS_NOTSTARTED .


def jobReactivateRunningJobs(self):
    """ Look through the jobs table and reactivate all that are already in the
    running state by setting their _eng_allocate_new_workers fields to True;
    used by Nupic Scheduler as part of its failure-recovery procedure.
    """

    # Get a database connection and cursor
    with ConnectionFactory.get() as conn:

      query = 'UPDATE %s SET _eng_cjm_conn_id=%%s, ' \
              '              _eng_allocate_new_workers=TRUE ' \
              '    WHERE status=%%s ' \
              % (self.jobsTableName,)
      conn.cursor.execute(query, [self._connectionID, self.STATUS_RUNNING])

    return

'              _eng_allocate_new_workers=TRUE '

Look through the jobs table and reactivate all that are already in the running state by setting their _eng_allocate_new_workers fields to True ; used by Nupic Scheduler as part of its failure - recovery procedure .


def jobGetDemand(self,):
    """ Look through the jobs table and get the demand - minimum and maximum
    number of workers requested, if new workers are to be allocated, if there
    are any untended dead workers, for all running jobs.

    Parameters:
    ----------------------------------------------------------------
    retval:      list of ClientJobsDAO._jobs.jobDemandNamedTuple nametuples
                  containing the demand - min and max workers,
                  allocate_new_workers, untended_dead_workers, num_failed_workers
                  for each running (STATUS_RUNNING) job. Empty list when there
                  isn't any demand.

    """
    rows = self._getMatchingRowsWithRetries(
      self._jobs, dict(status=self.STATUS_RUNNING),
      [self._jobs.pubToDBNameDict[f]
       for f in self._jobs.jobDemandNamedTuple._fields])

    return [self._jobs.jobDemandNamedTuple._make(r) for r in rows]

def jobGetDemand ( self , ) : rows = self . _getMatchingRowsWithRetries ( self . _jobs , dict ( status = self . STATUS_RUNNING ) , [ self . _jobs . pubToDBNameDict [ f ] for f in self . _jobs . jobDemandNamedTuple . _fields ] ) return [ self . _jobs . jobDemandNamedTuple . _make ( r ) for r in rows ]

Look through the jobs table and get the demand - minimum and maximum number of workers requested if new workers are to be allocated if there are any untended dead workers for all running jobs .


def jobCancelAllRunningJobs(self):
    """ Set cancel field of all currently-running jobs to true.
    """

    # Get a database connection and cursor
    with ConnectionFactory.get() as conn:

      query = 'UPDATE %s SET cancel=TRUE WHERE status<>%%s ' \
              % (self.jobsTableName,)
      conn.cursor.execute(query, [self.STATUS_COMPLETED])

    return

Set cancel field of all currently - running jobs to true .


def jobCountCancellingJobs(self,):
    """ Look through the jobs table and count the running jobs whose
    cancel field is true.

    Parameters:
    ----------------------------------------------------------------
    retval:      A count of running jobs with the cancel field set to true.
    """
    with ConnectionFactory.get() as conn:
      query = 'SELECT COUNT(job_id) '\
              'FROM %s ' \
              'WHERE (status<>%%s AND cancel is TRUE)' \
              % (self.jobsTableName,)

      conn.cursor.execute(query, [self.STATUS_COMPLETED])
      rows = conn.cursor.fetchall()

    return rows[0][0]

Look through the jobs table and count the running jobs whose cancel field is true .


def jobGetCancellingJobs(self,):
    """ Look through the jobs table and get the list of running jobs whose
    cancel field is true.

    Parameters:
    ----------------------------------------------------------------
    retval:      A (possibly empty) sequence of running job IDs with cancel field
                  set to true
    """
    with ConnectionFactory.get() as conn:
      query = 'SELECT job_id '\
              'FROM %s ' \
              'WHERE (status<>%%s AND cancel is TRUE)' \
              % (self.jobsTableName,)
      conn.cursor.execute(query, [self.STATUS_COMPLETED])
      rows = conn.cursor.fetchall()

    return tuple(r[0] for r in rows)

Look through the jobs table and get the list of running jobs whose cancel field is true .


def partitionAtIntervals(data, intervals):
    """ Generator to allow iterating slices at dynamic intervals

    Parameters:
    ----------------------------------------------------------------
    data:       Any data structure that supports slicing (i.e. list or tuple)
    *intervals: Iterable of intervals.  The sum of intervals should be less
                than, or equal to the length of data.

    """
    assert sum(intervals) <= len(data)

    start = 0
    for interval in intervals:
      end = start + interval
      yield data[start:end]
      start = end

    raise StopIteration

def partitionAtIntervals ( data , intervals ) : assert sum ( intervals ) <= len ( data ) start = 0 for interval in intervals : end = start + interval yield data [ start : end ] start = end raise StopIteration


def jobInfoWithModels(self, jobID):
    """ Get all info about a job, with model details, if available.

    Parameters:
    ----------------------------------------------------------------
    job:    jobID of the job to query
    retval: A sequence of two-tuples if the jobID exists in the jobs
             table (exeption is raised if it doesn't exist). Each two-tuple
             contains an instance of jobInfoNamedTuple as the first element and
             an instance of modelInfoNamedTuple as the second element. NOTE: In
             the case where there are no matching model rows, a sequence of one
             two-tuple will still be returned, but the modelInfoNamedTuple
             fields will be None, and the jobInfoNamedTuple fields will be
             populated.
    """

    # Get a database connection and cursor
    combinedResults = None

    with ConnectionFactory.get() as conn:
      # NOTE: Since we're using a LEFT JOIN on the models table, there need not
      # be a matching row in the models table, but the matching row from the
      # jobs table will still be returned (along with all fields from the models
      # table with values of None in case there were no matchings models)
      query = ' '.join([
        'SELECT %s.*, %s.*' % (self.jobsTableName, self.modelsTableName),
        'FROM %s' % self.jobsTableName,
        'LEFT JOIN %s USING(job_id)' % self.modelsTableName,
        'WHERE job_id=%s'])

      conn.cursor.execute(query, (jobID,))

      if conn.cursor.rowcount > 0:
        combinedResults = [
          ClientJobsDAO._combineResults(
            result, self._jobs.jobInfoNamedTuple,
            self._models.modelInfoNamedTuple
          ) for result in conn.cursor.fetchall()]

    if combinedResults is not None:
      return combinedResults

    raise RuntimeError("jobID=%s not found within the jobs table" % (jobID))

Get all info about a job with model details if available .


def jobInfo(self, jobID):
    """ Get all info about a job

    Parameters:
    ----------------------------------------------------------------
    job:    jobID of the job to query
    retval:  namedtuple containing the job info.

    """
    row = self._getOneMatchingRowWithRetries(
      self._jobs, dict(job_id=jobID),
      [self._jobs.pubToDBNameDict[n]
       for n in self._jobs.jobInfoNamedTuple._fields])

    if row is None:
      raise RuntimeError("jobID=%s not found within the jobs table" % (jobID))

    # Create a namedtuple with the names to values
    return self._jobs.jobInfoNamedTuple._make(row)


def jobSetStatus(self, jobID, status, useConnectionID=True,):
    """ Change the status on the given job

    Parameters:
    ----------------------------------------------------------------
    job:        jobID of the job to change status
    status:     new status string (ClientJobsDAO.STATUS_xxxxx)

    useConnectionID: True if the connection id of the calling function
    must be the same as the connection that created the job. Set
    to False for hypersearch workers
    """
    # Get a database connection and cursor
    with ConnectionFactory.get() as conn:
      query = 'UPDATE %s SET status=%%s, ' \
              '              _eng_last_update_time=UTC_TIMESTAMP() ' \
              '          WHERE job_id=%%s' \
              % (self.jobsTableName,)
      sqlParams = [status, jobID]

      if useConnectionID:
        query += ' AND _eng_cjm_conn_id=%s'
        sqlParams.append(self._connectionID)

      result = conn.cursor.execute(query, sqlParams)

      if result != 1:
        raise RuntimeError("Tried to change the status of job %d to %s, but "
                           "this job belongs to some other CJM" % (
                            jobID, status))


def jobSetCompleted(self, jobID, completionReason, completionMsg,
                      useConnectionID = True):
    """ Change the status on the given job to completed

    Parameters:
    ----------------------------------------------------------------
    job:                 jobID of the job to mark as completed
    completionReason:    completionReason string
    completionMsg:       completionMsg string

    useConnectionID: True if the connection id of the calling function
    must be the same as the connection that created the job. Set
    to False for hypersearch workers
    """

    # Get a database connection and cursor
    with ConnectionFactory.get() as conn:
      query = 'UPDATE %s SET status=%%s, ' \
              '              completion_reason=%%s, ' \
              '              completion_msg=%%s, ' \
              '              end_time=UTC_TIMESTAMP(), ' \
              '              _eng_last_update_time=UTC_TIMESTAMP() ' \
              '          WHERE job_id=%%s' \
              % (self.jobsTableName,)
      sqlParams = [self.STATUS_COMPLETED, completionReason, completionMsg,
                   jobID]

      if useConnectionID:
        query += ' AND _eng_cjm_conn_id=%s'
        sqlParams.append(self._connectionID)

      result = conn.cursor.execute(query, sqlParams)

      if result != 1:
        raise RuntimeError("Tried to change the status of jobID=%s to "
                           "completed, but this job could not be found or "
                           "belongs to some other CJM" % (jobID))

'              end_time=UTC_TIMESTAMP(), '


def jobCancel(self, jobID):
    """ Cancel the given job. This will update the cancel field in the
    jobs table and will result in the job being cancelled.

    Parameters:
    ----------------------------------------------------------------
    jobID:                 jobID of the job to mark as completed

    to False for hypersearch workers
    """
    self._logger.info('Canceling jobID=%s', jobID)
    # NOTE: jobSetFields does retries on transient mysql failures
    self.jobSetFields(jobID, {"cancel" : True}, useConnectionID=False)

def
jobCancel
(
self
,
jobID
)
:
self
.
_logger
.
info
(
'Canceling jobID=%s'
,
jobID
)
self
.
jobSetFields
(
jobID
,
{
"cancel"
:
True
}
,
useConnectionID
=
False
)

Cancel the given job . This will update the cancel field in the jobs table and will result in the job being cancelled .


def jobGetModelIDs(self, jobID):
    """Fetch all the modelIDs that correspond to a given jobID; empty sequence
    if none"""

    rows = self._getMatchingRowsWithRetries(self._models, dict(job_id=jobID),
                                            ['model_id'])
    return [r[0] for r in rows]

def jobGetModelIDs ( self , jobID ) : rows = self . _getMatchingRowsWithRetries ( self . _models , dict ( job_id = jobID ) , [ 'model_id' ] ) return [ r [ 0 ] for r in rows ]

Fetch all the modelIDs that correspond to a given jobID ; empty sequence if none


def getActiveJobCountForClientInfo(self, clientInfo):
    """ Return the number of jobs for the given clientInfo and a status that is
    not completed.
    """
    with ConnectionFactory.get() as conn:
      query = 'SELECT count(job_id) ' \
              'FROM %s ' \
              'WHERE client_info = %%s ' \
              ' AND status != %%s' %  self.jobsTableName
      conn.cursor.execute(query, [clientInfo, self.STATUS_COMPLETED])
      activeJobCount = conn.cursor.fetchone()[0]

    return activeJobCount

def
getActiveJobCountForClientInfo
(
self
,
clientInfo
)
:
with
ConnectionFactory
.
get
(
)
as
conn
:
query
=
'SELECT count(job_id) '
'FROM %s '
'WHERE client_info = %%s '
' AND status != %%s'
%
self
.
jobsTableName
conn
.
cursor
.
execute
(
query
,
[
clientInfo
,
self
.
STATUS_COMPLETED
]
)
activeJobCount
=
conn
.
cursor
.
fetchone
(
)
[
0
]
return
activeJobCount

Return the number of jobs for the given clientInfo and a status that is not completed .


def getActiveJobCountForClientKey(self, clientKey):
    """ Return the number of jobs for the given clientKey and a status that is
    not completed.
    """
    with ConnectionFactory.get() as conn:
      query = 'SELECT count(job_id) ' \
              'FROM %s ' \
              'WHERE client_key = %%s ' \
              ' AND status != %%s' %  self.jobsTableName
      conn.cursor.execute(query, [clientKey, self.STATUS_COMPLETED])
      activeJobCount = conn.cursor.fetchone()[0]

    return activeJobCount

def
getActiveJobCountForClientKey
(
self
,
clientKey
)
:
with
ConnectionFactory
.
get
(
)
as
conn
:
query
=
'SELECT count(job_id) '
'FROM %s '
'WHERE client_key = %%s '
' AND status != %%s'
%
self
.
jobsTableName
conn
.
cursor
.
execute
(
query
,
[
clientKey
,
self
.
STATUS_COMPLETED
]
)
activeJobCount
=
conn
.
cursor
.
fetchone
(
)
[
0
]
return
activeJobCount

Return the number of jobs for the given clientKey and a status that is not completed .


def getActiveJobsForClientInfo(self, clientInfo, fields=[]):
    """ Fetch jobIDs for jobs in the table with optional fields given a
    specific clientInfo """

    # Form the sequence of field name strings that will go into the
    #  request
    dbFields = [self._jobs.pubToDBNameDict[x] for x in fields]
    dbFieldsStr = ','.join(['job_id'] + dbFields)

    with ConnectionFactory.get() as conn:
      query = 'SELECT %s FROM %s ' \
              'WHERE client_info = %%s ' \
              ' AND status != %%s' % (dbFieldsStr, self.jobsTableName)
      conn.cursor.execute(query, [clientInfo, self.STATUS_COMPLETED])
      rows = conn.cursor.fetchall()

    return rows

def
getActiveJobsForClientInfo
(
self
,
clientInfo
,
fields
=
[
]
)
:
dbFields
=
[
self
.
_jobs
.
pubToDBNameDict
[
x
]
for
x
in
fields
]
dbFieldsStr
=
','
.
join
(
[
'job_id'
]
+
dbFields
)
with
ConnectionFactory
.
get
(
)
as
conn
:
query
=
'SELECT %s FROM %s '
'WHERE client_info = %%s '
' AND status != %%s'
%
(
dbFieldsStr
,
self
.
jobsTableName
)
conn
.
cursor
.
execute
(
query
,
[
clientInfo
,
self
.
STATUS_COMPLETED
]
)
rows
=
conn
.
cursor
.
fetchall
(
)
return
rows


def jobUpdateResults(self, jobID, results):
    """ Update the results string and last-update-time fields of a model.

    Parameters:
    ----------------------------------------------------------------
    jobID:      job ID of model to modify
    results:    new results (json dict string)
    """
    with ConnectionFactory.get() as conn:
      query = 'UPDATE %s SET _eng_last_update_time=UTC_TIMESTAMP(), ' \
              '              results=%%s ' \
              '          WHERE job_id=%%s' % (self.jobsTableName,)
      conn.cursor.execute(query, [results, jobID])

Update the results string and last - update - time fields of a model .


def modelsClearAll(self):
    """ Delete all models from the models table

    Parameters:
    ----------------------------------------------------------------
    """
    self._logger.info('Deleting all rows from models table %r',
                      self.modelsTableName)
    with ConnectionFactory.get() as conn:
      query = 'DELETE FROM %s' % (self.modelsTableName)
      conn.cursor.execute(query)


def modelsInfo(self, modelIDs):
    """ Get ALL info for a set of models

    WARNING!!!: The order of the results are NOT necessarily in the same order as
    the order of the model IDs passed in!!!

    Parameters:
    ----------------------------------------------------------------
    modelIDs:    list of model IDs
    retval:      list of nametuples containing all the fields stored for each
                    model.
    """
    assert isinstance(modelIDs, self._SEQUENCE_TYPES), (
      "wrong modelIDs type: %s") % (type(modelIDs),)
    assert modelIDs, "modelIDs is empty"

    rows = self._getMatchingRowsWithRetries(
      self._models, dict(model_id=modelIDs),
      [self._models.pubToDBNameDict[f]
       for f in self._models.modelInfoNamedTuple._fields])

    results = [self._models.modelInfoNamedTuple._make(r) for r in rows]

    # NOTE: assetion will also fail if modelIDs contains duplicates
    assert len(results) == len(modelIDs), "modelIDs not found: %s" % (
      set(modelIDs) - set(r.modelId for r in results))

    return results

def
modelsInfo
(
self
,
modelIDs
)
:
assert
isinstance
(
modelIDs
,
self
.
_SEQUENCE_TYPES
)
,
(
"wrong modelIDs type: %s"
)
%
(
type
(
modelIDs
)
,
)
assert
modelIDs
,
"modelIDs is empty"
rows
=
self
.
_getMatchingRowsWithRetries
(
self
.
_models
,
dict
(
model_id
=
modelIDs
)
,
[
self
.
_models
.
pubToDBNameDict
[
f
]
for
f
in
self
.
_models
.
modelInfoNamedTuple
.
_fields
]
)
results
=
[
self
.
_models
.
modelInfoNamedTuple
.
_make
(
r
)
for
r
in
rows
]
assert
len
(
results
)
==
len
(
modelIDs
)
,
"modelIDs not found: %s"
%
(
set
(
modelIDs
)
set
(
r
.
modelId
for
r
in
results
)
)
return
results


def modelsGetFieldsForJob(self, jobID, fields, ignoreKilled=False):
    """ Gets the specified fields for all the models for a single job. This is
    similar to modelsGetFields

    Parameters:
    ----------------------------------------------------------------
    jobID:              jobID for the models to be searched
    fields:             A list  of fields to return
    ignoreKilled:       (True/False). If True, this will ignore models that
                        have been killed

    Returns: a (possibly empty) list of tuples as follows
      [
        (model_id1, [field1, ..., fieldn]),
        (model_id2, [field1, ..., fieldn]),
        (model_id3, [field1, ..., fieldn])
                    ...
      ]

    NOTE: since there is a window of time between a job getting inserted into
     jobs table and the job's worker(s) starting up and creating models, an
     empty-list result is one of the normal outcomes.
    """

    assert len(fields) >= 1, 'fields is empty'

    # Form the sequence of field name strings that will go into the
    #  request
    dbFields = [self._models.pubToDBNameDict[x] for x in fields]
    dbFieldsStr = ','.join(dbFields)

    query = 'SELECT model_id, %s FROM %s ' \
              '          WHERE job_id=%%s ' \
              % (dbFieldsStr, self.modelsTableName)
    sqlParams = [jobID]

    if ignoreKilled:
      query += ' AND (completion_reason IS NULL OR completion_reason != %s)'
      sqlParams.append(self.CMPL_REASON_KILLED)

    # Get a database connection and cursor
    with ConnectionFactory.get() as conn:
      conn.cursor.execute(query, sqlParams)
      rows = conn.cursor.fetchall()

    if rows is None:
      # fetchall is defined to return a (possibly-empty) sequence of
      # sequences; however, we occasionally see None returned and don't know
      # why...
      self._logger.error("Unexpected None result from cursor.fetchall; "
                         "query=%r; Traceback=%r",
                         query, traceback.format_exc())

    return [(r[0], list(r[1:])) for r in rows]

Gets the specified fields for all the models for a single job . This is similar to modelsGetFields


def modelsGetFieldsForCheckpointed(self, jobID, fields):
    """
    Gets fields from all models in a job that have been checkpointed. This is
    used to figure out whether or not a new model should be checkpointed.

    Parameters:
    -----------------------------------------------------------------------
    jobID:                    The jobID for the models to be searched
    fields:                   A list of fields to return

    Returns: a (possibly-empty) list of tuples as follows
      [
        (model_id1, [field1, ..., fieldn]),
        (model_id2, [field1, ..., fieldn]),
        (model_id3, [field1, ..., fieldn])
                    ...
      ]
    """

    assert len(fields) >= 1, "fields is empty"

    # Get a database connection and cursor
    with ConnectionFactory.get() as conn:
      dbFields = [self._models.pubToDBNameDict[f] for f in fields]
      dbFieldStr = ", ".join(dbFields)

      query = 'SELECT model_id, {fields} from {models}' \
              '   WHERE job_id=%s AND model_checkpoint_id IS NOT NULL'.format(
        fields=dbFieldStr, models=self.modelsTableName)

      conn.cursor.execute(query, [jobID])
      rows = conn.cursor.fetchall()

    return [(r[0], list(r[1:])) for r in rows]

Gets fields from all models in a job that have been checkpointed . This is used to figure out whether or not a new model should be checkpointed .


def modelsGetParams(self, modelIDs):
    """ Get the params and paramsHash for a set of models.

    WARNING!!!: The order of the results are NOT necessarily in the same order as
    the order of the model IDs passed in!!!

    Parameters:
    ----------------------------------------------------------------
    modelIDs:    list of model IDs
    retval:      list of result namedtuples defined in
                  ClientJobsDAO._models.getParamsNamedTuple. Each tuple
                  contains: (modelId, params, engParamsHash)
    """
    assert isinstance(modelIDs, self._SEQUENCE_TYPES), (
      "Wrong modelIDs type: %r") % (type(modelIDs),)
    assert len(modelIDs) >= 1, "modelIDs is empty"

    rows = self._getMatchingRowsWithRetries(
      self._models, {'model_id' : modelIDs},
      [self._models.pubToDBNameDict[f]
       for f in self._models.getParamsNamedTuple._fields])

    # NOTE: assertion will also fail when modelIDs contains duplicates
    assert len(rows) == len(modelIDs), "Didn't find modelIDs: %r" % (
      (set(modelIDs) - set(r[0] for r in rows)),)

    # Return the params and params hashes as a namedtuple
    return [self._models.getParamsNamedTuple._make(r) for r in rows]

def
modelsGetParams
(
self
,
modelIDs
)
:
assert
isinstance
(
modelIDs
,
self
.
_SEQUENCE_TYPES
)
,
(
"Wrong modelIDs type: %r"
)
%
(
type
(
modelIDs
)
,
)
assert
len
(
modelIDs
)
>=
1
,
"modelIDs is empty"
rows
=
self
.
_getMatchingRowsWithRetries
(
self
.
_models
,
{
'model_id'
:
modelIDs
}
,
[
self
.
_models
.
pubToDBNameDict
[
f
]
for
f
in
self
.
_models
.
getParamsNamedTuple
.
_fields
]
)
assert
len
(
rows
)
==
len
(
modelIDs
)
,
"Didn't find modelIDs: %r"
%
(
(
set
(
modelIDs
)
set
(
r
[
0
]
for
r
in
rows
)
)
,
)
return
[
self
.
_models
.
getParamsNamedTuple
.
_make
(
r
)
for
r
in
rows
]

Get the params and paramsHash for a set of models .


def modelsGetResultAndStatus(self, modelIDs):
    """ Get the results string and other status fields for a set of models.

    WARNING!!!: The order of the results are NOT necessarily in the same order
    as the order of the model IDs passed in!!!

    For each model, this returns a tuple containing:
     (modelID, results, status, updateCounter, numRecords, completionReason,
         completionMsg, engParamsHash

    Parameters:
    ----------------------------------------------------------------
    modelIDs:    list of model IDs
    retval:      list of result tuples. Each tuple contains:
                    (modelID, results, status, updateCounter, numRecords,
                      completionReason, completionMsg, engParamsHash)
    """
    assert isinstance(modelIDs, self._SEQUENCE_TYPES), (
      "Wrong modelIDs type: %r") % type(modelIDs)
    assert len(modelIDs) >= 1, "modelIDs is empty"

    rows = self._getMatchingRowsWithRetries(
      self._models, {'model_id' : modelIDs},
      [self._models.pubToDBNameDict[f]
       for f in self._models.getResultAndStatusNamedTuple._fields])

    # NOTE: assertion will also fail when modelIDs contains duplicates
    assert len(rows) == len(modelIDs), "Didn't find modelIDs: %r" % (
      (set(modelIDs) - set(r[0] for r in rows)),)

    # Return the results as a list of namedtuples
    return [self._models.getResultAndStatusNamedTuple._make(r) for r in rows]

def
modelsGetResultAndStatus
(
self
,
modelIDs
)
:
assert
isinstance
(
modelIDs
,
self
.
_SEQUENCE_TYPES
)
,
(
"Wrong modelIDs type: %r"
)
%
type
(
modelIDs
)
assert
len
(
modelIDs
)
>=
1
,
"modelIDs is empty"
rows
=
self
.
_getMatchingRowsWithRetries
(
self
.
_models
,
{
'model_id'
:
modelIDs
}
,
[
self
.
_models
.
pubToDBNameDict
[
f
]
for
f
in
self
.
_models
.
getResultAndStatusNamedTuple
.
_fields
]
)
assert
len
(
rows
)
==
len
(
modelIDs
)
,
"Didn't find modelIDs: %r"
%
(
(
set
(
modelIDs
)
set
(
r
[
0
]
for
r
in
rows
)
)
,
)
return
[
self
.
_models
.
getResultAndStatusNamedTuple
.
_make
(
r
)
for
r
in
rows
]

Get the results string and other status fields for a set of models .


def modelAdoptNextOrphan(self, jobId, maxUpdateInterval):
    """ Look through the models table for an orphaned model, which is a model
    that is not completed yet, whose _eng_last_update_time is more than
    maxUpdateInterval seconds ago.

    If one is found, change its _eng_worker_conn_id to the current worker's
    and return the model id.

    Parameters:
    ----------------------------------------------------------------
    retval:    modelId of the model we adopted, or None if none found
    """

    @g_retrySQL
    def findCandidateModelWithRetries():
      modelID = None
      with ConnectionFactory.get() as conn:
        # TODO: may need a table index on job_id/status for speed
        query = 'SELECT model_id FROM %s ' \
                '   WHERE  status=%%s ' \
                '          AND job_id=%%s ' \
                '          AND TIMESTAMPDIFF(SECOND, ' \
                '                            _eng_last_update_time, ' \
                '                            UTC_TIMESTAMP()) > %%s ' \
                '   LIMIT 1 ' \
                % (self.modelsTableName,)
        sqlParams = [self.STATUS_RUNNING, jobId, maxUpdateInterval]
        numRows = conn.cursor.execute(query, sqlParams)
        rows = conn.cursor.fetchall()

      assert numRows <= 1, "Unexpected numRows: %r" % numRows
      if numRows == 1:
        (modelID,) = rows[0]

      return modelID

    @g_retrySQL
    def adoptModelWithRetries(modelID):
      adopted = False
      with ConnectionFactory.get() as conn:
        query = 'UPDATE %s SET _eng_worker_conn_id=%%s, ' \
                  '            _eng_last_update_time=UTC_TIMESTAMP() ' \
                  '        WHERE model_id=%%s ' \
                  '              AND status=%%s' \
                  '              AND TIMESTAMPDIFF(SECOND, ' \
                  '                                _eng_last_update_time, ' \
                  '                                UTC_TIMESTAMP()) > %%s ' \
                  '        LIMIT 1 ' \
                  % (self.modelsTableName,)
        sqlParams = [self._connectionID, modelID, self.STATUS_RUNNING,
                     maxUpdateInterval]
        numRowsAffected = conn.cursor.execute(query, sqlParams)

        assert numRowsAffected <= 1, 'Unexpected numRowsAffected=%r' % (
          numRowsAffected,)

        if numRowsAffected == 1:
          adopted = True
        else:
          # Discern between transient failure during update and someone else
          # claiming this model
          (status, connectionID) = self._getOneMatchingRowNoRetries(
            self._models, conn, {'model_id':modelID},
            ['status', '_eng_worker_conn_id'])
          adopted = (status == self.STATUS_RUNNING and
                     connectionID == self._connectionID)
      return adopted

    adoptedModelID = None
    while True:
      modelID = findCandidateModelWithRetries()
      if modelID is None:
        break
      if adoptModelWithRetries(modelID):
        adoptedModelID = modelID
        break

    return adoptedModelID

'                            _eng_last_update_time, '

'                            UTC_TIMESTAMP()) > %%s '

'              AND TIMESTAMPDIFF(SECOND, '

'                                _eng_last_update_time, '

'                                UTC_TIMESTAMP()) > %%s '

Look through the models table for an orphaned model which is a model that is not completed yet whose _eng_last_update_time is more than maxUpdateInterval seconds ago .


def _initEphemerals(self):
    """
    Initialize attributes that are not saved with the checkpoint.
    """
    self._firstComputeCall = True
    self._accuracy = None
    self._protoScores = None
    self._categoryDistances = None

    self._knn = knn_classifier.KNNClassifier(**self.knnParams)

    for x in ('_partitions', '_useAuxiliary', '_doSphering',
              '_scanInfo', '_protoScores'):
      if not hasattr(self, x):
        setattr(self, x, None)

def _initEphemerals ( self ) : self . _firstComputeCall = True self . _accuracy = None self . _protoScores = None self . _categoryDistances = None self . _knn = knn_classifier . KNNClassifier ( ** self . knnParams ) for x in ( '_partitions' , '_useAuxiliary' , '_doSphering' , '_scanInfo' , '_protoScores' ) : if not hasattr ( self , x ) : setattr ( self , x , None )

Initialize attributes that are not saved with the checkpoint .