def writeToProto(self, proto):
    """
    Write state to proto object.

    :param proto: SDRClassifierRegionProto capnproto object
    """
    proto.implementation = self.implementation
    proto.steps = self.steps
    proto.alpha = self.alpha
    proto.verbosity = self.verbosity
    proto.maxCategoryCount = self.maxCategoryCount
    proto.learningMode = self.learningMode
    proto.inferenceMode = self.inferenceMode
    proto.recordNum = self.recordNum

    self._sdrClassifier.write(proto.sdrClassifier)

def writeToProto ( self , proto ) : proto . implementation = self . implementation proto . steps = self . steps proto . alpha = self . alpha proto . verbosity = self . verbosity proto . maxCategoryCount = self . maxCategoryCount proto . learningMode = self . learningMode proto . inferenceMode = self . inferenceMode proto . recordNum = self . recordNum self . _sdrClassifier . write ( proto . sdrClassifier )


def readFromProto(cls, proto):
    """
    Read state from proto object.

    :param proto: SDRClassifierRegionProto capnproto object
    """
    instance = cls()

    instance.implementation = proto.implementation
    instance.steps = proto.steps
    instance.stepsList = [int(i) for i in proto.steps.split(",")]
    instance.alpha = proto.alpha
    instance.verbosity = proto.verbosity
    instance.maxCategoryCount = proto.maxCategoryCount

    instance._sdrClassifier = SDRClassifierFactory.read(proto)

    instance.learningMode = proto.learningMode
    instance.inferenceMode = proto.inferenceMode
    instance.recordNum = proto.recordNum

    return instance

def readFromProto ( cls , proto ) : instance = cls ( ) instance . implementation = proto . implementation instance . steps = proto . steps instance . stepsList = [ int ( i ) for i in proto . steps . split ( "," ) ] instance . alpha = proto . alpha instance . verbosity = proto . verbosity instance . maxCategoryCount = proto . maxCategoryCount instance . _sdrClassifier = SDRClassifierFactory . read ( proto ) instance . learningMode = proto . learningMode instance . inferenceMode = proto . inferenceMode instance . recordNum = proto . recordNum return instance


def run(self):
    """ Runs the OPF Model

    Parameters:
    -------------------------------------------------------------------------
    retval:  (completionReason, completionMsg)
              where completionReason is one of the ClientJobsDAO.CMPL_REASON_XXX
                equates.
    """
    # -----------------------------------------------------------------------
    # Load the experiment's description.py module
    descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir(
      self._experimentDir)
    expIface = helpers.getExperimentDescriptionInterfaceFromModule(
      descriptionPyModule)
    expIface.normalizeStreamSources()

    modelDescription = expIface.getModelDescription()
    self._modelControl = expIface.getModelControl()

    # -----------------------------------------------------------------------
    # Create the input data stream for this task
    streamDef = self._modelControl['dataset']

    from nupic.data.stream_reader import StreamReader
    readTimeout = 0

    self._inputSource = StreamReader(streamDef, isBlocking=False,
                                     maxTimeout=readTimeout)

    # -----------------------------------------------------------------------
    #Get field statistics from the input source
    fieldStats = self._getFieldStats()
    # -----------------------------------------------------------------------
    # Construct the model instance
    self._model = ModelFactory.create(modelDescription)
    self._model.setFieldStatistics(fieldStats)
    self._model.enableLearning()
    self._model.enableInference(self._modelControl.get("inferenceArgs", None))

    # -----------------------------------------------------------------------
    # Instantiate the metrics
    self.__metricMgr = MetricsManager(self._modelControl.get('metrics',None),
                                      self._model.getFieldInfo(),
                                      self._model.getInferenceType())

    self.__loggedMetricPatterns = self._modelControl.get("loggedMetrics", [])

    self._optimizedMetricLabel = self.__getOptimizedMetricLabel()
    self._reportMetricLabels = matchPatterns(self._reportKeyPatterns,
                                              self._getMetricLabels())

    # -----------------------------------------------------------------------
    # Initialize periodic activities (e.g., for model result updates)
    self._periodic = self._initPeriodicActivities()

    # -----------------------------------------------------------------------
    # Create our top-level loop-control iterator
    numIters = self._modelControl.get('iterationCount', -1)

    # Are we asked to turn off learning for a certain # of iterations near the
    #  end?
    learningOffAt = None
    iterationCountInferOnly = self._modelControl.get('iterationCountInferOnly', 0)
    if iterationCountInferOnly == -1:
      self._model.disableLearning()
    elif iterationCountInferOnly > 0:
      assert numIters > iterationCountInferOnly, "when iterationCountInferOnly " \
        "is specified, iterationCount must be greater than " \
        "iterationCountInferOnly."
      learningOffAt = numIters - iterationCountInferOnly

    self.__runTaskMainLoop(numIters, learningOffAt=learningOffAt)

    # -----------------------------------------------------------------------
    # Perform final operations for model
    self._finalize()

    return (self._cmpReason, None)


def __runTaskMainLoop(self, numIters, learningOffAt=None):
    """ Main loop of the OPF Model Runner.

    Parameters:
    -----------------------------------------------------------------------

    recordIterator:    Iterator for counting number of records (see _runTask)
    learningOffAt:     If not None, learning is turned off when we reach this
                        iteration number

    """

    ## Reset sequence states in the model, so it starts looking for a new
    ## sequence
    self._model.resetSequenceStates()

    self._currentRecordIndex = -1
    while True:

      # If killed by a terminator, stop running
      if self._isKilled:
        break

      # If job stops or hypersearch ends, stop running
      if self._isCanceled:
        break

      # If the process is about to be killed, set as orphaned
      if self._isInterrupted.isSet():
        self.__setAsOrphaned()
        break

      # If model is mature, stop running ONLY IF  we are not the best model
      # for the job. Otherwise, keep running so we can keep returning
      # predictions to the user
      if self._isMature:
        if not self._isBestModel:
          self._cmpReason = self._jobsDAO.CMPL_REASON_STOPPED
          break
        else:
          self._cmpReason = self._jobsDAO.CMPL_REASON_EOF

      # Turn off learning?
        if learningOffAt is not None \
                  and self._currentRecordIndex == learningOffAt:
          self._model.disableLearning()

      # Read input record. Note that any failure here is a critical JOB failure
      #  and results in the job being immediately canceled and marked as
      #  failed. The runModelXXX code in hypesearch.utils, if it sees an
      #  exception of type utils.JobFailException, will cancel the job and
      #  copy the error message into the job record.
      try:
        inputRecord = self._inputSource.getNextRecordDict()
        if self._currentRecordIndex < 0:
          self._inputSource.setTimeout(10)
      except Exception, e:
        raise utils.JobFailException(ErrorCodes.streamReading, str(e.args),
                                     traceback.format_exc())

      if inputRecord is None:
        # EOF
        self._cmpReason = self._jobsDAO.CMPL_REASON_EOF
        break

      if inputRecord:
        # Process input record
        self._currentRecordIndex += 1

        result = self._model.run(inputRecord=inputRecord)

        # Compute metrics.
        result.metrics = self.__metricMgr.update(result)
        # If there are None, use defaults. see MetricsManager.getMetrics()
        # TODO remove this when JAVA API server is gone
        if not result.metrics:
          result.metrics = self.__metricMgr.getMetrics()

        # Write the result to the output cache. Don't write encodings, if they
        # were computed
        if InferenceElement.encodings in result.inferences:
          result.inferences.pop(InferenceElement.encodings)
        result.sensorInput.dataEncodings = None
        self._writePrediction(result)

        # Run periodic activities
        self._periodic.tick()

        if numIters >= 0 and self._currentRecordIndex >= numIters-1:
          break

      else:
        # Input source returned an empty record.
        #
        # NOTE: This is okay with Stream-based Source (when it times out
        # waiting for next record), but not okay with FileSource, which should
        # always return either with a valid record or None for EOF.
        raise ValueError("Got an empty record from FileSource: %r" %
                         inputRecord)


"Finished: modelID=%r; %r records processed. Performing final activities"

Run final activities after a model has run. These include recording and
    logging the final score

Run final activities after a model has run . These include recording and logging the final score


def __createModelCheckpoint(self):
    """ Create a checkpoint from the current model, and store it in a dir named
    after checkpoint GUID, and finally store the GUID in the Models DB """

    if self._model is None or self._modelCheckpointGUID is None:
      return

    # Create an output store, if one doesn't exist already
    if self._predictionLogger is None:
      self._createPredictionLogger()

    predictions = StringIO.StringIO()
    self._predictionLogger.checkpoint(
      checkpointSink=predictions,
      maxRows=int(Configuration.get('nupic.model.checkpoint.maxPredictionRows')))

    self._model.save(os.path.join(self._experimentDir, str(self._modelCheckpointGUID)))
    self._jobsDAO.modelSetFields(modelID,
                                 {'modelCheckpointId':str(self._modelCheckpointGUID)},
                                 ignoreUnchanged=True)

    self._logger.info("Checkpointed Hypersearch Model: modelID: %r, "
                      "checkpointID: %r", self._modelID, checkpointID)
    return

Create a checkpoint from the current model and store it in a dir named after checkpoint GUID and finally store the GUID in the Models DB


def __deleteModelCheckpoint(self, modelID):
    """
    Delete the stored checkpoint for the specified modelID. This function is
    called if the current model is now the best model, making the old model's
    checkpoint obsolete

    Parameters:
    -----------------------------------------------------------------------
    modelID:      The modelID for the checkpoint to delete. This is NOT the
                  unique checkpointID
    """

    checkpointID = \
        self._jobsDAO.modelsGetFields(modelID, ['modelCheckpointId'])[0]

    if checkpointID is None:
      return

    try:
      shutil.rmtree(os.path.join(self._experimentDir, str(self._modelCheckpointGUID)))
    except:
      self._logger.warn("Failed to delete model checkpoint %s. "\
                        "Assuming that another worker has already deleted it",
                        checkpointID)
      return

    self._jobsDAO.modelSetFields(modelID,
                                 {'modelCheckpointId':None},
                                 ignoreUnchanged=True)
    return

Delete the stored checkpoint for the specified modelID . This function is called if the current model is now the best model making the old model s checkpoint obsolete


def __getOptimizedMetricLabel(self):
    """ Get the label for the metric being optimized. This function also caches
    the label in the instance variable self._optimizedMetricLabel

    Parameters:
    -----------------------------------------------------------------------
    metricLabels:   A sequence of all the labels being computed for this model

    Returns:        The label for the metric being optmized over
    """
    matchingKeys = matchPatterns([self._optimizeKeyPattern],
                                  self._getMetricLabels())

    if len(matchingKeys) == 0:
      raise Exception("None of the generated metrics match the specified "
                      "optimization pattern: %s. Available metrics are %s" % \
                       (self._optimizeKeyPattern, self._getMetricLabels()))
    elif len(matchingKeys) > 1:
      raise Exception("The specified optimization pattern '%s' matches more "
              "than one metric: %s" % (self._optimizeKeyPattern, matchingKeys))

    return matchingKeys[0]

Get the label for the metric being optimized . This function also caches the label in the instance variable self . _optimizedMetricLabel


def _getFieldStats(self):
    """
    Method which returns a dictionary of field statistics received from the
    input source.

    Returns:

      fieldStats: dict of dicts where the first level is the field name and
        the second level is the statistic. ie. fieldStats['pounds']['min']

    """

    fieldStats = dict()
    fieldNames = self._inputSource.getFieldNames()
    for field in fieldNames:
      curStats = dict()
      curStats['min'] = self._inputSource.getFieldMin(field)
      curStats['max'] = self._inputSource.getFieldMax(field)
      fieldStats[field] = curStats
    return fieldStats

def _getFieldStats ( self ) : fieldStats = dict ( ) fieldNames = self . _inputSource . getFieldNames ( ) for field in fieldNames : curStats = dict ( ) curStats [ 'min' ] = self . _inputSource . getFieldMin ( field ) curStats [ 'max' ] = self . _inputSource . getFieldMax ( field ) fieldStats [ field ] = curStats return fieldStats

Method which returns a dictionary of field statistics received from the input source .


def _updateModelDBResults(self):
    """ Retrieves the current results and updates the model's record in
    the Model database.
    """

    # -----------------------------------------------------------------------
    # Get metrics
    metrics = self._getMetrics()

    # -----------------------------------------------------------------------
    # Extract report metrics that match the requested report REs
    reportDict = dict([(k,metrics[k]) for k in self._reportMetricLabels])

    # -----------------------------------------------------------------------
    # Extract the report item that matches the optimize key RE
    # TODO cache optimizedMetricLabel sooner
    metrics = self._getMetrics()
    optimizeDict = dict()
    if self._optimizeKeyPattern is not None:
      optimizeDict[self._optimizedMetricLabel] = \
                                      metrics[self._optimizedMetricLabel]

    # -----------------------------------------------------------------------
    # Update model results
    results = json.dumps((metrics , optimizeDict))
    self._jobsDAO.modelUpdateResults(self._modelID,  results=results,
                              metricValue=optimizeDict.values()[0],
                              numRecords=(self._currentRecordIndex + 1))

    self._logger.debug(
      "Model Results: modelID=%s; numRecords=%s; results=%s" % \
        (self._modelID, self._currentRecordIndex + 1, results))

    return

Retrieves the current results and updates the model s record in the Model database .


def __checkIfBestCompletedModel(self):
    """
    Reads the current "best model" for the job and returns whether or not the
    current model is better than the "best model" stored for the job

    Returns: (isBetter, storedBest, origResultsStr)

    isBetter:
      True if the current model is better than the stored "best model"
    storedResults:
      A dict of the currently stored results in the jobs table record
    origResultsStr:
      The json-encoded string that currently resides in the "results" field
      of the jobs record (used to create atomicity)
    """

    jobResultsStr = self._jobsDAO.jobGetFields(self._jobID, ['results'])[0]

    if jobResultsStr is None:
        jobResults = {}
    else:
      jobResults = json.loads(jobResultsStr)

    isSaved = jobResults.get('saved', False)
    bestMetric = jobResults.get('bestValue', None)

    currentMetric = self._getMetrics()[self._optimizedMetricLabel]
    self._isBestModel = (not isSaved) \
                        or (currentMetric < bestMetric)

    return self._isBestModel, jobResults, jobResultsStr

def __checkIfBestCompletedModel ( self ) : jobResultsStr = self . _jobsDAO . jobGetFields ( self . _jobID , [ 'results' ] ) [ 0 ] if jobResultsStr is None : jobResults = { } else : jobResults = json . loads ( jobResultsStr ) isSaved = jobResults . get ( 'saved' , False ) bestMetric = jobResults . get ( 'bestValue' , None ) currentMetric = self . _getMetrics ( ) [ self . _optimizedMetricLabel ] self . _isBestModel = ( not isSaved ) or ( currentMetric < bestMetric ) return self . _isBestModel , jobResults , jobResultsStr

Reads the current best model for the job and returns whether or not the current model is better than the best model stored for the job


def _writePrediction(self, result):
    """
    Writes the results of one iteration of a model. The results are written to
    this ModelRunner's in-memory cache unless this model is the "best model" for
    the job. If this model is the "best model", the predictions are written out
    to a permanent store via a prediction output stream instance

    Parameters:
    -----------------------------------------------------------------------
    result:      A opf_utils.ModelResult object, which contains the input and
                  output for this iteration
    """
    self.__predictionCache.append(result)

    if self._isBestModel:
     self.__flushPredictionCache()

def _writePrediction ( self , result ) : self . __predictionCache . append ( result ) if self . _isBestModel : self . __flushPredictionCache ( )

Writes the results of one iteration of a model . The results are written to this ModelRunner s in - memory cache unless this model is the best model for the job . If this model is the best model the predictions are written out to a permanent store via a prediction output stream instance


def __flushPredictionCache(self):
    """
    Writes the contents of this model's in-memory prediction cache to a permanent
    store via the prediction output stream instance
    """

    if not self.__predictionCache:
      return

    # Create an output store, if one doesn't exist already
    if self._predictionLogger is None:
      self._createPredictionLogger()

    startTime = time.time()
    self._predictionLogger.writeRecords(self.__predictionCache,
                                        progressCB=self.__writeRecordsCallback)
    self._logger.info("Flushed prediction cache; numrows=%s; elapsed=%s sec.",
                      len(self.__predictionCache), time.time() - startTime)
    self.__predictionCache.clear()

Writes the contents of this model s in - memory prediction cache to a permanent store via the prediction output stream instance


def __deleteOutputCache(self, modelID):
    """
    Delete's the output cache associated with the given modelID. This actually
    clears up the resources associated with the cache, rather than deleting al
    the records in the cache

    Parameters:
    -----------------------------------------------------------------------
    modelID:      The id of the model whose output cache is being deleted

    """

    # If this is our output, we should close the connection
    if modelID == self._modelID and self._predictionLogger is not None:
      self._predictionLogger.close()
      del self.__predictionCache
      self._predictionLogger = None
      self.__predictionCache = None

def __deleteOutputCache ( self , modelID ) : if modelID == self . _modelID and self . _predictionLogger is not None : self . _predictionLogger . close ( ) del self . __predictionCache self . _predictionLogger = None self . __predictionCache = None

Delete s the output cache associated with the given modelID . This actually clears up the resources associated with the cache rather than deleting al the records in the cache


def _initPeriodicActivities(self):
    """ Creates and returns a PeriodicActivityMgr instance initialized with
    our periodic activities

    Parameters:
    -------------------------------------------------------------------------
    retval:             a PeriodicActivityMgr instance
    """

    # Activity to update the metrics for this model
    # in the models table
    updateModelDBResults = PeriodicActivityRequest(repeating=True,
                                                 period=100,
                                                 cb=self._updateModelDBResults)

    updateJobResults = PeriodicActivityRequest(repeating=True,
                                               period=100,
                                               cb=self.__updateJobResultsPeriodic)

    checkCancelation = PeriodicActivityRequest(repeating=True,
                                               period=50,
                                               cb=self.__checkCancelation)

    checkMaturity = PeriodicActivityRequest(repeating=True,
                                            period=10,
                                            cb=self.__checkMaturity)

    # Do an initial update of the job record after 2 iterations to make
    # sure that it is populated with something without having to wait too long
    updateJobResultsFirst = PeriodicActivityRequest(repeating=False,
                                               period=2,
                                               cb=self.__updateJobResultsPeriodic)

    periodicActivities = [updateModelDBResults,
                          updateJobResultsFirst,
                          updateJobResults,
                          checkCancelation]

    if self._isMaturityEnabled:
      periodicActivities.append(checkMaturity)

    return PeriodicActivityMgr(requestedActivities=periodicActivities)

def _initPeriodicActivities ( self ) : updateModelDBResults = PeriodicActivityRequest ( repeating = True , period = 100 , cb = self . _updateModelDBResults ) updateJobResults = PeriodicActivityRequest ( repeating = True , period = 100 , cb = self . __updateJobResultsPeriodic ) checkCancelation = PeriodicActivityRequest ( repeating = True , period = 50 , cb = self . __checkCancelation ) checkMaturity = PeriodicActivityRequest ( repeating = True , period = 10 , cb = self . __checkMaturity ) updateJobResultsFirst = PeriodicActivityRequest ( repeating = False , period = 2 , cb = self . __updateJobResultsPeriodic ) periodicActivities = [ updateModelDBResults , updateJobResultsFirst , updateJobResults , checkCancelation ] if self . _isMaturityEnabled : periodicActivities . append ( checkMaturity ) return PeriodicActivityMgr ( requestedActivities = periodicActivities )


def __checkCancelation(self):
    """ Check if the cancelation flag has been set for this model
    in the Model DB"""

    # Update a hadoop job counter at least once every 600 seconds so it doesn't
    #  think our map task is dead
    print >>sys.stderr, "reporter:counter:HypersearchWorker,numRecords,50"

    # See if the job got cancelled
    jobCancel = self._jobsDAO.jobGetFields(self._jobID, ['cancel'])[0]
    if jobCancel:
      self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED
      self._isCanceled = True
      self._logger.info("Model %s canceled because Job %s was stopped.",
                        self._modelID, self._jobID)
    else:
      stopReason = self._jobsDAO.modelsGetFields(self._modelID, ['engStop'])[0]

      if stopReason is None:
        pass

      elif stopReason == ClientJobsDAO.STOP_REASON_KILLED:
        self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED
        self._isKilled = True
        self._logger.info("Model %s canceled because it was killed by hypersearch",
                          self._modelID)

      elif stopReason == ClientJobsDAO.STOP_REASON_STOPPED:
        self._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED
        self._isCanceled = True
        self._logger.info("Model %s stopped because hypersearch ended", self._modelID)
      else:
        raise RuntimeError ("Unexpected stop reason encountered: %s" % (stopReason))


def __checkMaturity(self):
    """ Save the current metric value and see if the model's performance has
    'leveled off.' We do this by looking at some number of previous number of
    recordings """

    if self._currentRecordIndex+1 < self._MIN_RECORDS_TO_BE_BEST:
      return

    # If we are already mature, don't need to check anything
    if self._isMature:
      return

    metric = self._getMetrics()[self._optimizedMetricLabel]
    self._metricRegression.addPoint(x=self._currentRecordIndex, y=metric)

   # Perform a linear regression to see if the error is leveled off
    #pctChange = self._metricRegression.getPctChange()
    #if pctChange  is not None and abs(pctChange ) <= self._MATURITY_MAX_CHANGE:
    pctChange, absPctChange = self._metricRegression.getPctChanges()
    if pctChange  is not None and absPctChange <= self._MATURITY_MAX_CHANGE:
      self._jobsDAO.modelSetFields(self._modelID,
                                   {'engMatured':True})

      # TODO: Don't stop if we are currently the best model. Also, if we
      # are still running after maturity, we have to periodically check to
      # see if we are still the best model. As soon we lose to some other
      # model, then we should stop at that point.
      self._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED
      self._isMature = True

      self._logger.info("Model %d has matured (pctChange=%s, n=%d). \n"\
                        "Scores = %s\n"\
                         "Stopping execution",self._modelID, pctChange,
                                              self._MATURITY_NUM_POINTS,
                                              self._metricRegression._window)

Save the current metric value and see if the model s performance has leveled off . We do this by looking at some number of previous number of recordings


def __setAsOrphaned(self):
    """
    Sets the current model as orphaned. This is called when the scheduler is
    about to kill the process to reallocate the worker to a different process.
    """
    cmplReason = ClientJobsDAO.CMPL_REASON_ORPHAN
    cmplMessage = "Killed by Scheduler"
    self._jobsDAO.modelSetCompleted(self._modelID, cmplReason, cmplMessage)

def
__setAsOrphaned
(
self
)
:
cmplReason
=
ClientJobsDAO
.
CMPL_REASON_ORPHAN
cmplMessage
=
"Killed by Scheduler"
self
.
_jobsDAO
.
modelSetCompleted
(
self
.
_modelID
,
cmplReason
,
cmplMessage
)

Sets the current model as orphaned . This is called when the scheduler is about to kill the process to reallocate the worker to a different process .


def readStateFromDB(self):
    """Set our state to that obtained from the engWorkerState field of the
    job record.

    Parameters:
    ---------------------------------------------------------------------
    stateJSON:    JSON encoded state from job record

    """
    self._priorStateJSON = self._hsObj._cjDAO.jobGetFields(self._hsObj._jobID,
                                                    ['engWorkerState'])[0]

    # Init if no prior state yet
    if self._priorStateJSON is None:
      swarms = dict()

      # Fast Swarm, first and only sprint has one swarm for each field
      # in fixedFields
      if self._hsObj._fixedFields is not None:
        print self._hsObj._fixedFields
        encoderSet = []
        for field in self._hsObj._fixedFields:
            if field =='_classifierInput':
              continue
            encoderName = self.getEncoderKeyFromName(field)
            assert encoderName in self._hsObj._encoderNames, "The field '%s' " \
              " specified in the fixedFields list is not present in this " \
              " model." % (field)
            encoderSet.append(encoderName)
        encoderSet.sort()
        swarms['.'.join(encoderSet)] = {
                                'status': 'active',
                                'bestModelId': None,
                                'bestErrScore': None,
                                'sprintIdx': 0,
                                }
      # Temporal prediction search, first sprint has N swarms of 1 field each,
      #  the predicted field may or may not be that one field.
      elif self._hsObj._searchType == HsSearchType.temporal:
        for encoderName in self._hsObj._encoderNames:
          swarms[encoderName] = {
                                  'status': 'active',
                                  'bestModelId': None,
                                  'bestErrScore': None,
                                  'sprintIdx': 0,
                                  }

      # Classification prediction search, first sprint has N swarms of 1 field
      #  each where this field can NOT be the predicted field.
      elif self._hsObj._searchType == HsSearchType.classification:
        for encoderName in self._hsObj._encoderNames:
          if encoderName == self._hsObj._predictedFieldEncoder:
            continue
          swarms[encoderName] = {
                                  'status': 'active',
                                  'bestModelId': None,
                                  'bestErrScore': None,
                                  'sprintIdx': 0,
                                  }

      # Legacy temporal. This is either a model that uses reconstruction or
      #  an older multi-step model that doesn't have a separate
      #  'classifierOnly' encoder for the predicted field. Here, the predicted
      #  field must ALWAYS be present and the first sprint tries the predicted
      #  field only
      elif self._hsObj._searchType == HsSearchType.legacyTemporal:
        swarms[self._hsObj._predictedFieldEncoder] = {
                       'status': 'active',
                       'bestModelId': None,
                       'bestErrScore': None,
                       'sprintIdx': 0,
                       }

      else:
        raise RuntimeError("Unsupported search type: %s" % \
                            (self._hsObj._searchType))

      # Initialize the state.
      self._state = dict(
        # The last time the state was updated by a worker.
        lastUpdateTime = time.time(),

        # Set from within setSwarmState() if we detect that the sprint we just
        #  completed did worse than a prior sprint. This stores the index of
        #  the last good sprint.
        lastGoodSprint = None,

        # Set from within setSwarmState() if lastGoodSprint is True and all
        #  sprints have completed.
        searchOver = False,

        # This is a summary of the active swarms - this information can also
        #  be obtained from the swarms entry that follows, but is summarized here
        #  for easier reference when viewing the state as presented by
        #  log messages and prints of the hsState data structure (by
        #  permutations_runner).
        activeSwarms = swarms.keys(),

        # All the swarms that have been created so far.
        swarms = swarms,

        # All the sprints that have completed or are in progress.
        sprints = [{'status': 'active',
                    'bestModelId': None,
                    'bestErrScore': None}],

        # The list of encoders we have "blacklisted" because they
        #  performed so poorly.
        blackListedEncoders = [],
        )

      # This will do nothing if the value of engWorkerState is not still None.
      self._hsObj._cjDAO.jobSetFieldIfEqual(
          self._hsObj._jobID, 'engWorkerState', json.dumps(self._state), None)

      self._priorStateJSON = self._hsObj._cjDAO.jobGetFields(
          self._hsObj._jobID, ['engWorkerState'])[0]
      assert (self._priorStateJSON is not None)

    # Read state from the database
    self._state = json.loads(self._priorStateJSON)
    self._dirty = False

Set our state to that obtained from the engWorkerState field of the job record .


def getFieldContributions(self):
    """Return the field contributions statistics.

    Parameters:
    ---------------------------------------------------------------------
    retval:   Dictionary where the keys are the field names and the values
                are how much each field contributed to the best score.
    """

    #in the fast swarm, there is only 1 sprint and field contributions are
    #not defined
    if self._hsObj._fixedFields is not None:
      return dict(), dict()
    # Get the predicted field encoder name
    predictedEncoderName = self._hsObj._predictedFieldEncoder

    # -----------------------------------------------------------------------
    # Collect all the single field scores
    fieldScores = []
    for swarmId, info in self._state['swarms'].iteritems():
      encodersUsed = swarmId.split('.')
      if len(encodersUsed) != 1:
        continue
      field = self.getEncoderNameFromKey(encodersUsed[0])
      bestScore = info['bestErrScore']

      # If the bestScore is None, this swarm hasn't completed yet (this could
      #  happen if we're exiting because of maxModels), so look up the best
      #  score so far
      if bestScore is None:
        (_modelId, bestScore) = \
            self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)

      fieldScores.append((bestScore, field))

    # -----------------------------------------------------------------------
    # If we only have 1 field that was tried in the first sprint, then use that
    #  as the base and get the contributions from the fields in the next sprint.
    if self._hsObj._searchType == HsSearchType.legacyTemporal:
      assert(len(fieldScores)==1)
      (baseErrScore, baseField) = fieldScores[0]

      for swarmId, info in self._state['swarms'].iteritems():
        encodersUsed = swarmId.split('.')
        if len(encodersUsed) != 2:
          continue

        fields = [self.getEncoderNameFromKey(name) for name in encodersUsed]
        fields.remove(baseField)

        fieldScores.append((info['bestErrScore'], fields[0]))

    # The first sprint tried a bunch of fields, pick the worst performing one
    #  (within the top self._hsObj._maxBranching ones) as the base
    else:
      fieldScores.sort(reverse=True)

      # If maxBranching was specified, pick the worst performing field within
      #  the top maxBranching+1 fields as our base, which will give that field
      #  a contribution of 0.
      if self._hsObj._maxBranching > 0 \
              and len(fieldScores) > self._hsObj._maxBranching:
        baseErrScore = fieldScores[-self._hsObj._maxBranching-1][0]
      else:
        baseErrScore = fieldScores[0][0]

    # -----------------------------------------------------------------------
    # Prepare and return the fieldContributions dict
    pctFieldContributionsDict = dict()
    absFieldContributionsDict = dict()

    # If we have no base score, can't compute field contributions. This can
    #  happen when we exit early due to maxModels or being cancelled
    if baseErrScore is not None:

      # If the base error score is 0, we can't compute a percent difference
      #  off of it, so move it to a very small float
      if abs(baseErrScore) < 0.00001:
        baseErrScore = 0.00001
      for (errScore, field) in fieldScores:
        if errScore is not None:
          pctBetter = (baseErrScore - errScore) * 100.0 / baseErrScore
        else:
          pctBetter = 0.0
          errScore = baseErrScore   # for absFieldContribution

        pctFieldContributionsDict[field] = pctBetter
        absFieldContributionsDict[field] = baseErrScore - errScore

    self.logger.debug("FieldContributions: %s" % (pctFieldContributionsDict))
    return pctFieldContributionsDict, absFieldContributionsDict

def
getFieldContributions
(
self
)
:
if
self
.
_hsObj
.
_fixedFields
is
not
:
return
dict
(
)
,
dict
(
)
predictedEncoderName
=
self
.
_hsObj
.
_predictedFieldEncoder
fieldScores
=
[
]
for
swarmId
,
info
in
self
.
_state
[
'swarms'
]
.
iteritems
(
)
:
encodersUsed
=
swarmId
.
split
(
'.'
)
if
len
(
encodersUsed
)
!=
1
:
continue
field
=
self
.
getEncoderNameFromKey
(
encodersUsed
[
0
]
)
bestScore
=
info
[
'bestErrScore'
]
if
bestScore
is
:
(
_modelId
,
bestScore
)
=
self
.
_hsObj
.
_resultsDB
.
bestModelIdAndErrScore
(
swarmId
)
fieldScores
.
append
(
(
bestScore
,
field
)
)
if
self
.
_hsObj
.
_searchType
==
HsSearchType
.
legacyTemporal
:
assert
(
len
(
fieldScores
)
==
1
)
(
baseErrScore
,
baseField
)
=
fieldScores
[
0
]
for
swarmId
,
info
in
self
.
_state
[
'swarms'
]
.
iteritems
(
)
:
encodersUsed
=
swarmId
.
split
(
'.'
)
if
len
(
encodersUsed
)
!=
2
:
continue
fields
=
[
self
.
getEncoderNameFromKey
(
name
)
for
name
in
encodersUsed
]
fields
.
remove
(
baseField
)
fieldScores
.
append
(
(
info
[
'bestErrScore'
]
,
fields
[
0
]
)
)
else
:
fieldScores
.
sort
(
reverse
=
True
)
if
self
.
_hsObj
.
_maxBranching
>
0
and
len
(
fieldScores
)
>
self
.
_hsObj
.
_maxBranching
:
baseErrScore
=
fieldScores
[
self
.
_hsObj
.
_maxBranching
1
]
[
0
]
else
:
baseErrScore
=
fieldScores
[
0
]
[
0
]
pctFieldContributionsDict
=
dict
(
)
absFieldContributionsDict
=
dict
(
)
if
baseErrScore
is
not
:
if
abs
(
baseErrScore
)
<
0.00001
:
baseErrScore
=
0.00001
for
(
errScore
,
field
)
in
fieldScores
:
if
errScore
is
not
:
pctBetter
=
(
baseErrScore
errScore
)
*
100.0
/
baseErrScore
else
:
pctBetter
=
0.0
errScore
=
baseErrScore
pctFieldContributionsDict
[
field
]
=
pctBetter
absFieldContributionsDict
[
field
]
=
baseErrScore
errScore
self
.
logger
.
debug
(
"FieldContributions: %s"
%
(
pctFieldContributionsDict
)
)
return
pctFieldContributionsDict
,
absFieldContributionsDict

Return the field contributions statistics .


def getAllSwarms(self, sprintIdx):
    """Return the list of all swarms in the given sprint.

    Parameters:
    ---------------------------------------------------------------------
    retval:   list of active swarm Ids in the given sprint
    """
    swarmIds = []
    for swarmId, info in self._state['swarms'].iteritems():
      if info['sprintIdx'] == sprintIdx:
        swarmIds.append(swarmId)

    return swarmIds

def getAllSwarms ( self , sprintIdx ) : swarmIds = [ ] for swarmId , info in self . _state [ 'swarms' ] . iteritems ( ) : if info [ 'sprintIdx' ] == sprintIdx : swarmIds . append ( swarmId ) return swarmIds

Return the list of all swarms in the given sprint .


def getCompletedSwarms(self):
    """Return the list of all completed swarms.

    Parameters:
    ---------------------------------------------------------------------
    retval:   list of active swarm Ids
    """
    swarmIds = []
    for swarmId, info in self._state['swarms'].iteritems():
      if info['status'] == 'completed':
        swarmIds.append(swarmId)

    return swarmIds

def getCompletedSwarms ( self ) : swarmIds = [ ] for swarmId , info in self . _state [ 'swarms' ] . iteritems ( ) : if info [ 'status' ] == 'completed' : swarmIds . append ( swarmId ) return swarmIds

Return the list of all completed swarms .


def getCompletingSwarms(self):
    """Return the list of all completing swarms.

    Parameters:
    ---------------------------------------------------------------------
    retval:   list of active swarm Ids
    """
    swarmIds = []
    for swarmId, info in self._state['swarms'].iteritems():
      if info['status'] == 'completing':
        swarmIds.append(swarmId)

    return swarmIds

def getCompletingSwarms ( self ) : swarmIds = [ ] for swarmId , info in self . _state [ 'swarms' ] . iteritems ( ) : if info [ 'status' ] == 'completing' : swarmIds . append ( swarmId ) return swarmIds

Return the list of all completing swarms .


def bestModelInSprint(self, sprintIdx):
    """Return the best model ID and it's errScore from the given sprint,
    which may still be in progress. This returns the best score from all models
    in the sprint which have matured so far.

    Parameters:
    ---------------------------------------------------------------------
    retval:   (modelId, errScore)
    """
    # Get all the swarms in this sprint
    swarms = self.getAllSwarms(sprintIdx)

    # Get the best model and score from each swarm
    bestModelId = None
    bestErrScore = numpy.inf
    for swarmId in swarms:
      (modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)
      if errScore < bestErrScore:
        bestModelId = modelId
        bestErrScore = errScore

    return (bestModelId, bestErrScore)

def bestModelInSprint ( self , sprintIdx ) : swarms = self . getAllSwarms ( sprintIdx ) bestModelId = None bestErrScore = numpy . inf for swarmId in swarms : ( modelId , errScore ) = self . _hsObj . _resultsDB . bestModelIdAndErrScore ( swarmId ) if errScore < bestErrScore : bestModelId = modelId bestErrScore = errScore return ( bestModelId , bestErrScore )

Return the best model ID and it s errScore from the given sprint which may still be in progress . This returns the best score from all models in the sprint which have matured so far .


def setSwarmState(self, swarmId, newStatus):
    """Change the given swarm's state to 'newState'. If 'newState' is
    'completed', then bestModelId and bestErrScore must be provided.

    Parameters:
    ---------------------------------------------------------------------
    swarmId:      swarm Id
    newStatus:    new status, either 'active', 'completing', 'completed', or
                    'killed'
    """
    assert (newStatus in ['active', 'completing', 'completed', 'killed'])

    # Set the swarm status
    swarmInfo = self._state['swarms'][swarmId]
    if swarmInfo['status'] == newStatus:
      return

    # If some other worker noticed it as completed, setting it to completing
    #  is obviously old information....
    if swarmInfo['status'] == 'completed' and newStatus == 'completing':
      return

    self._dirty = True
    swarmInfo['status'] = newStatus
    if newStatus == 'completed':
      (modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)
      swarmInfo['bestModelId'] = modelId
      swarmInfo['bestErrScore'] = errScore

    # If no longer active, remove it from the activeSwarms entry
    if newStatus != 'active' and swarmId in self._state['activeSwarms']:
      self._state['activeSwarms'].remove(swarmId)

    # If new status is 'killed', kill off any running particles in that swarm
    if newStatus=='killed':
      self._hsObj.killSwarmParticles(swarmId)

    # In case speculative particles are enabled, make sure we generate a new
    #  swarm at this time if all of the swarms in the current sprint have
    #  completed. This will insure that we don't mark the sprint as completed
    #  before we've created all the possible swarms.
    sprintIdx = swarmInfo['sprintIdx']
    self.isSprintActive(sprintIdx)

    # Update the sprint status. Check all the swarms that belong to this sprint.
    #  If they are all completed, the sprint is completed.
    sprintInfo = self._state['sprints'][sprintIdx]

    statusCounts = dict(active=0, completing=0, completed=0, killed=0)
    bestModelIds = []
    bestErrScores = []
    for info in self._state['swarms'].itervalues():
      if info['sprintIdx'] != sprintIdx:
        continue
      statusCounts[info['status']] += 1
      if info['status'] == 'completed':
        bestModelIds.append(info['bestModelId'])
        bestErrScores.append(info['bestErrScore'])

    if statusCounts['active'] > 0:
      sprintStatus = 'active'
    elif statusCounts['completing'] > 0:
      sprintStatus = 'completing'
    else:
      sprintStatus = 'completed'
    sprintInfo['status'] = sprintStatus

    # If the sprint is complete, get the best model from all of its swarms and
    #  store that as the sprint best
    if sprintStatus == 'completed':
      if len(bestErrScores) > 0:
        whichIdx = numpy.array(bestErrScores).argmin()
        sprintInfo['bestModelId'] = bestModelIds[whichIdx]
        sprintInfo['bestErrScore'] = bestErrScores[whichIdx]
      else:
        # This sprint was empty, most likely because all particles were
        #  killed. Give it a huge error score
        sprintInfo['bestModelId'] = 0
        sprintInfo['bestErrScore'] = numpy.inf

      # See if our best err score got NO BETTER as compared to a previous
      #  sprint. If so, stop exploring subsequent sprints (lastGoodSprint
      #  is no longer None).
      bestPrior = numpy.inf
      for idx in range(sprintIdx):
        if self._state['sprints'][idx]['status'] == 'completed':
          (_, errScore) = self.bestModelInCompletedSprint(idx)
          if errScore is None:
            errScore = numpy.inf
        else:
          errScore = numpy.inf
        if errScore < bestPrior:
          bestPrior = errScore

      if sprintInfo['bestErrScore'] >= bestPrior:
        self._state['lastGoodSprint'] = sprintIdx-1

      # If ALL sprints up to the last good one are done, the search is now over
      if self._state['lastGoodSprint'] is not None \
            and not self.anyGoodSprintsActive():
        self._state['searchOver'] = True

def setSwarmState ( self , swarmId , newStatus ) : assert ( newStatus in [ 'active' , 'completing' , 'completed' , 'killed' ] ) swarmInfo = self . _state [ 'swarms' ] [ swarmId ] if swarmInfo [ 'status' ] == newStatus : return if swarmInfo [ 'status' ] == 'completed' and newStatus == 'completing' : return self . _dirty = True swarmInfo [ 'status' ] = newStatus if newStatus == 'completed' : ( modelId , errScore ) = self . _hsObj . _resultsDB . bestModelIdAndErrScore ( swarmId ) swarmInfo [ 'bestModelId' ] = modelId swarmInfo [ 'bestErrScore' ] = errScore if newStatus != 'active' and swarmId in self . _state [ 'activeSwarms' ] : self . _state [ 'activeSwarms' ] . remove ( swarmId ) if newStatus == 'killed' : self . _hsObj . killSwarmParticles ( swarmId ) sprintIdx = swarmInfo [ 'sprintIdx' ] self . isSprintActive ( sprintIdx ) sprintInfo = self . _state [ 'sprints' ] [ sprintIdx ] statusCounts = dict ( active = 0 , completing = 0 , completed = 0 , killed = 0 ) bestModelIds = [ ] bestErrScores = [ ] for info in self . _state [ 'swarms' ] . itervalues ( ) : if info [ 'sprintIdx' ] != sprintIdx : continue statusCounts [ info [ 'status' ] ] += 1 if info [ 'status' ] == 'completed' : bestModelIds . append ( info [ 'bestModelId' ] ) bestErrScores . append ( info [ 'bestErrScore' ] ) if statusCounts [ 'active' ] > 0 : sprintStatus = 'active' elif statusCounts [ 'completing' ] > 0 : sprintStatus = 'completing' else : sprintStatus = 'completed' sprintInfo [ 'status' ] = sprintStatus if sprintStatus == 'completed' : if len ( bestErrScores ) > 0 : whichIdx = numpy . array ( bestErrScores ) . argmin ( ) sprintInfo [ 'bestModelId' ] = bestModelIds [ whichIdx ] sprintInfo [ 'bestErrScore' ] = bestErrScores [ whichIdx ] else : sprintInfo [ 'bestModelId' ] = 0 sprintInfo [ 'bestErrScore' ] = numpy . inf bestPrior = numpy . inf for idx in range ( sprintIdx ) : if self . _state [ 'sprints' ] [ idx ] [ 'status' ] == 'completed' : ( _ , errScore ) = self . bestModelInCompletedSprint ( idx ) if errScore is None : errScore = numpy . inf else : errScore = numpy . inf if errScore < bestPrior : bestPrior = errScore if sprintInfo [ 'bestErrScore' ] >= bestPrior : self . _state [ 'lastGoodSprint' ] = sprintIdx - 1 if self . _state [ 'lastGoodSprint' ] is not None and not self . anyGoodSprintsActive ( ) : self . _state [ 'searchOver' ] = True

Change the given swarm s state to newState . If newState is completed then bestModelId and bestErrScore must be provided .


def anyGoodSprintsActive(self):
    """Return True if there are any more good sprints still being explored.
    A 'good' sprint is one that is earlier than where we detected an increase
    in error from sprint to subsequent sprint.
    """
    if self._state['lastGoodSprint'] is not None:
      goodSprints = self._state['sprints'][0:self._state['lastGoodSprint']+1]
    else:
      goodSprints = self._state['sprints']

    for sprint in goodSprints:
      if sprint['status'] == 'active':
        anyActiveSprints = True
        break
    else:
      anyActiveSprints = False

    return anyActiveSprints

def anyGoodSprintsActive ( self ) : if self . _state [ 'lastGoodSprint' ] is not None : goodSprints = self . _state [ 'sprints' ] [ 0 : self . _state [ 'lastGoodSprint' ] + 1 ] else : goodSprints = self . _state [ 'sprints' ] for sprint in goodSprints : if sprint [ 'status' ] == 'active' : anyActiveSprints = True break else : anyActiveSprints = False return anyActiveSprints

Return True if there are any more good sprints still being explored . A good sprint is one that is earlier than where we detected an increase in error from sprint to subsequent sprint .


def isSprintCompleted(self, sprintIdx):
    """Return True if the given sprint has completed."""
    numExistingSprints = len(self._state['sprints'])
    if sprintIdx >= numExistingSprints:
      return False

    return (self._state['sprints'][sprintIdx]['status'] == 'completed')

def isSprintCompleted ( self , sprintIdx ) : numExistingSprints = len ( self . _state [ 'sprints' ] ) if sprintIdx >= numExistingSprints : return False return ( self . _state [ 'sprints' ] [ sprintIdx ] [ 'status' ] == 'completed' )

Return True if the given sprint has completed .


def addEncoder(self, name, encoder):
    """
    Adds one encoder.

    :param name: (string) name of encoder, should be unique
    :param encoder: (:class:`.Encoder`) the encoder to add
    """
    self.encoders.append((name, encoder, self.width))
    for d in encoder.getDescription():
      self.description.append((d[0], d[1] + self.width))
    self.width += encoder.getWidth()

def addEncoder ( self , name , encoder ) : self . encoders . append ( ( name , encoder , self . width ) ) for d in encoder . getDescription ( ) : self . description . append ( ( d [ 0 ] , d [ 1 ] + self . width ) ) self . width += encoder . getWidth ( )


def invariant(self):
    """Verify the validity of the node spec object

    The type of each sub-object is verified and then
    the validity of each node spec item is verified by calling
    it invariant() method. It also makes sure that there is at most
    one default input and one default output.
    """
    # Verify the description and singleNodeOnly attributes
    assert isinstance(self.description, str)
    assert isinstance(self.singleNodeOnly, bool)

    # Make sure that all items dicts are really dicts
    assert isinstance(self.inputs, dict)
    assert isinstance(self.outputs, dict)
    assert isinstance(self.parameters, dict)
    assert isinstance(self.commands, dict)

    # Verify all item dicts
    hasDefaultInput = False
    for k, v in self.inputs.items():
      assert isinstance(k, str)
      assert isinstance(v, InputSpec)
      v.invariant()
      if v.isDefaultInput:
        assert not hasDefaultInput
        hasDefaultInput = True

    hasDefaultOutput = False
    for k, v in self.outputs.items():
      assert isinstance(k, str)
      assert isinstance(v, OutputSpec)
      v.invariant()
      if v.isDefaultOutput:
        assert not hasDefaultOutput
        hasDefaultOutput = True

    for k, v in self.parameters.items():
      assert isinstance(k, str)
      assert isinstance(v, ParameterSpec)
      v.invariant()

    for k, v in self.commands.items():
      assert isinstance(k, str)
      assert isinstance(v, CommandSpec)
      v.invariant()

def invariant ( self ) : assert isinstance ( self . description , str ) assert isinstance ( self . singleNodeOnly , bool ) assert isinstance ( self . inputs , dict ) assert isinstance ( self . outputs , dict ) assert isinstance ( self . parameters , dict ) assert isinstance ( self . commands , dict ) hasDefaultInput = False for k , v in self . inputs . items ( ) : assert isinstance ( k , str ) assert isinstance ( v , InputSpec ) v . invariant ( ) if v . isDefaultInput : assert not hasDefaultInput hasDefaultInput = True hasDefaultOutput = False for k , v in self . outputs . items ( ) : assert isinstance ( k , str ) assert isinstance ( v , OutputSpec ) v . invariant ( ) if v . isDefaultOutput : assert not hasDefaultOutput hasDefaultOutput = True for k , v in self . parameters . items ( ) : assert isinstance ( k , str ) assert isinstance ( v , ParameterSpec ) v . invariant ( ) for k , v in self . commands . items ( ) : assert isinstance ( k , str ) assert isinstance ( v , CommandSpec ) v . invariant ( )


def toDict(self):
    """Convert the information of the node spec to a plain dict of basic types

    The description and singleNodeOnly attributes are placed directly in
    the result dicts. The inputs, outputs, parameters and commands dicts
    contain Spec item objects (InputSpec, OutputSpec, etc). Each such object
    is converted also to a plain dict using the internal items2dict() function
    (see bellow).
    """

    def items2dict(items):
      """Convert a dict of node spec items to a plain dict

      Each node spec item object will be converted to a dict of its
      attributes. The entire items dict will become a dict of dicts (same keys).
      """
      d = {}
      for k, v in items.items():
        d[k] = v.__dict__

      return d

    self.invariant()
    return dict(description=self.description,
                singleNodeOnly=self.singleNodeOnly,
                inputs=items2dict(self.inputs),
                outputs=items2dict(self.outputs),
                parameters=items2dict(self.parameters),
                commands=items2dict(self.commands))

def toDict ( self ) : def items2dict ( items ) : d = { } for k , v in items . items ( ) : d [ k ] = v . __dict__ return d self . invariant ( ) return dict ( description = self . description , singleNodeOnly = self . singleNodeOnly , inputs = items2dict ( self . inputs ) , outputs = items2dict ( self . outputs ) , parameters = items2dict ( self . parameters ) , commands = items2dict ( self . commands ) )


def updateResultsForJob(self, forceUpdate=True):
    """ Chooses the best model for a given job.

    Parameters
    -----------------------------------------------------------------------
    forceUpdate:  (True/False). If True, the update will ignore all the
                  restrictions on the minimum time to update and the minimum
                  number of records to update. This should typically only be
                  set to true if the model has completed running
    """
    updateInterval = time.time() - self._lastUpdateAttemptTime
    if updateInterval < self._MIN_UPDATE_INTERVAL and not forceUpdate:
      return

    self.logger.info("Attempting model selection for jobID=%d: time=%f"\
                     "  lastUpdate=%f"%(self._jobID,
                                        time.time(),
                                        self._lastUpdateAttemptTime))

    timestampUpdated = self._cjDB.jobUpdateSelectionSweep(self._jobID,
                                                          self._MIN_UPDATE_INTERVAL)
    if not timestampUpdated:
      self.logger.info("Unable to update selection sweep timestamp: jobID=%d" \
                       " updateTime=%f"%(self._jobID, self._lastUpdateAttemptTime))
      if not forceUpdate:
        return

    self._lastUpdateAttemptTime = time.time()
    self.logger.info("Succesfully updated selection sweep timestamp jobid=%d updateTime=%f"\
                     %(self._jobID, self._lastUpdateAttemptTime))

    minUpdateRecords = self._MIN_UPDATE_THRESHOLD

    jobResults = self._getJobResults()
    if forceUpdate or jobResults is None:
      minUpdateRecords = 0

    candidateIDs, bestMetric = self._cjDB.modelsGetCandidates(self._jobID, minUpdateRecords)

    self.logger.info("Candidate models=%s, metric=%s, jobID=%s"\
                     %(candidateIDs, bestMetric, self._jobID))

    if len(candidateIDs) == 0:
      return

    self._jobUpdateCandidate(candidateIDs[0], bestMetric, results=jobResults)

Chooses the best model for a given job .


def createEncoder():
  """Create the encoder instance for our test and return it."""
  consumption_encoder = ScalarEncoder(21, 0.0, 100.0, n=50, name="consumption",
      clipInput=True)
  time_encoder = DateEncoder(timeOfDay=(21, 9.5), name="timestamp_timeOfDay")

  encoder = MultiEncoder()
  encoder.addEncoder("consumption", consumption_encoder)
  encoder.addEncoder("timestamp", time_encoder)

  return encoder

def createEncoder ( ) : consumption_encoder = ScalarEncoder ( 21 , 0.0 , 100.0 , n = 50 , name = "consumption" , clipInput = True ) time_encoder = DateEncoder ( timeOfDay = ( 21 , 9.5 ) , name = "timestamp_timeOfDay" ) encoder = MultiEncoder ( ) encoder . addEncoder ( "consumption" , consumption_encoder ) encoder . addEncoder ( "timestamp" , time_encoder ) return encoder

Create the encoder instance for our test and return it .


def __validateExperimentControl(self, control):
    """ Validates control dictionary for the experiment context"""
    # Validate task list
    taskList = control.get('tasks', None)
    if taskList is not None:
      taskLabelsList = []

      for task in taskList:
        validateOpfJsonValue(task, "opfTaskSchema.json")
        validateOpfJsonValue(task['taskControl'], "opfTaskControlSchema.json")

        taskLabel = task['taskLabel']

        assert isinstance(taskLabel, types.StringTypes), \
               "taskLabel type: %r" % type(taskLabel)
        assert len(taskLabel) > 0, "empty string taskLabel not is allowed"

        taskLabelsList.append(taskLabel.lower())

      taskLabelDuplicates = filter(lambda x: taskLabelsList.count(x) > 1,
                                   taskLabelsList)
      assert len(taskLabelDuplicates) == 0, \
             "Duplcate task labels are not allowed: %s" % taskLabelDuplicates

    return


def _matchReportKeys(reportKeyREs=[], allReportKeys=[]):
  """
  Extract all items from the 'allKeys' list whose key matches one of the regular
  expressions passed in 'reportKeys'.

  Parameters:
  ----------------------------------------------------------------------------
  reportKeyREs:     List of regular expressions
  allReportKeys:    List of all keys

  retval:         list of keys from allReportKeys that match the regular expressions
                    in 'reportKeyREs'
                  If an invalid regular expression was included in 'reportKeys',
                    then BadKeyError() is raised
  """

  matchingReportKeys = []

  # Extract the report items of interest
  for keyRE in reportKeyREs:
    # Find all keys that match this regular expression
    matchObj = re.compile(keyRE)
    found = False
    for keyName in allReportKeys:
      match = matchObj.match(keyName)
      if match and match.end() == len(keyName):
        matchingReportKeys.append(keyName)
        found = True
    if not found:
      raise _BadKeyError(keyRE)

  return matchingReportKeys

def _matchReportKeys ( reportKeyREs = [ ] , allReportKeys = [ ] ) : matchingReportKeys = [ ] for keyRE in reportKeyREs : matchObj = re . compile ( keyRE ) found = False for keyName in allReportKeys : match = matchObj . match ( keyName ) if match and match . end ( ) == len ( keyName ) : matchingReportKeys . append ( keyName ) found = True if not found : raise _BadKeyError ( keyRE ) return matchingReportKeys

Extract all items from the allKeys list whose key matches one of the regular expressions passed in reportKeys .


def _getReportItem(itemName, results):
  """
  Get a specific item by name out of the results dict.

  The format of itemName is a string of dictionary keys separated by colons,
  each key being one level deeper into the results dict. For example,
  'key1:key2' would fetch results['key1']['key2'].

  If itemName is not found in results, then None is returned

  """

  subKeys = itemName.split(':')
  subResults = results
  for subKey in subKeys:
    subResults = subResults[subKey]

  return subResults

def _getReportItem ( itemName , results ) : subKeys = itemName . split ( ':' ) subResults = results for subKey in subKeys : subResults = subResults [ subKey ] return subResults

Get a specific item by name out of the results dict .


def _handleModelRunnerException(jobID, modelID, jobsDAO, experimentDir, logger,
                                e):
  """ Perform standard handling of an exception that occurs while running
  a model.

  Parameters:
  -------------------------------------------------------------------------
  jobID:                ID for this hypersearch job in the jobs table
  modelID:              model ID
  jobsDAO:              ClientJobsDAO instance
  experimentDir:        directory containing the experiment
  logger:               the logger to use
  e:                    the exception that occurred
  retval:               (completionReason, completionMsg)
  """

  msg = StringIO.StringIO()
  print >>msg, "Exception occurred while running model %s: %r (%s)" % (
    modelID, e, type(e))
  traceback.print_exc(None, msg)

  completionReason = jobsDAO.CMPL_REASON_ERROR
  completionMsg = msg.getvalue()
  logger.error(completionMsg)

  # Write results to the model database for the error case. Ignore
  # InvalidConnectionException, as this is usually caused by orphaned models
  #
  # TODO: do we really want to set numRecords to 0? Last updated value might
  #       be useful for debugging
  if type(e) is not InvalidConnectionException:
    jobsDAO.modelUpdateResults(modelID,  results=None, numRecords=0)

  # TODO: Make sure this wasn't the best model in job. If so, set the best
  # appropriately

  # If this was an exception that should mark the job as failed, do that
  # now.
  if type(e) == JobFailException:
    workerCmpReason = jobsDAO.jobGetFields(jobID,
        ['workerCompletionReason'])[0]
    if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:
      jobsDAO.jobSetFields(jobID, fields=dict(
          cancel=True,
          workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,
          workerCompletionMsg = ": ".join(str(i) for i in e.args)),
          useConnectionID=False,
          ignoreUnchanged=True)

  return (completionReason, completionMsg)

Perform standard handling of an exception that occurs while running a model .


def runModelGivenBaseAndParams(modelID, jobID, baseDescription, params,
            predictedField, reportKeys, optimizeKey, jobsDAO,
            modelCheckpointGUID, logLevel=None, predictionCacheMaxRecords=None):
  """ This creates an experiment directory with a base.py description file
  created from 'baseDescription' and a description.py generated from the
  given params dict and then runs the experiment.

  Parameters:
  -------------------------------------------------------------------------
  modelID:              ID for this model in the models table
  jobID:                ID for this hypersearch job in the jobs table
  baseDescription:      Contents of a description.py with the base experiment
                                          description
  params:               Dictionary of specific parameters to override within
                                  the baseDescriptionFile.
  predictedField:       Name of the input field for which this model is being
                                    optimized
  reportKeys:           Which metrics of the experiment to store into the
                                    results dict of the model's database entry
  optimizeKey:          Which metric we are optimizing for
  jobsDAO               Jobs data access object - the interface to the
                                  jobs database which has the model's table.
  modelCheckpointGUID:  A persistent, globally-unique identifier for
                                  constructing the model checkpoint key
  logLevel:             override logging level to this value, if not None

  retval:               (completionReason, completionMsg)
  """
  from nupic.swarming.ModelRunner import OPFModelRunner

  # The logger for this method
  logger = logging.getLogger('com.numenta.nupic.hypersearch.utils')

  # --------------------------------------------------------------------------
  # Create a temp directory for the experiment and the description files
  experimentDir = tempfile.mkdtemp()
  try:
    logger.info("Using experiment directory: %s" % (experimentDir))

    # Create the decription.py from the overrides in params
    paramsFilePath = os.path.join(experimentDir, 'description.py')
    paramsFile = open(paramsFilePath, 'wb')
    paramsFile.write(_paramsFileHead())

    items = params.items()
    items.sort()
    for (key,value) in items:
      quotedKey = _quoteAndEscape(key)
      if isinstance(value, basestring):

        paramsFile.write("  %s : '%s',\n" % (quotedKey , value))
      else:
        paramsFile.write("  %s : %s,\n" % (quotedKey , value))

    paramsFile.write(_paramsFileTail())
    paramsFile.close()

    # Write out the base description
    baseParamsFile = open(os.path.join(experimentDir, 'base.py'), 'wb')
    baseParamsFile.write(baseDescription)
    baseParamsFile.close()

    # Store the experiment's sub-description file into the model table
    #  for reference
    fd = open(paramsFilePath)
    expDescription = fd.read()
    fd.close()
    jobsDAO.modelSetFields(modelID, {'genDescription': expDescription})

    # Run the experiment now
    try:
      runner = OPFModelRunner(
        modelID=modelID,
        jobID=jobID,
        predictedField=predictedField,
        experimentDir=experimentDir,
        reportKeyPatterns=reportKeys,
        optimizeKeyPattern=optimizeKey,
        jobsDAO=jobsDAO,
        modelCheckpointGUID=modelCheckpointGUID,
        logLevel=logLevel,
        predictionCacheMaxRecords=predictionCacheMaxRecords)

      signal.signal(signal.SIGINT, runner.handleWarningSignal)

      (completionReason, completionMsg) = runner.run()

    except InvalidConnectionException:
      raise
    except Exception, e:

      (completionReason, completionMsg) = _handleModelRunnerException(jobID,
                                     modelID, jobsDAO, experimentDir, logger, e)

  finally:
    # delete our temporary directory tree
    shutil.rmtree(experimentDir)
    signal.signal(signal.SIGINT, signal.default_int_handler)

  # Return completion reason and msg
  return (completionReason, completionMsg)

def
runModelGivenBaseAndParams
(
modelID
,
jobID
,
baseDescription
,
params
,
predictedField
,
reportKeys
,
optimizeKey
,
jobsDAO
,
modelCheckpointGUID
,
logLevel
=
,
predictionCacheMaxRecords
=
)
:
from
nupic
.
swarming
.
ModelRunner
import
OPFModelRunner
logger
=
logging
.
getLogger
(
'com.numenta.nupic.hypersearch.utils'
)
experimentDir
=
tempfile
.
mkdtemp
(
)
try
:
logger
.
info
(
"Using experiment directory: %s"
%
(
experimentDir
)
)
paramsFilePath
=
os
.
path
.
join
(
experimentDir
,
'description.py'
)
paramsFile
=
open
(
paramsFilePath
,
'wb'
)
paramsFile
.
write
(
_paramsFileHead
(
)
)
items
=
params
.
items
(
)
items
.
sort
(
)
for
(
key
,
value
)
in
items
:
quotedKey
=
_quoteAndEscape
(
key
)
if
isinstance
(
value
,
basestring
)
:
paramsFile
.
write
(
"  %s : '%s',\n"
%
(
quotedKey
,
value
)
)
else
:
paramsFile
.
write
(
"  %s : %s,\n"
%
(
quotedKey
,
value
)
)
paramsFile
.
write
(
_paramsFileTail
(
)
)
paramsFile
.
close
(
)
baseParamsFile
=
open
(
os
.
path
.
join
(
experimentDir
,
'base.py'
)
,
'wb'
)
baseParamsFile
.
write
(
baseDescription
)
baseParamsFile
.
close
(
)
fd
=
open
(
paramsFilePath
)
expDescription
=
fd
.
read
(
)
fd
.
close
(
)
jobsDAO
.
modelSetFields
(
modelID
,
{
'genDescription'
:
expDescription
}
)
try
:
runner
=
OPFModelRunner
(
modelID
=
modelID
,
jobID
=
jobID
,
predictedField
=
predictedField
,
experimentDir
=
experimentDir
,
reportKeyPatterns
=
reportKeys
,
optimizeKeyPattern
=
optimizeKey
,
jobsDAO
=
jobsDAO
,
modelCheckpointGUID
=
modelCheckpointGUID
,
logLevel
=
logLevel
,
predictionCacheMaxRecords
=
predictionCacheMaxRecords
)
signal
.
signal
(
signal
.
SIGINT
,
runner
.
handleWarningSignal
)
(
completionReason
,
completionMsg
)
=
runner
.
run
(
)
except
InvalidConnectionException
:
raise
except
Exception
,
e
:
(
completionReason
,
completionMsg
)
=
_handleModelRunnerException
(
jobID
,
modelID
,
jobsDAO
,
experimentDir
,
logger
,
e
)
finally
:
shutil
.
rmtree
(
experimentDir
)
signal
.
signal
(
signal
.
SIGINT
,
signal
.
default_int_handler
)
return
(
completionReason
,
completionMsg
)

This creates an experiment directory with a base . py description file created from baseDescription and a description . py generated from the given params dict and then runs the experiment .


def rCopy(d, f=identityConversion, discardNoneKeys=True, deepCopy=True):
  """Recursively copies a dict and returns the result.

  Args:
    d: The dict to copy.
    f: A function to apply to values when copying that takes the value and the
        list of keys from the root of the dict to the value and returns a value
        for the new dict.
    discardNoneKeys: If True, discard key-value pairs when f returns None for
        the value.
    deepCopy: If True, all values in returned dict are true copies (not the
        same object).
  Returns:
    A new dict with keys and values from d replaced with the result of f.
  """
  # Optionally deep copy the dict.
  if deepCopy:
    d = copy.deepcopy(d)

  newDict = {}
  toCopy = [(k, v, newDict, ()) for k, v in d.iteritems()]
  while len(toCopy) > 0:
    k, v, d, prevKeys = toCopy.pop()
    prevKeys = prevKeys + (k,)
    if isinstance(v, dict):
      d[k] = dict()
      toCopy[0:0] = [(innerK, innerV, d[k], prevKeys)
                     for innerK, innerV in v.iteritems()]
    else:
      #print k, v, prevKeys
      newV = f(v, prevKeys)
      if not discardNoneKeys or newV is not None:
        d[k] = newV
  return newDict

def rCopy ( d , f = identityConversion , discardNoneKeys = True , deepCopy = True ) : if deepCopy : d = copy . deepcopy ( d ) newDict = { } toCopy = [ ( k , v , newDict , ( ) ) for k , v in d . iteritems ( ) ] while len ( toCopy ) > 0 : k , v , d , prevKeys = toCopy . pop ( ) prevKeys = prevKeys + ( k , ) if isinstance ( v , dict ) : d [ k ] = dict ( ) toCopy [ 0 : 0 ] = [ ( innerK , innerV , d [ k ] , prevKeys ) for innerK , innerV in v . iteritems ( ) ] else : newV = f ( v , prevKeys ) if not discardNoneKeys or newV is not None : d [ k ] = newV return newDict

Recursively copies a dict and returns the result .


def rApply(d, f):
  """Recursively applies f to the values in dict d.

  Args:
    d: The dict to recurse over.
    f: A function to apply to values in d that takes the value and a list of
        keys from the root of the dict to the value.
  """
  remainingDicts = [(d, ())]
  while len(remainingDicts) > 0:
    current, prevKeys = remainingDicts.pop()
    for k, v in current.iteritems():
      keys = prevKeys + (k,)
      if isinstance(v, dict):
        remainingDicts.insert(0, (v, keys))
      else:
        f(v, keys)

def rApply ( d , f ) : remainingDicts = [ ( d , ( ) ) ] while len ( remainingDicts ) > 0 : current , prevKeys = remainingDicts . pop ( ) for k , v in current . iteritems ( ) : keys = prevKeys + ( k , ) if isinstance ( v , dict ) : remainingDicts . insert ( 0 , ( v , keys ) ) else : f ( v , keys )

Recursively applies f to the values in dict d .


def clippedObj(obj, maxElementSize=64):
  """
  Return a clipped version of obj suitable for printing, This
  is useful when generating log messages by printing data structures, but
  don't want the message to be too long.

  If passed in a dict, list, or namedtuple, each element of the structure's
  string representation will be limited to 'maxElementSize' characters. This
  will return a new object where the string representation of each element
  has been truncated to fit within maxElementSize.
  """

  # Is it a named tuple?
  if hasattr(obj, '_asdict'):
    obj = obj._asdict()

  # Printing a dict?
  if isinstance(obj, dict):
    objOut = dict()
    for key,val in obj.iteritems():
      objOut[key] = clippedObj(val)

  # Printing a list?
  elif hasattr(obj, '__iter__'):
    objOut = []
    for val in obj:
      objOut.append(clippedObj(val))

  # Some other object
  else:
    objOut = str(obj)
    if len(objOut) > maxElementSize:
      objOut = objOut[0:maxElementSize] + '...'

  return objOut

def clippedObj ( obj , maxElementSize = 64 ) : if hasattr ( obj , '_asdict' ) : obj = obj . _asdict ( ) if isinstance ( obj , dict ) : objOut = dict ( ) for key , val in obj . iteritems ( ) : objOut [ key ] = clippedObj ( val ) elif hasattr ( obj , '__iter__' ) : objOut = [ ] for val in obj : objOut . append ( clippedObj ( val ) ) else : objOut = str ( obj ) if len ( objOut ) > maxElementSize : objOut = objOut [ 0 : maxElementSize ] + '...' return objOut

Return a clipped version of obj suitable for printing This is useful when generating log messages by printing data structures but don t want the message to be too long .


def loadJsonValueFromFile(inputFilePath):
  """ Loads a json value from a file and converts it to the corresponding python
  object.

  inputFilePath:
                  Path of the json file;

  Returns:
                  python value that represents the loaded json value

  """
  with open(inputFilePath) as fileObj:
    value = json.load(fileObj)

  return value

def loadJsonValueFromFile ( inputFilePath ) : with open ( inputFilePath ) as fileObj : value = json . load ( fileObj ) return value

Loads a json value from a file and converts it to the corresponding python object .


def tick(self):
    """ Activity tick handler; services all activities

    Returns:      True if controlling iterator says it's okay to keep going;
                  False to stop
    """

    # Run activities whose time has come
    for act in self.__activities:
      if not act.iteratorHolder[0]:
        continue

      try:
        next(act.iteratorHolder[0])
      except StopIteration:
        act.cb()
        if act.repeating:
          act.iteratorHolder[0] = iter(xrange(act.period))
        else:
          act.iteratorHolder[0] = None

    return True

def tick ( self ) : for act in self . __activities : if not act . iteratorHolder [ 0 ] : continue try : next ( act . iteratorHolder [ 0 ] ) except StopIteration : act . cb ( ) if act . repeating : act . iteratorHolder [ 0 ] = iter ( xrange ( act . period ) ) else : act . iteratorHolder [ 0 ] = None return True

Activity tick handler ; services all activities


def rUpdate(original, updates):
  """Recursively updates the values in original with the values from updates."""
  # Keep a list of the sub-dictionaries that need to be updated to avoid having
  # to use recursion (which could fail for dictionaries with a lot of nesting.
  dictPairs = [(original, updates)]
  while len(dictPairs) > 0:
    original, updates = dictPairs.pop()
    for k, v in updates.iteritems():
      if k in original and isinstance(original[k], dict) and isinstance(v, dict):
        dictPairs.append((original[k], v))
      else:
        original[k] = v

def rUpdate ( original , updates ) : dictPairs = [ ( original , updates ) ] while len ( dictPairs ) > 0 : original , updates = dictPairs . pop ( ) for k , v in updates . iteritems ( ) : if k in original and isinstance ( original [ k ] , dict ) and isinstance ( v , dict ) : dictPairs . append ( ( original [ k ] , v ) ) else : original [ k ] = v

Recursively updates the values in original with the values from updates .


def dictDiffAndReport(da, db):
  """ Compares two python dictionaries at the top level and report differences,
  if any, to stdout

  da:             first dictionary
  db:             second dictionary

  Returns:        The same value as returned by dictDiff() for the given args
  """
  differences = dictDiff(da, db)

  if not differences:
    return differences

  if differences['inAButNotInB']:
    print ">>> inAButNotInB: %s" % differences['inAButNotInB']

  if differences['inBButNotInA']:
    print ">>> inBButNotInA: %s" % differences['inBButNotInA']

  for key in differences['differentValues']:
    print ">>> da[%s] != db[%s]" % (key, key)
    print "da[%s] = %r" % (key, da[key])
    print "db[%s] = %r" % (key, db[key])

  return differences

def
dictDiffAndReport
(
da
,
db
)
:
differences
=
dictDiff
(
da
,
db
)
if
not
differences
:
return
differences
if
differences
[
'inAButNotInB'
]
:
print
">>> inAButNotInB: %s"
%
differences
[
'inAButNotInB'
]
if
differences
[
'inBButNotInA'
]
:
print
">>> inBButNotInA: %s"
%
differences
[
'inBButNotInA'
]
for
key
in
differences
[
'differentValues'
]
:
print
">>> da[%s] != db[%s]"
%
(
key
,
key
)
print
"da[%s] = %r"
%
(
key
,
da
[
key
]
)
print
"db[%s] = %r"
%
(
key
,
db
[
key
]
)
return
differences

Compares two python dictionaries at the top level and report differences if any to stdout


def dictDiff(da, db):
  """ Compares two python dictionaries at the top level and return differences

  da:             first dictionary
  db:             second dictionary

  Returns:        None if dictionaries test equal; otherwise returns a
                  dictionary as follows:
                  {
                    'inAButNotInB':
                        <sequence of keys that are in da but not in db>
                    'inBButNotInA':
                        <sequence of keys that are in db but not in da>
                    'differentValues':
                        <sequence of keys whose corresponding values differ
                         between da and db>
                  }
  """
  different = False

  resultDict = dict()

  resultDict['inAButNotInB'] = set(da) - set(db)
  if resultDict['inAButNotInB']:
    different = True

  resultDict['inBButNotInA'] = set(db) - set(da)
  if resultDict['inBButNotInA']:
    different = True

  resultDict['differentValues'] = []
  for key in (set(da) - resultDict['inAButNotInB']):
    comparisonResult = da[key] == db[key]
    if isinstance(comparisonResult, bool):
      isEqual = comparisonResult
    else:
      # This handles numpy arrays (but only at the top level)
      isEqual = comparisonResult.all()
    if not isEqual:
      resultDict['differentValues'].append(key)
      different = True

  assert (((resultDict['inAButNotInB'] or resultDict['inBButNotInA'] or
          resultDict['differentValues']) and different) or not different)

  return resultDict if different else None

def dictDiff ( da , db ) : different = False resultDict = dict ( ) resultDict [ 'inAButNotInB' ] = set ( da ) - set ( db ) if resultDict [ 'inAButNotInB' ] : different = True resultDict [ 'inBButNotInA' ] = set ( db ) - set ( da ) if resultDict [ 'inBButNotInA' ] : different = True resultDict [ 'differentValues' ] = [ ] for key in ( set ( da ) - resultDict [ 'inAButNotInB' ] ) : comparisonResult = da [ key ] == db [ key ] if isinstance ( comparisonResult , bool ) : isEqual = comparisonResult else : isEqual = comparisonResult . all ( ) if not isEqual : resultDict [ 'differentValues' ] . append ( key ) different = True assert ( ( ( resultDict [ 'inAButNotInB' ] or resultDict [ 'inBButNotInA' ] or resultDict [ 'differentValues' ] ) and different ) or not different ) return resultDict if different else None


def getSpec(cls):
    """Return the Spec for IdentityRegion.
    """
    spec = {
        "description":IdentityRegion.__doc__,
        "singleNodeOnly":True,
        "inputs":{
          "in":{
            "description":"The input vector.",
            "dataType":"Real32",
            "count":0,
            "required":True,
            "regionLevel":False,
            "isDefaultInput":True,
            "requireSplitterMap":False},
        },
        "outputs":{
          "out":{
            "description":"A copy of the input vector.",
            "dataType":"Real32",
            "count":0,
            "regionLevel":True,
            "isDefaultOutput":True},
        },

        "parameters":{
          "dataWidth":{
            "description":"Size of inputs",
            "accessMode":"Read",
            "dataType":"UInt32",
            "count":1,
            "constraints":""},
        },
    }

    return spec

def
getSpec
(
cls
)
:
spec
=
{
"description"
:
IdentityRegion
.
__doc__
,
"singleNodeOnly"
:
True
,
"inputs"
:
{
"in"
:
{
"description"
:
"The input vector."
,
"dataType"
:
"Real32"
,
"count"
:
0
,
"required"
:
True
,
"regionLevel"
:
False
,
"isDefaultInput"
:
True
,
"requireSplitterMap"
:
False
}
,
}
,
"outputs"
:
{
"out"
:
{
"description"
:
"A copy of the input vector."
,
"dataType"
:
"Real32"
,
"count"
:
0
,
"regionLevel"
:
True
,
"isDefaultOutput"
:
True
}
,
}
,
"parameters"
:
{
"dataWidth"
:
{
"description"
:
"Size of inputs"
,
"accessMode"
:
"Read"
,
"dataType"
:
"UInt32"
,
"count"
:
1
,
"constraints"
:
""
}
,
}
,
}
return
spec


def _setRandomEncoderResolution(minResolution=0.001):
  """
  Given model params, figure out the correct resolution for the
  RandomDistributed encoder. Modifies params in place.
  """
  encoder = (
    model_params.MODEL_PARAMS["modelParams"]["sensorParams"]["encoders"]["value"]
  )

  if encoder["type"] == "RandomDistributedScalarEncoder":
    rangePadding = abs(_INPUT_MAX - _INPUT_MIN) * 0.2
    minValue = _INPUT_MIN - rangePadding
    maxValue = _INPUT_MAX + rangePadding
    resolution = max(minResolution,
                     (maxValue - minValue) / encoder.pop("numBuckets")
                    )
    encoder["resolution"] = resolution

def _setRandomEncoderResolution ( minResolution = 0.001 ) : encoder = ( model_params . MODEL_PARAMS [ "modelParams" ] [ "sensorParams" ] [ "encoders" ] [ "value" ] ) if encoder [ "type" ] == "RandomDistributedScalarEncoder" : rangePadding = abs ( _INPUT_MAX - _INPUT_MIN ) * 0.2 minValue = _INPUT_MIN - rangePadding maxValue = _INPUT_MAX + rangePadding resolution = max ( minResolution , ( maxValue - minValue ) / encoder . pop ( "numBuckets" ) ) encoder [ "resolution" ] = resolution

Given model params figure out the correct resolution for the RandomDistributed encoder . Modifies params in place .


def removeLabels(self, start=None, end=None, labelFilter=None):
    """
    Remove labels from each record with record ROWID in range from
    start to end, noninclusive of end. Removes all records if labelFilter is
    None, otherwise only removes the labels eqaul to labelFilter.

    This will recalculate all points from end to the last record stored in the
    internal cache of this classifier.
    """

    if len(self.saved_states) == 0:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for "
        "'removeLabels'. Model has no saved records.")

    startID = self.saved_states[0].ROWID

    clippedStart = 0 if start is None else max(0, start - startID)
    clippedEnd = len(self.saved_states) if end is None else \
      max(0, min( len( self.saved_states) , end - startID))

    if clippedEnd <= clippedStart:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for "
        "'removeLabels'.", debugInfo={
          'requestRange': {
            'startRecordID': start,
            'endRecordID': end
          },
          'clippedRequestRange': {
            'startRecordID': clippedStart,
            'endRecordID': clippedEnd
          },
          'validRange': {
            'startRecordID': startID,
            'endRecordID': self.saved_states[len(self.saved_states)-1].ROWID
          },
          'numRecordsStored': len(self.saved_states)
        })

    # Remove records within the cache
    recordsToDelete = []
    for state in self.saved_states[clippedStart:clippedEnd]:
      if labelFilter is not None:
        if labelFilter in state.anomalyLabel:
          state.anomalyLabel.remove(labelFilter)
      else:
        state.anomalyLabel = []
      state.setByUser = False
      recordsToDelete.append(state)
    self._deleteRecordsFromKNN(recordsToDelete)

    # Remove records not in cache
    self._deleteRangeFromKNN(start, end)

    # Recompute [clippedEnd, ...)
    for state in self.saved_states[clippedEnd:]:
      self._updateState(state)

    return {'status': 'success'}

Remove labels from each record with record ROWID in range from start to end noninclusive of end . Removes all records if labelFilter is None otherwise only removes the labels eqaul to labelFilter .


def _addRecordToKNN(self, record):
    """
    This method will add the record to the KNN classifier.
    """
    classifier = self.htm_prediction_model._getAnomalyClassifier()
    knn = classifier.getSelf()._knn

    prototype_idx = classifier.getSelf().getParameter('categoryRecencyList')
    category = self._labelListToCategoryNumber(record.anomalyLabel)

    # If record is already in the classifier, overwrite its labeling
    if record.ROWID in prototype_idx:
      knn.prototypeSetCategory(record.ROWID, category)
      return

    # Learn this pattern in the knn
    pattern = self._getStateAnomalyVector(record)
    rowID = record.ROWID
    knn.learn(pattern, category, rowID=rowID)

def _addRecordToKNN ( self , record ) : classifier = self . htm_prediction_model . _getAnomalyClassifier ( ) knn = classifier . getSelf ( ) . _knn prototype_idx = classifier . getSelf ( ) . getParameter ( 'categoryRecencyList' ) category = self . _labelListToCategoryNumber ( record . anomalyLabel ) if record . ROWID in prototype_idx : knn . prototypeSetCategory ( record . ROWID , category ) return pattern = self . _getStateAnomalyVector ( record ) rowID = record . ROWID knn . learn ( pattern , category , rowID = rowID )

This method will add the record to the KNN classifier .


def _deleteRecordsFromKNN(self, recordsToDelete):
    """
    This method will remove the given records from the classifier.

    parameters
    ------------
    recordsToDelete - list of records to delete from the classififier
    """
    classifier = self.htm_prediction_model._getAnomalyClassifier()
    knn = classifier.getSelf()._knn

    prototype_idx = classifier.getSelf().getParameter('categoryRecencyList')

    idsToDelete = [r.ROWID for r in recordsToDelete if \
      not r.setByUser and r.ROWID in prototype_idx]

    nProtos = knn._numPatterns
    knn.removeIds(idsToDelete)
    assert knn._numPatterns == nProtos - len(idsToDelete)

def _deleteRecordsFromKNN ( self , recordsToDelete ) : classifier = self . htm_prediction_model . _getAnomalyClassifier ( ) knn = classifier . getSelf ( ) . _knn prototype_idx = classifier . getSelf ( ) . getParameter ( 'categoryRecencyList' ) idsToDelete = [ r . ROWID for r in recordsToDelete if not r . setByUser and r . ROWID in prototype_idx ] nProtos = knn . _numPatterns knn . removeIds ( idsToDelete ) assert knn . _numPatterns == nProtos - len ( idsToDelete )

This method will remove the given records from the classifier .


def _deleteRangeFromKNN(self, start=0, end=None):
    """
    This method will remove any stored records within the range from start to
    end. Noninclusive of end.

    parameters
    ------------
    start - integer representing the ROWID of the start of the deletion range,
    end - integer representing the ROWID of the end of the deletion range,
      if None, it will default to end.
    """
    classifier = self.htm_prediction_model._getAnomalyClassifier()
    knn = classifier.getSelf()._knn

    prototype_idx = numpy.array(
      classifier.getSelf().getParameter('categoryRecencyList'))

    if end is None:
      end = prototype_idx.max() + 1

    idsIdxToDelete = numpy.logical_and(prototype_idx >= start,
                                       prototype_idx < end)
    idsToDelete = prototype_idx[idsIdxToDelete]

    nProtos = knn._numPatterns
    knn.removeIds(idsToDelete.tolist())
    assert knn._numPatterns == nProtos - len(idsToDelete)

def _deleteRangeFromKNN ( self , start = 0 , end = None ) : classifier = self . htm_prediction_model . _getAnomalyClassifier ( ) knn = classifier . getSelf ( ) . _knn prototype_idx = numpy . array ( classifier . getSelf ( ) . getParameter ( 'categoryRecencyList' ) ) if end is None : end = prototype_idx . max ( ) + 1 idsIdxToDelete = numpy . logical_and ( prototype_idx >= start , prototype_idx < end ) idsToDelete = prototype_idx [ idsIdxToDelete ] nProtos = knn . _numPatterns knn . removeIds ( idsToDelete . tolist ( ) ) assert knn . _numPatterns == nProtos - len ( idsToDelete )

This method will remove any stored records within the range from start to end . Noninclusive of end .


def _recomputeRecordFromKNN(self, record):
    """
    return the classified labeling of record
    """
    inputs = {
      "categoryIn": [None],
      "bottomUpIn": self._getStateAnomalyVector(record),
    }

    outputs = {"categoriesOut": numpy.zeros((1,)),
               "bestPrototypeIndices":numpy.zeros((1,)),
               "categoryProbabilitiesOut":numpy.zeros((1,))}

    # Run inference only to capture state before learning
    classifier = self.htm_prediction_model._getAnomalyClassifier()
    knn = classifier.getSelf()._knn

    # Only use points before record to classify and after the wait period.
    classifier_indexes = \
      numpy.array(classifier.getSelf().getParameter('categoryRecencyList'))
    valid_idx = numpy.where(
        (classifier_indexes >= self._autoDetectWaitRecords) &
        (classifier_indexes < record.ROWID)
      )[0].tolist()

    if len(valid_idx) == 0:
      return None

    classifier.setParameter('inferenceMode', True)
    classifier.setParameter('learningMode', False)
    classifier.getSelf().compute(inputs, outputs)
    classifier.setParameter('learningMode', True)

    classifier_distances = classifier.getSelf().getLatestDistances()
    valid_distances = classifier_distances[valid_idx]
    if valid_distances.min() <= self._classificationMaxDist:
      classifier_indexes_prev = classifier_indexes[valid_idx]
      rowID = classifier_indexes_prev[valid_distances.argmin()]
      indexID = numpy.where(classifier_indexes == rowID)[0][0]
      category = classifier.getSelf().getCategoryList()[indexID]
      return category
    return None

def _recomputeRecordFromKNN ( self , record ) : inputs = { "categoryIn" : [ None ] , "bottomUpIn" : self . _getStateAnomalyVector ( record ) , } outputs = { "categoriesOut" : numpy . zeros ( ( 1 , ) ) , "bestPrototypeIndices" : numpy . zeros ( ( 1 , ) ) , "categoryProbabilitiesOut" : numpy . zeros ( ( 1 , ) ) } classifier = self . htm_prediction_model . _getAnomalyClassifier ( ) knn = classifier . getSelf ( ) . _knn classifier_indexes = numpy . array ( classifier . getSelf ( ) . getParameter ( 'categoryRecencyList' ) ) valid_idx = numpy . where ( ( classifier_indexes >= self . _autoDetectWaitRecords ) & ( classifier_indexes < record . ROWID ) ) [ 0 ] . tolist ( ) if len ( valid_idx ) == 0 : return None classifier . setParameter ( 'inferenceMode' , True ) classifier . setParameter ( 'learningMode' , False ) classifier . getSelf ( ) . compute ( inputs , outputs ) classifier . setParameter ( 'learningMode' , True ) classifier_distances = classifier . getSelf ( ) . getLatestDistances ( ) valid_distances = classifier_distances [ valid_idx ] if valid_distances . min ( ) <= self . _classificationMaxDist : classifier_indexes_prev = classifier_indexes [ valid_idx ] rowID = classifier_indexes_prev [ valid_distances . argmin ( ) ] indexID = numpy . where ( classifier_indexes == rowID ) [ 0 ] [ 0 ] category = classifier . getSelf ( ) . getCategoryList ( ) [ indexID ] return category return None


def _constructClassificationRecord(self):
    """
    Construct a _HTMClassificationRecord based on the current state of the
    htm_prediction_model of this classifier.

    ***This will look into the internals of the model and may depend on the
    SP, TM, and KNNClassifier***
    """
    model = self.htm_prediction_model
    sp = model._getSPRegion()
    tm = model._getTPRegion()
    tpImp = tm.getSelf()._tfdr

    # Count the number of unpredicted columns
    activeColumns = sp.getOutputData("bottomUpOut").nonzero()[0]
    score = numpy.in1d(activeColumns, self._prevPredictedColumns).sum()
    score = (self._activeColumnCount - score)/float(self._activeColumnCount)

    spSize = sp.getParameter('activeOutputCount')
    tpSize = tm.getParameter('cellsPerColumn') * tm.getParameter('columnCount')

    classificationVector = numpy.array([])

    if self._vectorType == 'tpc':
      # Classification Vector: [---TM Cells---]
      classificationVector = numpy.zeros(tpSize)
      activeCellMatrix = tpImp.getLearnActiveStateT().reshape(tpSize, 1)
      activeCellIdx = numpy.where(activeCellMatrix > 0)[0]
      if activeCellIdx.shape[0] > 0:
        classificationVector[numpy.array(activeCellIdx, dtype=numpy.uint16)] = 1
    elif self._vectorType == 'sp_tpe':
      # Classification Vecotr: [---SP---|---(TM-SP)----]
      classificationVector = numpy.zeros(spSize+spSize)
      if activeColumns.shape[0] > 0:
        classificationVector[activeColumns] = 1.0

      errorColumns = numpy.setdiff1d(self._prevPredictedColumns, activeColumns)
      if errorColumns.shape[0] > 0:
        errorColumnIndexes = ( numpy.array(errorColumns, dtype=numpy.uint16) +
          spSize )
        classificationVector[errorColumnIndexes] = 1.0
    else:
      raise TypeError("Classification vector type must be either 'tpc' or"
        " 'sp_tpe', current value is %s" % (self._vectorType))

    # Store the state for next time step
    numPredictedCols = len(self._prevPredictedColumns)
    predictedColumns = tm.getOutputData("topDownOut").nonzero()[0]
    self._prevPredictedColumns = copy.deepcopy(predictedColumns)

    if self._anomalyVectorLength is None:
      self._anomalyVectorLength = len(classificationVector)

    result = _CLAClassificationRecord(
      ROWID=int(model.getParameter('__numRunCalls') - 1), #__numRunCalls called
        #at beginning of model.run
      anomalyScore=score,
      anomalyVector=classificationVector.nonzero()[0].tolist(),
      anomalyLabel=[]
    )
    return result

Construct a _HTMClassificationRecord based on the current state of the htm_prediction_model of this classifier .


def compute(self):
    """
    Run an iteration of this anomaly classifier
    """
    result = self._constructClassificationRecord()

    # Classify this point after waiting the classification delay
    if result.ROWID >= self._autoDetectWaitRecords:
      self._updateState(result)

    # Save new classification record and keep history as moving window
    self.saved_states.append(result)
    if len(self.saved_states) > self._history_length:
      self.saved_states.pop(0)

    return result

def compute ( self ) : result = self . _constructClassificationRecord ( ) if result . ROWID >= self . _autoDetectWaitRecords : self . _updateState ( result ) self . saved_states . append ( result ) if len ( self . saved_states ) > self . _history_length : self . saved_states . pop ( 0 ) return result


def setAutoDetectWaitRecords(self, waitRecords):
    """
    Sets the autoDetectWaitRecords.
    """
    if not isinstance(waitRecords, int):
      raise HTMPredictionModelInvalidArgument("Invalid argument type \'%s\'. WaitRecord "
        "must be a number." % (type(waitRecords)))

    if len(self.saved_states) > 0 and waitRecords < self.saved_states[0].ROWID:
      raise HTMPredictionModelInvalidArgument("Invalid value. autoDetectWaitRecord value "
        "must be valid record within output stream. Current minimum ROWID in "
        "output stream is %d." % (self.saved_states[0].ROWID))

    self._autoDetectWaitRecords = waitRecords

    # Update all the states in the classifier's cache
    for state in self.saved_states:
      self._updateState(state)


def _allocateSpatialFDR(self, rfInput):
    """Allocate the spatial pooler instance."""
    if self._sfdr:
      return

    # Retrieve the necessary extra arguments that were handled automatically
    autoArgs = dict((name, getattr(self, name))
                     for name in self._spatialArgNames)

    # Instantiate the spatial pooler class.
    if ( (self.SpatialClass == CPPSpatialPooler) or
         (self.SpatialClass == PYSpatialPooler) ):

      autoArgs['columnDimensions'] = [self.columnCount]
      autoArgs['inputDimensions'] = [self.inputWidth]
      autoArgs['potentialRadius'] = self.inputWidth

      self._sfdr = self.SpatialClass(
        **autoArgs
      )

def _allocateSpatialFDR ( self , rfInput ) : if self . _sfdr : return autoArgs = dict ( ( name , getattr ( self , name ) ) for name in self . _spatialArgNames ) if ( ( self . SpatialClass == CPPSpatialPooler ) or ( self . SpatialClass == PYSpatialPooler ) ) : autoArgs [ 'columnDimensions' ] = [ self . columnCount ] autoArgs [ 'inputDimensions' ] = [ self . inputWidth ] autoArgs [ 'potentialRadius' ] = self . inputWidth self . _sfdr = self . SpatialClass ( ** autoArgs )


def compute(self, inputs, outputs):
    """
    Run one iteration, profiling it if requested.

    :param inputs: (dict) mapping region input names to numpy.array values
    :param outputs: (dict) mapping region output names to numpy.arrays that
           should be populated with output values by this method
    """

    # Uncomment this to find out who is generating divide by 0, or other numpy warnings
    # numpy.seterr(divide='raise', invalid='raise', over='raise')

    # Modify this line to turn on profiling for a given node. The results file
    #  ('hotshot.stats') will be sensed and printed out by the vision framework's
    #  RunInference.py script at the end of inference.
    # Also uncomment the hotshot import at the top of this file.
    if False and self.learningMode \
        and self._iterations > 0 and self._iterations <= 10:

      import hotshot
      if self._iterations == 10:
        print "\n  Collecting and sorting internal node profiling stats generated by hotshot..."
        stats = hotshot.stats.load("hotshot.stats")
        stats.strip_dirs()
        stats.sort_stats('time', 'calls')
        stats.print_stats()

      # The guts of the compute are contained in the _compute() call so that we
      # can profile it if requested.
      if self._profileObj is None:
        print "\n  Preparing to capture profile using hotshot..."
        if os.path.exists('hotshot.stats'):
          # There is an old hotshot stats profile left over, remove it.
          os.remove('hotshot.stats')
        self._profileObj = hotshot.Profile("hotshot.stats", 1, 1)
                                          # filename, lineevents, linetimings
      self._profileObj.runcall(self._compute, *[inputs, outputs])
    else:
      self._compute(inputs, outputs)

Run one iteration profiling it if requested .