def extract_packing_plan(self, topology):
    """
    Returns the representation of packing plan that will
    be returned from Tracker.
    """
    packingPlan = {
        "id": "",
        "container_plans": []
    }

    if not topology.packing_plan:
      return packingPlan

    container_plans = topology.packing_plan.container_plans

    containers = []
    for container_plan in container_plans:
      instances = []
      for instance_plan in container_plan.instance_plans:
        instance_resources = {"cpu": instance_plan.resource.cpu,
                              "ram": instance_plan.resource.ram,
                              "disk": instance_plan.resource.disk}
        instance = {"component_name" : instance_plan.component_name,
                    "task_id" : instance_plan.task_id,
                    "component_index": instance_plan.component_index,
                    "instance_resources": instance_resources}
        instances.append(instance)
      required_resource = {"cpu": container_plan.requiredResource.cpu,
                           "ram": container_plan.requiredResource.ram,
                           "disk": container_plan.requiredResource.disk}
      scheduled_resource = {}
      if container_plan.scheduledResource:
        scheduled_resource = {"cpu": container_plan.scheduledResource.cpu,
                              "ram": container_plan.scheduledResource.ram,
                              "disk": container_plan.scheduledResource.disk}
      container = {"id": container_plan.id,
                   "instances": instances,
                   "required_resources": required_resource,
                   "scheduled_resources": scheduled_resource}
      containers.append(container)

    packingPlan["id"] = topology.packing_plan.id
    packingPlan["container_plans"] = containers
    return json.dumps(packingPlan)

def extract_packing_plan ( self , topology ) : packingPlan = { "id" : "" , "container_plans" : [ ] } if not topology . packing_plan : return packingPlan container_plans = topology . packing_plan . container_plans containers = [ ] for container_plan in container_plans : instances = [ ] for instance_plan in container_plan . instance_plans : instance_resources = { "cpu" : instance_plan . resource . cpu , "ram" : instance_plan . resource . ram , "disk" : instance_plan . resource . disk } instance = { "component_name" : instance_plan . component_name , "task_id" : instance_plan . task_id , "component_index" : instance_plan . component_index , "instance_resources" : instance_resources } instances . append ( instance ) required_resource = { "cpu" : container_plan . requiredResource . cpu , "ram" : container_plan . requiredResource . ram , "disk" : container_plan . requiredResource . disk } scheduled_resource = { } if container_plan . scheduledResource : scheduled_resource = { "cpu" : container_plan . scheduledResource . cpu , "ram" : container_plan . scheduledResource . ram , "disk" : container_plan . scheduledResource . disk } container = { "id" : container_plan . id , "instances" : instances , "required_resources" : required_resource , "scheduled_resources" : scheduled_resource } containers . append ( container ) packingPlan [ "id" ] = topology . packing_plan . id packingPlan [ "container_plans" ] = containers return json . dumps ( packingPlan )

Returns the representation of packing plan that will be returned from Tracker .


def setTopologyInfo(self, topology):
    """
    Extracts info from the stored proto states and
    convert it into representation that is exposed using
    the API.
    This method is called on any change for the topology.
    For example, when a container moves and its host or some
    port changes. All the information is parsed all over
    again and cache is updated.
    """
    # Execution state is the most basic info.
    # If there is no execution state, just return
    # as the rest of the things don't matter.
    if not topology.execution_state:
      Log.info("No execution state found for: " + topology.name)
      return

    Log.info("Setting topology info for topology: " + topology.name)
    has_physical_plan = True
    if not topology.physical_plan:
      has_physical_plan = False

    Log.info("Setting topology info for topology: " + topology.name)
    has_packing_plan = True
    if not topology.packing_plan:
      has_packing_plan = False

    has_tmaster_location = True
    if not topology.tmaster:
      has_tmaster_location = False

    has_scheduler_location = True
    if not topology.scheduler_location:
      has_scheduler_location = False

    topologyInfo = {
        "name": topology.name,
        "id": topology.id,
        "logical_plan": None,
        "physical_plan": None,
        "packing_plan": None,
        "execution_state": None,
        "tmaster_location": None,
        "scheduler_location": None,
    }

    executionState = self.extract_execution_state(topology)
    executionState["has_physical_plan"] = has_physical_plan
    executionState["has_packing_plan"] = has_packing_plan
    executionState["has_tmaster_location"] = has_tmaster_location
    executionState["has_scheduler_location"] = has_scheduler_location
    executionState["status"] = topology.get_status()

    topologyInfo["metadata"] = self.extract_metadata(topology)
    topologyInfo["runtime_state"] = self.extract_runtime_state(topology)

    topologyInfo["execution_state"] = executionState
    topologyInfo["logical_plan"] = self.extract_logical_plan(topology)
    topologyInfo["physical_plan"] = self.extract_physical_plan(topology)
    topologyInfo["packing_plan"] = self.extract_packing_plan(topology)
    topologyInfo["tmaster_location"] = self.extract_tmaster(topology)
    topologyInfo["scheduler_location"] = self.extract_scheduler_location(topology)

    self.topologyInfos[(topology.name, topology.state_manager_name)] = topologyInfo

def
setTopologyInfo
(
self
,
topology
)
:
if
not
topology
.
execution_state
:
Log
.
info
(
"No execution state found for: "
+
topology
.
name
)
return
Log
.
info
(
"Setting topology info for topology: "
+
topology
.
name
)
has_physical_plan
=
True
if
not
topology
.
physical_plan
:
has_physical_plan
=
False
Log
.
info
(
"Setting topology info for topology: "
+
topology
.
name
)
has_packing_plan
=
True
if
not
topology
.
packing_plan
:
has_packing_plan
=
False
has_tmaster_location
=
True
if
not
topology
.
tmaster
:
has_tmaster_location
=
False
has_scheduler_location
=
True
if
not
topology
.
scheduler_location
:
has_scheduler_location
=
False
topologyInfo
=
{
"name"
:
topology
.
name
,
"id"
:
topology
.
id
,
"logical_plan"
:
,
"physical_plan"
:
,
"packing_plan"
:
,
"execution_state"
:
,
"tmaster_location"
:
,
"scheduler_location"
:
,
}
executionState
=
self
.
extract_execution_state
(
topology
)
executionState
[
"has_physical_plan"
]
=
has_physical_plan
executionState
[
"has_packing_plan"
]
=
has_packing_plan
executionState
[
"has_tmaster_location"
]
=
has_tmaster_location
executionState
[
"has_scheduler_location"
]
=
has_scheduler_location
executionState
[
"status"
]
=
topology
.
get_status
(
)
topologyInfo
[
"metadata"
]
=
self
.
extract_metadata
(
topology
)
topologyInfo
[
"runtime_state"
]
=
self
.
extract_runtime_state
(
topology
)
topologyInfo
[
"execution_state"
]
=
executionState
topologyInfo
[
"logical_plan"
]
=
self
.
extract_logical_plan
(
topology
)
topologyInfo
[
"physical_plan"
]
=
self
.
extract_physical_plan
(
topology
)
topologyInfo
[
"packing_plan"
]
=
self
.
extract_packing_plan
(
topology
)
topologyInfo
[
"tmaster_location"
]
=
self
.
extract_tmaster
(
topology
)
topologyInfo
[
"scheduler_location"
]
=
self
.
extract_scheduler_location
(
topology
)
self
.
topologyInfos
[
(
topology
.
name
,
topology
.
state_manager_name
)
]
=
topologyInfo

Extracts info from the stored proto states and convert it into representation that is exposed using the API . This method is called on any change for the topology . For example when a container moves and its host or some port changes . All the information is parsed all over again and cache is updated .


def getTopologyInfo(self, topologyName, cluster, role, environ):
    """
    Returns the JSON representation of a topology
    by its name, cluster, environ, and an optional role parameter.
    Raises exception if no such topology is found.
    """
    # Iterate over the values to filter the desired topology.
    for (topology_name, _), topologyInfo in self.topologyInfos.items():
      executionState = topologyInfo["execution_state"]
      if (topologyName == topology_name and
          cluster == executionState["cluster"] and
          environ == executionState["environ"]):
        # If role is specified, first try to match "role" field. If "role" field
        # does not exist, try to match "submission_user" field.
        if not role or executionState.get("role") == role:
          return topologyInfo
    if role is not None:
      Log.info("Could not find topology info for topology: %s," \
               "cluster: %s, role: %s, and environ: %s",
               topologyName, cluster, role, environ)
    else:
      Log.info("Could not find topology info for topology: %s," \
               "cluster: %s and environ: %s", topologyName, cluster, environ)
    raise Exception("No topology found")

Returns the JSON representation of a topology by its name cluster environ and an optional role parameter . Raises exception if no such topology is found .


def load_configs(self):
    """load config files"""
    self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY])
    if EXTRA_LINKS_KEY in self.configs:
      for extra_link in self.configs[EXTRA_LINKS_KEY]:
        self.extra_links.append(self.validate_extra_link(extra_link))

def load_configs ( self ) : self . statemgr_config . set_state_locations ( self . configs [ STATEMGRS_KEY ] ) if EXTRA_LINKS_KEY in self . configs : for extra_link in self . configs [ EXTRA_LINKS_KEY ] : self . extra_links . append ( self . validate_extra_link ( extra_link ) )


def validate_extra_link(self, extra_link):
    """validate extra link"""
    if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link:
      raise Exception("Invalid extra.links format. " +
                      "Extra link must include a 'name' and 'formatter' field")

    self.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY])
    return extra_link


def validated_formatter(self, url_format):
    """validate visualization url format"""
    # We try to create a string by substituting all known
    # parameters. If an unknown parameter is present, an error
    # will be thrown
    valid_parameters = {
        "${CLUSTER}": "cluster",
        "${ENVIRON}": "environ",
        "${TOPOLOGY}": "topology",
        "${ROLE}": "role",
        "${USER}": "user",
    }
    dummy_formatted_url = url_format
    for key, value in valid_parameters.items():
      dummy_formatted_url = dummy_formatted_url.replace(key, value)

    # All $ signs must have been replaced
    if '$' in dummy_formatted_url:
      raise Exception("Invalid viz.url.format: %s" % (url_format))

    # No error is thrown, so the format is valid.
    return url_format

def
validated_formatter
(
self
,
url_format
)
:
valid_parameters
=
{
"${CLUSTER}"
:
"cluster"
,
"${ENVIRON}"
:
"environ"
,
"${TOPOLOGY}"
:
"topology"
,
"${ROLE}"
:
"role"
,
"${USER}"
:
"user"
,
}
dummy_formatted_url
=
url_format
for
key
,
value
in
valid_parameters
.
items
(
)
:
dummy_formatted_url
=
dummy_formatted_url
.
replace
(
key
,
value
)
if
'$'
in
dummy_formatted_url
:
raise
Exception
(
"Invalid viz.url.format: %s"
%
(
url_format
)
)
return
url_format


def emit(self, tup, tup_id=None, stream=Stream.DEFAULT_STREAM_ID,
           direct_task=None, need_task_ids=False):
    """Emits a new tuple from this Spout

    It is compatible with StreamParse API.

    :type tup: list or tuple
    :param tup: the new output Tuple to send from this spout,
                should contain only serializable data.
    :type tup_id: str or object
    :param tup_id: the ID for the Tuple. Leave this blank for an unreliable emit.
                   (Same as messageId in Java)
    :type stream: str
    :param stream: the ID of the stream this Tuple should be emitted to.
                   Leave empty to emit to the default stream.
    :type direct_task: int
    :param direct_task: the task to send the Tuple to if performing a direct emit.
    :type need_task_ids: bool
    :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.
    """
    # first check whether this tuple is sane
    self.pplan_helper.check_output_schema(stream, tup)

    # get custom grouping target task ids; get empty list if not custom grouping
    custom_target_task_ids = self.pplan_helper.choose_tasks_for_custom_grouping(stream, tup)

    self.pplan_helper.context.invoke_hook_emit(tup, stream, None)

    data_tuple = tuple_pb2.HeronDataTuple()
    data_tuple.key = 0

    if direct_task is not None:
      if not isinstance(direct_task, int):
        raise TypeError("direct_task argument needs to be an integer, given: %s"
                        % str(type(direct_task)))
      # performing emit-direct
      data_tuple.dest_task_ids.append(direct_task)
    elif custom_target_task_ids is not None:
      # for custom grouping
      for task_id in custom_target_task_ids:
        data_tuple.dest_task_ids.append(task_id)

    if tup_id is not None:
      tuple_info = TupleHelper.make_root_tuple_info(stream, tup_id)
      if self.acking_enabled:
        # this message is rooted
        root = data_tuple.roots.add()
        root.taskid = self.pplan_helper.my_task_id
        root.key = tuple_info.key
        self.in_flight_tuples[tuple_info.key] = tuple_info
      else:
        self.immediate_acks.append(tuple_info)

    tuple_size_in_bytes = 0

    start_time = time.time()

    # Serialize
    for obj in tup:
      serialized = self.serializer.serialize(obj)
      data_tuple.values.append(serialized)
      tuple_size_in_bytes += len(serialized)

    serialize_latency_ns = (time.time() - start_time) * system_constants.SEC_TO_NS
    self.spout_metrics.serialize_data_tuple(stream, serialize_latency_ns)

    super(SpoutInstance, self).admit_data_tuple(stream_id=stream, data_tuple=data_tuple,
                                                tuple_size_in_bytes=tuple_size_in_bytes)
    self.total_tuples_emitted += 1
    self.spout_metrics.update_emit_count(stream)
    if need_task_ids:
      sent_task_ids = custom_target_task_ids or []
      if direct_task is not None:
        sent_task_ids.append(direct_task)
      return sent_task_ids


def _is_continue_to_work(self):
    """Checks whether we still need to do more work

    When the topology state is RUNNING:
    1. if the out_queue is not full and ack is not enabled, we could wake up next time to
       produce more tuples and push to the out_queue
    2. if the out_queue is not full but the acking is enabled, we need to make sure that
      the number of pending tuples is smaller than max_spout_pending
    3. if there are more to read, we will wake up itself next time.
    """
    if not self._is_topology_running():
      return False

    max_spout_pending = \
      self.pplan_helper.context.get_cluster_config().get(api_constants.TOPOLOGY_MAX_SPOUT_PENDING)

    if not self.acking_enabled and self.output_helper.is_out_queue_available():
      return True
    elif self.acking_enabled and self.output_helper.is_out_queue_available() and \
        len(self.in_flight_tuples) < max_spout_pending:
      return True
    elif self.acking_enabled and not self.in_stream.is_empty():
      return True
    else:
      return False

def _is_continue_to_work ( self ) : if not self . _is_topology_running ( ) : return False max_spout_pending = self . pplan_helper . context . get_cluster_config ( ) . get ( api_constants . TOPOLOGY_MAX_SPOUT_PENDING ) if not self . acking_enabled and self . output_helper . is_out_queue_available ( ) : return True elif self . acking_enabled and self . output_helper . is_out_queue_available ( ) and len ( self . in_flight_tuples ) < max_spout_pending : return True elif self . acking_enabled and not self . in_stream . is_empty ( ) : return True else : return False


def to_table(components, topo_info):
  """ normalize raw logical plan info to table """
  inputs, outputs = defaultdict(list), defaultdict(list)
  for ctype, component in components.items():
    if ctype == 'bolts':
      for component_name, component_info in component.items():
        for input_stream in component_info['inputs']:
          input_name = input_stream['component_name']
          inputs[component_name].append(input_name)
          outputs[input_name].append(component_name)
  info = []
  spouts_instance = topo_info['physical_plan']['spouts']
  bolts_instance = topo_info['physical_plan']['bolts']
  for ctype, component in components.items():
    # stages is an int so keep going
    if ctype == "stages":
      continue
    for component_name, component_info in component.items():
      row = [ctype[:-1], component_name]
      if ctype == 'spouts':
        row.append(len(spouts_instance[component_name]))
      else:
        row.append(len(bolts_instance[component_name]))
      row.append(','.join(inputs.get(component_name, ['-'])))
      row.append(','.join(outputs.get(component_name, ['-'])))
      info.append(row)
  header = ['type', 'name', 'parallelism', 'input', 'output']
  return info, header

def to_table ( components , topo_info ) : inputs , outputs = defaultdict ( list ) , defaultdict ( list ) for ctype , component in components . items ( ) : if ctype == 'bolts' : for component_name , component_info in component . items ( ) : for input_stream in component_info [ 'inputs' ] : input_name = input_stream [ 'component_name' ] inputs [ component_name ] . append ( input_name ) outputs [ input_name ] . append ( component_name ) info = [ ] spouts_instance = topo_info [ 'physical_plan' ] [ 'spouts' ] bolts_instance = topo_info [ 'physical_plan' ] [ 'bolts' ] for ctype , component in components . items ( ) : if ctype == "stages" : continue for component_name , component_info in component . items ( ) : row = [ ctype [ : - 1 ] , component_name ] if ctype == 'spouts' : row . append ( len ( spouts_instance [ component_name ] ) ) else : row . append ( len ( bolts_instance [ component_name ] ) ) row . append ( ',' . join ( inputs . get ( component_name , [ '-' ] ) ) ) row . append ( ',' . join ( outputs . get ( component_name , [ '-' ] ) ) ) info . append ( row ) header = [ 'type' , 'name' , 'parallelism' , 'input' , 'output' ] return info , header


def filter_bolts(table, header):
  """ filter to keep bolts """
  bolts_info = []
  for row in table:
    if row[0] == 'bolt':
      bolts_info.append(row)
  return bolts_info, header

def filter_bolts ( table , header ) : bolts_info = [ ] for row in table : if row [ 0 ] == 'bolt' : bolts_info . append ( row ) return bolts_info , header


def filter_spouts(table, header):
  """ filter to keep spouts """
  spouts_info = []
  for row in table:
    if row[0] == 'spout':
      spouts_info.append(row)
  return spouts_info, header

def filter_spouts ( table , header ) : spouts_info = [ ] for row in table : if row [ 0 ] == 'spout' : spouts_info . append ( row ) return spouts_info , header


def _get_topologies_with_watch(self, callback, isWatching):
    """
    Helper function to get topologies with
    a callback. The future watch is placed
    only if isWatching is True.
    """
    path = self.get_topologies_path()
    if isWatching:
      LOG.info("Adding children watch for path: " + path)

    # pylint: disable=unused-variable
    @self.client.ChildrenWatch(path)
    def watch_topologies(topologies):
      """ callback to watch topologies """
      callback(topologies)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

def
_get_topologies_with_watch
(
self
,
callback
,
isWatching
)
:
path
=
self
.
get_topologies_path
(
)
if
isWatching
:
LOG
.
info
(
"Adding children watch for path: "
+
path
)
@
self
.
client
.
ChildrenWatch
(
path
)
def
watch_topologies
(
topologies
)
:
callback
(
topologies
)
return
isWatching

Helper function to get topologies with a callback . The future watch is placed only if isWatching is True .


def _get_packing_plan_with_watch(self, topologyName, callback, isWatching):
    """
    Helper function to get packing_plan with
    a callback. The future watch is placed
    only if isWatching is True.
    """
    path = self.get_packing_plan_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-argument,unused-variable
    @self.client.DataWatch(path)
    def watch_packing_plan(data, stats):
      """ watch the packing plan for updates """
      if data:
        packing_plan = PackingPlan()
        packing_plan.ParseFromString(data)
        callback(packing_plan)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

def
_get_packing_plan_with_watch
(
self
,
topologyName
,
callback
,
isWatching
)
:
path
=
self
.
get_packing_plan_path
(
topologyName
)
if
isWatching
:
LOG
.
info
(
"Adding data watch for path: "
+
path
)
@
self
.
client
.
DataWatch
(
path
)
def
watch_packing_plan
(
data
,
stats
)
:
if
data
:
packing_plan
=
PackingPlan
(
)
packing_plan
.
ParseFromString
(
data
)
callback
(
packing_plan
)
else
:
callback
(
)
return
isWatching

Helper function to get packing_plan with a callback . The future watch is placed only if isWatching is True .


def get_pplan(self, topologyName, callback=None):
    """ get physical plan """
    isWatching = False

    # Temp dict used to return result
    # if callback is not provided.
    ret = {
        "result": None
    }
    if callback:
      isWatching = True
    else:
      def callback(data):
        """
        Custom callback to get the topologies right now.
        """
        ret["result"] = data

    self._get_pplan_with_watch(topologyName, callback, isWatching)

    # The topologies are now populated with the data.
    return ret["result"]

def get_pplan ( self , topologyName , callback = None ) : isWatching = False ret = { "result" : None } if callback : isWatching = True else : def callback ( data ) : ret [ "result" ] = data self . _get_pplan_with_watch ( topologyName , callback , isWatching ) return ret [ "result" ]


def create_pplan(self, topologyName, pplan):
    """ create physical plan """
    if not pplan or not pplan.IsInitialized():
      raise_(StateException("Physical Plan protobuf not init properly",
                            StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[2])

    path = self.get_pplan_path(topologyName)
    LOG.info("Adding topology: {0} to path: {1}".format(
        topologyName, path))
    pplanString = pplan.SerializeToString()
    try:
      self.client.create(path, value=pplanString, makepath=True)
      return True
    except NoNodeError:
      raise_(StateException("NoNodeError while creating pplan",
                            StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2])
    except NodeExistsError:
      raise_(StateException("NodeExistsError while creating pplan",
                            StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[2])
    except ZookeeperError:
      raise_(StateException("Zookeeper while creating pplan",
                            StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2])
    except Exception:
      # Just re raise the exception.
      raise


def get_execution_state(self, topologyName, callback=None):
    """ get execution state """
    isWatching = False

    # Temp dict used to return result
    # if callback is not provided.
    ret = {
        "result": None
    }
    if callback:
      isWatching = True
    else:
      def callback(data):
        """
        Custom callback to get the topologies right now.
        """
        ret["result"] = data

    self._get_execution_state_with_watch(topologyName, callback, isWatching)

    # The topologies are now populated with the data.
    return ret["result"]

def get_execution_state ( self , topologyName , callback = None ) : isWatching = False ret = { "result" : None } if callback : isWatching = True else : def callback ( data ) : ret [ "result" ] = data self . _get_execution_state_with_watch ( topologyName , callback , isWatching ) return ret [ "result" ]


def _get_execution_state_with_watch(self, topologyName, callback, isWatching):
    """
    Helper function to get execution state with
    a callback. The future watch is placed
    only if isWatching is True.
    """
    path = self.get_execution_state_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-variable, unused-argument
    @self.client.DataWatch(path)
    def watch_execution_state(data, stats):
      """ invoke callback to watch execute state """
      if data:
        executionState = ExecutionState()
        executionState.ParseFromString(data)
        callback(executionState)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

def
_get_execution_state_with_watch
(
self
,
topologyName
,
callback
,
isWatching
)
:
path
=
self
.
get_execution_state_path
(
topologyName
)
if
isWatching
:
LOG
.
info
(
"Adding data watch for path: "
+
path
)
@
self
.
client
.
DataWatch
(
path
)
def
watch_execution_state
(
data
,
stats
)
:
if
data
:
executionState
=
ExecutionState
(
)
executionState
.
ParseFromString
(
data
)
callback
(
executionState
)
else
:
callback
(
)
return
isWatching

Helper function to get execution state with a callback . The future watch is placed only if isWatching is True .


def create_execution_state(self, topologyName, executionState):
    """ create execution state """
    if not executionState or not executionState.IsInitialized():
      raise_(StateException("Execution State protobuf not init properly",
                            StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[2])

    path = self.get_execution_state_path(topologyName)
    LOG.info("Adding topology: {0} to path: {1}".format(
        topologyName, path))
    executionStateString = executionState.SerializeToString()
    try:
      self.client.create(path, value=executionStateString, makepath=True)
      return True
    except NoNodeError:
      raise_(StateException("NoNodeError while creating execution state",
                            StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2])
    except NodeExistsError:
      raise_(StateException("NodeExistsError while creating execution state",
                            StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[2])
    except ZookeeperError:
      raise_(StateException("Zookeeper while creating execution state",
                            StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2])
    except Exception:
      # Just re raise the exception.
      raise


def get_scheduler_location(self, topologyName, callback=None):
    """ get scheduler location """
    isWatching = False

    # Temp dict used to return result
    # if callback is not provided.
    ret = {
        "result": None
    }
    if callback:
      isWatching = True
    else:
      def callback(data):
        """
        Custom callback to get the scheduler location right now.
        """
        ret["result"] = data

    self._get_scheduler_location_with_watch(topologyName, callback, isWatching)

    return ret["result"]

def get_scheduler_location ( self , topologyName , callback = None ) : isWatching = False ret = { "result" : None } if callback : isWatching = True else : def callback ( data ) : ret [ "result" ] = data self . _get_scheduler_location_with_watch ( topologyName , callback , isWatching ) return ret [ "result" ]


def _get_scheduler_location_with_watch(self, topologyName, callback, isWatching):
    """
    Helper function to get scheduler location with
    a callback. The future watch is placed
    only if isWatching is True.
    """
    path = self.get_scheduler_location_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-variable, unused-argument
    @self.client.DataWatch(path)
    def watch_scheduler_location(data, stats):
      """ invoke callback to watch scheduler location """
      if data:
        scheduler_location = SchedulerLocation()
        scheduler_location.ParseFromString(data)
        callback(scheduler_location)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

def
_get_scheduler_location_with_watch
(
self
,
topologyName
,
callback
,
isWatching
)
:
path
=
self
.
get_scheduler_location_path
(
topologyName
)
if
isWatching
:
LOG
.
info
(
"Adding data watch for path: "
+
path
)
@
self
.
client
.
DataWatch
(
path
)
def
watch_scheduler_location
(
data
,
stats
)
:
if
data
:
scheduler_location
=
SchedulerLocation
(
)
scheduler_location
.
ParseFromString
(
data
)
callback
(
scheduler_location
)
else
:
callback
(
)
return
isWatching

Helper function to get scheduler location with a callback . The future watch is placed only if isWatching is True .


def load(file_object):
  """
  Deserializes Java primitive data and objects serialized by ObjectOutputStream
  from a file-like object.
  """
  marshaller = JavaObjectUnmarshaller(file_object)
  marshaller.add_transformer(DefaultObjectTransformer())
  return marshaller.readObject()

def load ( file_object ) : marshaller = JavaObjectUnmarshaller ( file_object ) marshaller . add_transformer ( DefaultObjectTransformer ( ) ) return marshaller . readObject ( )

Deserializes Java primitive data and objects serialized by ObjectOutputStream from a file - like object .


def loads(string):
  """
  Deserializes Java objects and primitive data serialized by ObjectOutputStream
  from a string.
  """
  f = StringIO.StringIO(string)
  marshaller = JavaObjectUnmarshaller(f)
  marshaller.add_transformer(DefaultObjectTransformer())
  return marshaller.readObject()

def loads ( string ) : f = StringIO . StringIO ( string ) marshaller = JavaObjectUnmarshaller ( f ) marshaller . add_transformer ( DefaultObjectTransformer ( ) ) return marshaller . readObject ( )

Deserializes Java objects and primitive data serialized by ObjectOutputStream from a string .


def copy(self, new_object):
    """copy an object"""
    new_object.classdesc = self.classdesc

    for name in self.classdesc.fields_names:
      new_object.__setattr__(name, getattr(self, name))

def copy ( self , new_object ) : new_object . classdesc = self . classdesc for name in self . classdesc . fields_names : new_object . __setattr__ ( name , getattr ( self , name ) )


def validate_state_locations(self):
    """
    Names of all state locations must be unique.
    """
    names = map(lambda loc: loc["name"], self.locations)
    assert len(names) == len(set(names)), "Names of state locations must be unique"

Names of all state locations must be unique .


def initialize(self, config, context):
    """Implements Pulsar Spout's initialize method"""
    self.logger.info("Initializing PulsarSpout with the following")
    self.logger.info("Component-specific config: \n%s" % str(config))
    self.logger.info("Context: \n%s" % str(context))

    self.emit_count = 0
    self.ack_count = 0
    self.fail_count = 0

    if not PulsarSpout.serviceUrl in config or not PulsarSpout.topicName in config:
      self.logger.fatal("Need to specify both serviceUrl and topicName")
    self.pulsar_cluster = str(config[PulsarSpout.serviceUrl])
    self.topic = str(config[PulsarSpout.topicName])
    mode = config[api_constants.TOPOLOGY_RELIABILITY_MODE]
    if mode == api_constants.TopologyReliabilityMode.ATLEAST_ONCE:
      self.acking_timeout = 1000 * int(config[api_constants.TOPOLOGY_MESSAGE_TIMEOUT_SECS])
    else:
      self.acking_timeout = 30000
    if PulsarSpout.receiveTimeoutMs in config:
      self.receive_timeout_ms = config[PulsarSpout.receiveTimeoutMs]
    else:
      self.receive_timeout_ms = 10
    if PulsarSpout.deserializer in config:
      self.deserializer = config[PulsarSpout.deserializer]
      if not callable(self.deserializer):
        self.logger.fatal("Pulsar Message Deserializer needs to be callable")
    else:
      self.deserializer = self.default_deserializer

    # First generate the config
    self.logConfFileName = GenerateLogConfig(context)
    self.logger.info("Generated LogConf at %s" % self.logConfFileName)

    # We currently use the high level consumer API
    # For supporting effectively once, we will need to switch
    # to using lower level Reader API, when it becomes
    # available in python
    self.client = pulsar.Client(self.pulsar_cluster, log_conf_file_path=self.logConfFileName)
    self.logger.info("Setup Client with cluster %s" % self.pulsar_cluster)
    try:
      self.consumer = self.client.subscribe(self.topic, context.get_topology_name(),
                                            consumer_type=pulsar.ConsumerType.Failover,
                                            unacked_messages_timeout_ms=self.acking_timeout)
    except Exception as e:
      self.logger.fatal("Pulsar client subscription failed: %s" % str(e))

    self.logger.info("Subscribed to topic %s" % self.topic)

Implements Pulsar Spout s initialize method


def getInstanceJstack(self, topology_info, instance_id):
    """
    Fetches Instance jstack from heron-shell.
    """
    pid_response = yield getInstancePid(topology_info, instance_id)
    try:
      http_client = tornado.httpclient.AsyncHTTPClient()
      pid_json = json.loads(pid_response)
      pid = pid_json['stdout'].strip()
      if pid == '':
        raise Exception('Failed to get pid')
      endpoint = utils.make_shell_endpoint(topology_info, instance_id)
      url = "%s/jstack/%s" % (endpoint, pid)
      response = yield http_client.fetch(url)
      Log.debug("HTTP call for url: %s", url)
      raise tornado.gen.Return(response.body)
    except tornado.httpclient.HTTPError as e:
      raise Exception(str(e))

def
getInstanceJstack
(
self
,
topology_info
,
instance_id
)
:
pid_response
=
yield
getInstancePid
(
topology_info
,
instance_id
)
try
:
http_client
=
tornado
.
httpclient
.
AsyncHTTPClient
(
)
pid_json
=
json
.
loads
(
pid_response
)
pid
=
pid_json
[
'stdout'
]
.
strip
(
)
if
pid
==
''
:
raise
Exception
(
'Failed to get pid'
)
endpoint
=
utils
.
make_shell_endpoint
(
topology_info
,
instance_id
)
url
=
"%s/jstack/%s"
%
(
endpoint
,
pid
)
response
=
yield
http_client
.
fetch
(
url
)
Log
.
debug
(
"HTTP call for url: %s"
,
url
)
raise
tornado
.
gen
.
Return
(
response
.
body
)
except
tornado
.
httpclient
.
HTTPError
as
e
:
raise
Exception
(
str
(
e
)
)

Fetches Instance jstack from heron - shell .


def create_parser(subparsers):
  """ Create the parse for the update command """
  parser = subparsers.add_parser(
      'update',
      help='Update a topology',
      usage="%(prog)s [options] cluster/[role]/[env] <topology-name> "
      + "[--component-parallelism <name:value>] "
      + "[--container-number value] "
      + "[--runtime-config [component:]<name:value>]",
      add_help=True)

  args.add_titles(parser)
  args.add_cluster_role_env(parser)
  args.add_topology(parser)

  args.add_config(parser)
  args.add_dry_run(parser)
  args.add_service_url(parser)
  args.add_verbose(parser)

  # Special parameters for update command
  def parallelism_type(value):
    pattern = re.compile(r"^[\w\.-]+:[\d]+$")
    if not pattern.match(value):
      raise argparse.ArgumentTypeError(
          "Invalid syntax for component parallelism (<component_name:value>): %s" % value)
    return value

  parser.add_argument(
      '--component-parallelism',
      action='append',
      type=parallelism_type,
      required=False,
      help='Component name and the new parallelism value '
      + 'colon-delimited: <component_name>:<parallelism>')

  def runtime_config_type(value):
    pattern = re.compile(r"^([\w\.-]+:){1,2}[\w\.-]+$")
    if not pattern.match(value):
      raise argparse.ArgumentTypeError(
          "Invalid syntax for runtime config ([component:]<name:value>): %s"
          % value)
    return value

  parser.add_argument(
      '--runtime-config',
      action='append',
      type=runtime_config_type,
      required=False,
      help='Runtime configurations for topology and components '
      + 'colon-delimited: [component:]<name>:<value>')

  def container_number_type(value):
    pattern = re.compile(r"^\d+$")
    if not pattern.match(value):
      raise argparse.ArgumentTypeError(
          "Invalid syntax for container number (value): %s"
          % value)
    return value

  parser.add_argument(
      '--container-number',
      action='append',
      type=container_number_type,
      required=False,
      help='Number of containers <value>')

  parser.set_defaults(subcommand='update')
  return parser


def build_extra_args_dict(cl_args):
  """ Build extra args map """
  # Check parameters
  component_parallelism = cl_args['component_parallelism']
  runtime_configs = cl_args['runtime_config']
  container_number = cl_args['container_number']
  # Users need to provide either (component-parallelism || container_number) or runtime-config
  if (component_parallelism and runtime_configs) or (container_number and runtime_configs):
    raise Exception(
        "(component-parallelism or container_num) and runtime-config " +
        "can't be updated at the same time")

  dict_extra_args = {}

  nothing_set = True
  if component_parallelism:
    dict_extra_args.update({'component_parallelism': component_parallelism})
    nothing_set = False

  if container_number:
    dict_extra_args.update({'container_number': container_number})
    nothing_set = False

  if runtime_configs:
    dict_extra_args.update({'runtime_config': runtime_configs})
    nothing_set = False

  if nothing_set:
    raise Exception(
        "Missing arguments --component-parallelism or --runtime-config or --container-number")

  if cl_args['dry_run']:
    dict_extra_args.update({'dry_run': True})
    if 'dry_run_format' in cl_args:
      dict_extra_args.update({'dry_run_format': cl_args["dry_run_format"]})

  return dict_extra_args


def convert_args_dict_to_list(dict_extra_args):
  """ flatten extra args """
  list_extra_args = []
  if 'component_parallelism' in dict_extra_args:
    list_extra_args += ["--component_parallelism",
                        ','.join(dict_extra_args['component_parallelism'])]
  if 'runtime_config' in dict_extra_args:
    list_extra_args += ["--runtime_config",
                        ','.join(dict_extra_args['runtime_config'])]
  if 'container_number' in dict_extra_args:
    list_extra_args += ["--container_number",
                        ','.join(dict_extra_args['container_number'])]
  if 'dry_run' in dict_extra_args and dict_extra_args['dry_run']:
    list_extra_args += ['--dry_run']
  if 'dry_run_format' in dict_extra_args:
    list_extra_args += ['--dry_run_format', dict_extra_args['dry_run_format']]

  return list_extra_args

def convert_args_dict_to_list ( dict_extra_args ) : list_extra_args = [ ] if 'component_parallelism' in dict_extra_args : list_extra_args += [ "--component_parallelism" , ',' . join ( dict_extra_args [ 'component_parallelism' ] ) ] if 'runtime_config' in dict_extra_args : list_extra_args += [ "--runtime_config" , ',' . join ( dict_extra_args [ 'runtime_config' ] ) ] if 'container_number' in dict_extra_args : list_extra_args += [ "--container_number" , ',' . join ( dict_extra_args [ 'container_number' ] ) ] if 'dry_run' in dict_extra_args and dict_extra_args [ 'dry_run' ] : list_extra_args += [ '--dry_run' ] if 'dry_run_format' in dict_extra_args : list_extra_args += [ '--dry_run_format' , dict_extra_args [ 'dry_run_format' ] ] return list_extra_args


def run(command, parser, cl_args, unknown_args):
  """ run the update command """

  Log.debug("Update Args: %s", cl_args)

  # Build jar list
  extra_lib_jars = jars.packing_jars()
  action = "update topology%s" % (' in dry-run mode' if cl_args["dry_run"] else '')

  # Build extra args
  dict_extra_args = {}
  try:
    dict_extra_args = build_extra_args_dict(cl_args)
  except Exception as err:
    return SimpleResult(Status.InvocationError, err.message)

  # Execute
  if cl_args['deploy_mode'] == config.SERVER_MODE:
    return cli_helper.run_server(command, cl_args, action, dict_extra_args)
  else:
    # Convert extra argument to commandline format and then execute
    list_extra_args = convert_args_dict_to_list(dict_extra_args)
    return cli_helper.run_direct(command, cl_args, action, list_extra_args, extra_lib_jars)

def
run
(
command
,
parser
,
cl_args
,
unknown_args
)
:
Log
.
debug
(
"Update Args: %s"
,
cl_args
)
extra_lib_jars
=
jars
.
packing_jars
(
)
action
=
"update topology%s"
%
(
' in dry-run mode'
if
cl_args
[
"dry_run"
]
else
''
)
dict_extra_args
=
{
}
try
:
dict_extra_args
=
build_extra_args_dict
(
cl_args
)
except
Exception
as
err
:
return
SimpleResult
(
Status
.
InvocationError
,
err
.
message
)
if
cl_args
[
'deploy_mode'
]
==
config
.
SERVER_MODE
:
return
cli_helper
.
run_server
(
command
,
cl_args
,
action
,
dict_extra_args
)
else
:
list_extra_args
=
convert_args_dict_to_list
(
dict_extra_args
)
return
cli_helper
.
run_direct
(
command
,
cl_args
,
action
,
list_extra_args
,
extra_lib_jars
)


def getInstancePid(topology_info, instance_id):
  """
  This method is used by other modules, and so it
  is not a part of the class.
  Fetches Instance pid from heron-shell.
  """
  try:
    http_client = tornado.httpclient.AsyncHTTPClient()
    endpoint = utils.make_shell_endpoint(topology_info, instance_id)
    url = "%s/pid/%s" % (endpoint, instance_id)
    Log.debug("HTTP call for url: %s", url)
    response = yield http_client.fetch(url)
    raise tornado.gen.Return(response.body)
  except tornado.httpclient.HTTPError as e:
    raise Exception(str(e))

def
getInstancePid
(
topology_info
,
instance_id
)
:
try
:
http_client
=
tornado
.
httpclient
.
AsyncHTTPClient
(
)
endpoint
=
utils
.
make_shell_endpoint
(
topology_info
,
instance_id
)
url
=
"%s/pid/%s"
%
(
endpoint
,
instance_id
)
Log
.
debug
(
"HTTP call for url: %s"
,
url
)
response
=
yield
http_client
.
fetch
(
url
)
raise
tornado
.
gen
.
Return
(
response
.
body
)
except
tornado
.
httpclient
.
HTTPError
as
e
:
raise
Exception
(
str
(
e
)
)

This method is used by other modules and so it is not a part of the class . Fetches Instance pid from heron - shell .


def is_grouping_sane(cls, gtype):
    """Checks if a given gtype is sane"""
    if gtype == cls.SHUFFLE or gtype == cls.ALL or gtype == cls.LOWEST or gtype == cls.NONE:
      return True
    elif isinstance(gtype, cls.FIELDS):
      return gtype.gtype == topology_pb2.Grouping.Value("FIELDS") and \
             gtype.fields is not None
    elif isinstance(gtype, cls.CUSTOM):
      return gtype.gtype == topology_pb2.Grouping.Value("CUSTOM") and \
             gtype.python_serialized is not None
    else:
      #pylint: disable=fixme
      #TODO: DIRECT are not supported yet
      return False

def is_grouping_sane ( cls , gtype ) : if gtype == cls . SHUFFLE or gtype == cls . ALL or gtype == cls . LOWEST or gtype == cls . NONE : return True elif isinstance ( gtype , cls . FIELDS ) : return gtype . gtype == topology_pb2 . Grouping . Value ( "FIELDS" ) and gtype . fields is not None elif isinstance ( gtype , cls . CUSTOM ) : return gtype . gtype == topology_pb2 . Grouping . Value ( "CUSTOM" ) and gtype . python_serialized is not None else : return False


def custom(cls, customgrouper):
    """Custom grouping from a given implementation of ICustomGrouping

    :param customgrouper: The ICustomGrouping implemention to use
    """
    if customgrouper is None:
      raise TypeError("Argument to custom() must be ICustomGrouping instance or classpath")
    if not isinstance(customgrouper, ICustomGrouping) and not isinstance(customgrouper, str):
      raise TypeError("Argument to custom() must be ICustomGrouping instance or classpath")
    serialized = default_serializer.serialize(customgrouper)
    return cls.custom_serialized(serialized, is_java=False)


def custom_serialized(cls, serialized, is_java=True):
    """Custom grouping from a given serialized string

    This class is created for compatibility with ``custom_serialized(cls, java_serialized)`` method
    of StreamParse API, although its functionality is not yet implemented (Java-serialized).
    Currently only custom grouping implemented in Python is supported, and ``custom()`` method
    should be used to indicate its classpath, rather than directly to use this method.

    In the future, users can directly specify Java-serialized object with ``is_java=True`` in order
    to use a custom grouping implemented in Java for python topology.

    :param serialized: serialized classpath to custom grouping class to use (if python)
    :param is_java: indicate whether this is Java serialized, or python serialized
    """
    if not isinstance(serialized, bytes):
      raise TypeError("Argument to custom_serialized() must be "
                      "a serialized Python class as bytes, given: %s" % str(serialized))
    if not is_java:
      return cls.CUSTOM(gtype=topology_pb2.Grouping.Value("CUSTOM"),
                        python_serialized=serialized)
    else:
      raise NotImplementedError("Custom grouping implemented in Java for Python topology"
                                "is not yet supported.")


def register_metrics(self, metrics_collector, interval):
    """Registers its metrics to a given metrics collector with a given interval"""
    for field, metrics in self.metrics.items():
      metrics_collector.register_metric(field, metrics, interval)

def register_metrics ( self , metrics_collector , interval ) : for field , metrics in self . metrics . items ( ) : metrics_collector . register_metric ( field , metrics , interval )


def update_count(self, name, incr_by=1, key=None):
    """Update the value of CountMetric or MultiCountMetric

    :type name: str
    :param name: name of the registered metric to be updated.
    :type incr_by: int
    :param incr_by: specifies how much to increment. Default is 1.
    :type key: str or None
    :param key: specifies a key for MultiCountMetric. Needs to be `None` for updating CountMetric.
    """
    if name not in self.metrics:
      Log.error("In update_count(): %s is not registered in the metric", name)

    if key is None and isinstance(self.metrics[name], CountMetric):
      self.metrics[name].incr(incr_by)
    elif key is not None and isinstance(self.metrics[name], MultiCountMetric):
      self.metrics[name].incr(key, incr_by)
    else:
      Log.error("In update_count(): %s is registered but not supported with this method", name)


def update_reduced_metric(self, name, value, key=None):
    """Update the value of ReducedMetric or MultiReducedMetric

    :type name: str
    :param name: name of the registered metric to be updated.
    :param value: specifies a value to be reduced.
    :type key: str or None
    :param key: specifies a key for MultiReducedMetric. Needs to be `None` for updating
                ReducedMetric.
    """
    if name not in self.metrics:
      Log.error("In update_reduced_metric(): %s is not registered in the metric", name)

    if key is None and isinstance(self.metrics[name], ReducedMetric):
      self.metrics[name].update(value)
    elif key is not None and isinstance(self.metrics[name], MultiReducedMetric):
      self.metrics[name].update(key, value)
    else:
      Log.error("In update_count(): %s is registered but not supported with this method", name)


def update_received_packet(self, received_pkt_size_bytes):
    """Update received packet metrics"""
    self.update_count(self.RECEIVED_PKT_COUNT)
    self.update_count(self.RECEIVED_PKT_SIZE, incr_by=received_pkt_size_bytes)

def update_received_packet ( self , received_pkt_size_bytes ) : self . update_count ( self . RECEIVED_PKT_COUNT ) self . update_count ( self . RECEIVED_PKT_SIZE , incr_by = received_pkt_size_bytes )


def update_sent_packet(self, sent_pkt_size_bytes):
    """Update sent packet metrics"""
    self.update_count(self.SENT_PKT_COUNT)
    self.update_count(self.SENT_PKT_SIZE, incr_by=sent_pkt_size_bytes)

def update_sent_packet ( self , sent_pkt_size_bytes ) : self . update_count ( self . SENT_PKT_COUNT ) self . update_count ( self . SENT_PKT_SIZE , incr_by = sent_pkt_size_bytes )


def register_metrics(self, context):
    """Registers metrics to context

    :param context: Topology Context
    """
    sys_config = system_config.get_sys_config()
    interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC])
    collector = context.get_metrics_collector()
    super(ComponentMetrics, self).register_metrics(collector, interval)

def register_metrics ( self , context ) : sys_config = system_config . get_sys_config ( ) interval = float ( sys_config [ constants . HERON_METRICS_EXPORT_INTERVAL_SEC ] ) collector = context . get_metrics_collector ( ) super ( ComponentMetrics , self ) . register_metrics ( collector , interval )


def serialize_data_tuple(self, stream_id, latency_in_ns):
    """Apply update to serialization metrics"""
    self.update_count(self.TUPLE_SERIALIZATION_TIME_NS, incr_by=latency_in_ns, key=stream_id)

def serialize_data_tuple ( self , stream_id , latency_in_ns ) : self . update_count ( self . TUPLE_SERIALIZATION_TIME_NS , incr_by = latency_in_ns , key = stream_id )


def next_tuple(self, latency_in_ns):
    """Apply updates to the next tuple metrics"""
    self.update_reduced_metric(self.NEXT_TUPLE_LATENCY, latency_in_ns)
    self.update_count(self.NEXT_TUPLE_COUNT)

def next_tuple ( self , latency_in_ns ) : self . update_reduced_metric ( self . NEXT_TUPLE_LATENCY , latency_in_ns ) self . update_count ( self . NEXT_TUPLE_COUNT )


def execute_tuple(self, stream_id, source_component, latency_in_ns):
    """Apply updates to the execute metrics"""
    self.update_count(self.EXEC_COUNT, key=stream_id)
    self.update_reduced_metric(self.EXEC_LATENCY, latency_in_ns, stream_id)
    self.update_count(self.EXEC_TIME_NS, incr_by=latency_in_ns, key=stream_id)

    global_stream_id = source_component + "/" + stream_id
    self.update_count(self.EXEC_COUNT, key=global_stream_id)
    self.update_reduced_metric(self.EXEC_LATENCY, latency_in_ns, global_stream_id)
    self.update_count(self.EXEC_TIME_NS, incr_by=latency_in_ns, key=global_stream_id)

def execute_tuple ( self , stream_id , source_component , latency_in_ns ) : self . update_count ( self . EXEC_COUNT , key = stream_id ) self . update_reduced_metric ( self . EXEC_LATENCY , latency_in_ns , stream_id ) self . update_count ( self . EXEC_TIME_NS , incr_by = latency_in_ns , key = stream_id ) global_stream_id = source_component + "/" + stream_id self . update_count ( self . EXEC_COUNT , key = global_stream_id ) self . update_reduced_metric ( self . EXEC_LATENCY , latency_in_ns , global_stream_id ) self . update_count ( self . EXEC_TIME_NS , incr_by = latency_in_ns , key = global_stream_id )


def deserialize_data_tuple(self, stream_id, source_component, latency_in_ns):
    """Apply updates to the deserialization metrics"""
    self.update_count(self.TUPLE_DESERIALIZATION_TIME_NS, incr_by=latency_in_ns, key=stream_id)
    global_stream_id = source_component + "/" + stream_id
    self.update_count(self.TUPLE_DESERIALIZATION_TIME_NS, incr_by=latency_in_ns,
                      key=global_stream_id)

def deserialize_data_tuple ( self , stream_id , source_component , latency_in_ns ) : self . update_count ( self . TUPLE_DESERIALIZATION_TIME_NS , incr_by = latency_in_ns , key = stream_id ) global_stream_id = source_component + "/" + stream_id self . update_count ( self . TUPLE_DESERIALIZATION_TIME_NS , incr_by = latency_in_ns , key = global_stream_id )


def register_metric(self, name, metric, time_bucket_in_sec):
    """Registers a given metric

    :param name: name of the metric
    :param metric: IMetric object to be registered
    :param time_bucket_in_sec: time interval for update to the metrics manager
    """
    if name in self.metrics_map:
      raise RuntimeError("Another metric has already been registered with name: %s" % name)

    Log.debug("Register metric: %s, with interval: %s", name, str(time_bucket_in_sec))
    self.metrics_map[name] = metric

    if time_bucket_in_sec in self.time_bucket_in_sec_to_metrics_name:
      self.time_bucket_in_sec_to_metrics_name[time_bucket_in_sec].append(name)
    else:
      self.time_bucket_in_sec_to_metrics_name[time_bucket_in_sec] = [name]
      self._register_timer_task(time_bucket_in_sec)


def poll(self):
    """Poll from the buffer

    It is a non-blocking operation, and when the buffer is empty, it raises Queue.Empty exception
    """
    try:
      # non-blocking
      ret = self._buffer.get(block=False)
      if self._producer_callback is not None:
        self._producer_callback()
      return ret
    except Queue.Empty:
      Log.debug("%s: Empty in poll()" % str(self))
      raise Queue.Empty

def
poll
(
self
)
:
try
:
ret
=
self
.
_buffer
.
get
(
block
=
False
)
if
self
.
_producer_callback
is
not
:
self
.
_producer_callback
(
)
return
ret
except
Queue
.
Empty
:
Log
.
debug
(
"%s: Empty in poll()"
%
str
(
self
)
)
raise
Queue
.
Empty


def offer(self, item):
    """Offer to the buffer

    It is a non-blocking operation, and when the buffer is full, it raises Queue.Full exception
    """
    try:
      # non-blocking
      self._buffer.put(item, block=False)
      if self._consumer_callback is not None:
        self._consumer_callback()
      return True
    except Queue.Full:
      Log.debug("%s: Full in offer()" % str(self))
      raise Queue.Full

def
offer
(
self
,
item
)
:
try
:
self
.
_buffer
.
put
(
item
,
block
=
False
)
if
self
.
_consumer_callback
is
not
:
self
.
_consumer_callback
(
)
return
True
except
Queue
.
Full
:
Log
.
debug
(
"%s: Full in offer()"
%
str
(
self
)
)
raise
Queue
.
Full


def parse(version):
    """
    Parse version to major, minor, patch, pre-release, build parts.
    """
    match = _REGEX.match(version)
    if match is None:
        raise ValueError('%s is not valid SemVer string' % version)

    verinfo = match.groupdict()

    verinfo['major'] = int(verinfo['major'])
    verinfo['minor'] = int(verinfo['minor'])
    verinfo['patch'] = int(verinfo['patch'])

    return verinfo

def
parse
(
version
)
:
match
=
_REGEX
.
match
(
version
)
if
match
is
:
raise
ValueError
(
'%s is not valid SemVer string'
%
version
)
verinfo
=
match
.
groupdict
(
)
verinfo
[
'major'
]
=
int
(
verinfo
[
'major'
]
)
verinfo
[
'minor'
]
=
int
(
verinfo
[
'minor'
]
)
verinfo
[
'patch'
]
=
int
(
verinfo
[
'patch'
]
)
return
verinfo

Parse version to major minor patch pre - release build parts .


def get_all_zk_state_managers(conf):
  """
  Creates all the zookeeper state_managers and returns
  them in a list
  """
  state_managers = []
  state_locations = conf.get_state_locations_of_type("zookeeper")
  for location in state_locations:
    name = location['name']
    hostport = location['hostport']
    hostportlist = []
    for hostportpair in hostport.split(','):
      host = None
      port = None
      if ':' in hostport:
        hostandport = hostportpair.split(':')
        if len(hostandport) == 2:
          host = hostandport[0]
          port = int(hostandport[1])
      if not host or not port:
        raise Exception("Hostport for %s must be of the format 'host:port'." % (name))
      hostportlist.append((host, port))
    tunnelhost = location['tunnelhost']
    rootpath = location['rootpath']
    LOG.info("Connecting to zk hostports: " + str(hostportlist) + " rootpath: " + rootpath)
    state_manager = ZkStateManager(name, hostportlist, rootpath, tunnelhost)
    state_managers.append(state_manager)

  return state_managers


def get_all_file_state_managers(conf):
  """
  Returns all the file state_managers.
  """
  state_managers = []
  state_locations = conf.get_state_locations_of_type("file")
  for location in state_locations:
    name = location['name']
    rootpath = os.path.expanduser(location['rootpath'])
    LOG.info("Connecting to file state with rootpath: " + rootpath)
    state_manager = FileStateManager(name, rootpath)
    state_managers.append(state_manager)

  return state_managers


def incr(self, key, to_add=1):
    """Increments the value of a given key by ``to_add``"""
    if key not in self.value:
      self.value[key] = CountMetric()
    self.value[key].incr(to_add)

def incr ( self , key , to_add = 1 ) : if key not in self . value : self . value [ key ] = CountMetric ( ) self . value [ key ] . incr ( to_add )

Increments the value of a given key by to_add


def update(self, key, value):
    """Updates a value of a given key and apply reduction"""
    if key not in self.value:
      self.value[key] = ReducedMetric(self.reducer)

    self.value[key].update(value)

def update ( self , key , value ) : if key not in self . value : self . value [ key ] = ReducedMetric ( self . reducer ) self . value [ key ] . update ( value )


def add_key(self, key):
    """Adds a new key to this metric"""
    if key not in self.value:
      self.value[key] = ReducedMetric(self.reducer)

def add_key ( self , key ) : if key not in self . value : self . value [ key ] = ReducedMetric ( self . reducer )


def add_data_tuple(self, stream_id, new_data_tuple, tuple_size_in_bytes):
    """Add a new data tuple to the currently buffered set of tuples"""
    if (self.current_data_tuple_set is None) or \
        (self.current_data_tuple_set.stream.id != stream_id) or \
        (len(self.current_data_tuple_set.tuples) >= self.data_tuple_set_capacity) or \
        (self.current_data_tuple_size_in_bytes >= self.max_data_tuple_size_in_bytes):
      self._init_new_data_tuple(stream_id)

    added_tuple = self.current_data_tuple_set.tuples.add()
    added_tuple.CopyFrom(new_data_tuple)

    self.current_data_tuple_size_in_bytes += tuple_size_in_bytes
    self.total_data_emitted_in_bytes += tuple_size_in_bytes

def add_data_tuple ( self , stream_id , new_data_tuple , tuple_size_in_bytes ) : if ( self . current_data_tuple_set is None ) or ( self . current_data_tuple_set . stream . id != stream_id ) or ( len ( self . current_data_tuple_set . tuples ) >= self . data_tuple_set_capacity ) or ( self . current_data_tuple_size_in_bytes >= self . max_data_tuple_size_in_bytes ) : self . _init_new_data_tuple ( stream_id ) added_tuple = self . current_data_tuple_set . tuples . add ( ) added_tuple . CopyFrom ( new_data_tuple ) self . current_data_tuple_size_in_bytes += tuple_size_in_bytes self . total_data_emitted_in_bytes += tuple_size_in_bytes


def add_ckpt_state(self, ckpt_id, ckpt_state):
    """Add the checkpoint state message to be sent back the stmgr

    :param ckpt_id: The id of the checkpoint
    :ckpt_state: The checkpoint state
    """
    # first flush any buffered tuples
    self._flush_remaining()
    msg = ckptmgr_pb2.StoreInstanceStateCheckpoint()
    istate = ckptmgr_pb2.InstanceStateCheckpoint()
    istate.checkpoint_id = ckpt_id
    istate.state = ckpt_state
    msg.state.CopyFrom(istate)
    self._push_tuple_to_stream(msg)

def add_ckpt_state ( self , ckpt_id , ckpt_state ) : self . _flush_remaining ( ) msg = ckptmgr_pb2 . StoreInstanceStateCheckpoint ( ) istate = ckptmgr_pb2 . InstanceStateCheckpoint ( ) istate . checkpoint_id = ckpt_id istate . state = ckpt_state msg . state . CopyFrom ( istate ) self . _push_tuple_to_stream ( msg )


def valid_path(path):
  '''
  Check if an entry in the class path exists as either a directory or a file
  '''
  # check if the suffic of classpath suffix exists as directory
  if path.endswith('*'):
    Log.debug('Checking classpath entry suffix as directory: %s', path[:-1])
    if os.path.isdir(path[:-1]):
      return True
    return False

  # check if the classpath entry is a directory
  Log.debug('Checking classpath entry as directory: %s', path)
  if os.path.isdir(path):
    return True
  else:
    # check if the classpath entry is a file
    Log.debug('Checking classpath entry as file: %s', path)
    if os.path.isfile(path):
      return True

  return False


def valid_java_classpath(classpath):
  '''
  Given a java classpath, check whether the path entries are valid or not
  '''
  paths = classpath.split(':')
  for path_entry in paths:
    if not valid_path(path_entry.strip()):
      return False
  return True

def valid_java_classpath ( classpath ) : paths = classpath . split ( ':' ) for path_entry in paths : if not valid_path ( path_entry . strip ( ) ) : return False return True

Given a java classpath check whether the path entries are valid or not


def _get_deps_list(abs_path_to_pex):
  """Get a list of paths to included dependencies in the specified pex file

  Note that dependencies are located under `.deps` directory
  """
  pex = zipfile.ZipFile(abs_path_to_pex, mode='r')
  deps = list(set([re.match(egg_regex, i).group(1) for i in pex.namelist()
                   if re.match(egg_regex, i) is not None]))
  return deps

def _get_deps_list ( abs_path_to_pex ) : pex = zipfile . ZipFile ( abs_path_to_pex , mode = 'r' ) deps = list ( set ( [ re . match ( egg_regex , i ) . group ( 1 ) for i in pex . namelist ( ) if re . match ( egg_regex , i ) is not None ] ) ) return deps


def load_pex(path_to_pex, include_deps=True):
  """Loads pex file and its dependencies to the current python path"""
  abs_path_to_pex = os.path.abspath(path_to_pex)
  Log.debug("Add a pex to the path: %s" % abs_path_to_pex)
  if abs_path_to_pex not in sys.path:
    sys.path.insert(0, os.path.dirname(abs_path_to_pex))

  # add dependencies to path
  if include_deps:
    for dep in _get_deps_list(abs_path_to_pex):
      to_join = os.path.join(os.path.dirname(abs_path_to_pex), dep)
      if to_join not in sys.path:
        Log.debug("Add a new dependency to the path: %s" % dep)
        sys.path.insert(0, to_join)

  Log.debug("Python path: %s" % str(sys.path))

def
load_pex
(
path_to_pex
,
include_deps
=
True
)
:
abs_path_to_pex
=
os
.
path
.
abspath
(
path_to_pex
)
Log
.
debug
(
"Add a pex to the path: %s"
%
abs_path_to_pex
)
if
abs_path_to_pex
not
in
sys
.
path
:
sys
.
path
.
insert
(
0
,
os
.
path
.
dirname
(
abs_path_to_pex
)
)
if
include_deps
:
for
dep
in
_get_deps_list
(
abs_path_to_pex
)
:
to_join
=
os
.
path
.
join
(
os
.
path
.
dirname
(
abs_path_to_pex
)
,
dep
)
if
to_join
not
in
sys
.
path
:
Log
.
debug
(
"Add a new dependency to the path: %s"
%
dep
)
sys
.
path
.
insert
(
0
,
to_join
)
Log
.
debug
(
"Python path: %s"
%
str
(
sys
.
path
)
)


def resolve_heron_suffix_issue(abs_pex_path, class_path):
  """Resolves duplicate package suffix problems

  When dynamically loading a pex file and a corresponding python class (bolt/spout/topology),
  if the top level package in which to-be-loaded classes reside is named 'heron', the path conflicts
  with this Heron Instance pex package (heron.instance.src.python...), making the Python
  interpreter unable to find the target class in a given pex file.
  This function resolves this issue by individually loading packages with suffix `heron` to
  avoid this issue.

  However, if a dependent module/class that is not directly specified under ``class_path``
  and has conflicts with other native heron packages, there is a possibility that
  such a class/module might not be imported correctly. For example, if a given ``class_path`` was
  ``heron.common.src.module.Class``, but it has a dependent module (such as by import statement),
  ``heron.common.src.python.dep_module.DepClass`` for example, pex_loader does not guarantee that
  ``DepClass` is imported correctly. This is because ``heron.common.src.python.dep_module`` is not
  explicitly added to sys.path, while ``heron.common.src.python`` module exists as the native heron
  package, from which ``dep_module`` cannot be found, so Python interpreter may raise ImportError.

  The best way to avoid this issue is NOT to dynamically load a pex file whose top level package
  name is ``heron``. Note that this method is included because some of the example topologies and
  tests have to have a pex with its top level package name of ``heron``.
  """
  # import top-level package named `heron` of a given pex file
  importer = zipimport.zipimporter(abs_pex_path)
  importer.load_module("heron")

  # remove 'heron' and the classname
  to_load_lst = class_path.split('.')[1:-1]
  loaded = ['heron']
  loaded_mod = None
  for to_load in to_load_lst:
    sub_importer = zipimport.zipimporter(os.path.join(abs_pex_path, '/'.join(loaded)))
    loaded_mod = sub_importer.load_module(to_load)
    loaded.append(to_load)

  return loaded_mod

def resolve_heron_suffix_issue ( abs_pex_path , class_path ) : importer = zipimport . zipimporter ( abs_pex_path ) importer . load_module ( "heron" ) to_load_lst = class_path . split ( '.' ) [ 1 : - 1 ] loaded = [ 'heron' ] loaded_mod = None for to_load in to_load_lst : sub_importer = zipimport . zipimporter ( os . path . join ( abs_pex_path , '/' . join ( loaded ) ) ) loaded_mod = sub_importer . load_module ( to_load ) loaded . append ( to_load ) return loaded_mod


def import_and_get_class(path_to_pex, python_class_name):
  """Imports and load a class from a given pex file path and python class name

  For example, if you want to get a class called `Sample` in
  /some-path/sample.pex/heron/examples/src/python/sample.py,
  ``path_to_pex`` needs to be ``/some-path/sample.pex``, and
  ``python_class_name`` needs to be ``heron.examples.src.python.sample.Sample``
  """
  abs_path_to_pex = os.path.abspath(path_to_pex)

  Log.debug("Add a pex to the path: %s" % abs_path_to_pex)
  Log.debug("In import_and_get_class with cls_name: %s" % python_class_name)
  split = python_class_name.split('.')
  from_path = '.'.join(split[:-1])
  import_name = python_class_name.split('.')[-1]

  Log.debug("From path: %s, import name: %s" % (from_path, import_name))

  # Resolve duplicate package suffix problem (heron.), if the top level package name is heron
  if python_class_name.startswith("heron."):
    try:
      mod = resolve_heron_suffix_issue(abs_path_to_pex, python_class_name)
      return getattr(mod, import_name)
    except:
      Log.error("Could not resolve class %s with special handling" % python_class_name)

  mod = __import__(from_path, fromlist=[import_name], level=-1)
  Log.debug("Imported module: %s" % str(mod))
  return getattr(mod, import_name)


def new_source(self, source):
    """Adds a new source to the computation DAG"""

    source_streamlet = None
    if callable(source):
      source_streamlet = SupplierStreamlet(source)
    elif isinstance(source, Generator):
      source_streamlet = GeneratorStreamlet(source)
    else:
      raise RuntimeError("Builder's new source has to be either a Generator or a function")

    self._sources.append(source_streamlet)
    return source_streamlet


def build(self, bldr):
    """Builds the topology and returns the builder"""
    stage_names = sets.Set()
    for source in self._sources:
      source._build(bldr, stage_names)
    for source in self._sources:
      if not source._all_built():
        raise RuntimeError("Topology cannot be fully built! Are all sources added?")


def __replace(config, wildcards, config_file):
  """For each kvp in config, do wildcard substitution on the values"""
  for config_key in config:
    config_value = config[config_key]
    original_value = config_value
    if isinstance(config_value, str):
      for token in wildcards:
        if wildcards[token]:
          config_value = config_value.replace(token, wildcards[token])
      found = re.findall(r'\${[A-Z_]+}', config_value)
      if found:
        raise ValueError("%s=%s in file %s contains unsupported or unset wildcard tokens: %s" %
                         (config_key, original_value, config_file, ", ".join(found)))
      config[config_key] = config_value
  return config

For each kvp in config do wildcard substitution on the values


def get_command_handlers():
  '''
  Create a map of command names and handlers
  '''
  return {
      'activate': activate,
      'config': hconfig,
      'deactivate': deactivate,
      'help': cli_help,
      'kill': kill,
      'restart': restart,
      'submit': submit,
      'update': update,
      'version': version
  }

def get_command_handlers ( ) : return { 'activate' : activate , 'config' : hconfig , 'deactivate' : deactivate , 'help' : cli_help , 'kill' : kill , 'restart' : restart , 'submit' : submit , 'update' : update , 'version' : version }


def setDefault(self, constant, start, end):
    """ set default time """
    starttime = start / 60 * 60
    if starttime < start:
      starttime += 60
    endtime = end / 60 * 60
    while starttime <= endtime:
      # STREAMCOMP-1559
      # Second check is a work around, because the response from tmaster
      # contains value 0, if it is queries for the current timestamp,
      # since the bucket is created in the tmaster, but is not filled
      # by the metrics.
      if starttime not in self.timeline or self.timeline[starttime] == 0:
        self.timeline[starttime] = constant
      starttime += 60

def setDefault ( self , constant , start , end ) : starttime = start / 60 * 60 if starttime < start : starttime += 60 endtime = end / 60 * 60 while starttime <= endtime : if starttime not in self . timeline or self . timeline [ starttime ] == 0 : self . timeline [ starttime ] = constant starttime += 60


def process(self, tup):
    """Process a single tuple of input

    We add the (time, tuple) pair into our current_tuples. And then look for expiring
    elemnents
    """
    curtime = int(time.time())
    self.current_tuples.append((tup, curtime))
    self._expire(curtime)

def process ( self , tup ) : curtime = int ( time . time ( ) ) self . current_tuples . append ( ( tup , curtime ) ) self . _expire ( curtime )


def process_tick(self, tup):
    """Called every slide_interval
    """
    curtime = int(time.time())
    window_info = WindowContext(curtime - self.window_duration, curtime)
    tuple_batch = []
    for (tup, tm) in self.current_tuples:
      tuple_batch.append(tup)
    self.processWindow(window_info, tuple_batch)
    self._expire(curtime)

def process_tick ( self , tup ) : curtime = int ( time . time ( ) ) window_info = WindowContext ( curtime - self . window_duration , curtime ) tuple_batch = [ ] for ( tup , tm ) in self . current_tuples : tuple_batch . append ( tup ) self . processWindow ( window_info , tuple_batch ) self . _expire ( curtime )


def process_tick(self, tup):
    """Called every window_duration
    """
    curtime = int(time.time())
    window_info = WindowContext(curtime - self.window_duration, curtime)
    self.processWindow(window_info, list(self.current_tuples))
    for tup in self.current_tuples:
      self.ack(tup)
    self.current_tuples.clear()

def process_tick ( self , tup ) : curtime = int ( time . time ( ) ) window_info = WindowContext ( curtime - self . window_duration , curtime ) self . processWindow ( window_info , list ( self . current_tuples ) ) for tup in self . current_tuples : self . ack ( tup ) self . current_tuples . clear ( )


def getStmgrsRegSummary(self, tmaster, callback=None):
    """
    Get summary of stream managers registration summary
    """
    if not tmaster or not tmaster.host or not tmaster.stats_port:
      return
    reg_request = tmaster_pb2.StmgrsRegistrationSummaryRequest()
    request_str = reg_request.SerializeToString()
    port = str(tmaster.stats_port)
    host = tmaster.host
    url = "http://{0}:{1}/stmgrsregistrationsummary".format(host, port)
    request = tornado.httpclient.HTTPRequest(url,
                                             body=request_str,
                                             method='POST',
                                             request_timeout=5)
    Log.debug('Making HTTP call to fetch stmgrsregistrationsummary url: %s', url)
    try:
      client = tornado.httpclient.AsyncHTTPClient()
      result = yield client.fetch(request)
      Log.debug("HTTP call complete.")
    except tornado.httpclient.HTTPError as e:
      raise Exception(str(e))
    # Check the response code - error if it is in 400s or 500s
    responseCode = result.code
    if responseCode >= 400:
      message = "Error in getting exceptions from Tmaster, code: " + responseCode
      Log.error(message)
      raise tornado.gen.Return({
          "message": message
      })
    # Parse the response from tmaster.
    reg_response = tmaster_pb2.StmgrsRegistrationSummaryResponse()
    reg_response.ParseFromString(result.body)
    # Send response
    ret = {}
    for stmgr in reg_response.registered_stmgrs:
      ret[stmgr] = True
    for stmgr in reg_response.absent_stmgrs:
      ret[stmgr] = False
    raise tornado.gen.Return(ret)


def setup(executor):
  """Set up log, process and signal handlers"""
  # pylint: disable=unused-argument
  def signal_handler(signal_to_handle, frame):
    # We would do nothing here but just exit
    # Just catch the SIGTERM and then cleanup(), registered with atexit, would invoke
    Log.info('signal_handler invoked with signal %s', signal_to_handle)
    executor.stop_state_manager_watches()
    sys.exit(signal_to_handle)

  def cleanup():
    """Handler to trigger when receiving the SIGTERM signal
    Do cleanup inside this method, including:
    1. Terminate all children processes
    """
    Log.info('Executor terminated; exiting all process in executor.')

    # Kill child processes first and wait for log collection to finish
    for pid in executor.processes_to_monitor.keys():
      os.kill(pid, signal.SIGTERM)
    time.sleep(5)

    # We would not wait or check whether process spawned dead or not
    os.killpg(0, signal.SIGTERM)

  # Redirect stdout and stderr to files in append mode
  # The filename format is heron-executor-<container_id>.stdxxx
  shardid = executor.shard
  log.configure(logfile='heron-executor-%s.stdout' % shardid)

  pid = os.getpid()
  sid = os.getsid(pid)

  # POSIX prohibits the change of the process group ID of a session leader
  if pid <> sid:
    Log.info('Set up process group; executor becomes leader')
    os.setpgrp() # create new process group, become its leader

  Log.info('Register the SIGTERM signal handler')
  signal.signal(signal.SIGTERM, signal_handler)

  Log.info('Register the atexit clean up')
  atexit.register(cleanup)


def main():
  """Register exit handlers, initialize the executor and run it."""
  # Since Heron on YARN runs as headless users, pex compiled
  # binaries should be exploded into the container working
  # directory. In order to do this, we need to set the
  # PEX_ROOT shell environment before forking the processes
  shell_env = os.environ.copy()
  shell_env["PEX_ROOT"] = os.path.join(os.path.abspath('.'), ".pex")

  # Instantiate the executor, bind it to signal handlers and launch it
  executor = HeronExecutor(sys.argv, shell_env)
  executor.initialize()

  start(executor)

def main ( ) : shell_env = os . environ . copy ( ) shell_env [ "PEX_ROOT" ] = os . path . join ( os . path . abspath ( '.' ) , ".pex" ) executor = HeronExecutor ( sys . argv , shell_env ) executor . initialize ( ) start ( executor )

Register exit handlers initialize the executor and run it .


def _get_healthmgr_cmd(self):
    ''' get the command to start the topology health manager processes '''
    healthmgr_main_class = 'org.apache.heron.healthmgr.HealthManager'

    healthmgr_cmd = [os.path.join(self.heron_java_home, 'bin/java'),
                     # We could not rely on the default -Xmx setting, which could be very big,
                     # for instance, the default -Xmx in Twitter mesos machine is around 18GB
                     '-Xmx1024M',
                     '-XX:+PrintCommandLineFlags',
                     '-verbosegc',
                     '-XX:+PrintGCDetails',
                     '-XX:+PrintGCTimeStamps',
                     '-XX:+PrintGCDateStamps',
                     '-XX:+PrintGCCause',
                     '-XX:+UseGCLogFileRotation',
                     '-XX:NumberOfGCLogFiles=5',
                     '-XX:GCLogFileSize=100M',
                     '-XX:+PrintPromotionFailure',
                     '-XX:+PrintTenuringDistribution',
                     '-XX:+PrintHeapAtGC',
                     '-XX:+HeapDumpOnOutOfMemoryError',
                     '-XX:+UseConcMarkSweepGC',
                     '-XX:+PrintCommandLineFlags',
                     '-Xloggc:log-files/gc.healthmgr.log',
                     '-Djava.net.preferIPv4Stack=true',
                     '-cp', self.health_manager_classpath,
                     healthmgr_main_class,
                     "--cluster", self.cluster,
                     "--role", self.role,
                     "--environment", self.environment,
                     "--topology_name", self.topology_name,
                     "--metricsmgr_port", self.metrics_manager_port]

    return Command(healthmgr_cmd, self.shell_env)


def _get_tmaster_processes(self):
    ''' get the command to start the tmaster processes '''
    retval = {}
    tmaster_cmd_lst = [
        self.tmaster_binary,
        '--topology_name=%s' % self.topology_name,
        '--topology_id=%s' % self.topology_id,
        '--zkhostportlist=%s' % self.state_manager_connection,
        '--zkroot=%s' % self.state_manager_root,
        '--myhost=%s' % self.master_host,
        '--master_port=%s' % str(self.master_port),
        '--controller_port=%s' % str(self.tmaster_controller_port),
        '--stats_port=%s' % str(self.tmaster_stats_port),
        '--config_file=%s' % self.heron_internals_config_file,
        '--override_config_file=%s' % self.override_config_file,
        '--metrics_sinks_yaml=%s' % self.metrics_sinks_config_file,
        '--metricsmgr_port=%s' % str(self.metrics_manager_port),
        '--ckptmgr_port=%s' % str(self.checkpoint_manager_port)]

    tmaster_env = self.shell_env.copy() if self.shell_env is not None else {}
    tmaster_cmd = Command(tmaster_cmd_lst, tmaster_env)
    if os.environ.get('ENABLE_HEAPCHECK') is not None:
      tmaster_cmd.env.update({
          'LD_PRELOAD': "/usr/lib/libtcmalloc.so",
          'HEAPCHECK': "normal"
      })

    retval["heron-tmaster"] = tmaster_cmd

    if self.metricscache_manager_mode.lower() != "disabled":
      retval["heron-metricscache"] = self._get_metrics_cache_cmd()

    if self.health_manager_mode.lower() != "disabled":
      retval["heron-healthmgr"] = self._get_healthmgr_cmd()

    retval[self.metricsmgr_ids[0]] = self._get_metricsmgr_cmd(
        self.metricsmgr_ids[0],
        self.metrics_sinks_config_file,
        self.metrics_manager_port)

    if self.is_stateful_topology:
      retval.update(self._get_ckptmgr_process())

    return retval

def _get_tmaster_processes ( self ) : retval = { } tmaster_cmd_lst = [ self . tmaster_binary , '--topology_name=%s' % self . topology_name , '--topology_id=%s' % self . topology_id , '--zkhostportlist=%s' % self . state_manager_connection , '--zkroot=%s' % self . state_manager_root , '--myhost=%s' % self . master_host , '--master_port=%s' % str ( self . master_port ) , '--controller_port=%s' % str ( self . tmaster_controller_port ) , '--stats_port=%s' % str ( self . tmaster_stats_port ) , '--config_file=%s' % self . heron_internals_config_file , '--override_config_file=%s' % self . override_config_file , '--metrics_sinks_yaml=%s' % self . metrics_sinks_config_file , '--metricsmgr_port=%s' % str ( self . metrics_manager_port ) , '--ckptmgr_port=%s' % str ( self . checkpoint_manager_port ) ] tmaster_env = self . shell_env . copy ( ) if self . shell_env is not None else { } tmaster_cmd = Command ( tmaster_cmd_lst , tmaster_env ) if os . environ . get ( 'ENABLE_HEAPCHECK' ) is not None : tmaster_cmd . env . update ( { 'LD_PRELOAD' : "/usr/lib/libtcmalloc.so" , 'HEAPCHECK' : "normal" } ) retval [ "heron-tmaster" ] = tmaster_cmd if self . metricscache_manager_mode . lower ( ) != "disabled" : retval [ "heron-metricscache" ] = self . _get_metrics_cache_cmd ( ) if self . health_manager_mode . lower ( ) != "disabled" : retval [ "heron-healthmgr" ] = self . _get_healthmgr_cmd ( ) retval [ self . metricsmgr_ids [ 0 ] ] = self . _get_metricsmgr_cmd ( self . metricsmgr_ids [ 0 ] , self . metrics_sinks_config_file , self . metrics_manager_port ) if self . is_stateful_topology : retval . update ( self . _get_ckptmgr_process ( ) ) return retval


def _get_streaming_processes(self):
    '''
    Returns the processes to handle streams, including the stream-mgr and the user code containing
    the stream logic of the topology
    '''
    retval = {}
    instance_plans = self._get_instance_plans(self.packing_plan, self.shard)
    instance_info = []
    for instance_plan in instance_plans:
      global_task_id = instance_plan.task_id
      component_index = instance_plan.component_index
      component_name = instance_plan.component_name
      instance_id = "container_%s_%s_%d" % (str(self.shard), component_name, global_task_id)
      instance_info.append((instance_id, component_name, global_task_id, component_index))

    stmgr_cmd_lst = [
        self.stmgr_binary,
        '--topology_name=%s' % self.topology_name,
        '--topology_id=%s' % self.topology_id,
        '--topologydefn_file=%s' % self.topology_defn_file,
        '--zkhostportlist=%s' % self.state_manager_connection,
        '--zkroot=%s' % self.state_manager_root,
        '--stmgr_id=%s' % self.stmgr_ids[self.shard],
        '--instance_ids=%s' % ','.join(map(lambda x: x[0], instance_info)),
        '--myhost=%s' % self.master_host,
        '--data_port=%s' % str(self.master_port),
        '--local_data_port=%s' % str(self.tmaster_controller_port),
        '--metricsmgr_port=%s' % str(self.metrics_manager_port),
        '--shell_port=%s' % str(self.shell_port),
        '--config_file=%s' % self.heron_internals_config_file,
        '--override_config_file=%s' % self.override_config_file,
        '--ckptmgr_port=%s' % str(self.checkpoint_manager_port),
        '--ckptmgr_id=%s' % self.ckptmgr_ids[self.shard],
        '--metricscachemgr_mode=%s' % self.metricscache_manager_mode.lower()]

    stmgr_env = self.shell_env.copy() if self.shell_env is not None else {}
    stmgr_cmd = Command(stmgr_cmd_lst, stmgr_env)
    if os.environ.get('ENABLE_HEAPCHECK') is not None:
      stmgr_cmd.env.update({
          'LD_PRELOAD': "/usr/lib/libtcmalloc.so",
          'HEAPCHECK': "normal"
      })

    retval[self.stmgr_ids[self.shard]] = stmgr_cmd

    # metricsmgr_metrics_sink_config_file = 'metrics_sinks.yaml'

    retval[self.metricsmgr_ids[self.shard]] = self._get_metricsmgr_cmd(
        self.metricsmgr_ids[self.shard],
        self.metrics_sinks_config_file,
        self.metrics_manager_port
    )

    if self.is_stateful_topology:
      retval.update(self._get_ckptmgr_process())

    if self.pkg_type == 'jar' or self.pkg_type == 'tar':
      retval.update(self._get_java_instance_cmd(instance_info))
    elif self.pkg_type == 'pex':
      retval.update(self._get_python_instance_cmd(instance_info))
    elif self.pkg_type == 'so':
      retval.update(self._get_cpp_instance_cmd(instance_info))
    elif self.pkg_type == 'dylib':
      retval.update(self._get_cpp_instance_cmd(instance_info))
    else:
      raise ValueError("Unrecognized package type: %s" % self.pkg_type)

    return retval

def
_get_streaming_processes
(
self
)
:
retval
=
{
}
instance_plans
=
self
.
_get_instance_plans
(
self
.
packing_plan
,
self
.
shard
)
instance_info
=
[
]
for
instance_plan
in
instance_plans
:
global_task_id
=
instance_plan
.
task_id
component_index
=
instance_plan
.
component_index
component_name
=
instance_plan
.
component_name
instance_id
=
"container_%s_%s_%d"
%
(
str
(
self
.
shard
)
,
component_name
,
global_task_id
)
instance_info
.
append
(
(
instance_id
,
component_name
,
global_task_id
,
component_index
)
)
stmgr_cmd_lst
=
[
self
.
stmgr_binary
,
'--topology_name=%s'
%
self
.
topology_name
,
'--topology_id=%s'
%
self
.
topology_id
,
'--topologydefn_file=%s'
%
self
.
topology_defn_file
,
'--zkhostportlist=%s'
%
self
.
state_manager_connection
,
'--zkroot=%s'
%
self
.
state_manager_root
,
'--stmgr_id=%s'
%
self
.
stmgr_ids
[
self
.
shard
]
,
'--instance_ids=%s'
%
','
.
join
(
map
(
lambda
x
:
x
[
0
]
,
instance_info
)
)
,
'--myhost=%s'
%
self
.
master_host
,
'--data_port=%s'
%
str
(
self
.
master_port
)
,
'--local_data_port=%s'
%
str
(
self
.
tmaster_controller_port
)
,
'--metricsmgr_port=%s'
%
str
(
self
.
metrics_manager_port
)
,
'--shell_port=%s'
%
str
(
self
.
shell_port
)
,
'--config_file=%s'
%
self
.
heron_internals_config_file
,
'--override_config_file=%s'
%
self
.
override_config_file
,
'--ckptmgr_port=%s'
%
str
(
self
.
checkpoint_manager_port
)
,
'--ckptmgr_id=%s'
%
self
.
ckptmgr_ids
[
self
.
shard
]
,
'--metricscachemgr_mode=%s'
%
self
.
metricscache_manager_mode
.
lower
(
)
]
stmgr_env
=
self
.
shell_env
.
copy
(
)
if
self
.
shell_env
is
not
else
{
}
stmgr_cmd
=
Command
(
stmgr_cmd_lst
,
stmgr_env
)
if
os
.
environ
.
get
(
'ENABLE_HEAPCHECK'
)
is
not
:
stmgr_cmd
.
env
.
update
(
{
'LD_PRELOAD'
:
"/usr/lib/libtcmalloc.so"
,
'HEAPCHECK'
:
"normal"
}
)
retval
[
self
.
stmgr_ids
[
self
.
shard
]
]
=
stmgr_cmd
retval
[
self
.
metricsmgr_ids
[
self
.
shard
]
]
=
self
.
_get_metricsmgr_cmd
(
self
.
metricsmgr_ids
[
self
.
shard
]
,
self
.
metrics_sinks_config_file
,
self
.
metrics_manager_port
)
if
self
.
is_stateful_topology
:
retval
.
update
(
self
.
_get_ckptmgr_process
(
)
)
if
self
.
pkg_type
==
'jar'
or
self
.
pkg_type
==
'tar'
:
retval
.
update
(
self
.
_get_java_instance_cmd
(
instance_info
)
)
elif
self
.
pkg_type
==
'pex'
:
retval
.
update
(
self
.
_get_python_instance_cmd
(
instance_info
)
)
elif
self
.
pkg_type
==
'so'
:
retval
.
update
(
self
.
_get_cpp_instance_cmd
(
instance_info
)
)
elif
self
.
pkg_type
==
'dylib'
:
retval
.
update
(
self
.
_get_cpp_instance_cmd
(
instance_info
)
)
else
:
raise
ValueError
(
"Unrecognized package type: %s"
%
self
.
pkg_type
)
return
retval

Returns the processes to handle streams including the stream - mgr and the user code containing the stream logic of the topology


def _get_ckptmgr_process(self):
    ''' Get the command to start the checkpoint manager process'''

    ckptmgr_main_class = 'org.apache.heron.ckptmgr.CheckpointManager'

    ckptmgr_ram_mb = self.checkpoint_manager_ram / (1024 * 1024)
    ckptmgr_cmd = [os.path.join(self.heron_java_home, "bin/java"),
                   '-Xms%dM' % ckptmgr_ram_mb,
                   '-Xmx%dM' % ckptmgr_ram_mb,
                   '-XX:+PrintCommandLineFlags',
                   '-verbosegc',
                   '-XX:+PrintGCDetails',
                   '-XX:+PrintGCTimeStamps',
                   '-XX:+PrintGCDateStamps',
                   '-XX:+PrintGCCause',
                   '-XX:+UseGCLogFileRotation',
                   '-XX:NumberOfGCLogFiles=5',
                   '-XX:GCLogFileSize=100M',
                   '-XX:+PrintPromotionFailure',
                   '-XX:+PrintTenuringDistribution',
                   '-XX:+PrintHeapAtGC',
                   '-XX:+HeapDumpOnOutOfMemoryError',
                   '-XX:+UseConcMarkSweepGC',
                   '-XX:+UseConcMarkSweepGC',
                   '-Xloggc:log-files/gc.ckptmgr.log',
                   '-Djava.net.preferIPv4Stack=true',
                   '-cp',
                   self.checkpoint_manager_classpath,
                   ckptmgr_main_class,
                   '-t' + self.topology_name,
                   '-i' + self.topology_id,
                   '-c' + self.ckptmgr_ids[self.shard],
                   '-p' + self.checkpoint_manager_port,
                   '-f' + self.stateful_config_file,
                   '-o' + self.override_config_file,
                   '-g' + self.heron_internals_config_file]
    retval = {}
    retval[self.ckptmgr_ids[self.shard]] = Command(ckptmgr_cmd, self.shell_env)

    return retval


def _get_instance_plans(self, packing_plan, container_id):
    """
    For the given packing_plan, return the container plan with the given container_id. If protobufs
    supported maps, we could just get the plan by id, but it doesn't so we have a collection of
    containers to iterate over.
    """
    this_container_plan = None
    for container_plan in packing_plan.container_plans:
      if container_plan.id == container_id:
        this_container_plan = container_plan

    # When the executor runs in newly added container by `heron update`,
    # there is no plan for this container. In this situation,
    # return None to bypass instance processes.
    if this_container_plan is None:
      return None
    return this_container_plan.instance_plans

For the given packing_plan return the container plan with the given container_id . If protobufs supported maps we could just get the plan by id but it doesn t so we have a collection of containers to iterate over .


def _get_heron_support_processes(self):
    """ Get a map from all daemon services' name to the command to start them """
    retval = {}

    retval[self.heron_shell_ids[self.shard]] = Command([
        '%s' % self.heron_shell_binary,
        '--port=%s' % self.shell_port,
        '--log_file_prefix=%s/heron-shell-%s.log' % (self.log_dir, self.shard),
        '--secret=%s' % self.topology_id], self.shell_env)

    return retval

Get a map from all daemon services name to the command to start them


def _wait_process_std_out_err(self, name, process):
    ''' Wait for the termination of a process and log its stdout & stderr '''
    proc.stream_process_stdout(process, stdout_log_fn(name))
    process.wait()

def _wait_process_std_out_err ( self , name , process ) : proc . stream_process_stdout ( process , stdout_log_fn ( name ) ) process . wait ( )


def _start_processes(self, commands):
    """Start all commands and add them to the dict of processes to be monitored """
    Log.info("Start processes")
    processes_to_monitor = {}
    # First start all the processes
    for (name, command) in commands.items():
      p = self._run_process(name, command)
      processes_to_monitor[p.pid] = ProcessInfo(p, name, command)

      # Log down the pid file
      log_pid_for_process(name, p.pid)

    with self.process_lock:
      self.processes_to_monitor.update(processes_to_monitor)

def
_start_processes
(
self
,
commands
)
:
Log
.
info
(
"Start processes"
)
processes_to_monitor
=
{
}
for
(
name
,
command
)
in
commands
.
items
(
)
:
p
=
self
.
_run_process
(
name
,
command
)
processes_to_monitor
[
p
.
pid
]
=
ProcessInfo
(
p
,
name
,
command
)
log_pid_for_process
(
name
,
p
.
pid
)
with
self
.
process_lock
:
self
.
processes_to_monitor
.
update
(
processes_to_monitor
)


def start_process_monitor(self):
    """ Monitor all processes in processes_to_monitor dict,
    restarting any if they fail, up to max_runs times.
    """
    # Now wait for any child to die
    Log.info("Start process monitor")
    while True:
      if len(self.processes_to_monitor) > 0:
        (pid, status) = os.wait()

        with self.process_lock:
          if pid in self.processes_to_monitor.keys():
            old_process_info = self.processes_to_monitor[pid]
            name = old_process_info.name
            command = old_process_info.command
            Log.info("%s (pid=%s) exited with status %d. command=%s" % (name, pid, status, command))
            # Log the stdout & stderr of the failed process
            self._wait_process_std_out_err(name, old_process_info.process)

            # Just make it world readable
            if os.path.isfile("core.%d" % pid):
              os.system("chmod a+r core.%d" % pid)
            if old_process_info.attempts >= self.max_runs:
              Log.info("%s exited too many times" % name)
              sys.exit(1)
            time.sleep(self.interval_between_runs)
            p = self._run_process(name, command)
            del self.processes_to_monitor[pid]
            self.processes_to_monitor[p.pid] =\
              ProcessInfo(p, name, command, old_process_info.attempts + 1)

            # Log down the pid file
            log_pid_for_process(name, p.pid)

Monitor all processes in processes_to_monitor dict restarting any if they fail up to max_runs times .


def get_commands_to_run(self):
    """
    Prepare either TMaster or Streaming commands according to shard.
    The Shell command is attached to all containers. The empty container plan and non-exist
    container plan are bypassed.
    """
    # During shutdown the watch might get triggered with the empty packing plan
    if len(self.packing_plan.container_plans) == 0:
      return {}
    if self._get_instance_plans(self.packing_plan, self.shard) is None and self.shard != 0:
      retval = {}
      retval['heron-shell'] = Command([
          '%s' % self.heron_shell_binary,
          '--port=%s' % self.shell_port,
          '--log_file_prefix=%s/heron-shell-%s.log' % (self.log_dir, self.shard),
          '--secret=%s' % self.topology_id], self.shell_env)
      return retval

    if self.shard == 0:
      commands = self._get_tmaster_processes()
    else:
      self._untar_if_needed()
      commands = self._get_streaming_processes()

    # Attach daemon processes
    commands.update(self._get_heron_support_processes())
    return commands

Prepare either TMaster or Streaming commands according to shard . The Shell command is attached to all containers . The empty container plan and non - exist container plan are bypassed .


def launch(self):
    ''' Determines the commands to be run and compares them with the existing running commands.
    Then starts new ones required and kills old ones no longer required.
    '''
    with self.process_lock:
      current_commands = dict(map((lambda process: (process.name, process.command)),
                                  self.processes_to_monitor.values()))
      updated_commands = self.get_commands_to_run()

      # get the commands to kill, keep and start
      commands_to_kill, commands_to_keep, commands_to_start = \
          self.get_command_changes(current_commands, updated_commands)

      Log.info("current commands: %s" % sorted(current_commands.keys()))
      Log.info("new commands    : %s" % sorted(updated_commands.keys()))
      Log.info("commands_to_kill: %s" % sorted(commands_to_kill.keys()))
      Log.info("commands_to_keep: %s" % sorted(commands_to_keep.keys()))
      Log.info("commands_to_start: %s" % sorted(commands_to_start.keys()))

      self._kill_processes(commands_to_kill)
      self._start_processes(commands_to_start)
      Log.info("Launch complete - processes killed=%s kept=%s started=%s monitored=%s" %
               (len(commands_to_kill), len(commands_to_keep),
                len(commands_to_start), len(self.processes_to_monitor)))

Determines the commands to be run and compares them with the existing running commands . Then starts new ones required and kills old ones no longer required .


def start_state_manager_watches(self):
    """
    Receive updates to the packing plan from the statemgrs and update processes as needed.
    """
    Log.info("Start state manager watches")
    statemgr_config = StateMgrConfig()
    statemgr_config.set_state_locations(configloader.load_state_manager_locations(
        self.cluster, state_manager_config_file=self.state_manager_config_file,
        overrides={"heron.statemgr.connection.string": self.state_manager_connection}))
    try:
      self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config)
      for state_manager in self.state_managers:
        state_manager.start()
    except Exception as ex:
      Log.error("Found exception while initializing state managers: %s. Bailing out..." % ex)
      traceback.print_exc()
      sys.exit(1)

    # pylint: disable=unused-argument
    def on_packing_plan_watch(state_manager, new_packing_plan):
      Log.debug("State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s" %
                (self.shard, str(self.packing_plan), str(new_packing_plan)))

      if self.packing_plan != new_packing_plan:
        Log.info("PackingPlan change detected on shard %s, relaunching effected processes."
                 % self.shard)
        self.update_packing_plan(new_packing_plan)

        Log.info("Updating executor processes")
        self.launch()
      else:
        Log.info(
            "State watch triggered for PackingPlan update but plan not changed so not relaunching.")

    for state_manager in self.state_managers:
      # The callback function with the bound
      # state_manager as first variable.
      onPackingPlanWatch = functools.partial(on_packing_plan_watch, state_manager)
      state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch)
      Log.info("Registered state watch for packing plan changes with state manager %s." %
               str(state_manager))

Receive updates to the packing plan from the statemgrs and update processes as needed .


def run(self, name, config, builder):
    """Builds the topology and submits it"""
    if not isinstance(name, str):
      raise RuntimeError("Name has to be a string type")
    if not isinstance(config, Config):
      raise RuntimeError("config has to be a Config type")
    if not isinstance(builder, Builder):
      raise RuntimeError("builder has to be a Builder type")
    bldr = TopologyBuilder(name=name)
    builder.build(bldr)
    bldr.set_config(config._api_config)
    bldr.build_and_submit()

def
run
(
self
,
name
,
config
,
builder
)
:
if
not
isinstance
(
name
,
str
)
:
raise
RuntimeError
(
"Name has to be a string type"
)
if
not
isinstance
(
config
,
Config
)
:
raise
RuntimeError
(
"config has to be a Config type"
)
if
not
isinstance
(
builder
,
Builder
)
:
raise
RuntimeError
(
"builder has to be a Builder type"
)
bldr
=
TopologyBuilder
(
name
=
name
)
builder
.
build
(
bldr
)
bldr
.
set_config
(
config
.
_api_config
)
bldr
.
build_and_submit
(
)


def _modules_to_main(modList):
  """Force every module in modList to be placed into main"""
  if not modList:
    return

  main = sys.modules['__main__']
  for modname in modList:
    if isinstance(modname, str):
      try:
        mod = __import__(modname)
      except Exception:
        sys.stderr.write(
            'warning: could not import %s\n.  '
            'Your function may unexpectedly error due to this import failing;'
            'A version mismatch is likely.  Specific error was:\n' % modname)
        print_exec(sys.stderr)
      else:
        setattr(main, mod.__name__, mod)