def _load_class(cls, d):
  """
  Loads additional properties into class `cls`.
  """
  for k, v in d.items():
    if isinstance(k, tuple):
      typ, k = k
      if typ == 'property':
        v = property(*v)
      elif typ == 'staticmethod':
        v = staticmethod(v) # pylint: disable=redefined-variable-type
      elif typ == 'classmethod':
        v = classmethod(v)
    setattr(cls, k, v)
  return cls

def _load_class ( cls , d ) : for k , v in d . items ( ) : if isinstance ( k , tuple ) : typ , k = k if typ == 'property' : v = property ( * v ) elif typ == 'staticmethod' : v = staticmethod ( v ) elif typ == 'classmethod' : v = classmethod ( v ) setattr ( cls , k , v ) return cls

Loads additional properties into class cls .


def save_module(self, obj):
    """
    Save a module as an import
    """
    self.modules.add(obj)
    self.save_reduce(subimport, (obj.__name__,), obj=obj)

def save_module ( self , obj ) : self . modules . add ( obj ) self . save_reduce ( subimport , ( obj . __name__ , ) , obj = obj )


def save_file(self, obj): # pylint: disable=too-many-branches
    """Save a file"""
    try:
      import StringIO as pystringIO #we can't use cStringIO as it lacks the name attribute
    except ImportError:
      import io as pystringIO # pylint: disable=reimported

    if not hasattr(obj, 'name') or  not hasattr(obj, 'mode'):
      raise pickle.PicklingError("Cannot pickle files that do not map to an actual file")
    if obj is sys.stdout:
      return self.save_reduce(getattr, (sys, 'stdout'), obj=obj)
    if obj is sys.stderr:
      return self.save_reduce(getattr, (sys, 'stderr'), obj=obj)
    if obj is sys.stdin:
      raise pickle.PicklingError("Cannot pickle standard input")
    if  hasattr(obj, 'isatty') and obj.isatty():
      raise pickle.PicklingError("Cannot pickle files that map to tty objects")
    if 'r' not in obj.mode:
      raise pickle.PicklingError("Cannot pickle files that are not opened for reading")
    name = obj.name
    try:
      fsize = os.stat(name).st_size
    except OSError:
      raise pickle.PicklingError("Cannot pickle file %s as it cannot be stat" % name)

    if obj.closed:
      #create an empty closed string io
      retval = pystringIO.StringIO("")
      retval.close()
    elif not fsize: #empty file
      retval = pystringIO.StringIO("")
      try:
        tmpfile = file(name)
        tst = tmpfile.read(1)
      except IOError:
        raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
      tmpfile.close()
      if tst != '':
        raise pickle.PicklingError(
            "Cannot pickle file %s as it does not appear to map to a physical, real file" % name)
    else:
      try:
        tmpfile = file(name)
        contents = tmpfile.read()
        tmpfile.close()
      except IOError:
        raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
      retval = pystringIO.StringIO(contents)
      curloc = obj.tell()
      retval.seek(curloc)

    retval.name = name
    self.save(retval)
    self.memoize(obj)


def tail(filename, n):
  """Returns last n lines from the filename. No exception handling"""
  size = os.path.getsize(filename)
  with open(filename, "rb") as f:
   fm = mmap.mmap(f.fileno(), 0, mmap.MAP_SHARED, mmap.PROT_READ)
   try:
      for i in xrange(size - 1, -1, -1):
          if fm[i] == '\n':
             n -= 1
             if n == -1:
                break
      return fm[i + 1 if i else 0:].splitlines()
   finally:
        fm.close()

def tail ( filename , n ) : size = os . path . getsize ( filename ) with open ( filename , "rb" ) as f : fm = mmap . mmap ( f . fileno ( ) , 0 , mmap . MAP_SHARED , mmap . PROT_READ ) try : for i in xrange ( size - 1 , - 1 , - 1 ) : if fm [ i ] == '\n' : n -= 1 if n == - 1 : break return fm [ i + 1 if i else 0 : ] . splitlines ( ) finally : fm . close ( )

Returns last n lines from the filename . No exception handling


def get_serializer(context):
    """Returns a serializer for a given context"""
    cluster_config = context.get_cluster_config()
    serializer_clsname = cluster_config.get(constants.TOPOLOGY_SERIALIZER_CLASSNAME, None)
    if serializer_clsname is None:
      return PythonSerializer()
    else:
      try:
        topo_pex_path = context.get_topology_pex_path()
        pex_loader.load_pex(topo_pex_path)
        serializer_cls = pex_loader.import_and_get_class(topo_pex_path, serializer_clsname)
        serializer = serializer_cls()
        return serializer
      except Exception as e:
        raise RuntimeError("Error with loading custom serializer class: %s, with error message: %s"
                           % (serializer_clsname, str(e)))


def register_timer_task_in_sec(self, task, second):
    """Registers a new timer task

    :param task: function to be run at a specified second from now
    :param second: how many seconds to wait before the timer is triggered
    """
    # Python time is in float
    second_in_float = float(second)
    expiration = time.time() + second_in_float
    heappush(self.timer_tasks, (expiration, task))

def register_timer_task_in_sec ( self , task , second ) : second_in_float = float ( second ) expiration = time . time ( ) + second_in_float heappush ( self . timer_tasks , ( expiration , task ) )


def _get_next_timeout_interval(self):
    """Get the next timeout from now

    This should be used from do_wait().
    :returns (float) next_timeout, or 10.0 if there are no timer events
    """
    if len(self.timer_tasks) == 0:
      return sys.maxsize
    else:
      next_timeout_interval = self.timer_tasks[0][0] - time.time()
      return next_timeout_interval

def _get_next_timeout_interval ( self ) : if len ( self . timer_tasks ) == 0 : return sys . maxsize else : next_timeout_interval = self . timer_tasks [ 0 ] [ 0 ] - time . time ( ) return next_timeout_interval


def _trigger_timers(self):
    """Triggers expired timers"""
    current = time.time()
    while len(self.timer_tasks) > 0 and (self.timer_tasks[0][0] - current <= 0):
      task = heappop(self.timer_tasks)[1]
      task()

def _trigger_timers ( self ) : current = time . time ( ) while len ( self . timer_tasks ) > 0 and ( self . timer_tasks [ 0 ] [ 0 ] - current <= 0 ) : task = heappop ( self . timer_tasks ) [ 1 ] task ( )


def find_closing_braces(self, query):
    """Find the index of the closing braces for the opening braces
    at the start of the query string. Note that first character
    of input string must be an opening braces."""
    if query[0] != '(':
      raise Exception("Trying to find closing braces for no opening braces")
    num_open_braces = 0
    for i in range(len(query)):
      c = query[i]
      if c == '(':
        num_open_braces += 1
      elif c == ')':
        num_open_braces -= 1
      if num_open_braces == 0:
        return i
    raise Exception("No closing braces found")

Find the index of the closing braces for the opening braces at the start of the query string . Note that first character of input string must be an opening braces .


def get_sub_parts(self, query):
    """The subparts are seperated by a comma. Make sure
    that commas inside the part themselves are not considered."""
    parts = []
    num_open_braces = 0
    delimiter = ','
    last_starting_index = 0
    for i in range(len(query)):
      if query[i] == '(':
        num_open_braces += 1
      elif query[i] == ')':
        num_open_braces -= 1
      elif query[i] == delimiter and num_open_braces == 0:
        parts.append(query[last_starting_index: i].strip())
        last_starting_index = i + 1
    parts.append(query[last_starting_index:].strip())
    return parts

def get_sub_parts ( self , query ) : parts = [ ] num_open_braces = 0 delimiter = ',' last_starting_index = 0 for i in range ( len ( query ) ) : if query [ i ] == '(' : num_open_braces += 1 elif query [ i ] == ')' : num_open_braces -= 1 elif query [ i ] == delimiter and num_open_braces == 0 : parts . append ( query [ last_starting_index : i ] . strip ( ) ) last_starting_index = i + 1 parts . append ( query [ last_starting_index : ] . strip ( ) ) return parts

The subparts are seperated by a comma . Make sure that commas inside the part themselves are not considered .


def parse_query_string(self, query):
    """Returns a parse tree for the query, each of the node is a
    subclass of Operator. This is both a lexical as well as syntax analyzer step."""
    if not query:
      return None
    # Just braces do not matter
    if query[0] == '(':
      index = self.find_closing_braces(query)
      # This must be the last index, since this was an NOP starting brace
      if index != len(query) - 1:
        raise Exception("Invalid syntax")
      else:
        return self.parse_query_string(query[1:-1])
    start_index = query.find("(")
    # There must be a ( in the query
    if start_index < 0:
      # Otherwise it must be a constant
      try:
        constant = float(query)
        return constant
      except ValueError:
        raise Exception("Invalid syntax")
    token = query[:start_index]
    if token not in self.operators:
      raise Exception("Invalid token: " + token)

    # Get sub components
    rest_of_the_query = query[start_index:]
    braces_end_index = self.find_closing_braces(rest_of_the_query)
    if braces_end_index != len(rest_of_the_query) - 1:
      raise Exception("Invalid syntax")
    parts = self.get_sub_parts(rest_of_the_query[1:-1])

    # parts are simple strings in this case
    if token == "TS":
      # This will raise exception if parts are not syntactically correct
      return self.operators[token](parts)

    children = []
    for part in parts:
      children.append(self.parse_query_string(part))

    # Make a node for the current token
    node = self.operators[token](children)
    return node

def
parse_query_string
(
self
,
query
)
:
if
not
query
:
return
if
query
[
0
]
==
'('
:
index
=
self
.
find_closing_braces
(
query
)
if
index
!=
len
(
query
)
1
:
raise
Exception
(
"Invalid syntax"
)
else
:
return
self
.
parse_query_string
(
query
[
1
:
1
]
)
start_index
=
query
.
find
(
"("
)
if
start_index
<
0
:
try
:
constant
=
float
(
query
)
return
constant
except
ValueError
:
raise
Exception
(
"Invalid syntax"
)
token
=
query
[
:
start_index
]
if
token
not
in
self
.
operators
:
raise
Exception
(
"Invalid token: "
+
token
)
rest_of_the_query
=
query
[
start_index
:
]
braces_end_index
=
self
.
find_closing_braces
(
rest_of_the_query
)
if
braces_end_index
!=
len
(
rest_of_the_query
)
1
:
raise
Exception
(
"Invalid syntax"
)
parts
=
self
.
get_sub_parts
(
rest_of_the_query
[
1
:
1
]
)
if
token
==
"TS"
:
return
self
.
operators
[
token
]
(
parts
)
children
=
[
]
for
part
in
parts
:
children
.
append
(
self
.
parse_query_string
(
part
)
)
node
=
self
.
operators
[
token
]
(
children
)
return
node

Returns a parse tree for the query each of the node is a subclass of Operator . This is both a lexical as well as syntax analyzer step .


def process_incoming_tuples(self):
    """Should be called when tuple was buffered into in_stream

    This method is equivalent to ``addBoltTasks()`` but
    is designed for event-driven single-thread bolt.
    """
    # back-pressure
    if self.output_helper.is_out_queue_available():
      self._read_tuples_and_execute()
      self.output_helper.send_out_tuples()
    else:
      # update outqueue full count
      self.bolt_metrics.update_out_queue_full_count()

def process_incoming_tuples ( self ) : if self . output_helper . is_out_queue_available ( ) : self . _read_tuples_and_execute ( ) self . output_helper . send_out_tuples ( ) else : self . bolt_metrics . update_out_queue_full_count ( )


def ack(self, tup):
    """Indicate that processing of a Tuple has succeeded

    It is compatible with StreamParse API.
    """
    if not isinstance(tup, HeronTuple):
      Log.error("Only HeronTuple type is supported in ack()")
      return

    if self.acking_enabled:
      ack_tuple = tuple_pb2.AckTuple()
      ack_tuple.ackedtuple = int(tup.id)

      tuple_size_in_bytes = 0
      for rt in tup.roots:
        to_add = ack_tuple.roots.add()
        to_add.CopyFrom(rt)
        tuple_size_in_bytes += rt.ByteSize()
      super(BoltInstance, self).admit_control_tuple(ack_tuple, tuple_size_in_bytes, True)

    process_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS
    self.pplan_helper.context.invoke_hook_bolt_ack(tup, process_latency_ns)
    self.bolt_metrics.acked_tuple(tup.stream, tup.component, process_latency_ns)


def fail(self, tup):
    """Indicate that processing of a Tuple has failed

    It is compatible with StreamParse API.
    """
    if not isinstance(tup, HeronTuple):
      Log.error("Only HeronTuple type is supported in fail()")
      return

    if self.acking_enabled:
      fail_tuple = tuple_pb2.AckTuple()
      fail_tuple.ackedtuple = int(tup.id)

      tuple_size_in_bytes = 0
      for rt in tup.roots:
        to_add = fail_tuple.roots.add()
        to_add.CopyFrom(rt)
        tuple_size_in_bytes += rt.ByteSize()
      super(BoltInstance, self).admit_control_tuple(fail_tuple, tuple_size_in_bytes, False)

    fail_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS
    self.pplan_helper.context.invoke_hook_bolt_fail(tup, fail_latency_ns)
    self.bolt_metrics.failed_tuple(tup.stream, tup.component, fail_latency_ns)


def template_slave_hcl(cl_args, masters):
  '''
  Template slave config file
  '''
  slave_config_template = "%s/standalone/templates/slave.template.hcl" % cl_args["config_path"]
  slave_config_actual = "%s/standalone/resources/slave.hcl" % cl_args["config_path"]
  masters_in_quotes = ['"%s"' % master for master in masters]
  template_file(slave_config_template, slave_config_actual,
                {"<nomad_masters:master_port>": ", ".join(masters_in_quotes)})


def template_scheduler_yaml(cl_args, masters):
  '''
  Template scheduler.yaml
  '''
  single_master = masters[0]
  scheduler_config_actual = "%s/standalone/scheduler.yaml" % cl_args["config_path"]

  scheduler_config_template = "%s/standalone/templates/scheduler.template.yaml" \
                              % cl_args["config_path"]
  template_file(scheduler_config_template, scheduler_config_actual,
                {"<scheduler_uri>": "http://%s:4646" % single_master})


def template_uploader_yaml(cl_args, masters):
  '''
  Tempate uploader.yaml
  '''
  single_master = masters[0]
  uploader_config_template = "%s/standalone/templates/uploader.template.yaml" \
                             % cl_args["config_path"]
  uploader_config_actual = "%s/standalone/uploader.yaml" % cl_args["config_path"]

  template_file(uploader_config_template, uploader_config_actual,
                {"<http_uploader_uri>": "http://%s:9000/api/v1/file/upload" % single_master})


def template_apiserver_hcl(cl_args, masters, zookeepers):
  """
  template apiserver.hcl
  """
  single_master = masters[0]
  apiserver_config_template = "%s/standalone/templates/apiserver.template.hcl" \
                              % cl_args["config_path"]
  apiserver_config_actual = "%s/standalone/resources/apiserver.hcl" % cl_args["config_path"]

  replacements = {
      "<heron_apiserver_hostname>": '"%s"' % get_hostname(single_master, cl_args),
      "<heron_apiserver_executable>": '"%s/heron-apiserver"'
                                      % config.get_heron_bin_dir()
                                      if is_self(single_master)
                                      else '"%s/.heron/bin/heron-apiserver"'
                                      % get_remote_home(single_master, cl_args),
      "<zookeeper_host:zookeeper_port>": ",".join(
          ['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]),
      "<scheduler_uri>": "http://%s:4646" % single_master
  }

  template_file(apiserver_config_template, apiserver_config_actual, replacements)


def template_statemgr_yaml(cl_args, zookeepers):
  '''
  Template statemgr.yaml
  '''
  statemgr_config_file_template = "%s/standalone/templates/statemgr.template.yaml" \
                                  % cl_args["config_path"]
  statemgr_config_file_actual = "%s/standalone/statemgr.yaml" % cl_args["config_path"]

  template_file(statemgr_config_file_template, statemgr_config_file_actual,
                {"<zookeeper_host:zookeeper_port>": ",".join(
                    ['"%s"' % zk if ":" in zk else '"%s:2181"' % zk for zk in zookeepers])})


def template_heron_tools_hcl(cl_args, masters, zookeepers):
  '''
  template heron tools
  '''
  heron_tools_hcl_template = "%s/standalone/templates/heron_tools.template.hcl" \
                             % cl_args["config_path"]
  heron_tools_hcl_actual = "%s/standalone/resources/heron_tools.hcl" \
                             % cl_args["config_path"]

  single_master = masters[0]
  template_file(heron_tools_hcl_template, heron_tools_hcl_actual,
                {
                    "<zookeeper_host:zookeeper_port>": ",".join(
                        ['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]),
                    "<heron_tracker_executable>": '"%s/heron-tracker"' % config.get_heron_bin_dir(),
                    "<heron_tools_hostname>": '"%s"' % get_hostname(single_master, cl_args),
                    "<heron_ui_executable>": '"%s/heron-ui"' % config.get_heron_bin_dir()
                })


def print_cluster_info(cl_args):
  '''
  get cluster info for standalone cluster
  '''
  parsed_roles = read_and_parse_roles(cl_args)
  masters = list(parsed_roles[Role.MASTERS])
  slaves = list(parsed_roles[Role.SLAVES])
  zookeepers = list(parsed_roles[Role.ZOOKEEPERS])
  cluster = list(parsed_roles[Role.CLUSTER])

  # OrderedDicts are used here so that the key order can be
  # specified directly
  info = OrderedDict()
  info['numNodes'] = len(cluster)
  info['nodes'] = cluster
  roles = OrderedDict()
  roles['masters'] = masters
  roles['slaves'] = slaves
  roles['zookeepers'] = zookeepers
  urls = OrderedDict()
  urls['serviceUrl'] = get_service_url(cl_args)
  urls['heronUi'] = get_heron_ui_url(cl_args)
  urls['heronTracker'] = get_heron_tracker_url(cl_args)
  info['roles'] = roles
  info['urls'] = urls

  print json.dumps(info, indent=2)

def print_cluster_info ( cl_args ) : parsed_roles = read_and_parse_roles ( cl_args ) masters = list ( parsed_roles [ Role . MASTERS ] ) slaves = list ( parsed_roles [ Role . SLAVES ] ) zookeepers = list ( parsed_roles [ Role . ZOOKEEPERS ] ) cluster = list ( parsed_roles [ Role . CLUSTER ] ) info = OrderedDict ( ) info [ 'numNodes' ] = len ( cluster ) info [ 'nodes' ] = cluster roles = OrderedDict ( ) roles [ 'masters' ] = masters roles [ 'slaves' ] = slaves roles [ 'zookeepers' ] = zookeepers urls = OrderedDict ( ) urls [ 'serviceUrl' ] = get_service_url ( cl_args ) urls [ 'heronUi' ] = get_heron_ui_url ( cl_args ) urls [ 'heronTracker' ] = get_heron_tracker_url ( cl_args ) info [ 'roles' ] = roles info [ 'urls' ] = urls print json . dumps ( info , indent = 2 )


def add_additional_args(parsers):
  '''
  add additional parameters to parser
  '''
  for parser in parsers:
    cli_args.add_verbose(parser)
    cli_args.add_config(parser)
    parser.add_argument(
        '--heron-dir',
        default=config.get_heron_dir(),
        help='Path to Heron home directory')

def
add_additional_args
(
parsers
)
:
for
parser
in
parsers
:
cli_args
.
add_verbose
(
parser
)
cli_args
.
add_config
(
parser
)
parser
.
add_argument
(
'--heron-dir'
,
default
=
config
.
get_heron_dir
(
)
,
help
=
'Path to Heron home directory'
)


def stop_cluster(cl_args):
  '''
  teardown the cluster
  '''
  Log.info("Terminating cluster...")

  roles = read_and_parse_roles(cl_args)
  masters = roles[Role.MASTERS]
  slaves = roles[Role.SLAVES]
  dist_nodes = masters.union(slaves)

  # stop all jobs
  if masters:
    try:
      single_master = list(masters)[0]
      jobs = get_jobs(cl_args, single_master)
      for job in jobs:
        job_id = job["ID"]
        Log.info("Terminating job %s" % job_id)
        delete_job(cl_args, job_id, single_master)
    except:
      Log.debug("Error stopping jobs")
      Log.debug(sys.exc_info()[0])

  for node in dist_nodes:
    Log.info("Terminating processes on %s" % node)
    if not is_self(node):
      cmd = "ps aux | grep heron-nomad | awk '{print \$2}' " \
            "| xargs kill"
      cmd = ssh_remote_execute(cmd, node, cl_args)
    else:
      cmd = "ps aux | grep heron-nomad | awk '{print $2}' " \
            "| xargs kill"
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)

    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))

    Log.info("Cleaning up directories on %s" % node)
    cmd = "rm -rf /tmp/slave ; rm -rf /tmp/master"
    if not is_self(node):
      cmd = ssh_remote_execute(cmd, node, cl_args)
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)

    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))


def start_cluster(cl_args):
  '''
  Start a Heron standalone cluster
  '''
  roles = read_and_parse_roles(cl_args)
  masters = roles[Role.MASTERS]
  slaves = roles[Role.SLAVES]
  zookeepers = roles[Role.ZOOKEEPERS]
  Log.info("Roles:")
  Log.info(" - Master Servers: %s" % list(masters))
  Log.info(" - Slave Servers: %s" % list(slaves))
  Log.info(" - Zookeeper Servers: %s" % list(zookeepers))
  if not masters:
    Log.error("No master servers specified!")
    sys.exit(-1)
  if not slaves:
    Log.error("No slave servers specified!")
    sys.exit(-1)
  if not zookeepers:
    Log.error("No zookeeper servers specified!")
    sys.exit(-1)
  # make sure configs are templated
  update_config_files(cl_args)

  dist_nodes = list(masters.union(slaves))
  # if just local deployment
  if not (len(dist_nodes) == 1 and is_self(dist_nodes[0])):
    distribute_package(roles, cl_args)
  start_master_nodes(masters, cl_args)
  start_slave_nodes(slaves, cl_args)
  start_api_server(masters, cl_args)
  start_heron_tools(masters, cl_args)
  Log.info("Heron standalone cluster complete!")

def
start_cluster
(
cl_args
)
:
roles
=
read_and_parse_roles
(
cl_args
)
masters
=
roles
[
Role
.
MASTERS
]
slaves
=
roles
[
Role
.
SLAVES
]
zookeepers
=
roles
[
Role
.
ZOOKEEPERS
]
Log
.
info
(
"Roles:"
)
Log
.
info
(
" - Master Servers: %s"
%
list
(
masters
)
)
Log
.
info
(
" - Slave Servers: %s"
%
list
(
slaves
)
)
Log
.
info
(
" - Zookeeper Servers: %s"
%
list
(
zookeepers
)
)
if
not
masters
:
Log
.
error
(
"No master servers specified!"
)
sys
.
exit
(
1
)
if
not
slaves
:
Log
.
error
(
"No slave servers specified!"
)
sys
.
exit
(
1
)
if
not
zookeepers
:
Log
.
error
(
"No zookeeper servers specified!"
)
sys
.
exit
(
1
)
update_config_files
(
cl_args
)
dist_nodes
=
list
(
masters
.
union
(
slaves
)
)
if
not
(
len
(
dist_nodes
)
==
1
and
is_self
(
dist_nodes
[
0
]
)
)
:
distribute_package
(
roles
,
cl_args
)
start_master_nodes
(
masters
,
cl_args
)
start_slave_nodes
(
slaves
,
cl_args
)
start_api_server
(
masters
,
cl_args
)
start_heron_tools
(
masters
,
cl_args
)
Log
.
info
(
"Heron standalone cluster complete!"
)


def start_heron_tools(masters, cl_args):
  '''
  Start Heron tracker and UI
  '''
  single_master = list(masters)[0]
  wait_for_master_to_start(single_master)

  cmd = "%s run %s >> /tmp/heron_tools_start.log 2>&1 &" \
        % (get_nomad_path(cl_args), get_heron_tools_job_file(cl_args))
  Log.info("Starting Heron Tools on %s" % single_master)

  if not is_self(single_master):
    cmd = ssh_remote_execute(cmd, single_master, cl_args)
  Log.debug(cmd)
  pid = subprocess.Popen(cmd,
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)

  return_code = pid.wait()
  output = pid.communicate()
  Log.debug("return code: %s output: %s" % (return_code, output))
  if return_code != 0:
    Log.error("Failed to start Heron Tools on %s with error:\n%s" % (single_master, output[1]))
    sys.exit(-1)

  wait_for_job_to_start(single_master, "heron-tools")
  Log.info("Done starting Heron Tools")


def distribute_package(roles, cl_args):
  '''
  distribute Heron packages to all nodes
  '''
  Log.info("Distributing heron package to nodes (this might take a while)...")
  masters = roles[Role.MASTERS]
  slaves = roles[Role.SLAVES]

  tar_file = tempfile.NamedTemporaryFile(suffix=".tmp").name
  Log.debug("TAR file %s to %s" % (cl_args["heron_dir"], tar_file))
  make_tarfile(tar_file, cl_args["heron_dir"])
  dist_nodes = masters.union(slaves)

  scp_package(tar_file, dist_nodes, cl_args)


def wait_for_master_to_start(single_master):
  '''
  Wait for a nomad master to start
  '''
  i = 0
  while True:
    try:
      r = requests.get("http://%s:4646/v1/status/leader" % single_master)
      if r.status_code == 200:
        break
    except:
      Log.debug(sys.exc_info()[0])
      Log.info("Waiting for cluster to come up... %s" % i)
      time.sleep(1)
      if i > 10:
        Log.error("Failed to start Nomad Cluster!")
        sys.exit(-1)
    i = i + 1

def
wait_for_master_to_start
(
single_master
)
:
i
=
0
while
True
:
try
:
r
=
requests
.
get
(
"http://%s:4646/v1/status/leader"
%
single_master
)
if
r
.
status_code
==
200
:
break
except
:
Log
.
debug
(
sys
.
exc_info
(
)
[
0
]
)
Log
.
info
(
"Waiting for cluster to come up... %s"
%
i
)
time
.
sleep
(
1
)
if
i
>
10
:
Log
.
error
(
"Failed to start Nomad Cluster!"
)
sys
.
exit
(
1
)
i
=
i
+
1


def wait_for_job_to_start(single_master, job):
  '''
  Wait for a Nomad job to start
  '''
  i = 0
  while True:
    try:
      r = requests.get("http://%s:4646/v1/job/%s" % (single_master, job))
      if r.status_code == 200 and r.json()["Status"] == "running":
        break
      else:
        raise RuntimeError()
    except:
      Log.debug(sys.exc_info()[0])
      Log.info("Waiting for %s to come up... %s" % (job, i))
      time.sleep(1)
      if i > 20:
        Log.error("Failed to start Nomad Cluster!")
        sys.exit(-1)
    i = i + 1

def
wait_for_job_to_start
(
single_master
,
job
)
:
i
=
0
while
True
:
try
:
r
=
requests
.
get
(
"http://%s:4646/v1/job/%s"
%
(
single_master
,
job
)
)
if
r
.
status_code
==
200
and
r
.
json
(
)
[
"Status"
]
==
"running"
:
break
else
:
raise
RuntimeError
(
)
except
:
Log
.
debug
(
sys
.
exc_info
(
)
[
0
]
)
Log
.
info
(
"Waiting for %s to come up... %s"
%
(
job
,
i
)
)
time
.
sleep
(
1
)
if
i
>
20
:
Log
.
error
(
"Failed to start Nomad Cluster!"
)
sys
.
exit
(
1
)
i
=
i
+
1


def scp_package(package_file, destinations, cl_args):
  '''
  scp and extract package
  '''
  pids = []
  for dest in destinations:
    if is_self(dest):
      continue
    Log.info("Server: %s" % dest)
    file_path = "/tmp/heron.tar.gz"
    dest_file_path = "%s:%s" % (dest, file_path)

    remote_cmd = "rm -rf ~/.heron && mkdir ~/.heron " \
                 "&& tar -xzvf %s -C ~/.heron --strip-components 1" % (file_path)
    cmd = '%s && %s' \
          % (scp_cmd(package_file, dest_file_path, cl_args),
             ssh_remote_execute(remote_cmd, dest, cl_args))
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
    pids.append({"pid": pid, "dest": dest})

  errors = []
  for entry in pids:
    pid = entry["pid"]
    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))
    if return_code != 0:
      errors.append("Failed to scp package to %s with error:\n%s" % (entry["dest"], output[1]))

  if errors:
    for error in errors:
      Log.error(error)
    sys.exit(-1)

  Log.info("Done distributing packages")


def make_tarfile(output_filename, source_dir):
  '''
  Tar a directory
  '''
  with tarfile.open(output_filename, "w:gz") as tar:
    tar.add(source_dir, arcname=os.path.basename(source_dir))

def make_tarfile ( output_filename , source_dir ) : with tarfile . open ( output_filename , "w:gz" ) as tar : tar . add ( source_dir , arcname = os . path . basename ( source_dir ) )


def start_master_nodes(masters, cl_args):
  '''
  Start master nodes
  '''
  pids = []
  for master in masters:
    Log.info("Starting master on %s" % master)
    cmd = "%s agent -config %s >> /tmp/nomad_server_log 2>&1 &" \
          % (get_nomad_path(cl_args), get_nomad_master_config_file(cl_args))
    if not is_self(master):
      cmd = ssh_remote_execute(cmd, master, cl_args)
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
    pids.append({"pid": pid, "dest": master})

  errors = []
  for entry in pids:
    pid = entry["pid"]
    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))
    if return_code != 0:
      errors.append("Failed to start master on %s with error:\n%s" % (entry["dest"], output[1]))

  if errors:
    for error in errors:
      Log.error(error)
    sys.exit(-1)

  Log.info("Done starting masters")


def start_slave_nodes(slaves, cl_args):
  '''
  Star slave nodes
  '''
  pids = []
  for slave in slaves:
    Log.info("Starting slave on %s" % slave)
    cmd = "%s agent -config %s >> /tmp/nomad_client.log 2>&1 &" \
          % (get_nomad_path(cl_args), get_nomad_slave_config_file(cl_args))
    if not is_self(slave):
      cmd = ssh_remote_execute(cmd, slave, cl_args)
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
    pids.append({"pid": pid, "dest": slave})

  errors = []
  for entry in pids:
    pid = entry["pid"]
    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))
    if return_code != 0:
      errors.append("Failed to start slave on %s with error:\n%s" % (entry["dest"], output[1]))

  if errors:
    for error in errors:
      Log.error(error)
    sys.exit(-1)

  Log.info("Done starting slaves")


def read_and_parse_roles(cl_args):
  '''
  read config files to get roles
  '''
  roles = dict()

  with open(get_inventory_file(cl_args), 'r') as stream:
    try:
      roles = yaml.load(stream)
    except yaml.YAMLError as exc:
      Log.error("Error parsing inventory file: %s" % exc)
      sys.exit(-1)

  if Role.ZOOKEEPERS not in roles or not roles[Role.ZOOKEEPERS]:
    Log.error("Zookeeper servers node defined!")
    sys.exit(-1)

  if Role.CLUSTER not in roles or not roles[Role.CLUSTER]:
    Log.error("Heron cluster nodes defined!")
    sys.exit(-1)

  # Set roles
  roles[Role.MASTERS] = set([roles[Role.CLUSTER][0]])
  roles[Role.SLAVES] = set(roles[Role.CLUSTER])
  roles[Role.ZOOKEEPERS] = set(roles[Role.ZOOKEEPERS])
  roles[Role.CLUSTER] = set(roles[Role.CLUSTER])

  return roles

def
read_and_parse_roles
(
cl_args
)
:
roles
=
dict
(
)
with
open
(
get_inventory_file
(
cl_args
)
,
'r'
)
as
stream
:
try
:
roles
=
yaml
.
load
(
stream
)
except
yaml
.
YAMLError
as
exc
:
Log
.
error
(
"Error parsing inventory file: %s"
%
exc
)
sys
.
exit
(
1
)
if
Role
.
ZOOKEEPERS
not
in
roles
or
not
roles
[
Role
.
ZOOKEEPERS
]
:
Log
.
error
(
"Zookeeper servers node defined!"
)
sys
.
exit
(
1
)
if
Role
.
CLUSTER
not
in
roles
or
not
roles
[
Role
.
CLUSTER
]
:
Log
.
error
(
"Heron cluster nodes defined!"
)
sys
.
exit
(
1
)
roles
[
Role
.
MASTERS
]
=
set
(
[
roles
[
Role
.
CLUSTER
]
[
0
]
]
)
roles
[
Role
.
SLAVES
]
=
set
(
roles
[
Role
.
CLUSTER
]
)
roles
[
Role
.
ZOOKEEPERS
]
=
set
(
roles
[
Role
.
ZOOKEEPERS
]
)
roles
[
Role
.
CLUSTER
]
=
set
(
roles
[
Role
.
CLUSTER
]
)
return
roles


def get_remote_home(host, cl_args):
  '''
  get home directory of remote host
  '''
  cmd = "echo ~"
  if not is_self(host):
    cmd = ssh_remote_execute(cmd, host, cl_args)
  pid = subprocess.Popen(cmd,
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
  return_code = pid.wait()
  output = pid.communicate()

  if return_code != 0:
    Log.error("Failed to get home path for remote host %s with output:\n%s" % (host, output))
    sys.exit(-1)
  return output[0].strip("\n")


def get_hostname(ip_addr, cl_args):
  '''
  get host name of remote host
  '''
  if is_self(ip_addr):
    return get_self_hostname()
  cmd = "hostname"
  ssh_cmd = ssh_remote_execute(cmd, ip_addr, cl_args)
  pid = subprocess.Popen(ssh_cmd,
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
  return_code = pid.wait()
  output = pid.communicate()

  if return_code != 0:
    Log.error("Failed to get hostname for remote host %s with output:\n%s" % (ip_addr, output))
    sys.exit(-1)
  return output[0].strip("\n")


def is_self(addr):
  '''
  check if this host is this addr
  '''
  ips = []
  for i in netifaces.interfaces():
    entry = netifaces.ifaddresses(i)
    if netifaces.AF_INET in entry:
      for ipv4 in entry[netifaces.AF_INET]:
        if "addr" in ipv4:
          ips.append(ipv4["addr"])
  return addr in ips or addr == get_self_hostname()

def is_self ( addr ) : ips = [ ] for i in netifaces . interfaces ( ) : entry = netifaces . ifaddresses ( i ) if netifaces . AF_INET in entry : for ipv4 in entry [ netifaces . AF_INET ] : if "addr" in ipv4 : ips . append ( ipv4 [ "addr" ] ) return addr in ips or addr == get_self_hostname ( )


def log(self, message, level=None):
    """Log message, optionally providing a logging level

    It is compatible with StreamParse API.

    :type message: str
    :param message: the log message to send
    :type level: str
    :param level: the logging level,
                  one of: trace (=debug), debug, info, warn or error (default: info)
    """
    if level is None:
      _log_level = logging.INFO
    else:
      if level == "trace" or level == "debug":
        _log_level = logging.DEBUG
      elif level == "info":
        _log_level = logging.INFO
      elif level == "warn":
        _log_level = logging.WARNING
      elif level == "error":
        _log_level = logging.ERROR
      else:
        raise ValueError("%s is not supported as logging level" % str(level))

    self.logger.log(_log_level, message)

def
log
(
self
,
message
,
level
=
)
:
if
level
is
:
_log_level
=
logging
.
INFO
else
:
if
level
==
"trace"
or
level
==
"debug"
:
_log_level
=
logging
.
DEBUG
elif
level
==
"info"
:
_log_level
=
logging
.
INFO
elif
level
==
"warn"
:
_log_level
=
logging
.
WARNING
elif
level
==
"error"
:
_log_level
=
logging
.
ERROR
else
:
raise
ValueError
(
"%s is not supported as logging level"
%
str
(
level
)
)
self
.
logger
.
log
(
_log_level
,
message
)

Log message optionally providing a logging level


def dereference_symlinks(src):
    """
    Resolve all symbolic references that `src` points to.  Note that this
    is different than `os.path.realpath` as path components leading up to
    the final location may still be symbolic links.
    """
    while os.path.islink(src):
        src = os.path.join(os.path.dirname(src), os.readlink(src))

    return src

def dereference_symlinks ( src ) : while os . path . islink ( src ) : src = os . path . join ( os . path . dirname ( src ) , os . readlink ( src ) ) return src

Resolve all symbolic references that src points to . Note that this is different than os . path . realpath as path components leading up to the final location may still be symbolic links .


def to_table(result):
  ''' normalize raw result to table '''
  max_count = 20
  table, count = [], 0
  for role, envs_topos in result.items():
    for env, topos in envs_topos.items():
      for topo in topos:
        count += 1
        if count > max_count:
          continue
        else:
          table.append([role, env, topo])
  header = ['role', 'env', 'topology']
  rest_count = 0 if count <= max_count else count - max_count
  return table, header, rest_count

def to_table ( result ) : max_count = 20 table , count = [ ] , 0 for role , envs_topos in result . items ( ) : for env , topos in envs_topos . items ( ) : for topo in topos : count += 1 if count > max_count : continue else : table . append ( [ role , env , topo ] ) header = [ 'role' , 'env' , 'topology' ] rest_count = 0 if count <= max_count else count - max_count return table , header , rest_count


def add_context(self, err_context, succ_context=None):
    """ Prepend msg to add some context information

    :param pmsg: context info
    :return: None
    """
    self.err_context = err_context
    self.succ_context = succ_context

def add_context ( self , err_context , succ_context = None ) : self . err_context = err_context self . succ_context = succ_context


def is_host_port_reachable(self):
    """
    Returns true if the host is reachable. In some cases, it may not be reachable a tunnel
    must be used.
    """
    for hostport in self.hostportlist:
      try:
        socket.create_connection(hostport, StateManager.TIMEOUT_SECONDS)
        return True
      except:
        LOG.info("StateManager %s Unable to connect to host: %s port %i"
                 % (self.name, hostport[0], hostport[1]))
        continue
    return False

Returns true if the host is reachable . In some cases it may not be reachable a tunnel must be used .


def pick_unused_port(self):
    """ Pick an unused port. There is a slight chance that this wont work. """
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('127.0.0.1', 0))
    _, port = s.getsockname()
    s.close()
    return port

def pick_unused_port ( self ) : s = socket . socket ( socket . AF_INET , socket . SOCK_STREAM ) s . bind ( ( '127.0.0.1' , 0 ) ) _ , port = s . getsockname ( ) s . close ( ) return port

Pick an unused port . There is a slight chance that this wont work .


def establish_ssh_tunnel(self):
    """
    Establish an ssh tunnel for each local host and port
    that can be used to communicate with the state host.
    """
    localportlist = []
    for (host, port) in self.hostportlist:
      localport = self.pick_unused_port()
      self.tunnel.append(subprocess.Popen(
          ('ssh', self.tunnelhost, '-NL127.0.0.1:%d:%s:%d' % (localport, host, port))))
      localportlist.append(('127.0.0.1', localport))
    return localportlist

def establish_ssh_tunnel ( self ) : localportlist = [ ] for ( host , port ) in self . hostportlist : localport = self . pick_unused_port ( ) self . tunnel . append ( subprocess . Popen ( ( 'ssh' , self . tunnelhost , '-NL127.0.0.1:%d:%s:%d' % ( localport , host , port ) ) ) ) localportlist . append ( ( '127.0.0.1' , localport ) ) return localportlist

Establish an ssh tunnel for each local host and port that can be used to communicate with the state host .


def monitor(self):
    """
    Monitor the rootpath and call the callback
    corresponding to the change.
    This monitoring happens periodically. This function
    is called in a seperate thread from the main thread,
    because it sleeps for the intervals between each poll.
    """

    def trigger_watches_based_on_files(watchers, path, directory, ProtoClass):
      """
      For all the topologies in the watchers, check if the data
      in directory has changed. Trigger the callback if it has.
      """
      for topology, callbacks in watchers.items():
        file_path = os.path.join(path, topology)
        data = ""
        if os.path.exists(file_path):
          with open(os.path.join(path, topology)) as f:
            data = f.read()
        if topology not in directory or data != directory[topology]:
          proto_object = ProtoClass()
          proto_object.ParseFromString(data)
          for callback in callbacks:
            callback(proto_object)
          directory[topology] = data

    while not self.monitoring_thread_stop_signal:
      topologies_path = self.get_topologies_path()

      topologies = []
      if os.path.isdir(topologies_path):
        topologies = list(filter(
            lambda f: os.path.isfile(os.path.join(topologies_path, f)),
            os.listdir(topologies_path)))
      if set(topologies) != set(self.topologies_directory):
        for callback in self.topologies_watchers:
          callback(topologies)
      self.topologies_directory = topologies

      trigger_watches_based_on_files(
          self.topology_watchers, topologies_path, self.topologies_directory, Topology)

      # Get the directory name for execution state
      execution_state_path = os.path.dirname(self.get_execution_state_path(""))
      trigger_watches_based_on_files(
          self.execution_state_watchers, execution_state_path,
          self.execution_state_directory, ExecutionState)

      # Get the directory name for packing_plan
      packing_plan_path = os.path.dirname(self.get_packing_plan_path(""))
      trigger_watches_based_on_files(
          self.packing_plan_watchers, packing_plan_path, self.packing_plan_directory, PackingPlan)

      # Get the directory name for pplan
      pplan_path = os.path.dirname(self.get_pplan_path(""))
      trigger_watches_based_on_files(
          self.pplan_watchers, pplan_path,
          self.pplan_directory, PhysicalPlan)

      # Get the directory name for tmaster
      tmaster_path = os.path.dirname(self.get_tmaster_path(""))
      trigger_watches_based_on_files(
          self.tmaster_watchers, tmaster_path,
          self.tmaster_directory, TMasterLocation)

      # Get the directory name for scheduler location
      scheduler_location_path = os.path.dirname(self.get_scheduler_location_path(""))
      trigger_watches_based_on_files(
          self.scheduler_location_watchers, scheduler_location_path,
          self.scheduler_location_directory, SchedulerLocation)

      # Sleep for some time
      self.event.wait(timeout=5)

def monitor ( self ) : def trigger_watches_based_on_files ( watchers , path , directory , ProtoClass ) : for topology , callbacks in watchers . items ( ) : file_path = os . path . join ( path , topology ) data = "" if os . path . exists ( file_path ) : with open ( os . path . join ( path , topology ) ) as f : data = f . read ( ) if topology not in directory or data != directory [ topology ] : proto_object = ProtoClass ( ) proto_object . ParseFromString ( data ) for callback in callbacks : callback ( proto_object ) directory [ topology ] = data while not self . monitoring_thread_stop_signal : topologies_path = self . get_topologies_path ( ) topologies = [ ] if os . path . isdir ( topologies_path ) : topologies = list ( filter ( lambda f : os . path . isfile ( os . path . join ( topologies_path , f ) ) , os . listdir ( topologies_path ) ) ) if set ( topologies ) != set ( self . topologies_directory ) : for callback in self . topologies_watchers : callback ( topologies ) self . topologies_directory = topologies trigger_watches_based_on_files ( self . topology_watchers , topologies_path , self . topologies_directory , Topology ) execution_state_path = os . path . dirname ( self . get_execution_state_path ( "" ) ) trigger_watches_based_on_files ( self . execution_state_watchers , execution_state_path , self . execution_state_directory , ExecutionState ) packing_plan_path = os . path . dirname ( self . get_packing_plan_path ( "" ) ) trigger_watches_based_on_files ( self . packing_plan_watchers , packing_plan_path , self . packing_plan_directory , PackingPlan ) pplan_path = os . path . dirname ( self . get_pplan_path ( "" ) ) trigger_watches_based_on_files ( self . pplan_watchers , pplan_path , self . pplan_directory , PhysicalPlan ) tmaster_path = os . path . dirname ( self . get_tmaster_path ( "" ) ) trigger_watches_based_on_files ( self . tmaster_watchers , tmaster_path , self . tmaster_directory , TMasterLocation ) scheduler_location_path = os . path . dirname ( self . get_scheduler_location_path ( "" ) ) trigger_watches_based_on_files ( self . scheduler_location_watchers , scheduler_location_path , self . scheduler_location_directory , SchedulerLocation ) self . event . wait ( timeout = 5 )

Monitor the rootpath and call the callback corresponding to the change . This monitoring happens periodically . This function is called in a seperate thread from the main thread because it sleeps for the intervals between each poll .


def get_pplan(self, topologyName, callback=None):
    """
    Get physical plan of a topology
    """
    if callback:
      self.pplan_watchers[topologyName].append(callback)
    else:
      pplan_path = self.get_pplan_path(topologyName)
      with open(pplan_path) as f:
        data = f.read()
        pplan = PhysicalPlan()
        pplan.ParseFromString(data)
        return pplan

def get_pplan ( self , topologyName , callback = None ) : if callback : self . pplan_watchers [ topologyName ] . append ( callback ) else : pplan_path = self . get_pplan_path ( topologyName ) with open ( pplan_path ) as f : data = f . read ( ) pplan = PhysicalPlan ( ) pplan . ParseFromString ( data ) return pplan


def get_execution_state(self, topologyName, callback=None):
    """
    Get execution state
    """
    if callback:
      self.execution_state_watchers[topologyName].append(callback)
    else:
      execution_state_path = self.get_execution_state_path(topologyName)
      with open(execution_state_path) as f:
        data = f.read()
        executionState = ExecutionState()
        executionState.ParseFromString(data)
        return executionState

def get_execution_state ( self , topologyName , callback = None ) : if callback : self . execution_state_watchers [ topologyName ] . append ( callback ) else : execution_state_path = self . get_execution_state_path ( topologyName ) with open ( execution_state_path ) as f : data = f . read ( ) executionState = ExecutionState ( ) executionState . ParseFromString ( data ) return executionState


def get_scheduler_location(self, topologyName, callback=None):
    """
    Get scheduler location
    """
    if callback:
      self.scheduler_location_watchers[topologyName].append(callback)
    else:
      scheduler_location_path = self.get_scheduler_location_path(topologyName)
      with open(scheduler_location_path) as f:
        data = f.read()
        scheduler_location = SchedulerLocation()
        scheduler_location.ParseFromString(data)
        return scheduler_location

def get_scheduler_location ( self , topologyName , callback = None ) : if callback : self . scheduler_location_watchers [ topologyName ] . append ( callback ) else : scheduler_location_path = self . get_scheduler_location_path ( topologyName ) with open ( scheduler_location_path ) as f : data = f . read ( ) scheduler_location = SchedulerLocation ( ) scheduler_location . ParseFromString ( data ) return scheduler_location


def create_socket_options():
  """Creates SocketOptions object from a given sys_config dict"""
  sys_config = system_config.get_sys_config()
  opt_list = [const.INSTANCE_NETWORK_WRITE_BATCH_SIZE_BYTES,
              const.INSTANCE_NETWORK_WRITE_BATCH_TIME_MS,
              const.INSTANCE_NETWORK_READ_BATCH_SIZE_BYTES,
              const.INSTANCE_NETWORK_READ_BATCH_TIME_MS,
              const.INSTANCE_NETWORK_OPTIONS_SOCKET_RECEIVED_BUFFER_SIZE_BYTES,
              const.INSTANCE_NETWORK_OPTIONS_SOCKET_SEND_BUFFER_SIZE_BYTES]

  Log.debug("In create_socket_options()")
  try:
    value_lst = [int(sys_config[opt]) for opt in opt_list]
    sock_opt = SocketOptions(*value_lst)
    return sock_opt
  except ValueError as e:
    # couldn't convert to int
    raise ValueError("Invalid value in sys_config: %s" % str(e))
  except KeyError as e:
    # option key was not found
    raise KeyError("Incomplete sys_config: %s" % str(e))


def init_topology(mcs, classname, class_dict):
    """Initializes a topology protobuf"""
    if classname == 'Topology':
      # Base class can't initialize protobuf
      return
    heron_options = TopologyType.get_heron_options_from_env()
    initial_state = heron_options.get("cmdline.topology.initial.state", "RUNNING")
    tmp_directory = heron_options.get("cmdline.topologydefn.tmpdirectory")
    if tmp_directory is None:
      raise RuntimeError("Topology definition temp directory not specified")

    topology_name = heron_options.get("cmdline.topology.name", classname)
    topology_id = topology_name + str(uuid.uuid4())

    # create protobuf
    topology = topology_pb2.Topology()
    topology.id = topology_id
    topology.name = topology_name
    topology.state = topology_pb2.TopologyState.Value(initial_state)
    topology.topology_config.CopyFrom(TopologyType.get_topology_config_protobuf(class_dict))

    TopologyType.add_bolts_and_spouts(topology, class_dict)

    class_dict['topology_name'] = topology_name
    class_dict['topology_id'] = topology_id
    class_dict['protobuf_topology'] = topology
    class_dict['topologydefn_tmpdir'] = tmp_directory
    class_dict['heron_runtime_options'] = heron_options


def get_heron_options_from_env():
    """Retrieves heron options from the `HERON_OPTIONS` environment variable.

    Heron options have the following format:

        cmdline.topologydefn.tmpdirectory=/var/folders/tmpdir
        cmdline.topology.initial.state=PAUSED

    In this case, the returned map will contain:

        #!json
        {
          "cmdline.topologydefn.tmpdirectory": "/var/folders/tmpdir",
          "cmdline.topology.initial.state": "PAUSED"
        }

    Currently supports the following options natively:

    - `cmdline.topologydefn.tmpdirectory`: (required) the directory to which this
    topology's defn file is written
    - `cmdline.topology.initial.state`: (default: "RUNNING") the initial state of the topology
    - `cmdline.topology.name`: (default: class name) topology name on deployment

    Returns: map mapping from key to value
    """
    heron_options_raw = os.environ.get("HERON_OPTIONS")
    if heron_options_raw is None:
      raise RuntimeError("HERON_OPTIONS environment variable not found")

    options = {}
    for option_line in heron_options_raw.replace("%%%%", " ").split(','):
      key, sep, value = option_line.partition("=")
      if sep:
        options[key] = value
      else:
        raise ValueError("Invalid HERON_OPTIONS part %r" % option_line)
    return options

Retrieves heron options from the HERON_OPTIONS environment variable .


def add_spec(self, *specs):
    """Add specs to the topology

    :type specs: HeronComponentSpec
    :param specs: specs to add to the topology
    """
    for spec in specs:
      if not isinstance(spec, HeronComponentSpec):
        raise TypeError("Argument to add_spec needs to be HeronComponentSpec, given: %s"
                        % str(spec))
      if spec.name is None:
        raise ValueError("TopologyBuilder cannot take a spec without name")
      if spec.name == "config":
        raise ValueError("config is a reserved name")
      if spec.name in self._specs:
        raise ValueError("Attempting to add duplicate spec name: %r %r" % (spec.name, spec))

      self._specs[spec.name] = spec


def add_spout(self, name, spout_cls, par, config=None, optional_outputs=None):
    """Add a spout to the topology"""
    spout_spec = spout_cls.spec(name=name, par=par, config=config,
                                optional_outputs=optional_outputs)
    self.add_spec(spout_spec)
    return spout_spec

def add_spout ( self , name , spout_cls , par , config = None , optional_outputs = None ) : spout_spec = spout_cls . spec ( name = name , par = par , config = config , optional_outputs = optional_outputs ) self . add_spec ( spout_spec ) return spout_spec


def add_bolt(self, name, bolt_cls, par, inputs, config=None, optional_outputs=None):
    """Add a bolt to the topology"""
    bolt_spec = bolt_cls.spec(name=name, par=par, inputs=inputs, config=config,
                              optional_outputs=optional_outputs)
    self.add_spec(bolt_spec)
    return bolt_spec

def add_bolt ( self , name , bolt_cls , par , inputs , config = None , optional_outputs = None ) : bolt_spec = bolt_cls . spec ( name = name , par = par , inputs = inputs , config = config , optional_outputs = optional_outputs ) self . add_spec ( bolt_spec ) return bolt_spec


def set_config(self, config):
    """Set topology-wide configuration to the topology

    :type config: dict
    :param config: topology-wide config
    """
    if not isinstance(config, dict):
      raise TypeError("Argument to set_config needs to be dict, given: %s" % str(config))
    self._topology_config = config

Set topology - wide configuration to the topology


def build_and_submit(self):
    """Builds the topology and submits to the destination"""
    class_dict = self._construct_topo_class_dict()
    topo_cls = TopologyType(self.topology_name, (Topology,), class_dict)
    topo_cls.write()

def build_and_submit ( self ) : class_dict = self . _construct_topo_class_dict ( ) topo_cls = TopologyType ( self . topology_name , ( Topology , ) , class_dict ) topo_cls . write ( )


def queries_map():
  """map from query parameter to query name"""
  qs = _all_metric_queries()
  return dict(zip(qs[0], qs[1]) + zip(qs[2], qs[3]))

def queries_map ( ) : qs = _all_metric_queries ( ) return dict ( zip ( qs [ 0 ] , qs [ 1 ] ) + zip ( qs [ 2 ] , qs [ 3 ] ) )


def get_clusters():
  """Synced API call to get all cluster names"""
  instance = tornado.ioloop.IOLoop.instance()
  # pylint: disable=unnecessary-lambda
  try:
    return instance.run_sync(lambda: API.get_clusters())
  except Exception:
    Log.debug(traceback.format_exc())
    raise

def get_clusters ( ) : instance = tornado . ioloop . IOLoop . instance ( ) try : return instance . run_sync ( lambda : API . get_clusters ( ) ) except Exception : Log . debug ( traceback . format_exc ( ) ) raise


def get_logical_plan(cluster, env, topology, role):
  """Synced API call to get logical plans"""
  instance = tornado.ioloop.IOLoop.instance()
  try:
    return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role))
  except Exception:
    Log.debug(traceback.format_exc())
    raise

def get_logical_plan ( cluster , env , topology , role ) : instance = tornado . ioloop . IOLoop . instance ( ) try : return instance . run_sync ( lambda : API . get_logical_plan ( cluster , env , topology , role ) ) except Exception : Log . debug ( traceback . format_exc ( ) ) raise


def get_topology_info(*args):
  """Synced API call to get topology information"""
  instance = tornado.ioloop.IOLoop.instance()
  try:
    return instance.run_sync(lambda: API.get_topology_info(*args))
  except Exception:
    Log.debug(traceback.format_exc())
    raise

def get_topology_info ( * args ) : instance = tornado . ioloop . IOLoop . instance ( ) try : return instance . run_sync ( lambda : API . get_topology_info ( * args ) ) except Exception : Log . debug ( traceback . format_exc ( ) ) raise


def get_component_metrics(component, cluster, env, topology, role):
  """Synced API call to get component metrics"""
  all_queries = metric_queries()
  try:
    result = get_topology_metrics(cluster, env, topology, component, [],
                                  all_queries, [0, -1], role)
    return result["metrics"]
  except Exception:
    Log.debug(traceback.format_exc())
    raise

def get_component_metrics ( component , cluster , env , topology , role ) : all_queries = metric_queries ( ) try : result = get_topology_metrics ( cluster , env , topology , component , [ ] , all_queries , [ 0 , - 1 ] , role ) return result [ "metrics" ] except Exception : Log . debug ( traceback . format_exc ( ) ) raise


def configure(level=logging.INFO, logfile=None):
  """ Configure logger which dumps log on terminal

  :param level: logging level: info, warning, verbose...
  :type level: logging level
  :param logfile: log file name, default to None
  :type logfile: string
  :return: None
  :rtype: None
  """

  # Remove all the existing StreamHandlers to avoid duplicate
  for handler in Log.handlers:
    if isinstance(handler, logging.StreamHandler):
      Log.handlers.remove(handler)

  Log.setLevel(level)

  # if logfile is specified, FileHandler is used
  if logfile is not None:
    log_format = "[%(asctime)s] [%(levelname)s]: %(message)s"
    formatter = logging.Formatter(fmt=log_format, datefmt=date_format)
    file_handler = logging.FileHandler(logfile)
    file_handler.setFormatter(formatter)
    Log.addHandler(file_handler)
  # otherwise, use StreamHandler to output to stream (stdout, stderr...)
  else:
    log_format = "[%(asctime)s] %(log_color)s[%(levelname)s]%(reset)s: %(message)s"
    # pylint: disable=redefined-variable-type
    formatter = colorlog.ColoredFormatter(fmt=log_format, datefmt=date_format)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(formatter)
    Log.addHandler(stream_handler)


def init_rotating_logger(level, logfile, max_files, max_bytes):
  """Initializes a rotating logger

  It also makes sure that any StreamHandler is removed, so as to avoid stdout/stderr
  constipation issues
  """
  logging.basicConfig()

  root_logger = logging.getLogger()
  log_format = "[%(asctime)s] [%(levelname)s] %(filename)s: %(message)s"

  root_logger.setLevel(level)
  handler = RotatingFileHandler(logfile, maxBytes=max_bytes, backupCount=max_files)
  handler.setFormatter(logging.Formatter(fmt=log_format, datefmt=date_format))
  root_logger.addHandler(handler)

  for handler in root_logger.handlers:
    root_logger.debug("Associated handlers - " + str(handler))
    if isinstance(handler, logging.StreamHandler):
      root_logger.debug("Removing StreamHandler: " + str(handler))
      root_logger.handlers.remove(handler)


def set_logging_level(cl_args):
  """simply set verbose level based on command-line args

  :param cl_args: CLI arguments
  :type cl_args: dict
  :return: None
  :rtype: None
  """
  if 'verbose' in cl_args and cl_args['verbose']:
    configure(logging.DEBUG)
  else:
    configure(logging.INFO)

def set_logging_level ( cl_args ) : if 'verbose' in cl_args and cl_args [ 'verbose' ] : configure ( logging . DEBUG ) else : configure ( logging . INFO )

simply set verbose level based on command - line args


def _get_spout(self):
    """Returns Spout protobuf message"""
    spout = topology_pb2.Spout()
    spout.comp.CopyFrom(self._get_base_component())

    # Add output streams
    self._add_out_streams(spout)
    return spout

def _get_spout ( self ) : spout = topology_pb2 . Spout ( ) spout . comp . CopyFrom ( self . _get_base_component ( ) ) self . _add_out_streams ( spout ) return spout


def _get_bolt(self):
    """Returns Bolt protobuf message"""
    bolt = topology_pb2.Bolt()
    bolt.comp.CopyFrom(self._get_base_component())

    # Add streams
    self._add_in_streams(bolt)
    self._add_out_streams(bolt)
    return bolt

def _get_bolt ( self ) : bolt = topology_pb2 . Bolt ( ) bolt . comp . CopyFrom ( self . _get_base_component ( ) ) self . _add_in_streams ( bolt ) self . _add_out_streams ( bolt ) return bolt


def _get_base_component(self):
    """Returns Component protobuf message"""
    comp = topology_pb2.Component()
    comp.name = self.name
    comp.spec = topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME")
    comp.class_name = self.python_class_path
    comp.config.CopyFrom(self._get_comp_config())
    return comp

def _get_base_component ( self ) : comp = topology_pb2 . Component ( ) comp . name = self . name comp . spec = topology_pb2 . ComponentObjectSpec . Value ( "PYTHON_CLASS_NAME" ) comp . class_name = self . python_class_path comp . config . CopyFrom ( self . _get_comp_config ( ) ) return comp


def _get_comp_config(self):
    """Returns component-specific Config protobuf message

    It first adds ``topology.component.parallelism``, and is overriden by
    a user-defined component-specific configuration, specified by spec().
    """
    proto_config = topology_pb2.Config()

    # first add parallelism
    key = proto_config.kvs.add()
    key.key = TOPOLOGY_COMPONENT_PARALLELISM
    key.value = str(self.parallelism)
    key.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")

    # iterate through self.custom_config
    if self.custom_config is not None:
      sanitized = self._sanitize_config(self.custom_config)
      for key, value in sanitized.items():
        if isinstance(value, str):
          kvs = proto_config.kvs.add()
          kvs.key = key
          kvs.value = value
          kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
        else:
          # need to serialize
          kvs = proto_config.kvs.add()
          kvs.key = key
          kvs.serialized_value = default_serializer.serialize(value)
          kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE")

    return proto_config

def _get_comp_config ( self ) : proto_config = topology_pb2 . Config ( ) key = proto_config . kvs . add ( ) key . key = TOPOLOGY_COMPONENT_PARALLELISM key . value = str ( self . parallelism ) key . type = topology_pb2 . ConfigValueType . Value ( "STRING_VALUE" ) if self . custom_config is not None : sanitized = self . _sanitize_config ( self . custom_config ) for key , value in sanitized . items ( ) : if isinstance ( value , str ) : kvs = proto_config . kvs . add ( ) kvs . key = key kvs . value = value kvs . type = topology_pb2 . ConfigValueType . Value ( "STRING_VALUE" ) else : kvs = proto_config . kvs . add ( ) kvs . key = key kvs . serialized_value = default_serializer . serialize ( value ) kvs . type = topology_pb2 . ConfigValueType . Value ( "PYTHON_SERIALIZED_VALUE" ) return proto_config

Returns component - specific Config protobuf message


def _add_in_streams(self, bolt):
    """Adds inputs to a given protobuf Bolt message"""
    if self.inputs is None:
      return
    # sanitize inputs and get a map <GlobalStreamId -> Grouping>
    input_dict = self._sanitize_inputs()

    for global_streamid, gtype in input_dict.items():
      in_stream = bolt.inputs.add()
      in_stream.stream.CopyFrom(self._get_stream_id(global_streamid.component_id,
                                                    global_streamid.stream_id))
      if isinstance(gtype, Grouping.FIELDS):
        # it's a field grouping
        in_stream.gtype = gtype.gtype
        in_stream.grouping_fields.CopyFrom(self._get_stream_schema(gtype.fields))
      elif isinstance(gtype, Grouping.CUSTOM):
        # it's a custom grouping
        in_stream.gtype = gtype.gtype
        in_stream.custom_grouping_object = gtype.python_serialized
        in_stream.type = topology_pb2.CustomGroupingObjectType.Value("PYTHON_OBJECT")
      else:
        in_stream.gtype = gtype

def _add_in_streams ( self , bolt ) : if self . inputs is None : return input_dict = self . _sanitize_inputs ( ) for global_streamid , gtype in input_dict . items ( ) : in_stream = bolt . inputs . add ( ) in_stream . stream . CopyFrom ( self . _get_stream_id ( global_streamid . component_id , global_streamid . stream_id ) ) if isinstance ( gtype , Grouping . FIELDS ) : in_stream . gtype = gtype . gtype in_stream . grouping_fields . CopyFrom ( self . _get_stream_schema ( gtype . fields ) ) elif isinstance ( gtype , Grouping . CUSTOM ) : in_stream . gtype = gtype . gtype in_stream . custom_grouping_object = gtype . python_serialized in_stream . type = topology_pb2 . CustomGroupingObjectType . Value ( "PYTHON_OBJECT" ) else : in_stream . gtype = gtype


def _add_out_streams(self, spbl):
    """Adds outputs to a given protobuf Bolt or Spout message"""
    if self.outputs is None:
      return

    # sanitize outputs and get a map <stream_id -> out fields>
    output_map = self._sanitize_outputs()

    for stream_id, out_fields in output_map.items():
      out_stream = spbl.outputs.add()
      out_stream.stream.CopyFrom(self._get_stream_id(self.name, stream_id))
      out_stream.schema.CopyFrom(self._get_stream_schema(out_fields))

def _add_out_streams ( self , spbl ) : if self . outputs is None : return output_map = self . _sanitize_outputs ( ) for stream_id , out_fields in output_map . items ( ) : out_stream = spbl . outputs . add ( ) out_stream . stream . CopyFrom ( self . _get_stream_id ( self . name , stream_id ) ) out_stream . schema . CopyFrom ( self . _get_stream_schema ( out_fields ) )


def get_out_streamids(self):
    """Returns a set of output stream ids registered for this component"""
    if self.outputs is None:
      return set()

    if not isinstance(self.outputs, (list, tuple)):
      raise TypeError("Argument to outputs must be either list or tuple, given: %s"
                      % str(type(self.outputs)))
    ret_lst = []
    for output in self.outputs:
      if not isinstance(output, (str, Stream)):
        raise TypeError("Outputs must be a list of strings or Streams, given: %s" % str(output))
      ret_lst.append(Stream.DEFAULT_STREAM_ID if isinstance(output, str) else output.stream_id)
    return set(ret_lst)


def _get_stream_id(comp_name, stream_id):
    """Returns a StreamId protobuf message"""
    proto_stream_id = topology_pb2.StreamId()
    proto_stream_id.id = stream_id
    proto_stream_id.component_name = comp_name
    return proto_stream_id


def _get_stream_schema(fields):
    """Returns a StreamSchema protobuf message"""
    stream_schema = topology_pb2.StreamSchema()
    for field in fields:
      key = stream_schema.keys.add()
      key.key = field
      key.type = topology_pb2.Type.Value("OBJECT")

    return stream_schema

def _get_stream_schema ( fields ) : stream_schema = topology_pb2 . StreamSchema ( ) for field in fields : key = stream_schema . keys . add ( ) key . key = field key . type = topology_pb2 . Type . Value ( "OBJECT" ) return stream_schema


def component_id(self):
    """Returns component_id of this GlobalStreamId

    Note that if HeronComponentSpec is specified as componentId and its name is not yet
    available (i.e. when ``name`` argument was not given in ``spec()`` method in Bolt or Spout),
    this property returns a message with uuid. However, this is provided only for safety
    with __eq__(), __str__(), and __hash__() methods, and not meant to be called explicitly
    before TopologyType class finally sets the name attribute of HeronComponentSpec.
    """
    if isinstance(self._component_id, HeronComponentSpec):
      if self._component_id.name is None:
        # HeronComponentSpec instance's name attribute might not be available until
        # TopologyType metaclass finally sets it. This statement is to support __eq__(),
        # __hash__() and __str__() methods with safety, as raising Exception is not
        # appropriate this case.
        return "<No name available for HeronComponentSpec yet, uuid: %s>" % self._component_id.uuid
      return self._component_id.name
    elif isinstance(self._component_id, str):
      return self._component_id
    else:
      raise ValueError("Component Id for this GlobalStreamId is not properly set: <%s:%s>"
                       % (str(type(self._component_id)), str(self._component_id)))


def register_metric(self, name, metric, time_bucket_in_sec):
    """Registers a new metric to this context"""
    collector = self.get_metrics_collector()
    collector.register_metric(name, metric, time_bucket_in_sec)

def register_metric ( self , name , metric , time_bucket_in_sec ) : collector = self . get_metrics_collector ( ) collector . register_metric ( name , metric , time_bucket_in_sec )


def get_sources(self, component_id):
    """Returns the declared inputs to specified component

    :return: map <streamId namedtuple (same structure as protobuf msg) -> gtype>, or
             None if not found
    """
    # this is necessary because protobuf message is not hashable
    StreamId = namedtuple('StreamId', 'id, component_name')
    if component_id in self.inputs:
      ret = {}
      for istream in self.inputs.get(component_id):
        key = StreamId(id=istream.stream.id, component_name=istream.stream.component_name)
        ret[key] = istream.gtype
      return ret
    else:
      return None

def
get_sources
(
self
,
component_id
)
:
StreamId
=
namedtuple
(
'StreamId'
,
'id, component_name'
)
if
component_id
in
self
.
inputs
:
ret
=
{
}
for
istream
in
self
.
inputs
.
get
(
component_id
)
:
key
=
StreamId
(
id
=
istream
.
stream
.
id
,
component_name
=
istream
.
stream
.
component_name
)
ret
[
key
]
=
istream
.
gtype
return
ret
else
:
return


def get_component_tasks(self, component_id):
    """Returns the task ids allocated for the given component id"""
    ret = []
    for task_id, comp_id in self.task_to_component_map.items():
      if comp_id == component_id:
        ret.append(task_id)
    return ret

def get_component_tasks ( self , component_id ) : ret = [ ] for task_id , comp_id in self . task_to_component_map . items ( ) : if comp_id == component_id : ret . append ( task_id ) return ret


def add_task_hook(self, task_hook):
    """Registers a specified task hook to this context

    :type task_hook: heron.instance.src.python.utils.topology.ITaskHook
    :param task_hook: Implementation of ITaskHook
    """
    if not isinstance(task_hook, ITaskHook):
      raise TypeError("In add_task_hook(): attempt to add non ITaskHook instance, given: %s"
                      % str(type(task_hook)))
    self.task_hooks.append(task_hook)


def get_metrics_collector(self):
    """Returns this context's metrics collector"""
    if self.metrics_collector is None or not isinstance(self.metrics_collector, MetricsCollector):
      raise RuntimeError("Metrics collector is not registered in this context")
    return self.metrics_collector

Returns this context s metrics collector


def invoke_hook_spout_ack(self, message_id, complete_latency_ns):
    """invoke task hooks for every time spout acks a tuple

    :type message_id: str
    :param message_id: message id to which an acked tuple was anchored
    :type complete_latency_ns: float
    :param complete_latency_ns: complete latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      spout_ack_info = SpoutAckInfo(message_id=message_id,
                                    spout_task_id=self.get_task_id(),
                                    complete_latency_ms=complete_latency_ns *
                                    system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.spout_ack(spout_ack_info)

def invoke_hook_spout_ack ( self , message_id , complete_latency_ns ) : if len ( self . task_hooks ) > 0 : spout_ack_info = SpoutAckInfo ( message_id = message_id , spout_task_id = self . get_task_id ( ) , complete_latency_ms = complete_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . spout_ack ( spout_ack_info )


def invoke_hook_spout_fail(self, message_id, fail_latency_ns):
    """invoke task hooks for every time spout fails a tuple

    :type message_id: str
    :param message_id: message id to which a failed tuple was anchored
    :type fail_latency_ns: float
    :param fail_latency_ns: fail latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      spout_fail_info = SpoutFailInfo(message_id=message_id,
                                      spout_task_id=self.get_task_id(),
                                      fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.spout_fail(spout_fail_info)

def invoke_hook_spout_fail ( self , message_id , fail_latency_ns ) : if len ( self . task_hooks ) > 0 : spout_fail_info = SpoutFailInfo ( message_id = message_id , spout_task_id = self . get_task_id ( ) , fail_latency_ms = fail_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . spout_fail ( spout_fail_info )


def invoke_hook_bolt_execute(self, heron_tuple, execute_latency_ns):
    """invoke task hooks for every time bolt processes a tuple

    :type heron_tuple: HeronTuple
    :param heron_tuple: tuple that is executed
    :type execute_latency_ns: float
    :param execute_latency_ns: execute latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      bolt_execute_info = \
        BoltExecuteInfo(heron_tuple=heron_tuple,
                        executing_task_id=self.get_task_id(),
                        execute_latency_ms=execute_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.bolt_execute(bolt_execute_info)

def invoke_hook_bolt_execute ( self , heron_tuple , execute_latency_ns ) : if len ( self . task_hooks ) > 0 : bolt_execute_info = BoltExecuteInfo ( heron_tuple = heron_tuple , executing_task_id = self . get_task_id ( ) , execute_latency_ms = execute_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . bolt_execute ( bolt_execute_info )


def invoke_hook_bolt_ack(self, heron_tuple, process_latency_ns):
    """invoke task hooks for every time bolt acks a tuple

    :type heron_tuple: HeronTuple
    :param heron_tuple: tuple that is acked
    :type process_latency_ns: float
    :param process_latency_ns: process latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      bolt_ack_info = BoltAckInfo(heron_tuple=heron_tuple,
                                  acking_task_id=self.get_task_id(),
                                  process_latency_ms=process_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.bolt_ack(bolt_ack_info)

def invoke_hook_bolt_ack ( self , heron_tuple , process_latency_ns ) : if len ( self . task_hooks ) > 0 : bolt_ack_info = BoltAckInfo ( heron_tuple = heron_tuple , acking_task_id = self . get_task_id ( ) , process_latency_ms = process_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . bolt_ack ( bolt_ack_info )


def invoke_hook_bolt_fail(self, heron_tuple, fail_latency_ns):
    """invoke task hooks for every time bolt fails a tuple

    :type heron_tuple: HeronTuple
    :param heron_tuple: tuple that is failed
    :type fail_latency_ns: float
    :param fail_latency_ns: fail latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      bolt_fail_info = BoltFailInfo(heron_tuple=heron_tuple,
                                    failing_task_id=self.get_task_id(),
                                    fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.bolt_fail(bolt_fail_info)

def invoke_hook_bolt_fail ( self , heron_tuple , fail_latency_ns ) : if len ( self . task_hooks ) > 0 : bolt_fail_info = BoltFailInfo ( heron_tuple = heron_tuple , failing_task_id = self . get_task_id ( ) , fail_latency_ms = fail_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . bolt_fail ( bolt_fail_info )


def submit_fatjar(cl_args, unknown_args, tmp_dir):
  '''
  We use the packer to make a package for the jar and dump it
  to a well-known location. We then run the main method of class
  with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.

  This will run the jar file with the topology_class_name. The submitter
  inside will write out the topology defn file to a location that
  we specify. Then we write the topology defn file to a well known
  location. We then write to appropriate places in zookeeper
  and launch the scheduler jobs
  :param cl_args:
  :param unknown_args:
  :param tmp_dir:
  :return:
  '''
  # execute main of the topology to create the topology definition
  topology_file = cl_args['topology-file-name']

  main_class = cl_args['topology-class-name']

  res = execute.heron_class(
      class_name=main_class,
      lib_jars=config.get_heron_libs(jars.topology_jars()),
      extra_jars=[topology_file],
      args=tuple(unknown_args),
      java_defines=cl_args['topology_main_jvm_property'])

  result.render(res)

  if not result.is_successful(res):
    err_context = ("Failed to create topology definition " \
      "file when executing class '%s' of file '%s'") % (main_class, topology_file)
    res.add_context(err_context)
    return res

  results = launch_topologies(cl_args, topology_file, tmp_dir)

  return results

We use the packer to make a package for the jar and dump it to a well - known location . We then run the main method of class with the specified arguments . We pass arguments as an environment variable HERON_OPTIONS .


def submit_tar(cl_args, unknown_args, tmp_dir):
  '''
  Extract and execute the java files inside the tar and then add topology
  definition file created by running submitTopology

  We use the packer to make a package for the tar and dump it
  to a well-known location. We then run the main method of class
  with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
  This will run the jar file with the topology class name.

  The submitter inside will write out the topology defn file to a location
  that we specify. Then we write the topology defn file to a well known
  packer location. We then write to appropriate places in zookeeper
  and launch the aurora jobs
  :param cl_args:
  :param unknown_args:
  :param tmp_dir:
  :return:
  '''
  # execute main of the topology to create the topology definition
  topology_file = cl_args['topology-file-name']
  java_defines = cl_args['topology_main_jvm_property']
  main_class = cl_args['topology-class-name']
  res = execute.heron_tar(
      main_class,
      topology_file,
      tuple(unknown_args),
      tmp_dir,
      java_defines)

  result.render(res)

  if not result.is_successful(res):
    err_context = ("Failed to create topology definition " \
      "file when executing class '%s' of file '%s'") % (main_class, topology_file)
    res.add_context(err_context)
    return res

  return launch_topologies(cl_args, topology_file, tmp_dir)


def setup(self, context):
    """Implements TextFile Generator's setup method"""
    myindex = context.get_partition_index()
    self._files_to_consume = self._files[myindex::context.get_num_partitions()]
    self.logger.info("TextFileSpout files to consume %s" % self._files_to_consume)
    self._lines_to_consume = self._get_next_lines()
    self._emit_count = 0

def
setup
(
self
,
context
)
:
myindex
=
context
.
get_partition_index
(
)
self
.
_files_to_consume
=
self
.
_files
[
myindex
:
:
context
.
get_num_partitions
(
)
]
self
.
logger
.
info
(
"TextFileSpout files to consume %s"
%
self
.
_files_to_consume
)
self
.
_lines_to_consume
=
self
.
_get_next_lines
(
)
self
.
_emit_count
=
0

Implements TextFile Generator s setup method


def add_verbose(parser):
  """ add optional verbose argument"""
  parser.add_argument(
      '--verbose',
      metavar='(a boolean; default: "false")',
      type=bool,
      default=False)
  return parser

def
add_verbose
(
parser
)
:
parser
.
add_argument
(
'--verbose'
,
metavar
=
'(a boolean; default: "false")'
,
type
=
bool
,
default
=
False
)
return
parser


def add_tracker_url(parser):
  """ add optional tracker_url argument """
  parser.add_argument(
      '--tracker_url',
      metavar='(tracker url; default: "' + DEFAULT_TRACKER_URL + '")',
      type=str, default=DEFAULT_TRACKER_URL)
  return parser

def
add_tracker_url
(
parser
)
:
parser
.
add_argument
(
'--tracker_url'
,
metavar
=
'(tracker url; default: "'
+
DEFAULT_TRACKER_URL
+
'")'
,
type
=
str
,
default
=
DEFAULT_TRACKER_URL
)
return
parser


def hex_escape(bin_str):
  """
  Hex encode a binary string
  """
  printable = string.ascii_letters + string.digits + string.punctuation + ' '
  return ''.join(ch if ch in printable else r'0x{0:02x}'.format(ord(ch)) for ch in bin_str)

def
hex_escape
(
bin_str
)
:
printable
=
string
.
ascii_letters
+
string
.
digits
+
string
.
punctuation
+
' '
return
''
.
join
(
ch
if
ch
in
printable
else
r'0x{0:02x}'
.
format
(
ord
(
ch
)
)
for
ch
in
bin_str
)


def make_shell_endpoint(topologyInfo, instance_id):
  """
  Makes the http endpoint for the heron shell
  if shell port is present, otherwise returns None.
  """
  # Format: container_<id>_<instance_id>
  pplan = topologyInfo["physical_plan"]
  stmgrId = pplan["instances"][instance_id]["stmgrId"]
  host = pplan["stmgrs"][stmgrId]["host"]
  shell_port = pplan["stmgrs"][stmgrId]["shell_port"]
  return "http://%s:%d" % (host, shell_port)

def make_shell_endpoint ( topologyInfo , instance_id ) : pplan = topologyInfo [ "physical_plan" ] stmgrId = pplan [ "instances" ] [ instance_id ] [ "stmgrId" ] host = pplan [ "stmgrs" ] [ stmgrId ] [ "host" ] shell_port = pplan [ "stmgrs" ] [ stmgrId ] [ "shell_port" ] return "http://%s:%d" % ( host , shell_port )

Makes the http endpoint for the heron shell if shell port is present otherwise returns None .


def make_shell_logfiles_url(host, shell_port, _, instance_id=None):
  """
  Make the url for log-files in heron-shell
  from the info stored in stmgr.
  If no instance_id is provided, the link will
  be to the dir for the whole container.
  If shell port is not present, it returns None.
  """
  if not shell_port:
    return None
  if not instance_id:
    return "http://%s:%d/browse/log-files" % (host, shell_port)
  else:
    return "http://%s:%d/file/log-files/%s.log.0" % (host, shell_port, instance_id)

def make_shell_logfiles_url ( host , shell_port , _ , instance_id = None ) : if not shell_port : return None if not instance_id : return "http://%s:%d/browse/log-files" % ( host , shell_port ) else : return "http://%s:%d/file/log-files/%s.log.0" % ( host , shell_port , instance_id )

Make the url for log - files in heron - shell from the info stored in stmgr . If no instance_id is provided the link will be to the dir for the whole container . If shell port is not present it returns None .


def make_shell_logfile_data_url(host, shell_port, instance_id, offset, length):
  """
  Make the url for log-file data in heron-shell
  from the info stored in stmgr.
  """
  return "http://%s:%d/filedata/log-files/%s.log.0?offset=%s&length=%s" % \
    (host, shell_port, instance_id, offset, length)

Make the url for log - file data in heron - shell from the info stored in stmgr .


def create_packet(reqid, message):
    """Creates Outgoing Packet from a given reqid and message

    :param reqid: REQID object
    :param message: protocol buffer object
    """
    assert message.IsInitialized()
    packet = ''

    # calculate the totla size of the packet incl. header
    typename = message.DESCRIPTOR.full_name

    datasize = HeronProtocol.get_size_to_pack_string(typename) + \
               REQID.REQID_SIZE + HeronProtocol.get_size_to_pack_message(message)

    # first write out how much data is there as the header
    packet += HeronProtocol.pack_int(datasize)

    # next write the type string
    packet += HeronProtocol.pack_int(len(typename))
    packet += typename

    # reqid
    packet += reqid.pack()

    # add the proto
    packet += HeronProtocol.pack_int(message.ByteSize())
    packet += message.SerializeToString()
    return OutgoingPacket(packet)

def create_packet ( reqid , message ) : assert message . IsInitialized ( ) packet = '' typename = message . DESCRIPTOR . full_name datasize = HeronProtocol . get_size_to_pack_string ( typename ) + REQID . REQID_SIZE + HeronProtocol . get_size_to_pack_message ( message ) packet += HeronProtocol . pack_int ( datasize ) packet += HeronProtocol . pack_int ( len ( typename ) ) packet += typename packet += reqid . pack ( ) packet += HeronProtocol . pack_int ( message . ByteSize ( ) ) packet += message . SerializeToString ( ) return OutgoingPacket ( packet )


def send(self, dispatcher):
    """Sends this outgoing packet to dispatcher's socket"""
    if self.sent_complete:
      return

    sent = dispatcher.send(self.to_send)
    self.to_send = self.to_send[sent:]

def send ( self , dispatcher ) : if self . sent_complete : return sent = dispatcher . send ( self . to_send ) self . to_send = self . to_send [ sent : ]

Sends this outgoing packet to dispatcher s socket


def create_packet(header, data):
    """Creates an IncomingPacket object from header and data

    This method is for testing purposes
    """
    packet = IncomingPacket()
    packet.header = header
    packet.data = data

    if len(header) == HeronProtocol.HEADER_SIZE:
      packet.is_header_read = True
      if len(data) == packet.get_datasize():
        packet.is_complete = True

    return packet

def create_packet ( header , data ) : packet = IncomingPacket ( ) packet . header = header packet . data = data if len ( header ) == HeronProtocol . HEADER_SIZE : packet . is_header_read = True if len ( data ) == packet . get_datasize ( ) : packet . is_complete = True return packet


def read(self, dispatcher):
    """Reads incoming data from asyncore.dispatcher"""
    try:
      if not self.is_header_read:
        # try reading header
        to_read = HeronProtocol.HEADER_SIZE - len(self.header)
        self.header += dispatcher.recv(to_read)
        if len(self.header) == HeronProtocol.HEADER_SIZE:
          self.is_header_read = True
        else:
          Log.debug("Header read incomplete; read %d bytes of header" % len(self.header))
          return

      if self.is_header_read and not self.is_complete:
        # try reading data
        to_read = self.get_datasize() - len(self.data)
        self.data += dispatcher.recv(to_read)
        if len(self.data) == self.get_datasize():
          self.is_complete = True
    except socket.error as e:
      if e.errno == socket.errno.EAGAIN or e.errno == socket.errno.EWOULDBLOCK:
        # Try again later -> call continue_read later
        Log.debug("Try again error")
      else:
        # Fatal error
        Log.debug("Fatal error when reading IncomingPacket")
        raise RuntimeError("Fatal error occured in IncomingPacket.read()")

Reads incoming data from asyncore . dispatcher


def generate():
    """Generates a random REQID for request"""
    data_bytes = bytearray(random.getrandbits(8) for i in range(REQID.REQID_SIZE))
    return REQID(data_bytes)

def generate ( ) : data_bytes = bytearray ( random . getrandbits ( 8 ) for i in range ( REQID . REQID_SIZE ) ) return REQID ( data_bytes )


def yaml_config_reader(config_path):
  """Reads yaml config file and returns auto-typed config_dict"""
  if not config_path.endswith(".yaml"):
    raise ValueError("Config file not yaml")

  with open(config_path, 'r') as f:
    config = yaml.load(f)

  return config

def
yaml_config_reader
(
config_path
)
:
if
not
config_path
.
endswith
(
".yaml"
)
:
raise
ValueError
(
"Config file not yaml"
)
with
open
(
config_path
,
'r'
)
as
f
:
config
=
yaml
.
load
(
f
)
return
config

Reads yaml config file and returns auto - typed config_dict


def send_buffered_messages(self):
    """Send messages in out_stream to the Stream Manager"""
    while not self.out_stream.is_empty() and self._stmgr_client.is_registered:
      tuple_set = self.out_stream.poll()
      if isinstance(tuple_set, tuple_pb2.HeronTupleSet):
        tuple_set.src_task_id = self.my_pplan_helper.my_task_id
        self.gateway_metrics.update_sent_packet(tuple_set.ByteSize())
      self._stmgr_client.send_message(tuple_set)

def send_buffered_messages ( self ) : while not self . out_stream . is_empty ( ) and self . _stmgr_client . is_registered : tuple_set = self . out_stream . poll ( ) if isinstance ( tuple_set , tuple_pb2 . HeronTupleSet ) : tuple_set . src_task_id = self . my_pplan_helper . my_task_id self . gateway_metrics . update_sent_packet ( tuple_set . ByteSize ( ) ) self . _stmgr_client . send_message ( tuple_set )


def _handle_state_change_msg(self, new_helper):
    """Called when state change is commanded by stream manager"""
    assert self.my_pplan_helper is not None
    assert self.my_instance is not None and self.my_instance.py_class is not None

    if self.my_pplan_helper.get_topology_state() != new_helper.get_topology_state():
      # handle state change
      # update the pplan_helper
      self.my_pplan_helper = new_helper
      if new_helper.is_topology_running():
        if not self.is_instance_started:
          self.start_instance_if_possible()
        self.my_instance.py_class.invoke_activate()
      elif new_helper.is_topology_paused():
        self.my_instance.py_class.invoke_deactivate()
      else:
        raise RuntimeError("Unexpected TopologyState update: %s" % new_helper.get_topology_state())
    else:
      Log.info("Topology state remains the same.")

def
_handle_state_change_msg
(
self
,
new_helper
)
:
assert
self
.
my_pplan_helper
is
not
assert
self
.
my_instance
is
not
and
self
.
my_instance
.
py_class
is
not
if
self
.
my_pplan_helper
.
get_topology_state
(
)
!=
new_helper
.
get_topology_state
(
)
:
self
.
my_pplan_helper
=
new_helper
if
new_helper
.
is_topology_running
(
)
:
if
not
self
.
is_instance_started
:
self
.
start_instance_if_possible
(
)
self
.
my_instance
.
py_class
.
invoke_activate
(
)
elif
new_helper
.
is_topology_paused
(
)
:
self
.
my_instance
.
py_class
.
invoke_deactivate
(
)
else
:
raise
RuntimeError
(
"Unexpected TopologyState update: %s"
%
new_helper
.
get_topology_state
(
)
)
else
:
Log
.
info
(
"Topology state remains the same."
)


def check_output_schema(self, stream_id, tup):
    """Checks if a given stream_id and tuple matches with the output schema

    :type stream_id: str
    :param stream_id: stream id into which tuple is sent
    :type tup: list
    :param tup: tuple that is going to be sent
    """
    # do some checking to make sure that the number of fields match what's expected
    size = self._output_schema.get(stream_id, None)
    if size is None:
      raise RuntimeError("%s emitting to stream %s but was not declared in output fields"
                         % (self.my_component_name, stream_id))
    elif size != len(tup):
      raise RuntimeError("Number of fields emitted in stream %s does not match what's expected. "
                         "Expected: %s, Observed: %s" % (stream_id, size, len(tup)))


def get_topology_config(self):
    """Returns the topology config"""
    if self.pplan.topology.HasField("topology_config"):
      return self._get_dict_from_config(self.pplan.topology.topology_config)
    else:
      return {}

def get_topology_config ( self ) : if self . pplan . topology . HasField ( "topology_config" ) : return self . _get_dict_from_config ( self . pplan . topology . topology_config ) else : return { }


def set_topology_context(self, metrics_collector):
    """Sets a new topology context"""
    Log.debug("Setting topology context")
    cluster_config = self.get_topology_config()
    cluster_config.update(self._get_dict_from_config(self.my_component.config))
    task_to_component_map = self._get_task_to_comp_map()
    self.context = TopologyContextImpl(cluster_config, self.pplan.topology, task_to_component_map,
                                       self.my_task_id, metrics_collector,
                                       self.topology_pex_abs_path)

def
set_topology_context
(
self
,
metrics_collector
)
:
Log
.
debug
(
"Setting topology context"
)
cluster_config
=
self
.
get_topology_config
(
)
cluster_config
.
update
(
self
.
_get_dict_from_config
(
self
.
my_component
.
config
)
)
task_to_component_map
=
self
.
_get_task_to_comp_map
(
)
self
.
context
=
TopologyContextImpl
(
cluster_config
,
self
.
pplan
.
topology
,
task_to_component_map
,
self
.
my_task_id
,
metrics_collector
,
self
.
topology_pex_abs_path
)


def _get_dict_from_config(topology_config):
    """Converts Config protobuf message to python dictionary

    Values are converted according to the rules below:

    - Number string (e.g. "12" or "1.2") is appropriately converted to ``int`` or ``float``
    - Boolean string ("true", "True", "false" or "False") is converted to built-in boolean type
      (i.e. ``True`` or ``False``)
    - Normal string is inserted to dict as is
    - Serialized value is deserialized and inserted as a corresponding Python object
    """
    config = {}
    for kv in topology_config.kvs:
      if kv.HasField("value"):
        assert kv.type == topology_pb2.ConfigValueType.Value("STRING_VALUE")
        # value is string
        if PhysicalPlanHelper._is_number(kv.value):
          config[kv.key] = PhysicalPlanHelper._get_number(kv.value)
        elif kv.value.lower() in ("true", "false"):
          config[kv.key] = True if kv.value.lower() == "true" else False
        else:
          config[kv.key] = kv.value
      elif kv.HasField("serialized_value") and \
        kv.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"):
        # deserialize that
        config[kv.key] = default_serializer.deserialize(kv.serialized_value)
      else:
        assert kv.HasField("type")
        Log.error("Unsupported config <key:value> found: %s, with type: %s"
                  % (str(kv), str(kv.type)))
        continue

    return config


def _setup_custom_grouping(self, topology):
    """Checks whether there are any bolts that consume any of my streams using custom grouping"""
    for i in range(len(topology.bolts)):
      for in_stream in topology.bolts[i].inputs:
        if in_stream.stream.component_name == self.my_component_name and \
          in_stream.gtype == topology_pb2.Grouping.Value("CUSTOM"):
          # this bolt takes my output in custom grouping manner
          if in_stream.type == topology_pb2.CustomGroupingObjectType.Value("PYTHON_OBJECT"):
            custom_grouping_obj = default_serializer.deserialize(in_stream.custom_grouping_object)
            if isinstance(custom_grouping_obj, str):
              pex_loader.load_pex(self.topology_pex_abs_path)
              grouping_cls = \
                pex_loader.import_and_get_class(self.topology_pex_abs_path, custom_grouping_obj)
              custom_grouping_obj = grouping_cls()
            assert isinstance(custom_grouping_obj, ICustomGrouping)
            self.custom_grouper.add(in_stream.stream.id,
                                    self._get_taskids_for_component(topology.bolts[i].comp.name),
                                    custom_grouping_obj,
                                    self.my_component_name)

          elif in_stream.type == topology_pb2.CustomGroupingObjectType.Value("JAVA_OBJECT"):
            raise NotImplementedError("Java-serialized custom grouping is not yet supported "
                                      "for python topology")
          else:
            raise ValueError("Unrecognized custom grouping type found: %s" % str(in_stream.type))


def add(self, stream_id, task_ids, grouping, source_comp_name):
    """Adds the target component

    :type stream_id: str
    :param stream_id: stream id into which tuples are emitted
    :type task_ids: list of str
    :param task_ids: list of task ids to which tuples are emitted
    :type grouping: ICustomStreamGrouping object
    :param grouping: custom grouping to use
    :type source_comp_name: str
    :param source_comp_name: source component name
    """
    if stream_id not in self.targets:
      self.targets[stream_id] = []
    self.targets[stream_id].append(Target(task_ids, grouping, source_comp_name))

def add ( self , stream_id , task_ids , grouping , source_comp_name ) : if stream_id not in self . targets : self . targets [ stream_id ] = [ ] self . targets [ stream_id ] . append ( Target ( task_ids , grouping , source_comp_name ) )


def prepare(self, context):
    """Prepares the custom grouping for this component"""
    for stream_id, targets in self.targets.items():
      for target in targets:
        target.prepare(context, stream_id)

def prepare ( self , context ) : for stream_id , targets in self . targets . items ( ) : for target in targets : target . prepare ( context , stream_id )


def choose_tasks(self, stream_id, values):
    """Choose tasks for a given stream_id and values and Returns a list of target tasks"""
    if stream_id not in self.targets:
      return []

    ret = []
    for target in self.targets[stream_id]:
      ret.extend(target.choose_tasks(values))
    return ret

def choose_tasks ( self , stream_id , values ) : if stream_id not in self . targets : return [ ] ret = [ ] for target in self . targets [ stream_id ] : ret . extend ( target . choose_tasks ( values ) ) return ret


def format_mode(sres):
  """
  Format a line in the directory list based on the file's type and other attributes.
  """
  mode = sres.st_mode

  root = (mode & 0o700) >> 6
  group = (mode & 0o070) >> 3
  user = (mode & 0o7)

  def stat_type(md):
    ''' stat type'''
    if stat.S_ISDIR(md):
      return 'd'
    elif stat.S_ISSOCK(md):
      return 's'
    else:
      return '-'

  def triple(md):
    ''' triple '''
    return '%c%c%c' % (
        'r' if md & 0b100 else '-',
        'w' if md & 0b010 else '-',
        'x' if md & 0b001 else '-')

  return ''.join([stat_type(mode), triple(root), triple(group), triple(user)])

def format_mode ( sres ) : mode = sres . st_mode root = ( mode & 0o700 ) >> 6 group = ( mode & 0o070 ) >> 3 user = ( mode & 0o7 ) def stat_type ( md ) : if stat . S_ISDIR ( md ) : return 'd' elif stat . S_ISSOCK ( md ) : return 's' else : return '-' def triple ( md ) : return '%c%c%c' % ( 'r' if md & 0b100 else '-' , 'w' if md & 0b010 else '-' , 'x' if md & 0b001 else '-' ) return '' . join ( [ stat_type ( mode ) , triple ( root ) , triple ( group ) , triple ( user ) ] )

Format a line in the directory list based on the file s type and other attributes .


def format_mtime(mtime):
  """
  Format the date associated with a file to be displayed in directory listing.
  """
  now = datetime.now()
  dt = datetime.fromtimestamp(mtime)
  return '%s %2d %5s' % (
      dt.strftime('%b'), dt.day,
      dt.year if dt.year != now.year else dt.strftime('%H:%M'))

def
format_mtime
(
mtime
)
:
now
=
datetime
.
now
(
)
dt
=
datetime
.
fromtimestamp
(
mtime
)
return
'%s %2d %5s'
%
(
dt
.
strftime
(
'%b'
)
,
dt
.
day
,
dt
.
year
if
dt
.
year
!=
now
.
year
else
dt
.
strftime
(
'%H:%M'
)
)

Format the date associated with a file to be displayed in directory listing .


def format_prefix(filename, sres):
  """
  Prefix to a filename in the directory listing. This is to make the
  listing similar to an output of "ls -alh".
  """
  try:
    pwent = pwd.getpwuid(sres.st_uid)
    user = pwent.pw_name
  except KeyError:
    user = sres.st_uid

  try:
    grent = grp.getgrgid(sres.st_gid)
    group = grent.gr_name
  except KeyError:
    group = sres.st_gid

  return '%s %3d %10s %10s %10d %s' % (
      format_mode(sres),
      sres.st_nlink,
      user,
      group,
      sres.st_size,
      format_mtime(sres.st_mtime),
  )

def
format_prefix
(
filename
,
sres
)
:
try
:
pwent
=
pwd
.
getpwuid
(
sres
.
st_uid
)
user
=
pwent
.
pw_name
except
KeyError
:
user
=
sres
.
st_uid
try
:
grent
=
grp
.
getgrgid
(
sres
.
st_gid
)
group
=
grent
.
gr_name
except
KeyError
:
group
=
sres
.
st_gid
return
'%s %3d %10s %10s %10d %s'
%
(
format_mode
(
sres
)
,
sres
.
st_nlink
,
user
,
group
,
sres
.
st_size
,
format_mtime
(
sres
.
st_mtime
)
,
)

Prefix to a filename in the directory listing . This is to make the listing similar to an output of ls - alh .