def _prepare_summary_table(rows):
  """Create a new table that is a summary of the input rows.

  All with the same (job-name or job-id, status) go together.

  Args:
    rows: the input rows, a list of dictionaries.
  Returns:
    A new row set of summary information.
  """
  if not rows:
    return []

  # We either group on the job-name (if present) or fall back to the job-id
  key_field = 'job-name'
  if key_field not in rows[0]:
    key_field = 'job-id'

  # Group each of the rows based on (job-name or job-id, status)
  grouped = collections.defaultdict(lambda: collections.defaultdict(lambda: []))
  for row in rows:
    grouped[row.get(key_field, '')][row.get('status', '')] += [row]

  # Now that we have the rows grouped, create a summary table.
  # Use the original table as the driver in order to preserve the order.
  new_rows = []
  for job_key in sorted(grouped.keys()):
    group = grouped.get(job_key, None)
    canonical_status = ['RUNNING', 'SUCCESS', 'FAILURE', 'CANCEL']
    # Written this way to ensure that if somehow a new status is introduced,
    # it shows up in our output.
    for status in canonical_status + sorted(group.keys()):
      if status not in group:
        continue
      task_count = len(group[status])
      del group[status]
      if task_count:
        summary_row = collections.OrderedDict()
        summary_row[key_field] = job_key
        summary_row['status'] = status
        summary_row['task-count'] = task_count
        new_rows.append(summary_row)

  return new_rows

def _prepare_summary_table ( rows ) : if not rows : return [ ] key_field = 'job-name' if key_field not in rows [ 0 ] : key_field = 'job-id' grouped = collections . defaultdict ( lambda : collections . defaultdict ( lambda : [ ] ) ) for row in rows : grouped [ row . get ( key_field , '' ) ] [ row . get ( 'status' , '' ) ] += [ row ] new_rows = [ ] for job_key in sorted ( grouped . keys ( ) ) : group = grouped . get ( job_key , None ) canonical_status = [ 'RUNNING' , 'SUCCESS' , 'FAILURE' , 'CANCEL' ] for status in canonical_status + sorted ( group . keys ( ) ) : if status not in group : continue task_count = len ( group [ status ] ) del group [ status ] if task_count : summary_row = collections . OrderedDict ( ) summary_row [ key_field ] = job_key summary_row [ 'status' ] = status summary_row [ 'task-count' ] = task_count new_rows . append ( summary_row ) return new_rows

Create a new table that is a summary of the input rows .


def lookup_job_tasks(provider,
                     statuses,
                     user_ids=None,
                     job_ids=None,
                     job_names=None,
                     task_ids=None,
                     task_attempts=None,
                     labels=None,
                     create_time_min=None,
                     create_time_max=None,
                     max_tasks=0,
                     page_size=0,
                     summary_output=False):
  """Generate formatted jobs individually, in order of create-time.

  Args:
    provider: an instantiated dsub provider.
    statuses: a set of status strings that eligible jobs may match.
    user_ids: a set of user strings that eligible jobs may match.
    job_ids: a set of job-id strings eligible jobs may match.
    job_names: a set of job-name strings eligible jobs may match.
    task_ids: a set of task-id strings eligible tasks may match.
    task_attempts: a set of task-attempt strings eligible tasks may match.
    labels: set of LabelParam that all tasks must match.
    create_time_min: a timezone-aware datetime value for the earliest create
                     time of a task, inclusive.
    create_time_max: a timezone-aware datetime value for the most recent create
                     time of a task, inclusive.
    max_tasks: (int) maximum number of tasks to return per dstat job lookup.
    page_size: the page size to use for each query to the backend. May be
               ignored by some provider implementations.
    summary_output: (bool) summarize the job list.

  Yields:
    Individual task dictionaries with associated metadata
  """
  tasks_generator = provider.lookup_job_tasks(
      statuses,
      user_ids=user_ids,
      job_ids=job_ids,
      job_names=job_names,
      task_ids=task_ids,
      task_attempts=task_attempts,
      labels=labels,
      create_time_min=create_time_min,
      create_time_max=create_time_max,
      max_tasks=max_tasks,
      page_size=page_size)

  # Yield formatted tasks.
  for task in tasks_generator:
    yield _prepare_row(task, True, summary_output)

Generate formatted jobs individually in order of create - time .


def prepare_output(self, row):
    """Convert types of task fields."""
    date_fields = ['last-update', 'create-time', 'start-time', 'end-time']
    int_fields = ['task-attempt']

    for col in date_fields:
      if col in row:
        row[col] = self.default_format_date(row[col])

    for col in int_fields:
      if col in row and row[col] is not None:
        row[col] = int(row[col])

    return row

def prepare_output ( self , row ) : date_fields = [ 'last-update' , 'create-time' , 'start-time' , 'end-time' ] int_fields = [ 'task-attempt' ] for col in date_fields : if col in row : row [ col ] = self . default_format_date ( row [ col ] ) for col in int_fields : if col in row and row [ col ] is not None : row [ col ] = int ( row [ col ] ) return row


def trim_display_field(self, value, max_length):
    """Return a value for display; if longer than max length, use ellipsis."""
    if not value:
      return ''
    if len(value) > max_length:
      return value[:max_length - 3] + '...'
    return value

def trim_display_field ( self , value , max_length ) : if not value : return '' if len ( value ) > max_length : return value [ : max_length - 3 ] + '...' return value

Return a value for display ; if longer than max length use ellipsis .


def format_pairs(self, values):
    """Returns a string of comma-delimited key=value pairs."""
    return ', '.join(
        '%s=%s' % (key, value) for key, value in sorted(values.items()))

def
format_pairs
(
self
,
values
)
:
return
', '
.
join
(
'%s=%s'
%
(
key
,
value
)
for
key
,
value
in
sorted
(
values
.
items
(
)
)
)

Returns a string of comma - delimited key = value pairs .


def string_presenter(self, dumper, data):
    """Presenter to force yaml.dump to use multi-line string style."""
    if '\n' in data:
      return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
    else:
      return dumper.represent_scalar('tag:yaml.org,2002:str', data)

def string_presenter ( self , dumper , data ) : if '\n' in data : return dumper . represent_scalar ( 'tag:yaml.org,2002:str' , data , style = '|' ) else : return dumper . represent_scalar ( 'tag:yaml.org,2002:str' , data )

Presenter to force yaml . dump to use multi - line string style .


def get_zones(input_list):
  """Returns a list of zones based on any wildcard input.

  This function is intended to provide an easy method for producing a list
  of desired zones for a pipeline to run in.

  The Pipelines API default zone list is "any zone". The problem with
  "any zone" is that it can lead to incurring Cloud Storage egress charges
  if the GCE zone selected is in a different region than the GCS bucket.
  See https://cloud.google.com/storage/pricing#network-egress.

  A user with a multi-region US bucket would want to pipelines to run in
  a "us-*" zone.
  A user with a regional bucket in US would want to restrict pipelines to
  run in a zone in that region.

  Rarely does the specific zone matter for a pipeline.

  This function allows for a simple short-hand such as:
     [ "us-*" ]
     [ "us-central1-*" ]
  These examples will expand out to the full list of US and us-central1 zones
  respectively.

  Args:
    input_list: list of zone names/patterns

  Returns:
    A list of zones, with any wildcard zone specifications expanded.
  """
  if not input_list:
    return []

  output_list = []

  for zone in input_list:
    if zone.endswith('*'):
      prefix = zone[:-1]
      output_list.extend([z for z in _ZONES if z.startswith(prefix)])
    else:
      output_list.append(zone)

  return output_list

def get_zones ( input_list ) : if not input_list : return [ ] output_list = [ ] for zone in input_list : if zone . endswith ( '*' ) : prefix = zone [ : - 1 ] output_list . extend ( [ z for z in _ZONES if z . startswith ( prefix ) ] ) else : output_list . append ( zone ) return output_list

Returns a list of zones based on any wildcard input .


def parse_rfc3339_utc_string(rfc3339_utc_string):
  """Converts a datestamp from RFC3339 UTC to a datetime.

  Args:
    rfc3339_utc_string: a datetime string in RFC3339 UTC "Zulu" format

  Returns:
    A datetime.
  """

  # The timestamp from the Google Operations are all in RFC3339 format, but
  # they are sometimes formatted to millisconds, microseconds, sometimes
  # nanoseconds, and sometimes only seconds:
  # * 2016-11-14T23:05:56Z
  # * 2016-11-14T23:05:56.010Z
  # * 2016-11-14T23:05:56.010429Z
  # * 2016-11-14T23:05:56.010429380Z
  m = re.match(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).?(\d*)Z',
               rfc3339_utc_string)

  # It would be unexpected to get a different date format back from Google.
  # If we raise an exception here, we can break people completely.
  # Instead, let's just return None and people can report that some dates
  # are not showing up.
  # We might reconsider this approach in the future; it was originally
  # established when dates were only used for display.
  if not m:
    return None

  groups = m.groups()
  if len(groups[6]) not in (0, 3, 6, 9):
    return None

  # Create a UTC datestamp from parsed components
  # 1- Turn components 0-5 from strings to integers
  # 2- If the last component does not exist, set it to 0.
  #    If it does exist, make sure to interpret it as milliseconds.
  g = [int(val) for val in groups[:6]]

  fraction = groups[6]
  if not fraction:
    micros = 0
  elif len(fraction) == 3:
    micros = int(fraction) * 1000
  elif len(fraction) == 6:
    micros = int(fraction)
  elif len(fraction) == 9:
    # When nanoseconds are provided, we round
    micros = int(round(int(fraction) / 1000))
  else:
    assert False, 'Fraction length not 0, 6, or 9: {}'.len(fraction)

  try:
    return datetime(g[0], g[1], g[2], g[3], g[4], g[5], micros, tzinfo=pytz.utc)
  except ValueError as e:
    assert False, 'Could not parse RFC3339 datestring: {} exception: {}'.format(
        rfc3339_utc_string, e)

Converts a datestamp from RFC3339 UTC to a datetime .


def get_operation_full_job_id(op):
  """Returns the job-id or job-id.task-id for the operation."""
  job_id = op.get_field('job-id')
  task_id = op.get_field('task-id')
  if task_id:
    return '%s.%s' % (job_id, task_id)
  else:
    return job_id

def get_operation_full_job_id ( op ) : job_id = op . get_field ( 'job-id' ) task_id = op . get_field ( 'task-id' ) if task_id : return '%s.%s' % ( job_id , task_id ) else : return job_id

Returns the job - id or job - id . task - id for the operation .


def _cancel_batch(batch_fn, cancel_fn, ops):
  """Cancel a batch of operations.

  Args:
    batch_fn: API-specific batch function.
    cancel_fn: API-specific cancel function.
    ops: A list of operations to cancel.

  Returns:
    A list of operations canceled and a list of error messages.
  """

  # We define an inline callback which will populate a list of
  # successfully canceled operations as well as a list of operations
  # which were not successfully canceled.

  canceled = []
  failed = []

  def handle_cancel_response(request_id, response, exception):
    """Callback for the cancel response."""
    del response  # unused

    if exception:
      # We don't generally expect any failures here, except possibly trying
      # to cancel an operation that is already canceled or finished.
      #
      # If the operation is already finished, provide a clearer message than
      # "error 400: Bad Request".

      msg = 'error %s: %s' % (exception.resp.status, exception.resp.reason)
      if exception.resp.status == FAILED_PRECONDITION_CODE:
        detail = json.loads(exception.content)
        status = detail.get('error', {}).get('status')
        if status == FAILED_PRECONDITION_STATUS:
          msg = 'Not running'

      failed.append({'name': request_id, 'msg': msg})
    else:
      canceled.append({'name': request_id})

    return

  # Set up the batch object
  batch = batch_fn(callback=handle_cancel_response)

  # The callback gets a "request_id" which is the operation name.
  # Build a dict such that after the callback, we can lookup the operation
  # objects by name
  ops_by_name = {}
  for op in ops:
    op_name = op.get_field('internal-id')
    ops_by_name[op_name] = op
    batch.add(cancel_fn(name=op_name, body={}), request_id=op_name)

  # Cancel the operations
  batch.execute()

  # Iterate through the canceled and failed lists to build our return lists
  canceled_ops = [ops_by_name[op['name']] for op in canceled]
  error_messages = []
  for fail in failed:
    op = ops_by_name[fail['name']]
    error_messages.append("Error canceling '%s': %s" %
                          (get_operation_full_job_id(op), fail['msg']))

  return canceled_ops, error_messages

def
_cancel_batch
(
batch_fn
,
cancel_fn
,
ops
)
:
canceled
=
[
]
failed
=
[
]
def
handle_cancel_response
(
request_id
,
response
,
exception
)
:
del
response
if
exception
:
msg
=
'error %s: %s'
%
(
exception
.
resp
.
status
,
exception
.
resp
.
reason
)
if
exception
.
resp
.
status
==
FAILED_PRECONDITION_CODE
:
detail
=
json
.
loads
(
exception
.
content
)
status
=
detail
.
get
(
'error'
,
{
}
)
.
get
(
'status'
)
if
status
==
FAILED_PRECONDITION_STATUS
:
msg
=
'Not running'
failed
.
append
(
{
'name'
:
request_id
,
'msg'
:
msg
}
)
else
:
canceled
.
append
(
{
'name'
:
request_id
}
)
return
batch
=
batch_fn
(
callback
=
handle_cancel_response
)
ops_by_name
=
{
}
for
op
in
ops
:
op_name
=
op
.
get_field
(
'internal-id'
)
ops_by_name
[
op_name
]
=
op
batch
.
add
(
cancel_fn
(
name
=
op_name
,
body
=
{
}
)
,
request_id
=
op_name
)
batch
.
execute
(
)
canceled_ops
=
[
ops_by_name
[
op
[
'name'
]
]
for
op
in
canceled
]
error_messages
=
[
]
for
fail
in
failed
:
op
=
ops_by_name
[
fail
[
'name'
]
]
error_messages
.
append
(
"Error canceling '%s': %s"
%
(
get_operation_full_job_id
(
op
)
,
fail
[
'msg'
]
)
)
return
canceled_ops
,
error_messages


def cancel(batch_fn, cancel_fn, ops):
  """Cancel operations.

  Args:
    batch_fn: API-specific batch function.
    cancel_fn: API-specific cancel function.
    ops: A list of operations to cancel.

  Returns:
    A list of operations canceled and a list of error messages.
  """

  # Canceling many operations one-by-one can be slow.
  # The Pipelines API doesn't directly support a list of operations to cancel,
  # but the requests can be performed in batch.

  canceled_ops = []
  error_messages = []

  max_batch = 256
  total_ops = len(ops)
  for first_op in range(0, total_ops, max_batch):
    batch_canceled, batch_messages = _cancel_batch(
        batch_fn, cancel_fn, ops[first_op:first_op + max_batch])
    canceled_ops.extend(batch_canceled)
    error_messages.extend(batch_messages)

  return canceled_ops, error_messages


def retry_api_check(exception):
  """Return True if we should retry. False otherwise.

  Args:
    exception: An exception to test for transience.

  Returns:
    True if we should retry. False otherwise.
  """
  if isinstance(exception, apiclient.errors.HttpError):
    if exception.resp.status in TRANSIENT_HTTP_ERROR_CODES:
      _print_error('Retrying...')
      return True

  if isinstance(exception, socket.error):
    if exception.errno in TRANSIENT_SOCKET_ERROR_CODES:
      _print_error('Retrying...')
      return True

  if isinstance(exception, oauth2client.client.AccessTokenRefreshError):
    _print_error('Retrying...')
    return True

  # For a given installation, this could be a permanent error, but has only
  # been observed as transient.
  if isinstance(exception, SSLError):
    _print_error('Retrying...')
    return True

  # This has been observed as a transient error:
  #   ServerNotFoundError: Unable to find the server at genomics.googleapis.com
  if isinstance(exception, ServerNotFoundError):
    _print_error('Retrying...')
    return True

  return False

def retry_api_check ( exception ) : if isinstance ( exception , apiclient . errors . HttpError ) : if exception . resp . status in TRANSIENT_HTTP_ERROR_CODES : _print_error ( 'Retrying...' ) return True if isinstance ( exception , socket . error ) : if exception . errno in TRANSIENT_SOCKET_ERROR_CODES : _print_error ( 'Retrying...' ) return True if isinstance ( exception , oauth2client . client . AccessTokenRefreshError ) : _print_error ( 'Retrying...' ) return True if isinstance ( exception , SSLError ) : _print_error ( 'Retrying...' ) return True if isinstance ( exception , ServerNotFoundError ) : _print_error ( 'Retrying...' ) return True return False

Return True if we should retry . False otherwise .


def retry_auth_check(exception):
  """Specific check for auth error codes.

  Return True if we should retry.

  False otherwise.
  Args:
    exception: An exception to test for transience.

  Returns:
    True if we should retry. False otherwise.
  """
  if isinstance(exception, apiclient.errors.HttpError):
    if exception.resp.status in HTTP_AUTH_ERROR_CODES:
      _print_error('Retrying...')
      return True

  return False

def retry_auth_check ( exception ) : if isinstance ( exception , apiclient . errors . HttpError ) : if exception . resp . status in HTTP_AUTH_ERROR_CODES : _print_error ( 'Retrying...' ) return True return False


def setup_service(api_name, api_version, credentials=None):
  """Configures genomics API client.

  Args:
    api_name: Name of the Google API (for example: "genomics")
    api_version: Version of the API (for example: "v2alpha1")
    credentials: Credentials to be used for the gcloud API calls.

  Returns:
    A configured Google Genomics API client with appropriate credentials.
  """
  if not credentials:
    credentials = oauth2client.client.GoogleCredentials.get_application_default(
    )
  return apiclient.discovery.build(
      api_name, api_version, credentials=credentials)

def setup_service ( api_name , api_version , credentials = None ) : if not credentials : credentials = oauth2client . client . GoogleCredentials . get_application_default ( ) return apiclient . discovery . build ( api_name , api_version , credentials = credentials )


def execute(api):
    """Executes operation.

    Args:
      api: The base API object

    Returns:
       A response body object
    """
    try:
      return api.execute()
    except Exception as exception:
      now = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
      _print_error('%s: Exception %s: %s' % (now, type(exception).__name__,
                                             str(exception)))
      # Re-raise exception to be handled by retry logic
      raise exception

def
execute
(
api
)
:
try
:
return
api
.
execute
(
)
except
Exception
as
exception
:
now
=
datetime
.
now
(
)
.
strftime
(
'%Y-%m-%d %H:%M:%S.%f'
)
_print_error
(
'%s: Exception %s: %s'
%
(
now
,
type
(
exception
)
.
__name__
,
str
(
exception
)
)
)
raise
exception


def _eval_arg_type(arg_type, T=Any, arg=None, sig=None):
    """Returns a type from a snippit of python source. Should normally be
    something just like 'str' or 'Object'.

        arg_type            the source to be evaluated
        T                         the default type
        arg                     context of where this type was extracted
        sig                     context from where the arg was extracted

    Returns a type or a Type
    """
    try:
        T = eval(arg_type)
    except Exception as e:
        raise ValueError('The type of {0} could not be evaluated in {1} for {2}: {3}' \
            .format(arg_type, arg, sig, text_type(e)))
    else:
        if type(T) not in (type, Type):
            raise TypeError('{0} is not a valid type in {1} for {2}' \
                .format(repr(T), arg, sig))
        return T

Returns a type from a snippit of python source . Should normally be something just like str or Object .


def jsonify_status_code(status_code, *args, **kw):
    """Returns a jsonified response with the specified HTTP status code.

    The positional and keyword arguments are passed directly to the
    :func:`flask.jsonify` function which creates the response.
    """
    is_batch = kw.pop('is_batch', False)
    if is_batch:
        response = flask_make_response(json.dumps(*args, **kw))
        response.mimetype = 'application/json'
        response.status_code = status_code
        return response
    response = jsonify(*args, **kw)
    response.status_code = status_code
    return response

def jsonify_status_code ( status_code , * args , ** kw ) : is_batch = kw . pop ( 'is_batch' , False ) if is_batch : response = flask_make_response ( json . dumps ( * args , ** kw ) ) response . mimetype = 'application/json' response . status_code = status_code return response response = jsonify ( * args , ** kw ) response . status_code = status_code return response

Returns a jsonified response with the specified HTTP status code .


def send_payload(self, params):
        """Performs the actual sending action and returns the result
        """
        data = json.dumps({
            'jsonrpc': self.version,
            'method': self.service_name,
            'params': params,
            'id': text_type(uuid.uuid4())
        })
        data_binary = data.encode('utf-8')
        url_request = Request(self.service_url, data_binary, headers=self.headers)
        return urlopen(url_request).read()

def send_payload ( self , params ) : data = json . dumps ( { 'jsonrpc' : self . version , 'method' : self . service_name , 'params' : params , 'id' : text_type ( uuid . uuid4 ( ) ) } ) data_binary = data . encode ( 'utf-8' ) url_request = Request ( self . service_url , data_binary , headers = self . headers ) return urlopen ( url_request ) . read ( )


def json_rpc_format(self):
        """Return the Exception data in a format for JSON-RPC
        """

        error = {
            'name': text_type(self.__class__.__name__),
            'code': self.code,
            'message': '{0}'.format(text_type(self.message)),
            'data': self.data
        }

        if current_app.config['DEBUG']:
            import sys, traceback
            error['stack'] = traceback.format_exc()
            error['executable'] = sys.executable

        return error

def json_rpc_format ( self ) : error = { 'name' : text_type ( self . __class__ . __name__ ) , 'code' : self . code , 'message' : '{0}' . format ( text_type ( self . message ) ) , 'data' : self . data } if current_app . config [ 'DEBUG' ] : import sys , traceback error [ 'stack' ] = traceback . format_exc ( ) error [ 'executable' ] = sys . executable return error

Return the Exception data in a format for JSON - RPC


def from_file(cls, file):
        """Try loading given config file.

        :param str file: full path to the config file to load
        """
        if not os.path.exists(file):
            raise ValueError("Config file not found.")

        try:
            config_parser = configparser.ConfigParser()
            config_parser.read(file)

            configuration = cls(file, config_parser)
            if not configuration.check_config_sanity():
                raise ValueError("Error in config file.")
            else:
                return configuration
        except configparser.Error:
            raise ValueError("Config file is invalid.")

def
from_file
(
cls
,
file
)
:
if
not
os
.
path
.
exists
(
file
)
:
raise
ValueError
(
"Config file not found."
)
try
:
config_parser
=
configparser
.
ConfigParser
(
)
config_parser
.
read
(
file
)
configuration
=
cls
(
file
,
config_parser
)
if
not
configuration
.
check_config_sanity
(
)
:
raise
ValueError
(
"Error in config file."
)
else
:
return
configuration
except
configparser
.
Error
:
raise
ValueError
(
"Config file is invalid."
)


def discover(cls):
        """Make a guess about the config file location an try loading it."""
        file = os.path.join(Config.config_dir, Config.config_name)
        return cls.from_file(file)

def discover ( cls ) : file = os . path . join ( Config . config_dir , Config . config_name ) return cls . from_file ( file )

Make a guess about the config file location an try loading it .


def create_config(cls, cfgfile, nick, twtfile, twturl, disclose_identity, add_news):
        """Create a new config file at the default location.

        :param str cfgfile: path to the config file
        :param str nick: nickname to use for own tweets
        :param str twtfile: path to the local twtxt file
        :param str twturl: URL to the remote twtxt file
        :param bool disclose_identity: if true the users id will be disclosed
        :param bool add_news: if true follow twtxt news feed
        """
        cfgfile_dir = os.path.dirname(cfgfile)
        if not os.path.exists(cfgfile_dir):
            os.makedirs(cfgfile_dir)

        cfg = configparser.ConfigParser()

        cfg.add_section("twtxt")
        cfg.set("twtxt", "nick", nick)
        cfg.set("twtxt", "twtfile", twtfile)
        cfg.set("twtxt", "twturl", twturl)
        cfg.set("twtxt", "disclose_identity", str(disclose_identity))
        cfg.set("twtxt", "character_limit", "140")
        cfg.set("twtxt", "character_warning", "140")

        cfg.add_section("following")
        if add_news:
            cfg.set("following", "twtxt", "https://buckket.org/twtxt_news.txt")

        conf = cls(cfgfile, cfg)
        conf.write_config()
        return conf

def create_config ( cls , cfgfile , nick , twtfile , twturl , disclose_identity , add_news ) : cfgfile_dir = os . path . dirname ( cfgfile ) if not os . path . exists ( cfgfile_dir ) : os . makedirs ( cfgfile_dir ) cfg = configparser . ConfigParser ( ) cfg . add_section ( "twtxt" ) cfg . set ( "twtxt" , "nick" , nick ) cfg . set ( "twtxt" , "twtfile" , twtfile ) cfg . set ( "twtxt" , "twturl" , twturl ) cfg . set ( "twtxt" , "disclose_identity" , str ( disclose_identity ) ) cfg . set ( "twtxt" , "character_limit" , "140" ) cfg . set ( "twtxt" , "character_warning" , "140" ) cfg . add_section ( "following" ) if add_news : cfg . set ( "following" , "twtxt" , "https://buckket.org/twtxt_news.txt" ) conf = cls ( cfgfile , cfg ) conf . write_config ( ) return conf

Create a new config file at the default location .


def write_config(self):
        """Writes `self.cfg` to `self.config_file`."""
        with open(self.config_file, "w") as config_file:
            self.cfg.write(config_file)

def write_config ( self ) : with open ( self . config_file , "w" ) as config_file : self . cfg . write ( config_file )

Writes self . cfg to self . config_file .


def validate_config_key(ctx, param, value):
    """Validate a configuration key according to `section.item`."""
    if not value:
        return value

    try:
        section, item = value.split(".", 1)
    except ValueError:
        raise click.BadArgumentUsage("Given key does not contain a section name.")
    else:
        return section, item

Validate a configuration key according to section . item .


def expand_mentions(text, embed_names=True):
    """Searches the given text for mentions and expands them.

    For example:
    "@source.nick" will be expanded to "@<source.nick source.url>".
    """
    if embed_names:
        mention_format = "@<{name} {url}>"
    else:
        mention_format = "@<{url}>"

    def handle_mention(match):
        source = get_source_by_name(match.group(1))
        if source is None:
            return "@{0}".format(match.group(1))
        return mention_format.format(
            name=source.nick,
            url=source.url)

    return short_mention_re.sub(handle_mention, text)

def
expand_mentions
(
text
,
embed_names
=
True
)
:
if
embed_names
:
mention_format
=
"@<{name} {url}>"
else
:
mention_format
=
"@<{url}>"
def
handle_mention
(
match
)
:
source
=
get_source_by_name
(
match
.
group
(
1
)
)
if
source
is
:
return
"@{0}"
.
format
(
match
.
group
(
1
)
)
return
mention_format
.
format
(
name
=
source
.
nick
,
url
=
source
.
url
)
return
short_mention_re
.
sub
(
handle_mention
,
text
)

Searches the given text for mentions and expands them .


def make_aware(dt):
    """Appends tzinfo and assumes UTC, if datetime object has no tzinfo already."""
    return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)

def make_aware ( dt ) : return dt if dt . tzinfo else dt . replace ( tzinfo = timezone . utc )

Appends tzinfo and assumes UTC if datetime object has no tzinfo already .


def from_file(cls, file, *args, **kwargs):
        """Try loading given cache file."""
        try:
            cache = shelve.open(file)
            return cls(file, cache, *args, **kwargs)
        except OSError as e:
            logger.debug("Loading {0} failed".format(file))
            raise e

def
from_file
(
cls
,
file
,
*
args
,
**
kwargs
)
:
try
:
cache
=
shelve
.
open
(
file
)
return
cls
(
file
,
cache
,
*
args
,
**
kwargs
)
except
OSError
as
e
:
logger
.
debug
(
"Loading {0} failed"
.
format
(
file
)
)
raise
e


def discover(cls, *args, **kwargs):
        """Make a guess about the cache file location an try loading it."""
        file = os.path.join(Cache.cache_dir, Cache.cache_name)
        return cls.from_file(file, *args, **kwargs)

def discover ( cls , * args , ** kwargs ) : file = os . path . join ( Cache . cache_dir , Cache . cache_name ) return cls . from_file ( file , * args , ** kwargs )

Make a guess about the cache file location an try loading it .


def is_cached(self, url):
        """Checks if specified URL is cached."""
        try:
            return True if url in self.cache else False
        except TypeError:
            return False

def is_cached ( self , url ) : try : return True if url in self . cache else False except TypeError : return False


def add_tweets(self, url, last_modified, tweets):
        """Adds new tweets to the cache."""
        try:
            self.cache[url] = {"last_modified": last_modified, "tweets": tweets}
            self.mark_updated()
            return True
        except TypeError:
            return False

def add_tweets ( self , url , last_modified , tweets ) : try : self . cache [ url ] = { "last_modified" : last_modified , "tweets" : tweets } self . mark_updated ( ) return True except TypeError : return False


def get_tweets(self, url, limit=None):
        """Retrieves tweets from the cache."""
        try:
            tweets = self.cache[url]["tweets"]
            self.mark_updated()
            return sorted(tweets, reverse=True)[:limit]
        except KeyError:
            return []

def get_tweets ( self , url , limit = None ) : try : tweets = self . cache [ url ] [ "tweets" ] self . mark_updated ( ) return sorted ( tweets , reverse = True ) [ : limit ] except KeyError : return [ ]


def remove_tweets(self, url):
        """Tries to remove cached tweets."""
        try:
            del self.cache[url]
            self.mark_updated()
            return True
        except KeyError:
            return False

def remove_tweets ( self , url ) : try : del self . cache [ url ] self . mark_updated ( ) return True except KeyError : return False


def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, cache, force_update):
    """Retrieve your personal timeline."""
    if source:
        source_obj = ctx.obj["conf"].get_source_by_nick(source)
        if not source_obj:
            logger.debug("Not following {0}, trying as URL".format(source))
            source_obj = Source(source, source)
        sources = [source_obj]
    else:
        sources = ctx.obj["conf"].following

    tweets = []

    if cache:
        try:
            with Cache.discover(update_interval=ctx.obj["conf"].timeline_update_interval) as cache:
                force_update = force_update or not cache.is_valid
                if force_update:
                    tweets = get_remote_tweets(sources, limit, timeout, cache)
                else:
                    logger.debug("Multiple calls to 'timeline' within {0} seconds. Skipping update".format(
                        cache.update_interval))
                    # Behold, almighty list comprehensions! (I might have gone overboard here…)
                    tweets = list(chain.from_iterable([cache.get_tweets(source.url) for source in sources]))
        except OSError as e:
            logger.debug(e)
            tweets = get_remote_tweets(sources, limit, timeout)
    else:
        tweets = get_remote_tweets(sources, limit, timeout)

    if twtfile and not source:
        source = Source(ctx.obj["conf"].nick, ctx.obj["conf"].twturl, file=twtfile)
        tweets.extend(get_local_tweets(source, limit))

    if not tweets:
        return

    tweets = sort_and_truncate_tweets(tweets, sorting, limit)

    if pager:
        click.echo_via_pager(style_timeline(tweets, porcelain))
    else:
        click.echo(style_timeline(tweets, porcelain))


def config(ctx, key, value, remove, edit):
    """Get or set config item."""
    conf = ctx.obj["conf"]

    if not edit and not key:
        raise click.BadArgumentUsage("You have to specify either a key or use --edit.")

    if edit:
        return click.edit(filename=conf.config_file)

    if remove:
        try:
            conf.cfg.remove_option(key[0], key[1])
        except Exception as e:
            logger.debug(e)
        else:
            conf.write_config()
        return

    if not value:
        try:
            click.echo(conf.cfg.get(key[0], key[1]))
        except Exception as e:
            logger.debug(e)
        return

    if not conf.cfg.has_section(key[0]):
        conf.cfg.add_section(key[0])

    conf.cfg.set(key[0], key[1], value)
    conf.write_config()


def relative_datetime(self):
        """Return human-readable relative time string."""
        now = datetime.now(timezone.utc)
        tense = "from now" if self.created_at > now else "ago"
        return "{0} {1}".format(humanize.naturaldelta(now - self.created_at), tense)

def
relative_datetime
(
self
)
:
now
=
datetime
.
now
(
timezone
.
utc
)
tense
=
"from now"
if
self
.
created_at
>
now
else
"ago"
return
"{0} {1}"
.
format
(
humanize
.
naturaldelta
(
now
self
.
created_at
)
,
tense
)

Return human - readable relative time string .


def save(url, *args, **kwargs):
    """ Parse the options, set defaults and then fire up PhantomJS. """

    device = heimdallDevice(kwargs.get('device', None))

    kwargs['width'] = kwargs.get('width', None) or device.width
    kwargs['height'] = kwargs.get('height', None) or device.height
    kwargs['user_agent'] = kwargs.get('user_agent', None) or device.user_agent

    screenshot_image = screenshot(url, **kwargs)

    if kwargs.get('optimize'):
        image = Image.open(screenshot_image.path)
        image.save(screenshot_image.path, optimize=True)

    return screenshot_image

def save ( url , * args , ** kwargs ) : device = heimdallDevice ( kwargs . get ( 'device' , None ) ) kwargs [ 'width' ] = kwargs . get ( 'width' , None ) or device . width kwargs [ 'height' ] = kwargs . get ( 'height' , None ) or device . height kwargs [ 'user_agent' ] = kwargs . get ( 'user_agent' , None ) or device . user_agent screenshot_image = screenshot ( url , ** kwargs ) if kwargs . get ( 'optimize' ) : image = Image . open ( screenshot_image . path ) image . save ( screenshot_image . path , optimize = True ) return screenshot_image

Parse the options set defaults and then fire up PhantomJS .


def screenshot(url, *args, **kwargs):
    """ Call PhantomJS with the specified flags and options. """

    phantomscript = os.path.join(os.path.dirname(__file__),
                                 'take_screenshot.js')

    directory = kwargs.get('save_dir', '/tmp')
    image_name = kwargs.get('image_name', None) or _image_name_from_url(url)
    ext = kwargs.get('format', 'png').lower()
    save_path = os.path.join(directory, image_name) + '.' + ext
    crop_to_visible = kwargs.get('crop_to_visible', False)

    cmd_args = [
        'phantomjs',
        '--ssl-protocol=any',
        phantomscript,
        url,
        '--width',
        str(kwargs['width']),
        '--height',
        str(kwargs['height']),
        '--useragent',
        str(kwargs['user_agent']),
        '--dir',
        directory,
        '--ext',
        ext,
        '--name',
        str(image_name),
    ]
    if crop_to_visible:
        cmd_args.append('--croptovisible')

    # TODO:
    # - quality
    # - renderafter
    # - maxexecutiontime
    # - resourcetimeout

    output = subprocess.Popen(cmd_args,
                              stdout=subprocess.PIPE).communicate()[0]

    return Screenshot(save_path, directory, image_name + '.' + ext, ext)

def screenshot ( url , * args , ** kwargs ) : phantomscript = os . path . join ( os . path . dirname ( __file__ ) , 'take_screenshot.js' ) directory = kwargs . get ( 'save_dir' , '/tmp' ) image_name = kwargs . get ( 'image_name' , None ) or _image_name_from_url ( url ) ext = kwargs . get ( 'format' , 'png' ) . lower ( ) save_path = os . path . join ( directory , image_name ) + '.' + ext crop_to_visible = kwargs . get ( 'crop_to_visible' , False ) cmd_args = [ 'phantomjs' , '--ssl-protocol=any' , phantomscript , url , '--width' , str ( kwargs [ 'width' ] ) , '--height' , str ( kwargs [ 'height' ] ) , '--useragent' , str ( kwargs [ 'user_agent' ] ) , '--dir' , directory , '--ext' , ext , '--name' , str ( image_name ) , ] if crop_to_visible : cmd_args . append ( '--croptovisible' ) output = subprocess . Popen ( cmd_args , stdout = subprocess . PIPE ) . communicate ( ) [ 0 ] return Screenshot ( save_path , directory , image_name + '.' + ext , ext )

Call PhantomJS with the specified flags and options .


def _image_name_from_url(url):
    """ Create a nice image name from the url. """

    find = r'https?://|[^\w]'
    replace = '_'
    return re.sub(find, replace, url).strip('_')

def _image_name_from_url ( url ) : find = r'https?://|[^\w]' replace = '_' return re . sub ( find , replace , url ) . strip ( '_' )


def worker(f):
    """
    Decorator. Abortable worker. If wrapped task will be cancelled by
    dispatcher, decorator will send ftp codes of successful interrupt.

    ::

        >>> @worker
        ... async def worker(self, connection, rest):
        ...     ...

    """
    @functools.wraps(f)
    async def wrapper(cls, connection, rest):
        try:
            await f(cls, connection, rest)
        except asyncio.CancelledError:
            connection.response("426", "transfer aborted")
            connection.response("226", "abort successful")

    return wrapper

def
worker
(
f
)
:
@
functools
.
wraps
(
f
)
async
def
wrapper
(
cls
,
connection
,
rest
)
:
try
:
await
f
(
cls
,
connection
,
rest
)
except
asyncio
.
CancelledError
:
connection
.
response
(
"426"
,
"transfer aborted"
)
connection
.
response
(
"226"
,
"abort successful"
)
return
wrapper

Decorator . Abortable worker . If wrapped task will be cancelled by dispatcher decorator will send ftp codes of successful interrupt .


def get_permissions(self, path):
        """
        Return nearest parent permission for `path`.

        :param path: path which permission you want to know
        :type path: :py:class:`str` or :py:class:`pathlib.PurePosixPath`

        :rtype: :py:class:`aioftp.Permission`
        """
        path = pathlib.PurePosixPath(path)
        parents = filter(lambda p: p.is_parent(path), self.permissions)
        perm = min(
            parents,
            key=lambda p: len(path.relative_to(p.path).parts),
            default=Permission(),
        )
        return perm

def get_permissions ( self , path ) : path = pathlib . PurePosixPath ( path ) parents = filter ( lambda p : p . is_parent ( path ) , self . permissions ) perm = min ( parents , key = lambda p : len ( path . relative_to ( p . path ) . parts ) , default = Permission ( ) , ) return perm

Return nearest parent permission for path .


def release(self):
        """
        Release, incrementing the internal counter by one.
        """
        if self.value is not None:
            self.value += 1
            if self.value > self.maximum_value:
                raise ValueError("Too many releases")

def
release
(
self
)
:
if
self
.
value
is
not
:
self
.
value
+=
1
if
self
.
value
>
self
.
maximum_value
:
raise
ValueError
(
"Too many releases"
)

Release incrementing the internal counter by one .


def register_memory():
    """Register an approximation of memory used by FTP server process
    and all of its children.
    """
    # XXX How to get a reliable representation of memory being used is
    # not clear. (rss - shared) seems kind of ok but we might also use
    # the private working set via get_memory_maps().private*.
    def get_mem(proc):
        if os.name == 'posix':
            mem = proc.memory_info_ex()
            counter = mem.rss
            if 'shared' in mem._fields:
                counter -= mem.shared
            return counter
        else:
            # TODO figure out what to do on Windows
            return proc.get_memory_info().rss

    if SERVER_PROC is not None:
        mem = get_mem(SERVER_PROC)
        for child in SERVER_PROC.children():
            mem += get_mem(child)
        server_memory.append(bytes2human(mem))

def register_memory ( ) : def get_mem ( proc ) : if os . name == 'posix' : mem = proc . memory_info_ex ( ) counter = mem . rss if 'shared' in mem . _fields : counter -= mem . shared return counter else : return proc . get_memory_info ( ) . rss if SERVER_PROC is not None : mem = get_mem ( SERVER_PROC ) for child in SERVER_PROC . children ( ) : mem += get_mem ( child ) server_memory . append ( bytes2human ( mem ) )

Register an approximation of memory used by FTP server process and all of its children .


def connect():
    """Connect to FTP server, login and return an ftplib.FTP instance."""
    ftp_class = ftplib.FTP if not SSL else ftplib.FTP_TLS
    ftp = ftp_class(timeout=TIMEOUT)
    ftp.connect(HOST, PORT)
    ftp.login(USER, PASSWORD)
    if SSL:
        ftp.prot_p()  # secure data connection
    return ftp

def connect ( ) : ftp_class = ftplib . FTP if not SSL else ftplib . FTP_TLS ftp = ftp_class ( timeout = TIMEOUT ) ftp . connect ( HOST , PORT ) ftp . login ( USER , PASSWORD ) if SSL : ftp . prot_p ( ) return ftp

Connect to FTP server login and return an ftplib . FTP instance .


def bytes_per_second(ftp, retr=True):
    """Return the number of bytes transmitted in 1 second."""
    tot_bytes = 0
    if retr:
        def request_file():
            ftp.voidcmd('TYPE I')
            conn = ftp.transfercmd("retr " + TESTFN)
            return conn

        with contextlib.closing(request_file()) as conn:
            register_memory()
            stop_at = time.time() + 1.0
            while stop_at > time.time():
                chunk = conn.recv(BUFFER_LEN)
                if not chunk:
                    a = time.time()
                    ftp.voidresp()
                    conn.close()
                    conn = request_file()
                    stop_at += time.time() - a
                tot_bytes += len(chunk)

        try:
            while chunk:
                chunk = conn.recv(BUFFER_LEN)
            ftp.voidresp()
            conn.close()
        except (ftplib.error_temp, ftplib.error_perm):
            pass
    else:
        ftp.voidcmd('TYPE I')
        with contextlib.closing(ftp.transfercmd("STOR " + TESTFN)) as conn:
            register_memory()
            chunk = b'x' * BUFFER_LEN
            stop_at = time.time() + 1
            while stop_at > time.time():
                tot_bytes += conn.send(chunk)
        ftp.voidresp()

    return tot_bytes

def
bytes_per_second
(
ftp
,
retr
=
True
)
:
tot_bytes
=
0
if
retr
:
def
request_file
(
)
:
ftp
.
voidcmd
(
'TYPE I'
)
conn
=
ftp
.
transfercmd
(
"retr "
+
TESTFN
)
return
conn
with
contextlib
.
closing
(
request_file
(
)
)
as
conn
:
register_memory
(
)
stop_at
=
time
.
time
(
)
+
1.0
while
stop_at
>
time
.
time
(
)
:
chunk
=
conn
.
recv
(
BUFFER_LEN
)
if
not
chunk
:
a
=
time
.
time
(
)
ftp
.
voidresp
(
)
conn
.
close
(
)
conn
=
request_file
(
)
stop_at
+=
time
.
time
(
)
a
tot_bytes
+=
len
(
chunk
)
try
:
while
chunk
:
chunk
=
conn
.
recv
(
BUFFER_LEN
)
ftp
.
voidresp
(
)
conn
.
close
(
)
except
(
ftplib
.
error_temp
,
ftplib
.
error_perm
)
:
pass
else
:
ftp
.
voidcmd
(
'TYPE I'
)
with
contextlib
.
closing
(
ftp
.
transfercmd
(
"STOR "
+
TESTFN
)
)
as
conn
:
register_memory
(
)
chunk
=
b'x'
*
BUFFER_LEN
stop_at
=
time
.
time
(
)
+
1
while
stop_at
>
time
.
time
(
)
:
tot_bytes
+=
conn
.
send
(
chunk
)
ftp
.
voidresp
(
)
return
tot_bytes

Return the number of bytes transmitted in 1 second .


def async_enterable(f):
    """
    Decorator. Bring coroutine result up, so it can be used as async context

    ::

        >>> async def foo():
        ...
        ...     ...
        ...     return AsyncContextInstance(...)
        ...
        ... ctx = await foo()
        ... async with ctx:
        ...
        ...     # do

    ::

        >>> @async_enterable
        ... async def foo():
        ...
        ...     ...
        ...     return AsyncContextInstance(...)
        ...
        ... async with foo() as ctx:
        ...
        ...     # do
        ...
        ... ctx = await foo()
        ... async with ctx:
        ...
        ...     # do

    """
    @functools.wraps(f)
    def wrapper(*args, **kwargs):

        class AsyncEnterableInstance:

            async def __aenter__(self):
                self.context = await f(*args, **kwargs)
                return await self.context.__aenter__()

            async def __aexit__(self, *args, **kwargs):
                await self.context.__aexit__(*args, **kwargs)

            def __await__(self):
                return f(*args, **kwargs).__await__()

        return AsyncEnterableInstance()

    return wrapper

def async_enterable ( f ) : @ functools . wraps ( f ) def wrapper ( * args , ** kwargs ) : class AsyncEnterableInstance : async def __aenter__ ( self ) : self . context = await f ( * args , ** kwargs ) return await self . context . __aenter__ ( ) async def __aexit__ ( self , * args , ** kwargs ) : await self . context . __aexit__ ( * args , ** kwargs ) def __await__ ( self ) : return f ( * args , ** kwargs ) . __await__ ( ) return AsyncEnterableInstance ( ) return wrapper

Decorator . Bring coroutine result up so it can be used as async context


def setlocale(name):
    """
    Context manager with threading lock for set locale on enter, and set it
    back to original state on exit.

    ::

        >>> with setlocale("C"):
        ...     ...
    """
    with LOCALE_LOCK:
        old_locale = locale.setlocale(locale.LC_ALL)
        try:
            yield locale.setlocale(locale.LC_ALL, name)
        finally:
            locale.setlocale(locale.LC_ALL, old_locale)

def setlocale ( name ) : with LOCALE_LOCK : old_locale = locale . setlocale ( locale . LC_ALL ) try : yield locale . setlocale ( locale . LC_ALL , name ) finally : locale . setlocale ( locale . LC_ALL , old_locale )

Context manager with threading lock for set locale on enter and set it back to original state on exit .


def append(self, data, start):
        """
        Count `data` for throttle

        :param data: bytes of data for count
        :type data: :py:class:`bytes`

        :param start: start of read/write time from
            :py:meth:`asyncio.BaseEventLoop.time`
        :type start: :py:class:`float`
        """
        if self._limit is not None and self._limit > 0:
            if self._start is None:
                self._start = start
            if start - self._start > self.reset_rate:
                self._sum -= round((start - self._start) * self._limit)
                self._start = start
            self._sum += len(data)

def append ( self , data , start ) : if self . _limit is not None and self . _limit > 0 : if self . _start is None : self . _start = start if start - self . _start > self . reset_rate : self . _sum -= round ( ( start - self . _start ) * self . _limit ) self . _start = start self . _sum += len ( data )


def limit(self, value):
        """
        Set throttle limit

        :param value: bytes per second
        :type value: :py:class:`int` or :py:class:`None`
        """
        self._limit = value
        self._start = None
        self._sum = 0

def limit ( self , value ) : self . _limit = value self . _start = None self . _sum = 0


def clone(self):
        """
        Clone throttles without memory
        """
        return StreamThrottle(
            read=self.read.clone(),
            write=self.write.clone()
        )

def clone ( self ) : return StreamThrottle ( read = self . read . clone ( ) , write = self . write . clone ( ) )


def append(self, name, data, start):
        """
        Update timeout for all throttles

        :param name: name of throttle to append to ("read" or "write")
        :type name: :py:class:`str`

        :param data: bytes of data for count
        :type data: :py:class:`bytes`

        :param start: start of read/write time from
            :py:meth:`asyncio.BaseEventLoop.time`
        :type start: :py:class:`float`
        """
        for throttle in self.throttles.values():
            getattr(throttle, name).append(data, start)

def append ( self , name , data , start ) : for throttle in self . throttles . values ( ) : getattr ( throttle , name ) . append ( data , start )


def check_codes(self, expected_codes, received_code, info):
        """
        Checks if any of expected matches received.

        :param expected_codes: tuple of expected codes
        :type expected_codes: :py:class:`tuple`

        :param received_code: received code for matching
        :type received_code: :py:class:`aioftp.Code`

        :param info: list of response lines from server
        :type info: :py:class:`list`

        :raises aioftp.StatusCodeError: if received code does not matches any
            expected code
        """
        if not any(map(received_code.matches, expected_codes)):
            raise errors.StatusCodeError(expected_codes, received_code, info)

def check_codes ( self , expected_codes , received_code , info ) : if not any ( map ( received_code . matches , expected_codes ) ) : raise errors . StatusCodeError ( expected_codes , received_code , info )

Checks if any of expected matches received .


def parse_directory_response(s):
        """
        Parsing directory server response.

        :param s: response line
        :type s: :py:class:`str`

        :rtype: :py:class:`pathlib.PurePosixPath`
        """
        seq_quotes = 0
        start = False
        directory = ""
        for ch in s:
            if not start:
                if ch == "\"":
                    start = True
            else:
                if ch == "\"":
                    seq_quotes += 1
                else:
                    if seq_quotes == 1:
                        break
                    elif seq_quotes == 2:
                        seq_quotes = 0
                        directory += '"'
                    directory += ch
        return pathlib.PurePosixPath(directory)

def parse_directory_response ( s ) : seq_quotes = 0 start = False directory = "" for ch in s : if not start : if ch == "\"" : start = True else : if ch == "\"" : seq_quotes += 1 else : if seq_quotes == 1 : break elif seq_quotes == 2 : seq_quotes = 0 directory += '"' directory += ch return pathlib . PurePosixPath ( directory )


def parse_list_line_windows(self, b):
        """
        Parsing Microsoft Windows `dir` output

        :param b: response line
        :type b: :py:class:`bytes` or :py:class:`str`

        :return: (path, info)
        :rtype: (:py:class:`pathlib.PurePosixPath`, :py:class:`dict`)
        """
        line = b.decode(encoding=self.encoding).rstrip("\r\n")
        date_time_end = line.index("M")
        date_time_str = line[:date_time_end + 1].strip().split(" ")
        date_time_str = " ".join([x for x in date_time_str if len(x) > 0])
        line = line[date_time_end + 1:].lstrip()
        with setlocale("C"):
            strptime = datetime.datetime.strptime
            date_time = strptime(date_time_str, "%m/%d/%Y %I:%M %p")
        info = {}
        info["modify"] = self.format_date_time(date_time)
        next_space = line.index(" ")
        if line.startswith("<DIR>"):
            info["type"] = "dir"
        else:
            info["type"] = "file"
            info["size"] = line[:next_space].replace(",", "")
            if not info["size"].isdigit():
                raise ValueError
        # This here could cause a problem if a filename started with
        # whitespace, but if we were to try to detect such a condition
        # we would have to make strong assumptions about the input format
        filename = line[next_space:].lstrip()
        if filename == "." or filename == "..":
            raise ValueError
        return pathlib.PurePosixPath(filename), info

def
parse_list_line_windows
(
self
,
b
)
:
line
=
b
.
decode
(
encoding
=
self
.
encoding
)
.
rstrip
(
"\r\n"
)
date_time_end
=
line
.
index
(
"M"
)
date_time_str
=
line
[
:
date_time_end
+
1
]
.
strip
(
)
.
split
(
" "
)
date_time_str
=
" "
.
join
(
[
x
for
x
in
date_time_str
if
len
(
x
)
>
0
]
)
line
=
line
[
date_time_end
+
1
:
]
.
lstrip
(
)
with
setlocale
(
"C"
)
:
strptime
=
datetime
.
datetime
.
strptime
date_time
=
strptime
(
date_time_str
,
"%m/%d/%Y %I:%M %p"
)
info
=
{
}
info
[
"modify"
]
=
self
.
format_date_time
(
date_time
)
next_space
=
line
.
index
(
" "
)
if
line
.
startswith
(
"<DIR>"
)
:
info
[
"type"
]
=
"dir"
else
:
info
[
"type"
]
=
"file"
info
[
"size"
]
=
line
[
:
next_space
]
.
replace
(
","
,
""
)
if
not
info
[
"size"
]
.
isdigit
(
)
:
raise
ValueError
filename
=
line
[
next_space
:
]
.
lstrip
(
)
if
filename
==
"."
or
filename
==
".."
:
raise
ValueError
return
pathlib
.
PurePosixPath
(
filename
)
,
info


def upload_stream(self, destination, *, offset=0):
        """
        Create stream for write data to `destination` file.

        :param destination: destination path of file on server side
        :type destination: :py:class:`str` or :py:class:`pathlib.PurePosixPath`

        :param offset: byte offset for stream start position
        :type offset: :py:class:`int`

        :rtype: :py:class:`aioftp.DataConnectionThrottleStreamIO`
        """
        return self.get_stream(
            "STOR " + str(destination),
            "1xx",
            offset=offset,
        )

def
upload_stream
(
self
,
destination
,
*
,
offset
=
0
)
:
return
self
.
get_stream
(
"STOR "
+
str
(
destination
)
,
"1xx"
,
offset
=
offset
,
)

Create stream for write data to destination file .


def jenks_breaks(values, nb_class):
    """
    Compute jenks natural breaks on a sequence of `values`, given `nb_class`,
    the number of desired class.

    Parameters
    ----------
    values : array-like
        The Iterable sequence of numbers (integer/float) to be used.
    nb_class : int
        The desired number of class (as some other functions requests
        a `k` value, `nb_class` is like `k` + 1). Have to be lesser than
        the length of `values` and greater than 2.

    Returns
    -------
    breaks : tuple of floats
        The computed break values, including minimum and maximum, in order
        to have all the bounds for building `nb_class` class,
        so the returned tuple has a length of `nb_class` + 1.

    Examples
    --------
    Using nb_class = 3, expecting 4 break values , including min and max :

    >>> jenks_breaks(
            [1.3, 7.1, 7.3, 2.3, 3.9, 4.1, 7.8, 1.2, 4.3, 7.3, 5.0, 4.3],
            nb_class = 3)  # Should output (1.2, 2.3, 5.0, 7.8)

    """

    if not isinstance(values, Iterable) or isinstance(values, (str, bytes)):
        raise TypeError("A sequence of numbers is expected")
    if isinstance(nb_class, float) and int(nb_class) == nb_class:
        nb_class = int(nb_class)
    if not isinstance(nb_class, int):
        raise TypeError(
            "Number of class have to be a positive integer: "
            "expected an instance of 'int' but found {}"
            .format(type(nb_class)))

    nb_values = len(values)
    if np and isinstance(values, np.ndarray):
        values = values[np.argwhere(np.isfinite(values)).reshape(-1)]
    else:
        values = [i for i in values if isfinite(i)]

    if len(values) != nb_values:
        warnings.warn('Invalid values encountered (NaN or Inf) were ignored')
        nb_values = len(values)

    if nb_class >= nb_values or nb_class < 2:
        raise ValueError("Number of class have to be an integer "
                         "greater than 2 and "
                         "smaller than the number of values to use")

    return jenks._jenks_breaks(values, nb_class)

Compute jenks natural breaks on a sequence of values given nb_class the number of desired class .


def grab(self, bbox=None):
        """Grabs an image directly to a buffer.

        :param bbox: Optional tuple or list containing (x1, y1, x2, y2) coordinates
            of sub-region to capture.
        :return: PIL RGB image
        :raises: ValueError, if image data does not have 3 channels (RGB), each with 8
            bits.
        :rtype: Image
        """
        w = Gdk.get_default_root_window()
        if bbox is not None:
            g = [bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1]]
        else:
            g = w.get_geometry()
        pb = Gdk.pixbuf_get_from_window(w, *g)
        if pb.get_bits_per_sample() != 8:
            raise ValueError('Expected 8 bits per pixel.')
        elif pb.get_n_channels() != 3:
            raise ValueError('Expected RGB image.')

        # Read the entire buffer into a python bytes object.
        # read_pixel_bytes: New in version 2.32.
        pixel_bytes = pb.read_pixel_bytes().get_data()  # type: bytes
        width, height = g[2], g[3]

        # Probably for SSE alignment reasons, the pixbuf has extra data in each line.
        # The args after "raw" help handle this; see
        # http://effbot.org/imagingbook/decoder.htm#the-raw-decoder
        return Image.frombytes(
            'RGB', (width, height), pixel_bytes, 'raw', 'RGB', pb.get_rowstride(), 1)

def
grab
(
self
,
bbox
=
)
:
w
=
Gdk
.
get_default_root_window
(
)
if
bbox
is
not
:
g
=
[
bbox
[
0
]
,
bbox
[
1
]
,
bbox
[
2
]
bbox
[
0
]
,
bbox
[
3
]
bbox
[
1
]
]
else
:
g
=
w
.
get_geometry
(
)
pb
=
Gdk
.
pixbuf_get_from_window
(
w
,
*
g
)
if
pb
.
get_bits_per_sample
(
)
!=
8
:
raise
ValueError
(
'Expected 8 bits per pixel.'
)
elif
pb
.
get_n_channels
(
)
!=
3
:
raise
ValueError
(
'Expected RGB image.'
)
pixel_bytes
=
pb
.
read_pixel_bytes
(
)
.
get_data
(
)
width
,
height
=
g
[
2
]
,
g
[
3
]
return
Image
.
frombytes
(
'RGB'
,
(
width
,
height
)
,
pixel_bytes
,
'raw'
,
'RGB'
,
pb
.
get_rowstride
(
)
,
1
)


def grab(bbox=None, childprocess=None, backend=None):
    """Copy the contents of the screen to PIL image memory.

    :param bbox: optional bounding box (x1,y1,x2,y2)
    :param childprocess: pyscreenshot can cause an error,
            if it is used on more different virtual displays
            and back-end is not in different process.
            Some back-ends are always different processes: scrot, imagemagick
            The default is False if the program was started inside IDLE,
            otherwise it is True.
    :param backend: back-end can be forced if set (examples:scrot, wx,..),
                    otherwise back-end is automatic
    """
    if childprocess is None:
        childprocess = childprocess_default_value()
    return _grab(
        to_file=False, childprocess=childprocess, backend=backend, bbox=bbox)

def grab ( bbox = None , childprocess = None , backend = None ) : if childprocess is None : childprocess = childprocess_default_value ( ) return _grab ( to_file = False , childprocess = childprocess , backend = backend , bbox = bbox )

Copy the contents of the screen to PIL image memory .


def backend_version(backend, childprocess=None):
    """Back-end version.

    :param backend: back-end (examples:scrot, wx,..)
    :param childprocess: see :py:func:`grab`
    :return: version as string
    """
    if childprocess is None:
        childprocess = childprocess_default_value()
    if not childprocess:
        return _backend_version(backend)
    else:
        return run_in_childprocess(_backend_version, None, backend)

def backend_version ( backend , childprocess = None ) : if childprocess is None : childprocess = childprocess_default_value ( ) if not childprocess : return _backend_version ( backend ) else : return run_in_childprocess ( _backend_version , None , backend )


def open(
    config, mode="continue", zoom=None, bounds=None, single_input_file=None,
    with_cache=False, debug=False
):
    """
    Open a Mapchete process.

    Parameters
    ----------
    config : MapcheteConfig object, config dict or path to mapchete file
        Mapchete process configuration
    mode : string
        * ``memory``: Generate process output on demand without reading
          pre-existing data or writing new data.
        * ``readonly``: Just read data without processing new data.
        * ``continue``: (default) Don't overwrite existing output.
        * ``overwrite``: Overwrite existing output.
    zoom : list or integer
        process zoom level or a pair of minimum and maximum zoom level
    bounds : tuple
        left, bottom, right, top process boundaries in output pyramid
    single_input_file : string
        single input file if supported by process
    with_cache : bool
        process output data cached in memory

    Returns
    -------
    Mapchete
        a Mapchete process object
    """
    return Mapchete(
        MapcheteConfig(
            config, mode=mode, zoom=zoom, bounds=bounds,
            single_input_file=single_input_file, debug=debug),
        with_cache=with_cache)

def open ( config , mode = "continue" , zoom = None , bounds = None , single_input_file = None , with_cache = False , debug = False ) : return Mapchete ( MapcheteConfig ( config , mode = mode , zoom = zoom , bounds = bounds , single_input_file = single_input_file , debug = debug ) , with_cache = with_cache )


def _get_zoom_level(zoom, process):
    """Determine zoom levels."""
    if zoom is None:
        return reversed(process.config.zoom_levels)
    if isinstance(zoom, int):
        return [zoom]
    elif len(zoom) == 2:
        return reversed(range(min(zoom), max(zoom)+1))
    elif len(zoom) == 1:
        return zoom

def _get_zoom_level ( zoom , process ) : if zoom is None : return reversed ( process . config . zoom_levels ) if isinstance ( zoom , int ) : return [ zoom ] elif len ( zoom ) == 2 : return reversed ( range ( min ( zoom ) , max ( zoom ) + 1 ) ) elif len ( zoom ) == 1 : return zoom


def _process_worker(process, process_tile):
    """Worker function running the process."""
    logger.debug((process_tile.id, "running on %s" % current_process().name))

    # skip execution if overwrite is disabled and tile exists
    if (
        process.config.mode == "continue" and
        process.config.output.tiles_exist(process_tile)
    ):
        logger.debug((process_tile.id, "tile exists, skipping"))
        return ProcessInfo(
            tile=process_tile,
            processed=False,
            process_msg="output already exists",
            written=False,
            write_msg="nothing written"
        )

    # execute on process tile
    else:
        with Timer() as t:
            try:
                output = process.execute(process_tile, raise_nodata=True)
            except MapcheteNodataTile:
                output = None
        processor_message = "processed in %s" % t
        logger.debug((process_tile.id, processor_message))
        writer_info = process.write(process_tile, output)
        return ProcessInfo(
            tile=process_tile,
            processed=True,
            process_msg=processor_message,
            written=writer_info.written,
            write_msg=writer_info.write_msg
        )

def
_process_worker
(
process
,
process_tile
)
:
logger
.
debug
(
(
process_tile
.
id
,
"running on %s"
%
current_process
(
)
.
name
)
)
if
(
process
.
config
.
mode
==
"continue"
and
process
.
config
.
output
.
tiles_exist
(
process_tile
)
)
:
logger
.
debug
(
(
process_tile
.
id
,
"tile exists, skipping"
)
)
return
ProcessInfo
(
tile
=
process_tile
,
processed
=
False
,
process_msg
=
"output already exists"
,
written
=
False
,
write_msg
=
"nothing written"
)
else
:
with
Timer
(
)
as
t
:
try
:
output
=
process
.
execute
(
process_tile
,
raise_nodata
=
True
)
except
MapcheteNodataTile
:
output
=
processor_message
=
"processed in %s"
%
t
logger
.
debug
(
(
process_tile
.
id
,
processor_message
)
)
writer_info
=
process
.
write
(
process_tile
,
output
)
return
ProcessInfo
(
tile
=
process_tile
,
processed
=
True
,
process_msg
=
processor_message
,
written
=
writer_info
.
written
,
write_msg
=
writer_info
.
write_msg
)


def get_process_tiles(self, zoom=None):
        """
        Yield process tiles.

        Tiles intersecting with the input data bounding boxes as well as
        process bounds, if provided, are considered process tiles. This is to
        avoid iterating through empty tiles.

        Parameters
        ----------
        zoom : integer
            zoom level process tiles should be returned from; if none is given,
            return all process tiles

        yields
        ------
        BufferedTile objects
        """
        if zoom or zoom == 0:
            for tile in self.config.process_pyramid.tiles_from_geom(
                self.config.area_at_zoom(zoom), zoom
            ):
                yield tile
        else:
            for zoom in reversed(self.config.zoom_levels):
                for tile in self.config.process_pyramid.tiles_from_geom(
                    self.config.area_at_zoom(zoom), zoom
                ):
                    yield tile

def get_process_tiles ( self , zoom = None ) : if zoom or zoom == 0 : for tile in self . config . process_pyramid . tiles_from_geom ( self . config . area_at_zoom ( zoom ) , zoom ) : yield tile else : for zoom in reversed ( self . config . zoom_levels ) : for tile in self . config . process_pyramid . tiles_from_geom ( self . config . area_at_zoom ( zoom ) , zoom ) : yield tile


def batch_process(
        self, zoom=None, tile=None, multi=cpu_count(), max_chunksize=1
    ):
        """
        Process a large batch of tiles.

        Parameters
        ----------
        process : MapcheteProcess
            process to be run
        zoom : list or int
            either single zoom level or list of minimum and maximum zoom level;
            None processes all (default: None)
        tile : tuple
            zoom, row and column of tile to be processed (cannot be used with
            zoom)
        multi : int
            number of workers (default: number of CPU cores)
        max_chunksize : int
            maximum number of process tiles to be queued for each worker;
            (default: 1)
        """
        list(self.batch_processor(zoom, tile, multi, max_chunksize))

def batch_process ( self , zoom = None , tile = None , multi = cpu_count ( ) , max_chunksize = 1 ) : list ( self . batch_processor ( zoom , tile , multi , max_chunksize ) )


def batch_processor(
        self, zoom=None, tile=None, multi=cpu_count(), max_chunksize=1
    ):
        """
        Process a large batch of tiles and yield report messages per tile.

        Parameters
        ----------
        zoom : list or int
            either single zoom level or list of minimum and maximum zoom level;
            None processes all (default: None)
        tile : tuple
            zoom, row and column of tile to be processed (cannot be used with
            zoom)
        multi : int
            number of workers (default: number of CPU cores)
        max_chunksize : int
            maximum number of process tiles to be queued for each worker;
            (default: 1)
        """
        if zoom and tile:
            raise ValueError("use either zoom or tile")

        # run single tile
        if tile:
            yield _run_on_single_tile(self, tile)
        # run concurrently
        elif multi > 1:
            for process_info in _run_with_multiprocessing(
                self, list(_get_zoom_level(zoom, self)), multi, max_chunksize
            ):
                yield process_info
        # run sequentially
        elif multi == 1:
            for process_info in _run_without_multiprocessing(
                self, list(_get_zoom_level(zoom, self))
            ):
                yield process_info

def
batch_processor
(
self
,
zoom
=
,
tile
=
,
multi
=
cpu_count
(
)
,
max_chunksize
=
1
)
:
if
zoom
and
tile
:
raise
ValueError
(
"use either zoom or tile"
)
if
tile
:
yield
_run_on_single_tile
(
self
,
tile
)
elif
multi
>
1
:
for
process_info
in
_run_with_multiprocessing
(
self
,
list
(
_get_zoom_level
(
zoom
,
self
)
)
,
multi
,
max_chunksize
)
:
yield
process_info
elif
multi
==
1
:
for
process_info
in
_run_without_multiprocessing
(
self
,
list
(
_get_zoom_level
(
zoom
,
self
)
)
)
:
yield
process_info

Process a large batch of tiles and yield report messages per tile .


def execute(self, process_tile, raise_nodata=False):
        """
        Run the Mapchete process.

        Execute, write and return data.

        Parameters
        ----------
        process_tile : Tile or tile index tuple
            Member of the process tile pyramid (not necessarily the output
            pyramid, if output has a different metatiling setting)

        Returns
        -------
        data : NumPy array or features
            process output
        """
        if self.config.mode not in ["memory", "continue", "overwrite"]:
            raise ValueError("process mode must be memory, continue or overwrite")
        if isinstance(process_tile, tuple):
            process_tile = self.config.process_pyramid.tile(*process_tile)
        elif isinstance(process_tile, BufferedTile):
            pass
        else:
            raise TypeError("process_tile must be tuple or BufferedTile")

        if process_tile.zoom not in self.config.zoom_levels:
            return self.config.output.empty(process_tile)

        return self._execute(process_tile, raise_nodata=raise_nodata)


def read(self, output_tile):
        """
        Read from written process output.

        Parameters
        ----------
        output_tile : BufferedTile or tile index tuple
            Member of the output tile pyramid (not necessarily the process
            pyramid, if output has a different metatiling setting)

        Returns
        -------
        data : NumPy array or features
            process output
        """
        if self.config.mode not in ["readonly", "continue", "overwrite"]:
            raise ValueError("process mode must be readonly, continue or overwrite")
        if isinstance(output_tile, tuple):
            output_tile = self.config.output_pyramid.tile(*output_tile)
        elif isinstance(output_tile, BufferedTile):
            pass
        else:
            raise TypeError("output_tile must be tuple or BufferedTile")

        return self.config.output.read(output_tile)


def write(self, process_tile, data):
        """
        Write data into output format.

        Parameters
        ----------
        process_tile : BufferedTile or tile index tuple
            process tile
        data : NumPy array or features
            data to be written
        """
        if isinstance(process_tile, tuple):
            process_tile = self.config.process_pyramid.tile(*process_tile)
        elif not isinstance(process_tile, BufferedTile):
            raise ValueError("invalid process_tile type: %s" % type(process_tile))
        if self.config.mode not in ["continue", "overwrite"]:
            raise ValueError("cannot write output in current process mode")

        if self.config.mode == "continue" and (
            self.config.output.tiles_exist(process_tile)
        ):
            message = "output exists, not overwritten"
            logger.debug((process_tile.id, message))
            return ProcessInfo(
                tile=process_tile,
                processed=False,
                process_msg=None,
                written=False,
                write_msg=message
            )
        elif data is None:
            message = "output empty, nothing written"
            logger.debug((process_tile.id, message))
            return ProcessInfo(
                tile=process_tile,
                processed=False,
                process_msg=None,
                written=False,
                write_msg=message
            )
        else:
            with Timer() as t:
                self.config.output.write(process_tile=process_tile, data=data)
            message = "output written in %s" % t
            logger.debug((process_tile.id, message))
            return ProcessInfo(
                tile=process_tile,
                processed=False,
                process_msg=None,
                written=True,
                write_msg=message
            )


def get_raw_output(self, tile, _baselevel_readonly=False):
        """
        Get output raw data.

        This function won't work with multiprocessing, as it uses the
        ``threading.Lock()`` class.

        Parameters
        ----------
        tile : tuple, Tile or BufferedTile
            If a tile index is given, a tile from the output pyramid will be
            assumed. Tile cannot be bigger than process tile!

        Returns
        -------
        data : NumPy array or features
            process output
        """
        if not isinstance(tile, (BufferedTile, tuple)):
            raise TypeError("'tile' must be a tuple or BufferedTile")
        if isinstance(tile, tuple):
            tile = self.config.output_pyramid.tile(*tile)
        if _baselevel_readonly:
            tile = self.config.baselevels["tile_pyramid"].tile(*tile.id)

        # Return empty data if zoom level is outside of process zoom levels.
        if tile.zoom not in self.config.zoom_levels:
            return self.config.output.empty(tile)

        # TODO implement reprojection
        if tile.crs != self.config.process_pyramid.crs:
            raise NotImplementedError(
                "reprojection between processes not yet implemented"
            )

        if self.config.mode == "memory":
            # Determine affected process Tile and check whether it is already
            # cached.
            process_tile = self.config.process_pyramid.intersecting(tile)[0]
            return self._extract(
                in_tile=process_tile,
                in_data=self._execute_using_cache(process_tile),
                out_tile=tile
            )

        # TODO: cases where tile intersects with multiple process tiles
        process_tile = self.config.process_pyramid.intersecting(tile)[0]

        # get output_tiles that intersect with current tile
        if tile.pixelbuffer > self.config.output.pixelbuffer:
            output_tiles = list(self.config.output_pyramid.tiles_from_bounds(
                tile.bounds, tile.zoom
            ))
        else:
            output_tiles = self.config.output_pyramid.intersecting(tile)

        if self.config.mode == "readonly" or _baselevel_readonly:
            if self.config.output.tiles_exist(process_tile):
                return self._read_existing_output(tile, output_tiles)
            else:
                return self.config.output.empty(tile)
        elif self.config.mode == "continue" and not _baselevel_readonly:
            if self.config.output.tiles_exist(process_tile):
                return self._read_existing_output(tile, output_tiles)
            else:
                return self._process_and_overwrite_output(tile, process_tile)
        elif self.config.mode == "overwrite" and not _baselevel_readonly:
            return self._process_and_overwrite_output(tile, process_tile)


def _extract(self, in_tile=None, in_data=None, out_tile=None):
        """Extract data from tile."""
        return self.config.output.extract_subset(
            input_data_tiles=[(in_tile, in_data)],
            out_tile=out_tile
        )

def _extract ( self , in_tile = None , in_data = None , out_tile = None ) : return self . config . output . extract_subset ( input_data_tiles = [ ( in_tile , in_data ) ] , out_tile = out_tile )


def read(self, **kwargs):
        """
        Read existing output data from a previous run.

        Returns
        -------
        process output : NumPy array (raster) or feature iterator (vector)
        """
        if self.tile.pixelbuffer > self.config.output.pixelbuffer:
            output_tiles = list(self.config.output_pyramid.tiles_from_bounds(
                self.tile.bounds, self.tile.zoom
            ))
        else:
            output_tiles = self.config.output_pyramid.intersecting(self.tile)
        return self.config.output.extract_subset(
            input_data_tiles=[
                (output_tile, self.config.output.read(output_tile))
                for output_tile in output_tiles
            ],
            out_tile=self.tile,
        )

def read ( self , ** kwargs ) : if self . tile . pixelbuffer > self . config . output . pixelbuffer : output_tiles = list ( self . config . output_pyramid . tiles_from_bounds ( self . tile . bounds , self . tile . zoom ) ) else : output_tiles = self . config . output_pyramid . intersecting ( self . tile ) return self . config . output . extract_subset ( input_data_tiles = [ ( output_tile , self . config . output . read ( output_tile ) ) for output_tile in output_tiles ] , out_tile = self . tile , )

Read existing output data from a previous run .


def open(self, input_id, **kwargs):
        """
        Open input data.

        Parameters
        ----------
        input_id : string
            input identifier from configuration file or file path
        kwargs : driver specific parameters (e.g. resampling)

        Returns
        -------
        tiled input data : InputTile
            reprojected input data within tile
        """
        if not isinstance(input_id, str):
            return input_id.open(self.tile, **kwargs)
        if input_id not in self.params["input"]:
            raise ValueError("%s not found in config as input file" % input_id)
        return self.params["input"][input_id].open(self.tile, **kwargs)

def
open
(
self
,
input_id
,
**
kwargs
)
:
if
not
isinstance
(
input_id
,
str
)
:
return
input_id
.
open
(
self
.
tile
,
**
kwargs
)
if
input_id
not
in
self
.
params
[
"input"
]
:
raise
ValueError
(
"%s not found in config as input file"
%
input_id
)
return
self
.
params
[
"input"
]
[
input_id
]
.
open
(
self
.
tile
,
**
kwargs
)


def hillshade(
        self, elevation, azimuth=315.0, altitude=45.0, z=1.0, scale=1.0
    ):
        """
        Calculate hillshading from elevation data.

        Parameters
        ----------
        elevation : array
            input elevation data
        azimuth : float
            horizontal angle of light source (315: North-West)
        altitude : float
            vertical angle of light source (90 would result in slope shading)
        z : float
            vertical exaggeration factor
        scale : float
            scale factor of pixel size units versus height units (insert 112000
            when having elevation values in meters in a geodetic projection)

        Returns
        -------
        hillshade : array
        """
        return commons_hillshade.hillshade(
            elevation, self, azimuth, altitude, z, scale)

def hillshade ( self , elevation , azimuth = 315.0 , altitude = 45.0 , z = 1.0 , scale = 1.0 ) : return commons_hillshade . hillshade ( elevation , self , azimuth , altitude , z , scale )

Calculate hillshading from elevation data .


def contours(
        self, elevation, interval=100, field='elev', base=0
    ):
        """
        Extract contour lines from elevation data.

        Parameters
        ----------
        elevation : array
            input elevation data
        interval : integer
            elevation value interval when drawing contour lines
        field : string
            output field name containing elevation value
        base : integer
            elevation base value the intervals are computed from

        Returns
        -------
        contours : iterable
            contours as GeoJSON-like pairs of properties and geometry
        """
        return commons_contours.extract_contours(
            elevation, self.tile, interval=interval, field=field, base=base)

def contours ( self , elevation , interval = 100 , field = 'elev' , base = 0 ) : return commons_contours . extract_contours ( elevation , self . tile , interval = interval , field = field , base = base )

Extract contour lines from elevation data .


def clip(
        self, array, geometries, inverted=False, clip_buffer=0
    ):
        """
        Clip array by geometry.

        Parameters
        ----------
        array : array
            raster data to be clipped
        geometries : iterable
            geometries used to clip source array
        inverted : bool
            invert clipping (default: False)
        clip_buffer : int
            buffer (in pixels) geometries before applying clip

        Returns
        -------
        clipped array : array
        """
        return commons_clip.clip_array_with_vector(
            array, self.tile.affine, geometries,
            inverted=inverted, clip_buffer=clip_buffer*self.tile.pixel_x_size)

def clip ( self , array , geometries , inverted = False , clip_buffer = 0 ) : return commons_clip . clip_array_with_vector ( array , self . tile . affine , geometries , inverted = inverted , clip_buffer = clip_buffer * self . tile . pixel_x_size )


def clip_array_with_vector(
    array, array_affine, geometries, inverted=False, clip_buffer=0
):
    """
    Clip input array with a vector list.

    Parameters
    ----------
    array : array
        input raster data
    array_affine : Affine
        Affine object describing the raster's geolocation
    geometries : iterable
        iterable of dictionaries, where every entry has a 'geometry' and
        'properties' key.
    inverted : bool
        invert clip (default: False)
    clip_buffer : integer
        buffer (in pixels) geometries before clipping

    Returns
    -------
    clipped array : array
    """
    # buffer input geometries and clean up
    buffered_geometries = []
    for feature in geometries:
        feature_geom = to_shape(feature["geometry"])
        if feature_geom.is_empty:
            continue
        if feature_geom.geom_type == "GeometryCollection":
            # for GeometryCollections apply buffer to every subgeometry
            # and make union
            buffered_geom = unary_union([
                g.buffer(clip_buffer) for g in feature_geom])
        else:
            buffered_geom = feature_geom.buffer(clip_buffer)
        if not buffered_geom.is_empty:
            buffered_geometries.append(buffered_geom)

    # mask raster by buffered geometries
    if buffered_geometries:
        if array.ndim == 2:
            return ma.masked_array(
                array, geometry_mask(
                    buffered_geometries, array.shape, array_affine,
                    invert=inverted))
        elif array.ndim == 3:
            mask = geometry_mask(
                buffered_geometries, (array.shape[1], array.shape[2]),
                array_affine, invert=inverted)
            return ma.masked_array(
                array, mask=np.stack((mask for band in array)))

    # if no geometries, return unmasked array
    else:
        fill = False if inverted else True
        return ma.masked_array(
            array, mask=np.full(array.shape, fill, dtype=bool))

def clip_array_with_vector ( array , array_affine , geometries , inverted = False , clip_buffer = 0 ) : buffered_geometries = [ ] for feature in geometries : feature_geom = to_shape ( feature [ "geometry" ] ) if feature_geom . is_empty : continue if feature_geom . geom_type == "GeometryCollection" : buffered_geom = unary_union ( [ g . buffer ( clip_buffer ) for g in feature_geom ] ) else : buffered_geom = feature_geom . buffer ( clip_buffer ) if not buffered_geom . is_empty : buffered_geometries . append ( buffered_geom ) if buffered_geometries : if array . ndim == 2 : return ma . masked_array ( array , geometry_mask ( buffered_geometries , array . shape , array_affine , invert = inverted ) ) elif array . ndim == 3 : mask = geometry_mask ( buffered_geometries , ( array . shape [ 1 ] , array . shape [ 2 ] ) , array_affine , invert = inverted ) return ma . masked_array ( array , mask = np . stack ( ( mask for band in array ) ) ) else : fill = False if inverted else True return ma . masked_array ( array , mask = np . full ( array . shape , fill , dtype = bool ) )


def pyramid(
    input_raster,
    output_dir,
    pyramid_type=None,
    output_format=None,
    resampling_method=None,
    scale_method=None,
    zoom=None,
    bounds=None,
    overwrite=False,
    debug=False
):
    """Create tile pyramid out of input raster."""
    bounds = bounds if bounds else None
    options = dict(
        pyramid_type=pyramid_type,
        scale_method=scale_method,
        output_format=output_format,
        resampling=resampling_method,
        zoom=zoom,
        bounds=bounds,
        overwrite=overwrite
    )
    raster2pyramid(input_raster, output_dir, options)

def pyramid ( input_raster , output_dir , pyramid_type = None , output_format = None , resampling_method = None , scale_method = None , zoom = None , bounds = None , overwrite = False , debug = False ) : bounds = bounds if bounds else None options = dict ( pyramid_type = pyramid_type , scale_method = scale_method , output_format = output_format , resampling = resampling_method , zoom = zoom , bounds = bounds , overwrite = overwrite ) raster2pyramid ( input_raster , output_dir , options )

Create tile pyramid out of input raster .


def raster2pyramid(input_file, output_dir, options):
    """Create a tile pyramid out of an input raster dataset."""
    pyramid_type = options["pyramid_type"]
    scale_method = options["scale_method"]
    output_format = options["output_format"]
    resampling = options["resampling"]
    zoom = options["zoom"]
    bounds = options["bounds"]
    mode = "overwrite" if options["overwrite"] else "continue"

    # Prepare process parameters
    minzoom, maxzoom = _get_zoom(zoom, input_file, pyramid_type)
    with rasterio.open(input_file, "r") as input_raster:
        output_bands = input_raster.count
        input_dtype = input_raster.dtypes[0]
        output_dtype = input_raster.dtypes[0]
        nodataval = input_raster.nodatavals[0]
        nodataval = nodataval if nodataval else 0
        if output_format == "PNG" and output_bands > 3:
            output_bands = 3
            output_dtype = 'uint8'
        scales_minmax = ()
        if scale_method == "dtype_scale":
            for index in range(1, output_bands+1):
                scales_minmax += (DTYPE_RANGES[input_dtype], )
        elif scale_method == "minmax_scale":
            for index in range(1, output_bands+1):
                band = input_raster.read(index)
                scales_minmax += ((band.min(), band.max()), )
        elif scale_method == "crop":
            for index in range(1, output_bands+1):
                scales_minmax += ((0, 255), )
        if input_dtype == "uint8":
            scale_method = None
            scales_minmax = ()
            for index in range(1, output_bands+1):
                scales_minmax += ((None, None), )

    # Create configuration
    config = dict(
        process="mapchete.processes.pyramid.tilify",
        output={
            "path": output_dir,
            "format": output_format,
            "bands": output_bands,
            "dtype": output_dtype
            },
        pyramid=dict(pixelbuffer=5, grid=pyramid_type),
        scale_method=scale_method,
        scales_minmax=scales_minmax,
        input={"raster": input_file},
        config_dir=os.getcwd(),
        zoom_levels=dict(min=minzoom, max=maxzoom),
        nodataval=nodataval,
        resampling=resampling,
        bounds=bounds,
        baselevel={"zoom": maxzoom, "resampling": resampling},
        mode=mode
    )

    # create process
    with mapchete.open(config, zoom=zoom, bounds=bounds) as mp:
        # prepare output directory
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        # run process
        mp.batch_process(zoom=[minzoom, maxzoom])

def raster2pyramid ( input_file , output_dir , options ) : pyramid_type = options [ "pyramid_type" ] scale_method = options [ "scale_method" ] output_format = options [ "output_format" ] resampling = options [ "resampling" ] zoom = options [ "zoom" ] bounds = options [ "bounds" ] mode = "overwrite" if options [ "overwrite" ] else "continue" minzoom , maxzoom = _get_zoom ( zoom , input_file , pyramid_type ) with rasterio . open ( input_file , "r" ) as input_raster : output_bands = input_raster . count input_dtype = input_raster . dtypes [ 0 ] output_dtype = input_raster . dtypes [ 0 ] nodataval = input_raster . nodatavals [ 0 ] nodataval = nodataval if nodataval else 0 if output_format == "PNG" and output_bands > 3 : output_bands = 3 output_dtype = 'uint8' scales_minmax = ( ) if scale_method == "dtype_scale" : for index in range ( 1 , output_bands + 1 ) : scales_minmax += ( DTYPE_RANGES [ input_dtype ] , ) elif scale_method == "minmax_scale" : for index in range ( 1 , output_bands + 1 ) : band = input_raster . read ( index ) scales_minmax += ( ( band . min ( ) , band . max ( ) ) , ) elif scale_method == "crop" : for index in range ( 1 , output_bands + 1 ) : scales_minmax += ( ( 0 , 255 ) , ) if input_dtype == "uint8" : scale_method = None scales_minmax = ( ) for index in range ( 1 , output_bands + 1 ) : scales_minmax += ( ( None , None ) , ) config = dict ( process = "mapchete.processes.pyramid.tilify" , output = { "path" : output_dir , "format" : output_format , "bands" : output_bands , "dtype" : output_dtype } , pyramid = dict ( pixelbuffer = 5 , grid = pyramid_type ) , scale_method = scale_method , scales_minmax = scales_minmax , input = { "raster" : input_file } , config_dir = os . getcwd ( ) , zoom_levels = dict ( min = minzoom , max = maxzoom ) , nodataval = nodataval , resampling = resampling , bounds = bounds , baselevel = { "zoom" : maxzoom , "resampling" : resampling } , mode = mode ) with mapchete . open ( config , zoom = zoom , bounds = bounds ) as mp : if not os . path . exists ( output_dir ) : os . makedirs ( output_dir ) mp . batch_process ( zoom = [ minzoom , maxzoom ] )

Create a tile pyramid out of an input raster dataset .


def _get_zoom(zoom, input_raster, pyramid_type):
    """Determine minimum and maximum zoomlevel."""
    if not zoom:
        minzoom = 1
        maxzoom = get_best_zoom_level(input_raster, pyramid_type)
    elif len(zoom) == 1:
        minzoom = zoom[0]
        maxzoom = zoom[0]
    elif len(zoom) == 2:
        if zoom[0] < zoom[1]:
            minzoom = zoom[0]
            maxzoom = zoom[1]
        else:
            minzoom = zoom[1]
            maxzoom = zoom[0]
    return minzoom, maxzoom

def _get_zoom ( zoom , input_raster , pyramid_type ) : if not zoom : minzoom = 1 maxzoom = get_best_zoom_level ( input_raster , pyramid_type ) elif len ( zoom ) == 1 : minzoom = zoom [ 0 ] maxzoom = zoom [ 0 ] elif len ( zoom ) == 2 : if zoom [ 0 ] < zoom [ 1 ] : minzoom = zoom [ 0 ] maxzoom = zoom [ 1 ] else : minzoom = zoom [ 1 ] maxzoom = zoom [ 0 ] return minzoom , maxzoom

Determine minimum and maximum zoomlevel .


def validate_values(config, values):
    """
    Validate whether value is found in config and has the right type.

    Parameters
    ----------
    config : dict
        configuration dictionary
    values : list
        list of (str, type) tuples of values and value types expected in config

    Returns
    -------
    True if config is valid.

    Raises
    ------
    Exception if value is not found or has the wrong type.
    """
    if not isinstance(config, dict):
        raise TypeError("config must be a dictionary")
    for value, vtype in values:
        if value not in config:
            raise ValueError("%s not given" % value)
        if not isinstance(config[value], vtype):
            raise TypeError("%s must be %s" % (value, vtype))
    return True

def
validate_values
(
config
,
values
)
:
if
not
isinstance
(
config
,
dict
)
:
raise
TypeError
(
"config must be a dictionary"
)
for
value
,
vtype
in
values
:
if
value
not
in
config
:
raise
ValueError
(
"%s not given"
%
value
)
if
not
isinstance
(
config
[
value
]
,
vtype
)
:
raise
TypeError
(
"%s must be %s"
%
(
value
,
vtype
)
)
return
True

Validate whether value is found in config and has the right type .


def get_hash(x):
    """Return hash of x."""
    if isinstance(x, str):
        return hash(x)
    elif isinstance(x, dict):
        return hash(yaml.dump(x))

def get_hash ( x ) : if isinstance ( x , str ) : return hash ( x ) elif isinstance ( x , dict ) : return hash ( yaml . dump ( x ) )


def get_zoom_levels(process_zoom_levels=None, init_zoom_levels=None):
    """Validate and return zoom levels."""
    process_zoom_levels = _validate_zooms(process_zoom_levels)
    if init_zoom_levels is None:
        return process_zoom_levels
    else:
        init_zoom_levels = _validate_zooms(init_zoom_levels)
        if not set(init_zoom_levels).issubset(set(process_zoom_levels)):
            raise MapcheteConfigError(
                "init zooms must be a subset of process zoom")
        return init_zoom_levels


def snap_bounds(bounds=None, pyramid=None, zoom=None):
    """
    Snaps bounds to tiles boundaries of specific zoom level.

    Parameters
    ----------
    bounds : bounds to be snapped
    pyramid : TilePyramid
    zoom : int

    Returns
    -------
    Bounds(left, bottom, right, top)
    """
    if not isinstance(bounds, (tuple, list)):
        raise TypeError("bounds must be either a tuple or a list")
    if len(bounds) != 4:
        raise ValueError("bounds has to have exactly four values")
    if not isinstance(pyramid, BufferedTilePyramid):
        raise TypeError("pyramid has to be a BufferedTilePyramid")

    bounds = Bounds(*bounds)
    lb = pyramid.tile_from_xy(bounds.left, bounds.bottom, zoom, on_edge_use="rt").bounds
    rt = pyramid.tile_from_xy(bounds.right, bounds.top, zoom, on_edge_use="lb").bounds
    return Bounds(lb.left, lb.bottom, rt.right, rt.top)

Snaps bounds to tiles boundaries of specific zoom level .


def clip_bounds(bounds=None, clip=None):
    """
    Clips bounds by clip.

    Parameters
    ----------
    bounds : bounds to be clipped
    clip : clip bounds

    Returns
    -------
    Bounds(left, bottom, right, top)
    """
    bounds = Bounds(*bounds)
    clip = Bounds(*clip)
    return Bounds(
        max(bounds.left, clip.left),
        max(bounds.bottom, clip.bottom),
        min(bounds.right, clip.right),
        min(bounds.top, clip.top)
    )

def clip_bounds ( bounds = None , clip = None ) : bounds = Bounds ( * bounds ) clip = Bounds ( * clip ) return Bounds ( max ( bounds . left , clip . left ) , max ( bounds . bottom , clip . bottom ) , min ( bounds . right , clip . right ) , min ( bounds . top , clip . top ) )


def _validate_zooms(zooms):
    """
    Return a list of zoom levels.

    Following inputs are converted:
    - int --> [int]
    - dict{min, max} --> range(min, max + 1)
    - [int] --> [int]
    - [int, int] --> range(smaller int, bigger int + 1)
    """
    if isinstance(zooms, dict):
        if any([a not in zooms for a in ["min", "max"]]):
            raise MapcheteConfigError("min and max zoom required")
        zmin = _validate_zoom(zooms["min"])
        zmax = _validate_zoom(zooms["max"])
        if zmin > zmax:
            raise MapcheteConfigError(
                "max zoom must not be smaller than min zoom")
        return list(range(zmin, zmax + 1))
    elif isinstance(zooms, list):
        if len(zooms) == 1:
            return zooms
        elif len(zooms) == 2:
            zmin, zmax = sorted([_validate_zoom(z) for z in zooms])
            return list(range(zmin, zmax + 1))
        else:
            return zooms
    else:
        return [_validate_zoom(zooms)]


def _raw_at_zoom(config, zooms):
    """Return parameter dictionary per zoom level."""
    params_per_zoom = {}
    for zoom in zooms:
        params = {}
        for name, element in config.items():
            if name not in _RESERVED_PARAMETERS:
                out_element = _element_at_zoom(name, element, zoom)
                if out_element is not None:
                    params[name] = out_element
        params_per_zoom[zoom] = params
    return params_per_zoom

def _raw_at_zoom ( config , zooms ) : params_per_zoom = { } for zoom in zooms : params = { } for name , element in config . items ( ) : if name not in _RESERVED_PARAMETERS : out_element = _element_at_zoom ( name , element , zoom ) if out_element is not None : params [ name ] = out_element params_per_zoom [ zoom ] = params return params_per_zoom

Return parameter dictionary per zoom level .