def _retry(func):
    '''
    Decorator for methods that need many retries, because of
    intermittent failures, such as AWS calls via boto, which has a
    non-back-off retry.
    '''
    def retry_func(self, *args, **kwargs):
        tries = 1
        while True:
            # If a handler allows execution to continue, then
            # fall through and do a back-off retry.
            try:
                return func(self, *args, **kwargs)
                break
            except OSError as exc:
                ## OSError: [Errno 24] Too many open files
                logger.error('assuming OSError unrecoverable')
                raise
            except FailedExtraction as exc:
                ## pass through exc to caller
                logger.error('FAIL(%d)', tries, exc_info=True)
                raise
            except FailedVerification as exc:
                logger.warn('FAIL(%d)', tries, exc_info=True)
                if tries >= self.config['tries']:
                    if self.config.get('suppress_failures'):
                        logger.warn('suppressing failure and breaking out of this loop; data may be corrupt, downstream will have to cope')
                        break
                    else:
                        raise
            except Exception as exc:
                logger.warn('FAIL(%d): having I/O trouble with S3', tries, exc_info=True)
                if tries >= self.config['tries']:
                    raise

            logger.warn('RETRYING (%d left)', self.config['tries'] - tries)
            time.sleep(3 * tries)
            tries += 1

    return retry_func

Decorator for methods that need many retries because of intermittent failures such as AWS calls via boto which has a non - back - off retry .


def verify_md5(md5_expected, data, other_errors=None):
    "return True if okay, raise Exception if not"  # O_o ?
    md5_recv = hashlib.md5(data).hexdigest()
    if md5_expected != md5_recv:
        if other_errors is not None:
            logger.critical('\n'.join(other_errors))
        raise FailedVerification('original md5 = %r != %r = received md5' \
                                 % (md5_expected, md5_recv))
    return True

return True if okay raise Exception if not


def get_bucket(config, bucket_name=None):
    '''This function is mostly about managing configuration, and then
    finally returns a boto.Bucket object.

    AWS credentials come first from config keys
    aws_access_key_id_path, aws_secret_access_key_path (paths to one
    line files); secondly from environment variables
    AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY; also from $HOME/.aws/credentials
    or the magic Amazon http://169.254.169.254/ service.  If credentials
    are not set in the config then behavior is the same as other AWS-based
    command-line tools.

    '''
    if not bucket_name:
        if 'bucket' not in config:
            raise ConfigurationError(
                'The "bucket" parameter is required for the s3 stages.')
        bucket_name = config['bucket']

    # get AWS credentials. first, from config; else, from env vars.
    # (boto will read environment variables and other normal places.)
    aws_access_key_id_path = config.get('aws_access_key_id_path')
    aws_secret_access_key_path = config.get('aws_secret_access_key_path')

    params = ()
    if aws_access_key_id_path and aws_secret_access_key_path:
        try:
            access = open(aws_access_key_id_path).read().strip()
            secret = open(aws_secret_access_key_path).read().strip()
            params = (access, secret)
        except:
            logger.error('failed reading aws credentials from configured file', exc_info=True)
            raise

    conn = S3Connection(*params)
    bucket = conn.get_bucket(bucket_name)
    return bucket

This function is mostly about managing configuration and then finally returns a boto . Bucket object .


def _decode(self, data):
        '''
        Given the raw data from s3, return a generator for the items
        contained in that data. A generator is necessary to support
        chunk files, but non-chunk files can be provided by a generator
        that yields exactly one item.

        Decoding works by case analysis on the config option
        ``input_format``. If an invalid ``input_format`` is given, then
        a ``ConfigurationError`` is raised.
        '''
        informat = self.config['input_format'].lower()
        if informat == 'spinn3r':
            return _generate_stream_items(data)
        elif informat == 'streamitem':
            ver = self.config['streamcorpus_version']
            if ver not in _message_versions:
                raise ConfigurationError(
                    'Not a valid streamcorpus version: %s '
                    '(choose from: %s)'
                    % (ver, ', '.join(_message_versions.keys())))

            message = _message_versions[ver]
            return streamcorpus.Chunk(data=data, message=message)
        elif informat == 'featurecollection' and FCChunk is not None:
            return FCChunk(data=data)
        else:
            raise ConfigurationError(
                'from_s3_chunks unknown input_format = %r'
                % informat)

Given the raw data from s3 return a generator for the items contained in that data . A generator is necessary to support chunk files but non - chunk files can be provided by a generator that yields exactly one item .


def get_chunk(self, bucket_name, key_path):
        '''return Chunk object full of records
        bucket_name may be None'''
        bucket = get_bucket(self.config, bucket_name=bucket_name)
        key = bucket.get_key(key_path)
        if key is None:
            raise FailedExtraction('Key "%s" does not exist.' % key_path)

        fh = StringIO()
        key.get_contents_to_file(fh)
        data = fh.getvalue()
        if not data:
            raise FailedExtraction('%s: no data (does the key exist?)'
                                   % key.key)

        chunk_type, compression, encryption = parse_file_extensions(key_path)
        if encryption == 'gpg':
            if not self.gpg_decryption_key_path:
                raise FailedExtraction('%s ends with ".gpg" but gpg_decryption_key_path=%s'
                                       % (key.key, self.gpg_decryption_key_path))

        _errors = []
        if compression or encryption:
            _errors, data = decrypt_and_uncompress(
                data,
                self.gpg_decryption_key_path,
                tmp_dir=self.config.get('tmp_dir_path'),
                compression=compression,
                )
            if not data:
                msg = 'decrypt_and_uncompress got no data for {0!r}, from {1} bytes' \
                    + ' downloaded, errors: {2}' \
                        .format(key_path, len(data), '\n'.join(_errors))
                logger.error(msg)
                raise FailedExtraction(msg)
            logger.info( '\n'.join(_errors) )

        if not self.config['compare_md5_in_file_name']:
            logger.warn('not checking md5 in file name, consider setting '
                        'from_s3_chunks:compare_md5_in_file_name')
        else:
            logger.info('Verifying md5 for "%s"...' % key.key)

            # The regex hammer.
            m = re.search('([a-z0-9]{32})(?:\.|$)', key.key)
            if m is None:
                raise FailedExtraction(
                    'Could not extract md5 from key "%s". '
                    'Perhaps you should disable compare_md5_in_file_name?'
                    % key.key)

            i_content_md5 = m.group(1)
            #import pdb; pdb.set_trace()
            verify_md5(i_content_md5, data, other_errors=_errors)
        return self._decode(data)


def stream_id_to_kvlayer_key(stream_id):
    '''Convert a text stream ID to a kvlayer key.

    The return tuple can be used directly as a key in the
    :data:`STREAM_ITEMS_TABLE` table.

    :param str stream_id: stream ID to convert
    :return: :mod:`kvlayer` key tuple
    :raise exceptions.KeyError: if `stream_id` is malformed

    '''
    # Reminder: stream_id is 1234567890-123456789abcdef...0
    # where the first part is the (decimal) epoch_ticks and the second
    # part is the (hex) doc_id
    parts = stream_id.split('-')
    if len(parts) != 2:
        raise KeyError('invalid stream_id ' + stream_id)
    epoch_ticks_s = parts[0]
    doc_id_s = parts[1]
    if not epoch_ticks_s.isdigit():
        raise KeyError('invalid stream_id ' + stream_id)
    if doc_id_s.lstrip(string.hexdigits) != '':
        raise KeyError('invalid stream_id ' + stream_id)
    return (base64.b16decode(doc_id_s.upper()), int(epoch_ticks_s))

def
stream_id_to_kvlayer_key
(
stream_id
)
:
parts
=
stream_id
.
split
(
'-'
)
if
len
(
parts
)
!=
2
:
raise
KeyError
(
'invalid stream_id '
+
stream_id
)
epoch_ticks_s
=
parts
[
0
]
doc_id_s
=
parts
[
1
]
if
not
epoch_ticks_s
.
isdigit
(
)
:
raise
KeyError
(
'invalid stream_id '
+
stream_id
)
if
doc_id_s
.
lstrip
(
string
.
hexdigits
)
!=
''
:
raise
KeyError
(
'invalid stream_id '
+
stream_id
)
return
(
base64
.
b16decode
(
doc_id_s
.
upper
(
)
)
,
int
(
epoch_ticks_s
)
)

Convert a text stream ID to a kvlayer key .


def kvlayer_key_to_stream_id(k):
    '''Convert a kvlayer key to a text stream ID.

    `k` should be of the same form produced by
    :func:`stream_id_to_kvlayer_key`.

    :param k: :mod:`kvlayer` key tuple
    :return: converted stream ID
    :returntype str:

    '''
    abs_url_hash, epoch_ticks = k
    return '{0}-{1}'.format(epoch_ticks,
                            base64.b16encode(abs_url_hash).lower())

def kvlayer_key_to_stream_id ( k ) : abs_url_hash , epoch_ticks = k return '{0}-{1}' . format ( epoch_ticks , base64 . b16encode ( abs_url_hash ) . lower ( ) )

Convert a kvlayer key to a text stream ID .


def key_for_stream_item(si):
    '''Get a kvlayer key from a stream item.

    The return tuple can be used directly as a key in the
    :data:`STREAM_ITEMS_TABLE` table.  Note that this recalculates the
    stream ID, and if the internal data on the stream item is inconsistent
    then this could return a different result from
    :func:`stream_id_to_kvlayer_key`.

    :param si: stream item to get key for
    :return: :mod:`kvlayer` key tuple

    '''
    # get binary 16 byte digest
    urlhash = hashlib.md5(si.abs_url).digest()
    return (urlhash, int(si.stream_time.epoch_ticks))

def key_for_stream_item ( si ) : urlhash = hashlib . md5 ( si . abs_url ) . digest ( ) return ( urlhash , int ( si . stream_time . epoch_ticks ) )


def main(argv=sys.argv):
    args = parse(argv)
    """Serve up some ponies."""
    hostname = args.listen
    port = args.port
    print(
        "Making all your dreams for a pony come true on http://{0}:{1}.\n"
        "Press Ctrl+C to quit.\n".format(hostname, port))

    # Hush, werkzeug.
    logging.getLogger('werkzeug').setLevel(logging.CRITICAL)

    plugin_manager.load_installed_plugins()
    app = make_app()
    run_simple(hostname, port, app)


def build_parser():
    """Build the parser that will have all available commands and options."""
    description = (
        'HTTPony (pronounced aych-tee-tee-pony) is a simple HTTP '
        'server that pretty prints HTTP requests to a terminal. It '
        'is a useful aide for developing clients that send HTTP '
        'requests. HTTPony acts as a sink for a client so that a '
        'developer can understand what the client is sending.')
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument(
        '-l', '--listen', help='set the IP address or hostname',
        default='localhost')
    parser.add_argument(
        '-p', '--port', help='set the port', default=8000, type=int)

    return parser

Build the parser that will have all available commands and options .


def add_xpaths_to_stream_item(si):
    '''Mutably tag tokens with xpath offsets.

    Given some stream item, this will tag all tokens from all taggings
    in the document that contain character offsets. Note that some
    tokens may not have computable xpath offsets, so an xpath offset
    for those tokens will not be set. (See the documentation and
    comments for ``char_offsets_to_xpaths`` for what it means for a
    token to have a computable xpath.)

    If a token can have its xpath offset computed, it is added to its
    set of offsets with a ``OffsetType.XPATH_CHARS`` key.
    '''
    def sentences_to_xpaths(sentences):
        tokens = sentences_to_char_tokens(sentences)
        offsets = char_tokens_to_char_offsets(tokens)
        return char_offsets_to_xpaths(html, offsets)

    def xprange_to_offset(xprange):
        return Offset(type=OffsetType.XPATH_CHARS,
                      first=xprange.start_offset, length=0,
                      xpath=xprange.start_xpath,
                      content_form='clean_html', value=None,
                      xpath_end=xprange.end_xpath,
                      xpath_end_offset=xprange.end_offset)

    html = unicode(si.body.clean_html, 'utf-8')
    for sentences in si.body.sentences.itervalues():
        tokens = sentences_to_char_tokens(sentences)
        for token, xprange in izip(tokens, sentences_to_xpaths(sentences)):
            if xprange is None:
                continue
            offset = xprange_to_offset(xprange)
            token.offsets[OffsetType.XPATH_CHARS] = offset

def add_xpaths_to_stream_item ( si ) : def sentences_to_xpaths ( sentences ) : tokens = sentences_to_char_tokens ( sentences ) offsets = char_tokens_to_char_offsets ( tokens ) return char_offsets_to_xpaths ( html , offsets ) def xprange_to_offset ( xprange ) : return Offset ( type = OffsetType . XPATH_CHARS , first = xprange . start_offset , length = 0 , xpath = xprange . start_xpath , content_form = 'clean_html' , value = None , xpath_end = xprange . end_xpath , xpath_end_offset = xprange . end_offset ) html = unicode ( si . body . clean_html , 'utf-8' ) for sentences in si . body . sentences . itervalues ( ) : tokens = sentences_to_char_tokens ( sentences ) for token , xprange in izip ( tokens , sentences_to_xpaths ( sentences ) ) : if xprange is None : continue offset = xprange_to_offset ( xprange ) token . offsets [ OffsetType . XPATH_CHARS ] = offset


def sentences_to_char_tokens(si_sentences):
    '''Convert stream item sentences to character ``Offset``s.'''
    for sentence in si_sentences:
        for token in sentence.tokens:
            if OffsetType.CHARS in token.offsets:
                yield token

def sentences_to_char_tokens ( si_sentences ) : for sentence in si_sentences : for token in sentence . tokens : if OffsetType . CHARS in token . offsets : yield token

Convert stream item sentences to character Offset s .


def char_tokens_to_char_offsets(si_tokens):
    '''Convert character ``Offset``s to character ranges.'''
    for token in si_tokens:
        offset = token.offsets[OffsetType.CHARS]
        yield offset.first, offset.first + offset.length

def char_tokens_to_char_offsets ( si_tokens ) : for token in si_tokens : offset = token . offsets [ OffsetType . CHARS ] yield offset . first , offset . first + offset . length

Convert character Offset s to character ranges .


def char_offsets_to_xpaths(html, char_offsets):
    '''Converts HTML and a sequence of char offsets to xpath offsets.

    Returns a generator of :class:`streamcorpus.XpathRange` objects
    in correspondences with the sequence of ``char_offsets`` given.
    Namely, each ``XpathRange`` should address precisely the same text
    as that ``char_offsets`` (sans the HTML).

    Depending on how ``char_offsets`` was tokenized, it's possible that
    some tokens cannot have their xpaths generated reliably. In this
    case, a ``None`` value is yielded instead of a ``XpathRange``.

    ``char_offsets`` must be a sorted and non-overlapping sequence of
    character ranges. They do not have to be contiguous.
    '''
    html = uni(html)
    parser = XpathTextCollector()
    prev_end = 0
    prev_progress = True
    for start, end in char_offsets:
        if start == end:
            # Zero length tokens shall have no quarter!
            # Note that this is a special case. If we let zero-length tokens
            # be handled normally, then it will be recorded as if the parser
            # did not make any progress. But of course, there is no progress
            # to be had!
            yield None
            continue
        # If we didn't make any progress on the previous token, then we'll
        # need to try and make progress before we can start tracking offsets
        # again. Otherwise the parser will report incorrect offset info.
        #
        # (The parser can fail to make progress when tokens are split at
        # weird boundaries, e.g., `&amp` followed by `;`. The parser won't
        # make progress after `&amp` but will once `;` is given.)
        #
        # Here, we feed the parser one character at a time between where the
        # last token ended and where the next token will start. In most cases,
        # this will be enough to nudge the parser along. Once done, we can pick
        # up where we left off and start handing out offsets again.
        #
        # If this still doesn't let us make progress, then we'll have to skip
        # this token too.
        if not prev_progress:
            for i in xrange(prev_end, start):
                parser.feed(html[i])
                prev_end += 1
                if parser.made_progress:
                    break
            if not parser.made_progress:
                yield None
                continue
        # Hand the parser everything from the end of the last token to the
        # start of this one. Then ask for the Xpath, which should be at the
        # start of `char_offsets`.
        if prev_end < start:
            parser.feed(html[prev_end:start])
            if not parser.made_progress:
                parser.feed(html[start:end])
                prev_progress = parser.made_progress
                prev_end = end
                yield None
                continue
        xstart = parser.xpath_offset()
        # print('START', xstart)
        # Hand it the actual token and ask for the ending offset.
        parser.feed(html[start:end])
        xend = parser.xpath_offset()
        # print('END', xend)
        prev_end = end

        # If we couldn't make progress then the xpaths generated are probably
        # incorrect. (If the parser doesn't make progress, then we can't rely
        # on the callbacks to have been called, which means we may not have
        # captured all state correctly.)
        #
        # Therefore, we simply give up and claim this token is not addressable.
        if not parser.made_progress:
            prev_progress = False
            yield None
        else:
            prev_progress = True
            yield XpathRange(xstart[0], xstart[1], xend[0], xend[1])
    parser.feed(html[prev_end:])
    parser.close()

def char_offsets_to_xpaths ( html , char_offsets ) : html = uni ( html ) parser = XpathTextCollector ( ) prev_end = 0 prev_progress = True for start , end in char_offsets : if start == end : yield None continue if not prev_progress : for i in xrange ( prev_end , start ) : parser . feed ( html [ i ] ) prev_end += 1 if parser . made_progress : break if not parser . made_progress : yield None continue if prev_end < start : parser . feed ( html [ prev_end : start ] ) if not parser . made_progress : parser . feed ( html [ start : end ] ) prev_progress = parser . made_progress prev_end = end yield None continue xstart = parser . xpath_offset ( ) parser . feed ( html [ start : end ] ) xend = parser . xpath_offset ( ) prev_end = end if not parser . made_progress : prev_progress = False yield None else : prev_progress = True yield XpathRange ( xstart [ 0 ] , xstart [ 1 ] , xend [ 0 ] , xend [ 1 ] ) parser . feed ( html [ prev_end : ] ) parser . close ( )

Converts HTML and a sequence of char offsets to xpath offsets .


def add_element(self, tag):
        '''Record that `tag` has been seen at this depth.

        If `tag` is :class:`TextElement`, it records a text node.

        '''
        # Collapse adjacent text nodes
        if tag is TextElement and self.last_tag is TextElement:
            return
        self.last_tag = tag
        if tag not in self.tags:
            self.tags[tag] = 1
        else:
            self.tags[tag] += 1

def add_element ( self , tag ) : if tag is TextElement and self . last_tag is TextElement : return self . last_tag = tag if tag not in self . tags : self . tags [ tag ] = 1 else : self . tags [ tag ] += 1

Record that tag has been seen at this depth .


def xpath_piece(self):
        '''Get an XPath fragment for this location.

        It is of the form ``tag[n]`` where `tag` is the most recent
        element added and n is its position.

        '''
        if self.last_tag is TextElement:
            return 'text()[{count}]'.format(count=self.text_index())
        else:
            return '{tag}[{count}]'.format(tag=self.last_tag,
                                           count=self.tags[self.last_tag])

def xpath_piece ( self ) : if self . last_tag is TextElement : return 'text()[{count}]' . format ( count = self . text_index ( ) ) else : return '{tag}[{count}]' . format ( tag = self . last_tag , count = self . tags [ self . last_tag ] )

Get an XPath fragment for this location .


def text_index(self):
        '''Returns the one-based index of the current text node.'''
        # This is the number of text nodes we've seen so far.
        # If we are currently in a text node, great; if not then add
        # one for the text node that's about to begin.
        i = self.tags.get(TextElement, 0)
        if self.last_tag is not TextElement:
            i += 1
        return i

def text_index ( self ) : i = self . tags . get ( TextElement , 0 ) if self . last_tag is not TextElement : i += 1 return i

Returns the one - based index of the current text node .


def descendants(elem):
    '''
    Yields all the elements descendant of elem in document order
    '''
    for child in elem.xml_children:
        if isinstance(child, element):
            yield child
            yield from descendants(child)

def descendants ( elem ) : for child in elem . xml_children : if isinstance ( child , element ) : yield child yield from descendants ( child )


def select_elements(source):
    '''
    Yields all the elements from the source
    source - if an element, yields all child elements in order; if any other iterator yields the elements from that iterator
    '''
    if isinstance(source, element):
        source = source.xml_children
    return filter(lambda x: isinstance(x, element), source)

def select_elements ( source ) : if isinstance ( source , element ) : source = source . xml_children return filter ( lambda x : isinstance ( x , element ) , source )

Yields all the elements from the source source - if an element yields all child elements in order ; if any other iterator yields the elements from that iterator


def select_name(source, name):
    '''
    Yields all the elements with the given name
    source - if an element, starts with all child elements in order; can also be any other iterator
    name - will yield only elements with this name
    '''
    return filter(lambda x: x.xml_name == name, select_elements(source))

def select_name ( source , name ) : return filter ( lambda x : x . xml_name == name , select_elements ( source ) )

Yields all the elements with the given name source - if an element starts with all child elements in order ; can also be any other iterator name - will yield only elements with this name


def select_name_pattern(source, pat):
    '''
    Yields elements from the source whose name matches the given regular expression pattern
    source - if an element, starts with all child elements in order; can also be any other iterator
    pat - re.pattern object
    '''
    return filter(lambda x: pat.match(x.xml_name) is not None, select_elements(source))

def select_name_pattern ( source , pat ) : return filter ( lambda x : pat . match ( x . xml_name ) is not None , select_elements ( source ) )

Yields elements from the source whose name matches the given regular expression pattern source - if an element starts with all child elements in order ; can also be any other iterator pat - re . pattern object


def select_attribute(source, name, val=None):
    '''
    Yields elements from the source having the given attrivute, optionally with the given attribute value
    source - if an element, starts with all child elements in order; can also be any other iterator
    name - attribute name to check
    val - if None check only for the existence of the attribute, otherwise compare the given value as well
    '''
    def check(x):
        if val is None:
            return name in x.xml_attributes
        else:
            return name in x.xml_attributes and x.xml_attributes[name] == val
    return filter(check, select_elements(source))

def select_attribute ( source , name , val = None ) : def check ( x ) : if val is None : return name in x . xml_attributes else : return name in x . xml_attributes and x . xml_attributes [ name ] == val return filter ( check , select_elements ( source ) )

Yields elements from the source having the given attrivute optionally with the given attribute value source - if an element starts with all child elements in order ; can also be any other iterator name - attribute name to check val - if None check only for the existence of the attribute otherwise compare the given value as well


def following_siblings(elem):
    '''
    Yields elements and text which have the same parent as elem, but come afterward in document order
    '''
    it = itertools.dropwhile(lambda x: x != elem, elem.xml_parent.xml_children)
    next(it) #Skip the element itself
    return it

def following_siblings ( elem ) : it = itertools . dropwhile ( lambda x : x != elem , elem . xml_parent . xml_children ) next ( it ) return it

Yields elements and text which have the same parent as elem but come afterward in document order


def make_pretty(elem, depth=0, indent='  '):
    '''
    Add text nodes as possible to all descendants of an element for spacing & indentation
    to make the MicroXML as printed easier for people to read. Will not modify the
    value of any text node which is not already entirely whitespace.

    Warning: even though this operaton avoids molesting text nodes which already have
    whitespace, it still makes changes which alter the text. Not all whitespace in XML is
    ignorable. In XML cues from the DTD indicate which whitespace can be ignored.
    No such cues are available for MicroXML, so use this function with care. That said,
    in many real world applications of XML and MicroXML, this function causes no problems.

    elem - target element whose descendant nodes are to be modified.
    returns - the same element, which has been updated in place

    >>> from amara3.uxml import tree
    >>> from amara3.uxml.treeutil import *
    >>> DOC = '<x>1</x><c><x>2</x><d><x>3</x></d></c><x>4</x><y>5</y>'
    >>> tb = tree.treebuilder()
    >>> root = tb.parse(DOC)
    >>> len(root.xml_children)
    4
    >>> make_pretty(root)
    <uxml.element (8763373718343) "a" with 9 children>
    >>> len(root.xml_children)
    9
    >>> root.xml_encode()
    '\n  \n    <x>1</x>\n  \n  <c>\n    <x>2</x>\n    <d>\n      <x>3</x>\n    </d>\n  </c>\n  <x>4</x>\n  <y>5</y>\n'
    '''
    depth += 1
    updated_child_list = []
    updated_child_ix = 0
    for child in elem.xml_children:
        if isinstance(child, element):
            if updated_child_ix % 2:
                updated_child_list.append(child)
                updated_child_ix += 1
            else:
                #It's the turn for text, but we have an element
                new_text = text('\n' + indent*depth, elem)
                updated_child_list.append(new_text)
                updated_child_list.append(child)
                updated_child_ix += 2
            make_pretty(child, depth)
        else:
            if child.xml_value.strip():
                #More to it than whitespace, so leave alone
                #Note: if only whitespace entities are used, will still be left alone
                updated_child_list.append(child)
                updated_child_ix += 1
            else:
                #Only whitespace, so replace with proper indentation
                new_text = text('\n' + indent*depth, elem)
                updated_child_list.append(new_text)
                updated_child_ix += 1
    #Trailing indentation might be needed
    if not(updated_child_ix % 2):
        new_text = text('\n' + indent*(depth-1), elem)
        updated_child_list.append(new_text)
        #updated_child_ix += 1 #About to be done, so not really needed
    elem.xml_children = updated_child_list
    return elem

def
make_pretty
(
elem
,
depth
=
0
,
indent
=
'  '
)
:
depth
+=
1
updated_child_list
=
[
]
updated_child_ix
=
0
for
child
in
elem
.
xml_children
:
if
isinstance
(
child
,
element
)
:
if
updated_child_ix
%
2
:
updated_child_list
.
append
(
child
)
updated_child_ix
+=
1
else
:
new_text
=
text
(
'\n'
+
indent
*
depth
,
elem
)
updated_child_list
.
append
(
new_text
)
updated_child_list
.
append
(
child
)
updated_child_ix
+=
2
make_pretty
(
child
,
depth
)
else
:
if
child
.
xml_value
.
strip
(
)
:
updated_child_list
.
append
(
child
)
updated_child_ix
+=
1
else
:
new_text
=
text
(
'\n'
+
indent
*
depth
,
elem
)
updated_child_list
.
append
(
new_text
)
updated_child_ix
+=
1
if
not
(
updated_child_ix
%
2
)
:
new_text
=
text
(
'\n'
+
indent
*
(
depth
1
)
,
elem
)
updated_child_list
.
append
(
new_text
)
elem
.
xml_children
=
updated_child_list
return
elem

Add text nodes as possible to all descendants of an element for spacing & indentation to make the MicroXML as printed easier for people to read . Will not modify the value of any text node which is not already entirely whitespace .


def call_inkscape(args_strings, inkscape_binpath=None):
    """Call inkscape CLI with arguments and returns its return value.

    Parameters
    ----------
    args_string: list of str

    inkscape_binpath: str

    Returns
    -------
    return_value
        Inkscape command CLI call return value.
    """
    log.debug('Looking for the binary file for inkscape.')

    if inkscape_binpath is None:
        inkscape_binpath = get_inkscape_binpath()

    if inkscape_binpath is None or not os.path.exists(inkscape_binpath):
        raise IOError(
            'Inkscape binary has not been found. Please check configuration.'
        )

    return call_command(inkscape_binpath, args_strings)

Call inkscape CLI with arguments and returns its return value .


def inkscape_export(input_file, output_file, export_flag="-A", dpi=90, inkscape_binpath=None):
    """ Call Inkscape to export the input_file to output_file using the
    specific export argument flag for the output file type.

    Parameters
    ----------

    input_file: str
        Path to the input file

    output_file: str
        Path to the output file

    export_flag: str
        Inkscape CLI flag to indicate the type of the output file

    Returns
    -------
    return_value
        Command call return value

    """
    if not os.path.exists(input_file):
        log.error('File {} not found.'.format(input_file))
        raise IOError((0, 'File not found.', input_file))

    if '=' not in export_flag:
        export_flag += ' '

    arg_strings = []
    arg_strings += ['--without-gui']
    arg_strings += ['--export-text-to-path']
    arg_strings += ['{}"{}"'.format(export_flag, output_file)]
    arg_strings += ['--export-dpi={}'.format(dpi)]
    arg_strings += ['"{}"'.format(input_file)]

    return call_inkscape(arg_strings, inkscape_binpath=inkscape_binpath)

def
inkscape_export
(
input_file
,
output_file
,
export_flag
=
"-A"
,
dpi
=
90
,
inkscape_binpath
=
)
:
if
not
os
.
path
.
exists
(
input_file
)
:
log
.
error
(
'File {} not found.'
.
format
(
input_file
)
)
raise
IOError
(
(
0
,
'File not found.'
,
input_file
)
)
if
'='
not
in
export_flag
:
export_flag
+=
' '
arg_strings
=
[
]
arg_strings
+=
[
'--without-gui'
]
arg_strings
+=
[
'--export-text-to-path'
]
arg_strings
+=
[
'{}"{}"'
.
format
(
export_flag
,
output_file
)
]
arg_strings
+=
[
'--export-dpi={}'
.
format
(
dpi
)
]
arg_strings
+=
[
'"{}"'
.
format
(
input_file
)
]
return
call_inkscape
(
arg_strings
,
inkscape_binpath
=
inkscape_binpath
)

Call Inkscape to export the input_file to output_file using the specific export argument flag for the output file type .


def svg2pdf(svg_file_path, pdf_file_path, dpi=150, command_binpath=None, support_unicode=False):
    """ Transform SVG file to PDF file
    """

    if support_unicode:
        return rsvg_export(svg_file_path, pdf_file_path, dpi=dpi, rsvg_binpath=command_binpath)

    return inkscape_export(svg_file_path, pdf_file_path, export_flag="-A",
                           dpi=dpi, inkscape_binpath=command_binpath)

def svg2pdf ( svg_file_path , pdf_file_path , dpi = 150 , command_binpath = None , support_unicode = False ) : if support_unicode : return rsvg_export ( svg_file_path , pdf_file_path , dpi = dpi , rsvg_binpath = command_binpath ) return inkscape_export ( svg_file_path , pdf_file_path , export_flag = "-A" , dpi = dpi , inkscape_binpath = command_binpath )


def svg2png(svg_file_path, png_file_path, dpi=150, inkscape_binpath=None):
    """ Transform SVG file to PNG file
    """
    return inkscape_export(svg_file_path, png_file_path, export_flag="-e",
                           dpi=dpi, inkscape_binpath=inkscape_binpath)

def svg2png ( svg_file_path , png_file_path , dpi = 150 , inkscape_binpath = None ) : return inkscape_export ( svg_file_path , png_file_path , export_flag = "-e" , dpi = dpi , inkscape_binpath = inkscape_binpath )


def get_environment_for(file_path):
    """Return a Jinja2 environment for where file_path is.

    Parameters
    ----------
    file_path: str

    Returns
    -------
    jinja_env: Jinja2.Environment

    """
    work_dir = os.path.dirname(os.path.abspath(file_path))

    if not os.path.exists(work_dir):
        raise IOError('Could not find folder for dirname of file {}.'.format(file_path))

    try:
        jinja_env = Environment(loader=FileSystemLoader(work_dir))
    except:
        raise
    else:
        return jinja_env

Return a Jinja2 environment for where file_path is .


def _setup_template_file(self, template_file_path):
        """ Setup self.template

        Parameters
        ----------
        template_file_path: str
            Document template file path.
        """
        try:
            template_file = template_file_path
            template_env = get_environment_for(template_file_path)
            template = template_env.get_template(os.path.basename(template_file))
        except:
            raise
        else:
            self._template_file = template_file
            self._template_env = template_env
            self.template = template

def _setup_template_file ( self , template_file_path ) : try : template_file = template_file_path template_env = get_environment_for ( template_file_path ) template = template_env . get_template ( os . path . basename ( template_file ) ) except : raise else : self . _template_file = template_file self . _template_env = template_env self . template = template


def fill(self, doc_contents):
        """ Fill the content of the document with the information in doc_contents.

        Parameters
        ----------
        doc_contents: dict
            Set of values to set the template document.

        Returns
        -------
        filled_doc: str
            The content of the document with the template information filled.
        """
        try:
            filled_doc = self.template.render(**doc_contents)
        except:
            log.exception('Error rendering Document '
                          'for {}.'.format(doc_contents))
            raise
        else:
            self.file_content_ = filled_doc
            return filled_doc

def
fill
(
self
,
doc_contents
)
:
try
:
filled_doc
=
self
.
template
.
render
(
**
doc_contents
)
except
:
log
.
exception
(
'Error rendering Document '
'for {}.'
.
format
(
doc_contents
)
)
raise
else
:
self
.
file_content_
=
filled_doc
return
filled_doc

Fill the content of the document with the information in doc_contents .


def save_content(self, file_path, encoding='utf-8'):
        """ Save the content of the .txt file in a text file.

        Parameters
        ----------
        file_path: str
            Path to the output file.
        """
        if self.file_content_ is None:
            msg = 'Template content has not been updated. \
                   Please fill the template before rendering it.'
            log.exception(msg)
            raise ValueError(msg)

        try:
            write_to_file(file_path, content=self.file_content_,
                          encoding=encoding)
        except Exception as exc:
            msg = 'Document of type {} got an error when \
                   writing content.'.format(self.__class__)
            log.exception(msg)
            raise Exception(msg) from exc

Save the content of the . txt file in a text file .


def from_template_file(cls, template_file_path, command=None):
        """ Factory function to create a specific document of the
        class given by the `command` or the extension of `template_file_path`.

        See get_doctype_by_command and get_doctype_by_extension.

        Parameters
        ----------
        template_file_path: str

        command: str

        Returns
        -------
        doc

        """
        # get template file extension
        ext = os.path.basename(template_file_path).split('.')[-1]

        try:
            doc_type = get_doctype_by_command(command)
        except ValueError:
            doc_type = get_doctype_by_extension(ext)
        except:
            raise
        else:
            return doc_type(template_file_path)

def from_template_file ( cls , template_file_path , command = None ) : ext = os . path . basename ( template_file_path ) . split ( '.' ) [ - 1 ] try : doc_type = get_doctype_by_command ( command ) except ValueError : doc_type = get_doctype_by_extension ( ext ) except : raise else : return doc_type ( template_file_path )

Factory function to create a specific document of the class given by the command or the extension of template_file_path .


def fill(self, doc_contents):
        """ Fill the content of the document with the information in doc_contents.
        This is different from the TextDocument fill function, because this will
        check for symbools in the values of `doc_content` and replace them
        to good XML codes before filling the template.

        Parameters
        ----------
        doc_contents: dict
            Set of values to set the template document.

        Returns
        -------
        filled_doc: str
            The content of the document with the template information filled.
        """
        for key, content in doc_contents.items():
            doc_contents[key] = replace_chars_for_svg_code(content)

        return super(SVGDocument, self).fill(doc_contents=doc_contents)

def fill ( self , doc_contents ) : for key , content in doc_contents . items ( ) : doc_contents [ key ] = replace_chars_for_svg_code ( content ) return super ( SVGDocument , self ) . fill ( doc_contents = doc_contents )

Fill the content of the document with the information in doc_contents . This is different from the TextDocument fill function because this will check for symbools in the values of doc_content and replace them to good XML codes before filling the template .


def render(self, file_path, **kwargs):
        """ Save the content of the .svg file in the chosen rendered format.

        Parameters
        ----------
        file_path: str
            Path to the output file.

        Kwargs
        ------
        file_type: str
            Choices: 'png', 'pdf', 'svg'
            Default: 'pdf'

        dpi: int
            Dots-per-inch for the png and pdf.
            Default: 150

        support_unicode: bool
            Whether to allow unicode to be encoded in the PDF.
            Default: False
        """
        temp = get_tempfile(suffix='.svg')
        self.save_content(temp.name)

        file_type = kwargs.get('file_type', 'pdf')
        dpi = kwargs.get('dpi', 150)
        support_unicode = kwargs.get('support_unicode', False)
        try:
            if file_type == 'svg':
                shutil.copyfile(temp.name, file_path)
            elif file_type == 'png':
                svg2png(temp.name, file_path, dpi=dpi)
            elif file_type == 'pdf':
                svg2pdf(temp.name, file_path, dpi=dpi, support_unicode=support_unicode)
        except:
            log.exception(
                'Error exporting file {} to {}'.format(file_path, file_type)
            )
            raise

def
render
(
self
,
file_path
,
**
kwargs
)
:
temp
=
get_tempfile
(
suffix
=
'.svg'
)
self
.
save_content
(
temp
.
name
)
file_type
=
kwargs
.
get
(
'file_type'
,
'pdf'
)
dpi
=
kwargs
.
get
(
'dpi'
,
150
)
support_unicode
=
kwargs
.
get
(
'support_unicode'
,
False
)
try
:
if
file_type
==
'svg'
:
shutil
.
copyfile
(
temp
.
name
,
file_path
)
elif
file_type
==
'png'
:
svg2png
(
temp
.
name
,
file_path
,
dpi
=
dpi
)
elif
file_type
==
'pdf'
:
svg2pdf
(
temp
.
name
,
file_path
,
dpi
=
dpi
,
support_unicode
=
support_unicode
)
except
:
log
.
exception
(
'Error exporting file {} to {}'
.
format
(
file_path
,
file_type
)
)
raise

Save the content of the . svg file in the chosen rendered format .


def render(self, file_path, **kwargs):
        """ Save the content of the .text file in the PDF.

        Parameters
        ----------
        file_path: str
            Path to the output file.
        """
        temp = get_tempfile(suffix='.tex')
        self.save_content(temp.name)

        try:
            self._render_function(temp.name, file_path, output_format='pdf')
        except:
            log.exception('Error exporting file {} to PDF.'.format(file_path))
            raise

def
render
(
self
,
file_path
,
**
kwargs
)
:
temp
=
get_tempfile
(
suffix
=
'.tex'
)
self
.
save_content
(
temp
.
name
)
try
:
self
.
_render_function
(
temp
.
name
,
file_path
,
output_format
=
'pdf'
)
except
:
log
.
exception
(
'Error exporting file {} to PDF.'
.
format
(
file_path
)
)
raise

Save the content of the . text file in the PDF .


def parse(source, handler):
    '''
    Convert XML 1.0 to MicroXML

    source - XML 1.0 input
    handler - MicroXML events handler

    Returns uxml, extras

    uxml - MicroXML element extracted from the source
    extras - information to be preserved but not part of MicroXML, e.g. namespaces
    '''
    h = expat_callbacks(handler)
    p = xml.parsers.expat.ParserCreate(namespace_separator=' ')

    p.StartElementHandler = h.start_element
    p.EndElementHandler = h.end_element
    p.CharacterDataHandler = h.char_data
    p.StartNamespaceDeclHandler = h.start_namespace
    p.EndNamespaceDeclHandler = h.end_namespace
    p.Parse(source)
    return p

def
parse
(
source
,
handler
)
:
h
=
expat_callbacks
(
handler
)
p
=
xml
.
parsers
.
expat
.
ParserCreate
(
namespace_separator
=
' '
)
p
.
StartElementHandler
=
h
.
start_element
p
.
EndElementHandler
=
h
.
end_element
p
.
CharacterDataHandler
=
h
.
char_data
p
.
StartNamespaceDeclHandler
=
h
.
start_namespace
p
.
EndNamespaceDeclHandler
=
h
.
end_namespace
p
.
Parse
(
source
)
return
p


def parse(source, prefixes=None, model=None, encoding=None, use_xhtml_ns=False):
    '''
    Parse an input source with HTML text into an Amara 3 tree

    >>> from amara3.uxml import html5
    >>> import urllib.request
    >>> with urllib.request.urlopen('http://uche.ogbuji.net/') as response:
    ...     html5.parse(response)

    #Warning: if you pass a string, you must make sure it's a byte string, not a Unicode object.  You might also want to wrap it with amara.lib.inputsource.text if it's not obviously XML or HTML (for example it could be confused with a file name)
    '''
    def get_tree_instance(namespaceHTMLElements, use_xhtml_ns=use_xhtml_ns):
        #use_xhtml_ns is a boolean, whether or not to use http://www.w3.org/1999/xhtml
        return treebuilder(use_xhtml_ns)
    parser = html5lib.HTMLParser(tree=get_tree_instance)
    #doc = parser.parse(inputsource(source, None).stream, encoding=encoding)
    #doc = parser.parse(source, encoding=encoding)
    doc = parser.parse(source)
    first_element = next((e for e in doc.root_nodes if isinstance(e, element)), None)
    return first_element

def parse ( source , prefixes = None , model = None , encoding = None , use_xhtml_ns = False ) : def get_tree_instance ( namespaceHTMLElements , use_xhtml_ns = use_xhtml_ns ) : return treebuilder ( use_xhtml_ns ) parser = html5lib . HTMLParser ( tree = get_tree_instance ) doc = parser . parse ( source ) first_element = next ( ( e for e in doc . root_nodes if isinstance ( e , element ) ) , None ) return first_element


def markup_fragment(source, encoding=None):
    '''
    Parse a fragment if markup in HTML mode, and return a bindery node

    Warning: if you pass a string, you must make sure it's a byte string, not a Unicode object.  You might also want to wrap it with amara.lib.inputsource.text if it's not obviously XML or HTML (for example it could be confused with a file name)

    from amara.lib import inputsource
    from amara.bindery import html
    doc = html.markup_fragment(inputsource.text('XXX
Spam!
Eggs!YYY'))

    See also: http://wiki.xml3k.org/Amara2/Tagsoup
    '''
    doc = parse(source, encoding=encoding)
    frag = doc.html.body
    return frag

def markup_fragment ( source , encoding = None ) : doc = parse ( source , encoding = encoding ) frag = doc . html . body return frag

Parse a fragment if markup in HTML mode and return a bindery node


def insertText(self, data, insertBefore=None):
        """Insert data as text in the current node, positioned before the
        start of node insertBefore or to the end of the node's text.
        """
        if insertBefore:
            self.insertBefore(tree.text(data), insertBefore)
        else:
            self.xml_append(tree.text(data))

def insertText ( self , data , insertBefore = None ) : if insertBefore : self . insertBefore ( tree . text ( data ) , insertBefore ) else : self . xml_append ( tree . text ( data ) )

Insert data as text in the current node positioned before the start of node insertBefore or to the end of the node s text .


def insertBefore(self, node, refNode):
        """Insert node as a child of the current node, before refNode in the
        list of child nodes. Raises ValueError if refNode is not a child of
        the current node"""
        offset = self.xml_children.index(refNode)
        self.xml_insert(node, offset)

def insertBefore ( self , node , refNode ) : offset = self . xml_children . index ( refNode ) self . xml_insert ( node , offset )

Insert node as a child of the current node before refNode in the list of child nodes . Raises ValueError if refNode is not a child of the current node


def cloneNode(self):
        """Return a shallow copy of the current node i.e. a node with the same
        name and attributes but with no parent or child nodes
        """
        attrs = self.xml_attributes.copy()
        return element(self.xml_name, attrs=attrs)

def cloneNode ( self ) : attrs = self . xml_attributes . copy ( ) return element ( self . xml_name , attrs = attrs )

Return a shallow copy of the current node i . e . a node with the same name and attributes but with no parent or child nodes


def execute(option):
    '''A script that melody calls with each valid set of options. This
    script runs the required code and returns the results.'''

    namelist_option = []
    makefile_option = []
    flags = ""
    for entry in option:
        key = entry.keys()[0]
        if key == "Problem Size":
            namelist_option.append({"SIZE": entry[key]})
        elif key == "F90":
            makefile_option.append(entry)
        else:
            flags += entry[key] + " "
    makefile_option.append({"F90FLAGS": flags})

    namelist = create_input(namelist_option, "namelist",
                            template_location="templates")

    makefile_include = create_input(makefile_option, "Makefile.include",
                                    template_location="templates")

    benchmark_base = "shallow"

    # save the input files in the appropriate place
    location = benchmark_base + "/original/namelist"
    my_file = open(location, 'w')
    my_file.write(namelist)
    my_file.flush()

    location = benchmark_base + "/common/Makefile.include"
    my_file = open(location, 'w')
    my_file.write(makefile_include)
    my_file.flush()

    # compile shallow if required
    base_path = benchmark_base + "/original"
    import subprocess
    make_process = subprocess.Popen(["make", "clean"], cwd=base_path,
                                    stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE)
    if make_process.wait() != 0:
        return False, []

    make_process = subprocess.Popen(["make"], cwd=base_path,
                                    stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE)
    if make_process.wait() != 0:
        return False, []

    # run shallow
    make_process = subprocess.Popen(["./shallow_base"], cwd=base_path,
                                    stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE)
    if make_process.wait() != 0:
        return False, []
    # _ = make_process.stderr.read()
    stdout = make_process.stdout.read()

    # determine if the results are correct. We will need to look at
    # the results from stdout but for the moment we assume they are
    # correct

    # extract the required outputs
    for line in stdout.split("\n"):
        if "Time-stepping" in line:
            total_time = line.split()[2]

    return True, total_time

def
execute
(
option
)
:
namelist_option
=
[
]
makefile_option
=
[
]
flags
=
""
for
entry
in
option
:
key
=
entry
.
keys
(
)
[
0
]
if
key
==
"Problem Size"
:
namelist_option
.
append
(
{
"SIZE"
:
entry
[
key
]
}
)
elif
key
==
"F90"
:
makefile_option
.
append
(
entry
)
else
:
flags
+=
entry
[
key
]
+
" "
makefile_option
.
append
(
{
"F90FLAGS"
:
flags
}
)
namelist
=
create_input
(
namelist_option
,
"namelist"
,
template_location
=
"templates"
)
makefile_include
=
create_input
(
makefile_option
,
"Makefile.include"
,
template_location
=
"templates"
)
benchmark_base
=
"shallow"
location
=
benchmark_base
+
"/original/namelist"
my_file
=
open
(
location
,
'w'
)
my_file
.
write
(
namelist
)
my_file
.
flush
(
)
location
=
benchmark_base
+
"/common/Makefile.include"
my_file
=
open
(
location
,
'w'
)
my_file
.
write
(
makefile_include
)
my_file
.
flush
(
)
base_path
=
benchmark_base
+
"/original"
import
subprocess
make_process
=
subprocess
.
Popen
(
[
"make"
,
"clean"
]
,
cwd
=
base_path
,
stderr
=
subprocess
.
PIPE
,
stdout
=
subprocess
.
PIPE
)
if
make_process
.
wait
(
)
!=
0
:
return
False
,
[
]
make_process
=
subprocess
.
Popen
(
[
"make"
]
,
cwd
=
base_path
,
stderr
=
subprocess
.
PIPE
,
stdout
=
subprocess
.
PIPE
)
if
make_process
.
wait
(
)
!=
0
:
return
False
,
[
]
make_process
=
subprocess
.
Popen
(
[
"./shallow_base"
]
,
cwd
=
base_path
,
stderr
=
subprocess
.
PIPE
,
stdout
=
subprocess
.
PIPE
)
if
make_process
.
wait
(
)
!=
0
:
return
False
,
[
]
stdout
=
make_process
.
stdout
.
read
(
)
for
line
in
stdout
.
split
(
"\n"
)
:
if
"Time-stepping"
in
line
:
total_time
=
line
.
split
(
)
[
2
]
return
True
,
total_time

A script that melody calls with each valid set of options . This script runs the required code and returns the results .


def strval(node, outermost=True):
    '''
    XPath-like string value of node
    '''
    if not isinstance(node, element):
        return node.xml_value if outermost else [node.xml_value]
    accumulator = []
    for child in node.xml_children:
        if isinstance(child, text):
            accumulator.append(child.xml_value)
        elif isinstance(child, element):
            accumulator.extend(strval(child, outermost=False))
    if outermost: accumulator = ''.join(accumulator)
    return accumulator

def strval ( node , outermost = True ) : if not isinstance ( node , element ) : return node . xml_value if outermost else [ node . xml_value ] accumulator = [ ] for child in node . xml_children : if isinstance ( child , text ) : accumulator . append ( child . xml_value ) elif isinstance ( child , element ) : accumulator . extend ( strval ( child , outermost = False ) ) if outermost : accumulator = '' . join ( accumulator ) return accumulator


def xml_insert(self, child, index=-1):
        '''
        Append a node as the last child

        child - the child to append. If a string, convert to a text node, for convenience
        '''
        if isinstance(child, str):
            child = text(child, parent=self)
        else:
            child._xml_parent = weakref.ref(self)
        if index == -1:
            self.xml_children.append(child)
        else:
            self.xml_children.insert(index, child)
        return

def xml_insert ( self , child , index = - 1 ) : if isinstance ( child , str ) : child = text ( child , parent = self ) else : child . _xml_parent = weakref . ref ( self ) if index == - 1 : self . xml_children . append ( child ) else : self . xml_children . insert ( index , child ) return


def parse_config(options):
    """
    Get settings from config file.
    """

    if os.path.exists(options.config):
        config = ConfigParser.ConfigParser()
        try:
            config.read(options.config)
        except Exception, err:
            if not options.quiet:
                sys.stderr.write("ERROR: Config file read {config} error. {err}".format(config=options.config, err=err))
            sys.exit(-1)

        try:
            configdata = {
                "secrets": config.get("GOOGLE", "secrets"),
                "credentials": config.get("nagios-notification-google-calendar", "credentials"),
                "start": config.get("nagios-notification-google-calendar", "start"),
                "end": config.get("nagios-notification-google-calendar", "end"),
                "message": config.get("nagios-notification-google-calendar", "message"),
            }
        except ConfigParser.NoOptionError, err:
            if not options.quiet:
                sys.stderr.write("ERROR: Config file missing option error. {err}\n".format(err=err))
            sys.exit(-1)

        # check mandatory config options supplied
        mandatories = ["secrets", "credentials", "start", "end", "message", ]
        if not all(configdata[mandatory] for mandatory in mandatories):
            if not options.quiet:
                sys.stdout.write("Mandatory config option missing\n")
            sys.exit(0)

        return configdata
    else:
        if not options.quiet:
            sys.stderr.write("ERROR: Config file {config} does not exist\n".format(config=options.config))
        sys.exit(0)


def get_google_credentials(options, config):
    """
    Get google API credentials for user.
    """

    try:
        if options.get_google_credentials:
            flow = flow_from_clientsecrets(config["secrets"], scope=SCOPE, redirect_uri="oob")
            sys.stdout.write("Follow this URL: {url} and grant access to calendar.\n".format(url=flow.step1_get_authorize_url()))
            token = raw_input("Enter token:")
            credentials = flow.step2_exchange(token)
            storage = Storage(os.path.join(config["credentials"], "{username}.json".format(username=options.username)))
            storage.put(credentials)
            credentials.set_store(storage)
        else:
            storage = Storage(os.path.join(config["credentials"], "{username}.json".format(username=options.username)))
            credentials = storage.get()
    except Exception, err:
        if not options.quiet:
            sys.stderr.write("ERROR: Getting google API credentials error. {err}\n".format(err=err))
        sys.exit(-1)

    return credentials


def create_event_datetimes(options, config):
    """
    Create event start and end datetimes.
    """

    now = datetime.datetime.now()

    return {
        "start": {
            "dateTime": (now + datetime.timedelta(minutes=int(config["start"]))).strftime(DT_FORMAT),
            "timeZone": options.timezone,
        },
        "end": {
            "dateTime": (now + datetime.timedelta(minutes=int(config["end"]))).strftime(DT_FORMAT),
            "timeZone": options.timezone,
        },
    }

def create_event_datetimes ( options , config ) : now = datetime . datetime . now ( ) return { "start" : { "dateTime" : ( now + datetime . timedelta ( minutes = int ( config [ "start" ] ) ) ) . strftime ( DT_FORMAT ) , "timeZone" : options . timezone , } , "end" : { "dateTime" : ( now + datetime . timedelta ( minutes = int ( config [ "end" ] ) ) ) . strftime ( DT_FORMAT ) , "timeZone" : options . timezone , } , }


def create_event(options, config, credentials):
    """
    Create event in calendar with sms reminder.
    """

    try:
        http = credentials.authorize(httplib2.Http())
        service = build("calendar", "v3", http=http)

        event = {
            "summary": options.message,
            "location": "",
            "reminders": {
                "useDefault": False,
                "overrides": [
                    {
                        "method": "sms",
                        "minutes": config["message"],
                    },
                ],
            }
        }
        event.update(create_event_datetimes(options, config))

        service.events().insert(calendarId=options.calendar, sendNotifications=True, body=event).execute()
    except Exception, err:
        if not options.quiet:
            sys.stderr.write("ERROR: Creating google calendar event error. {err}\n".format(err=err))
        sys.exit(-1)

Create event in calendar with sms reminder .


def main():
    """
    Processing notification call main function.
    """

    # getting info for creating event
    options = parse_options()
    config = parse_config(options)
    credentials = get_google_credentials(options, config)

    if not options.get_google_credentials:
        create_event(options, config, credentials)

def main ( ) : options = parse_options ( ) config = parse_config ( options ) credentials = get_google_credentials ( options , config ) if not options . get_google_credentials : create_event ( options , config , credentials )

Processing notification call main function .


def get_extension(filepath, check_if_exists=False):
    """Return the extension of fpath.

    Parameters
    ----------
    fpath: string
    File name or path

    check_if_exists: bool

    Returns
    -------
    str
    The extension of the file name or path
    """
    if check_if_exists:
        if not os.path.exists(filepath):
            err = 'File not found: ' + filepath
            log.error(err)
            raise IOError(err)

    try:
        rest, ext = os.path.splitext(filepath)
    except:
        raise
    else:
        return ext

def
get_extension
(
filepath
,
check_if_exists
=
False
)
:
if
check_if_exists
:
if
not
os
.
path
.
exists
(
filepath
)
:
err
=
'File not found: '
+
filepath
log
.
error
(
err
)
raise
IOError
(
err
)
try
:
rest
,
ext
=
os
.
path
.
splitext
(
filepath
)
except
:
raise
else
:
return
ext


def add_extension_if_needed(filepath, ext, check_if_exists=False):
    """Add the extension ext to fpath if it doesn't have it.

    Parameters
    ----------
    filepath: str
    File name or path

    ext: str
    File extension

    check_if_exists: bool

    Returns
    -------
    File name or path with extension added, if needed.
    """
    if not filepath.endswith(ext):
        filepath += ext

    if check_if_exists:
        if not os.path.exists(filepath):
            err = 'File not found: ' + filepath
            log.error(err)
            raise IOError(err)

    return filepath

def
add_extension_if_needed
(
filepath
,
ext
,
check_if_exists
=
False
)
:
if
not
filepath
.
endswith
(
ext
)
:
filepath
+=
ext
if
check_if_exists
:
if
not
os
.
path
.
exists
(
filepath
)
:
err
=
'File not found: '
+
filepath
log
.
error
(
err
)
raise
IOError
(
err
)
return
filepath

Add the extension ext to fpath if it doesn t have it .


def get_tempfile(suffix='.txt', dirpath=None):
    """ Return a temporary file with the given suffix within dirpath.
    If dirpath is None, will look for a temporary folder in your system.

    Parameters
    ----------
    suffix: str
        Temporary file name suffix

    dirpath: str
        Folder path where create the temporary file

    Returns
    -------
    temp_filepath: str
        The path to the temporary path
    """
    if dirpath is None:
        dirpath = get_temp_dir()

    return tempfile.NamedTemporaryFile(suffix=suffix, dir=dirpath)

def get_tempfile ( suffix = '.txt' , dirpath = None ) : if dirpath is None : dirpath = get_temp_dir ( ) return tempfile . NamedTemporaryFile ( suffix = suffix , dir = dirpath )

Return a temporary file with the given suffix within dirpath . If dirpath is None will look for a temporary folder in your system .


def cleanup(workdir, extension):
    """ Remove the files in workdir that have the given extension.

    Parameters
    ----------
    workdir:
        Folder path from where to clean the files.

    extension: str
        File extension without the dot, e.g., 'txt'
    """
    [os.remove(f) for f in glob(os.path.join(workdir, '*.' + extension))]

def cleanup ( workdir , extension ) : [ os . remove ( f ) for f in glob ( os . path . join ( workdir , '*.' + extension ) ) ]

Remove the files in workdir that have the given extension .


def csv_to_json(csv_filepath, json_filepath, fieldnames, ignore_first_line=True):
    """ Convert a CSV file in `csv_filepath` into a JSON file in `json_filepath`.

    Parameters
    ----------
    csv_filepath: str
        Path to the input CSV file.

    json_filepath: str
        Path to the output JSON file. Will be overwritten if exists.

    fieldnames: List[str]
        Names of the fields in the CSV file.

    ignore_first_line: bool
    """
    import csv
    import json

    csvfile = open(csv_filepath, 'r')
    jsonfile = open(json_filepath, 'w')

    reader = csv.DictReader(csvfile, fieldnames)
    rows = []
    if ignore_first_line:
        next(reader)

    for row in reader:
        rows.append(row)

    json.dump(rows, jsonfile)
    jsonfile.close()
    csvfile.close()

def csv_to_json ( csv_filepath , json_filepath , fieldnames , ignore_first_line = True ) : import csv import json csvfile = open ( csv_filepath , 'r' ) jsonfile = open ( json_filepath , 'w' ) reader = csv . DictReader ( csvfile , fieldnames ) rows = [ ] if ignore_first_line : next ( reader ) for row in reader : rows . append ( row ) json . dump ( rows , jsonfile ) jsonfile . close ( ) csvfile . close ( )

Convert a CSV file in csv_filepath into a JSON file in json_filepath .


def replace_file_content(filepath, old, new, max=1):
    """ Modify the content of `filepath`, replacing `old` for `new`.

    Parameters
    ----------
    filepath: str
        Path to the file to be modified. It will be overwritten.

    old: str
        This is old substring to be replaced.

    new: str
        This is new substring, which would replace old substring.

    max: int
        If larger than 0, Only the first `max` occurrences are replaced.
    """
    with open(filepath, 'r') as f:
        content = f.read()

    content = content.replace(old, new, max)
    with open(filepath, 'w') as f:
        f.write(content)

def replace_file_content ( filepath , old , new , max = 1 ) : with open ( filepath , 'r' ) as f : content = f . read ( ) content = content . replace ( old , new , max ) with open ( filepath , 'w' ) as f : f . write ( content )

Modify the content of filepath replacing old for new .


def parse(self):
        """
        Run all parsing functions.
        """
        for tag in self.soup.findAll('span'):
            self.create_italic(tag)
            self.create_strong(tag)
            self.create_underline(tag)
            self.unwrap_span(tag)

        for tag in self.soup.findAll('a'):
            self.remove_comments(tag)
            self.check_next(tag)

        if self.soup.body:
            for tag in self.soup.body.findAll():
                self.remove_empty(tag)
                self.remove_inline_comment(tag)
                self.parse_attrs(tag)
                for token, target in self.tokens:
                    self.find_token(tag, token, target)

                self.remove_blacklisted_tags(tag)

def parse ( self ) : for tag in self . soup . findAll ( 'span' ) : self . create_italic ( tag ) self . create_strong ( tag ) self . create_underline ( tag ) self . unwrap_span ( tag ) for tag in self . soup . findAll ( 'a' ) : self . remove_comments ( tag ) self . check_next ( tag ) if self . soup . body : for tag in self . soup . body . findAll ( ) : self . remove_empty ( tag ) self . remove_inline_comment ( tag ) self . parse_attrs ( tag ) for token , target in self . tokens : self . find_token ( tag , token , target ) self . remove_blacklisted_tags ( tag )


def check_next(self, tag):
        """
        If next tag is link with same href, combine them.
        """
        if (type(tag.next_sibling) == element.Tag and
                tag.next_sibling.name == 'a'):

            next_tag = tag.next_sibling
            if tag.get('href') and next_tag.get('href'):
                href = self._parse_href(tag.get('href'))
                next_href = self._parse_href(next_tag.get('href'))

                if href == next_href:
                    next_text = next_tag.get_text()
                    tag.append(next_text)
                    self.tags_blacklist.append(next_tag)

def check_next ( self , tag ) : if ( type ( tag . next_sibling ) == element . Tag and tag . next_sibling . name == 'a' ) : next_tag = tag . next_sibling if tag . get ( 'href' ) and next_tag . get ( 'href' ) : href = self . _parse_href ( tag . get ( 'href' ) ) next_href = self . _parse_href ( next_tag . get ( 'href' ) ) if href == next_href : next_text = next_tag . get_text ( ) tag . append ( next_text ) self . tags_blacklist . append ( next_tag )

If next tag is link with same href combine them .


def create_italic(self, tag):
        """
        See if span tag has italic style and wrap with em tag.
        """
        style = tag.get('style')
        if style and 'font-style:italic' in style:
            tag.wrap(self.soup.new_tag('em'))

def create_italic ( self , tag ) : style = tag . get ( 'style' ) if style and 'font-style:italic' in style : tag . wrap ( self . soup . new_tag ( 'em' ) )

See if span tag has italic style and wrap with em tag .


def create_strong(self, tag):
        """
        See if span tag has bold style and wrap with strong tag.
        """
        style = tag.get('style')
        if (style and
                ('font-weight:bold' in style or 'font-weight:700' in style)):
            tag.wrap(self.soup.new_tag('strong'))

def create_strong ( self , tag ) : style = tag . get ( 'style' ) if ( style and ( 'font-weight:bold' in style or 'font-weight:700' in style ) ) : tag . wrap ( self . soup . new_tag ( 'strong' ) )

See if span tag has bold style and wrap with strong tag .


def create_underline(self, tag):
        """
        See if span tag has underline style and wrap with u tag.
        """
        style = tag.get('style')
        if style and 'text-decoration:underline' in style:
            tag.wrap(self.soup.new_tag('u'))

def create_underline ( self , tag ) : style = tag . get ( 'style' ) if style and 'text-decoration:underline' in style : tag . wrap ( self . soup . new_tag ( 'u' ) )

See if span tag has underline style and wrap with u tag .


def parse_attrs(self, tag):
        """
        Reject attributes not defined in ATTR_WHITELIST.
        """
        if tag.name in ATTR_WHITELIST.keys():
            attrs = copy(tag.attrs)
            for attr, value in attrs.items():
                if attr in ATTR_WHITELIST[tag.name]:
                    tag.attrs[attr] = self._parse_attr(tag.name, attr, value)
                else:
                    del tag.attrs[attr]
        else:
            tag.attrs = {}

def parse_attrs ( self , tag ) : if tag . name in ATTR_WHITELIST . keys ( ) : attrs = copy ( tag . attrs ) for attr , value in attrs . items ( ) : if attr in ATTR_WHITELIST [ tag . name ] : tag . attrs [ attr ] = self . _parse_attr ( tag . name , attr , value ) else : del tag . attrs [ attr ] else : tag . attrs = { }

Reject attributes not defined in ATTR_WHITELIST .


def clean_linebreaks(self, tag):
        """
        get unicode string without any other content transformation.
        and clean extra spaces
        """
        stripped = tag.decode(formatter=None)
        stripped = re.sub('\s+', ' ', stripped)
        stripped = re.sub('\n', '', stripped)
        return stripped

def
clean_linebreaks
(
self
,
tag
)
:
stripped
=
tag
.
decode
(
formatter
=
)
stripped
=
re
.
sub
(
'\s+'
,
' '
,
stripped
)
stripped
=
re
.
sub
(
'\n'
,
''
,
stripped
)
return
stripped

get unicode string without any other content transformation . and clean extra spaces


def _parse_href(self, href):
        """
        Extract "real" URL from Google redirected url by getting `q`
        querystring parameter.
        """
        params = parse_qs(urlsplit(href).query)
        return params.get('q')

def _parse_href ( self , href ) : params = parse_qs ( urlsplit ( href ) . query ) return params . get ( 'q' )

Extract real URL from Google redirected url by getting q querystring parameter .


def _parse_attr(self, tagname, attr, value):
        """
        Parse attribute. Delegate to href parser for hrefs, otherwise return
        value.
        """
        if tagname == 'a' and attr == 'href':
            return self._parse_href(value)
        else:
            return value

def _parse_attr ( self , tagname , attr , value ) : if tagname == 'a' and attr == 'href' : return self . _parse_href ( value ) else : return value

Parse attribute . Delegate to href parser for hrefs otherwise return value .


def translate_key_values(adict, translations, default=''):
    """Modify the keys in adict to the ones in translations.
    Be careful, this will modify your input dictionary.
    The keys not present in translations will be left intact.

    Parameters
    ----------
    adict: a dictionary

    translations: iterable of 2-tuples
    Each 2-tuple must have the following format:
    (<adict existing key>, <desired key name for the existing key>)

    Returns
    -------
    Translated adict
    """
    for src_key, dst_key in translations:
        adict[dst_key] = adict.pop(src_key, default)
    return adict

def translate_key_values ( adict , translations , default = '' ) : for src_key , dst_key in translations : adict [ dst_key ] = adict . pop ( src_key , default ) return adict

Modify the keys in adict to the ones in translations . Be careful this will modify your input dictionary . The keys not present in translations will be left intact .


def to_json_str(self):
        """Convert data to json string representation.

        Returns:
          json representation as string.
        """
        adict = dict(vars(self), sort_keys=True)
        adict['type'] = self.__class__.__name__
        return json.dumps(adict)

def to_json_str ( self ) : adict = dict ( vars ( self ) , sort_keys = True ) adict [ 'type' ] = self . __class__ . __name__ return json . dumps ( adict )

Convert data to json string representation .


def find_file_match(folder_path, regex=''):
    """
    Returns absolute paths of files that match the regex within folder_path and
    all its children folders.

    Note: The regex matching is done using the match function
    of the re module.

    Parameters
    ----------
    folder_path: string

    regex: string

    Returns
    -------
    A list of strings.

    """
    outlist = []
    for root, dirs, files in os.walk(folder_path):
        outlist.extend([os.path.join(root, f) for f in files
                        if re.match(regex, f)])

    return outlist

def find_file_match ( folder_path , regex = '' ) : outlist = [ ] for root , dirs , files in os . walk ( folder_path ) : outlist . extend ( [ os . path . join ( root , f ) for f in files if re . match ( regex , f ) ] ) return outlist

Returns absolute paths of files that match the regex within folder_path and all its children folders .


def concat(ctx, *strings):
    '''
    Yields one string, concatenation of argument strings
    '''
    strings = flatten([ (s.compute(ctx) if callable(s) else s) for s in strings ])
    strings = (next(string_arg(ctx, s), '') for s in strings)
    #assert(all(map(lambda x: isinstance(x, str), strings)))
    #FIXME: Check arg types
    yield ''.join(strings)

def concat ( ctx , * strings ) : strings = flatten ( [ ( s . compute ( ctx ) if callable ( s ) else s ) for s in strings ] ) strings = ( next ( string_arg ( ctx , s ) , '' ) for s in strings ) yield '' . join ( strings )

Yields one string concatenation of argument strings


def starts_with(ctx, full, part):
    '''
    Yields one boolean, whether the first string starts with the second
    '''
    full = next(string_arg(ctx, full), '')
    part = next(string_arg(ctx, part), '')
    yield full.startswith(part)

def starts_with ( ctx , full , part ) : full = next ( string_arg ( ctx , full ) , '' ) part = next ( string_arg ( ctx , part ) , '' ) yield full . startswith ( part )

Yields one boolean whether the first string starts with the second


def contains(ctx, full, part):
    '''
    Yields one boolean, whether the first string contains the second
    '''
    full = next(string_arg(ctx, full), '')
    part = next(string_arg(ctx, part), '')
    yield part in full

def contains ( ctx , full , part ) : full = next ( string_arg ( ctx , full ) , '' ) part = next ( string_arg ( ctx , part ) , '' ) yield part in full

Yields one boolean whether the first string contains the second


def string_length(ctx, s=None):
    '''
    Yields one number
    '''
    if s is None:
        s = ctx.node
    elif callable(s):
        s = next(s.compute(ctx), '')
    yield len(s)

def string_length ( ctx , s = None ) : if s is None : s = ctx . node elif callable ( s ) : s = next ( s . compute ( ctx ) , '' ) yield len ( s )


def boolean(ctx, obj):
    '''
    Yields one boolean, false if the argument sequence is empty, otherwise

    * false if the first item is a boolean and false
    * false if the first item is a number and positive or negative zero or NaN
    * false if the first item is a string and ''
    * true in all other cases
    '''
    if hasattr(obj, 'compute'):
        obj = next(seq.compute(ctx), '')
    else:
        obj = seq
    yield next(to_boolean(obj), '')

def boolean ( ctx , obj ) : if hasattr ( obj , 'compute' ) : obj = next ( seq . compute ( ctx ) , '' ) else : obj = seq yield next ( to_boolean ( obj ) , '' )

Yields one boolean false if the argument sequence is empty otherwise


def foreach_(ctx, seq, expr):
    '''
    Yields the result of applying an expression to each item in the input sequence.

    * seq: input sequence
    * expr: expression to be converted to string, then dynamically evaluated for each item on the sequence to produce the result
    '''
    from . import context, parse as uxpathparse

    if hasattr(seq, 'compute'):
        seq = seq.compute(ctx)

    expr = next(string_arg(ctx, expr), '')

    pexpr = uxpathparse(expr)
    for item in seq:
        innerctx = ctx.copy(item=item)
        yield from pexpr.compute(innerctx)

def foreach_ ( ctx , seq , expr ) : from . import context , parse as uxpathparse if hasattr ( seq , 'compute' ) : seq = seq . compute ( ctx ) expr = next ( string_arg ( ctx , expr ) , '' ) pexpr = uxpathparse ( expr ) for item in seq : innerctx = ctx . copy ( item = item ) yield from pexpr . compute ( innerctx )

Yields the result of applying an expression to each item in the input sequence .


def lookup_(ctx, tableid, key):
    '''
    Yields a sequence of a single value, the result of looking up a value from the tables provided in the context, or an empty sequence if lookup is unsuccessful

    * tableid: id of the lookup table to use
    * expr: expression to be converted to string, then dynamically evaluated for each item on the sequence to produce the result
    '''
    tableid = next(string_arg(ctx, tableid), '')
    key = next(string_arg(ctx, key), '')
    #value = ctx.
    for item in seq:
        innerctx = ctx.copy(item=item)
        yield from pexpr.compute(innerctx)

def lookup_ ( ctx , tableid , key ) : tableid = next ( string_arg ( ctx , tableid ) , '' ) key = next ( string_arg ( ctx , key ) , '' ) for item in seq : innerctx = ctx . copy ( item = item ) yield from pexpr . compute ( innerctx )

Yields a sequence of a single value the result of looking up a value from the tables provided in the context or an empty sequence if lookup is unsuccessful


def replace_chars_for_svg_code(svg_content):
    """ Replace known special characters to SVG code.

    Parameters
    ----------
    svg_content: str

    Returns
    -------
    corrected_svg: str
        Corrected SVG content
    """
    result = svg_content
    svg_char = [
        ('&', '&'),
        ('>', '>'),
        ('<', '<'),
        ('"', '"'),
    ]

    for c, entity in svg_char:
        result = result.replace(c, entity)

    return result

def replace_chars_for_svg_code ( svg_content ) : result = svg_content svg_char = [ ( '&' , '&' ) , ( '>' , '>' ) , ( '<' , '<' ) , ( '"' , '"' ) , ] for c , entity in svg_char : result = result . replace ( c , entity ) return result

Replace known special characters to SVG code .


def _check_svg_file(svg_file):
    """ Try to read a SVG file if `svg_file` is a string.
    Raise an exception in case of error or return the svg object.

    If `svg_file` is a svgutils svg object, will just return it.

    Parameters
    ----------
    svg_file: str or svgutils.transform.SVGFigure object
        If a `str`: path to a '.svg' file,
        otherwise a svgutils svg object is expected.

    Returns
    -------
    svgutils svg object

    Raises
    ------
    Exception if any error happens.
    """
    if isinstance(svg_file, str):
        try:
            svg = sg.fromfile(svg_file)
        except Exception as exc:
            raise Exception('Error reading svg file {}.'.format(svg_file)) from exc
        else:
            return svg

    if isinstance(svg_file, sg.SVGFigure):
        return svg_file

    raise ValueError('Expected `svg_file` to be `str` or `svgutils.SVG`, got {}.'.format(type(svg_file)))

Try to read a SVG file if svg_file is a string . Raise an exception in case of error or return the svg object .


def merge_svg_files(svg_file1, svg_file2, x_coord, y_coord, scale=1):
    """ Merge `svg_file2` in `svg_file1` in the given positions `x_coord`, `y_coord` and `scale`.

    Parameters
    ----------
    svg_file1: str or svgutils svg document object
        Path to a '.svg' file.

    svg_file2: str or svgutils svg document object
        Path to a '.svg' file.

    x_coord: float
        Horizontal axis position of the `svg_file2` content.

    y_coord: float
        Vertical axis position of the `svg_file2` content.

    scale: float
        Scale to apply to `svg_file2` content.

    Returns
    -------
    `svg1` svgutils object with the content of 'svg_file2'
    """
    svg1 = _check_svg_file(svg_file1)
    svg2 = _check_svg_file(svg_file2)

    svg2_root = svg2.getroot()
    svg1.append([svg2_root])

    svg2_root.moveto(x_coord, y_coord, scale=scale)

    return svg1

Merge svg_file2 in svg_file1 in the given positions x_coord y_coord and scale .


def merge_pdfs(pdf_filepaths, out_filepath):
    """ Merge all the PDF files in `pdf_filepaths` in a new PDF file `out_filepath`.

    Parameters
    ----------
    pdf_filepaths: list of str
        Paths to PDF files.

    out_filepath: str
        Path to the result PDF file.

    Returns
    -------
    path: str
        The output file path.
    """
    merger = PdfFileMerger()
    for pdf in pdf_filepaths:
        merger.append(PdfFileReader(open(pdf, 'rb')))

    merger.write(out_filepath)

    return out_filepath

def merge_pdfs ( pdf_filepaths , out_filepath ) : merger = PdfFileMerger ( ) for pdf in pdf_filepaths : merger . append ( PdfFileReader ( open ( pdf , 'rb' ) ) ) merger . write ( out_filepath ) return out_filepath

Merge all the PDF files in pdf_filepaths in a new PDF file out_filepath .


def _embed_font_to_svg(filepath, font_files):
    """ Return the ElementTree of the SVG content in `filepath`
    with the font content embedded.
    """
    with open(filepath, 'r') as svgf:
        tree = etree.parse(svgf)

    if not font_files:
        return tree

    fontfaces = FontFaceGroup()
    for font_file in font_files:
        fontfaces.append(FontFace(font_file))

    for element in tree.iter():
        if element.tag.split("}")[1] == 'svg':
            break

    element.insert(0, fontfaces.xml_elem)

    return tree

def _embed_font_to_svg ( filepath , font_files ) : with open ( filepath , 'r' ) as svgf : tree = etree . parse ( svgf ) if not font_files : return tree fontfaces = FontFaceGroup ( ) for font_file in font_files : fontfaces . append ( FontFace ( font_file ) ) for element in tree . iter ( ) : if element . tag . split ( "}" ) [ 1 ] == 'svg' : break element . insert ( 0 , fontfaces . xml_elem ) return tree

Return the ElementTree of the SVG content in filepath with the font content embedded .


def embed_font_to_svg(filepath, outfile, font_files):
    """ Write ttf and otf font content from `font_files`
    in the svg file in `filepath` and write the result in
    `outfile`.

    Parameters
    ----------
    filepath: str
        The SVG file whose content must be modified.

    outfile: str
        The file path where the result will be written.

    font_files: iterable of str
        List of paths to .ttf or .otf files.
    """
    tree = _embed_font_to_svg(filepath, font_files)
    tree.write(outfile, encoding='utf-8', pretty_print=True)

def embed_font_to_svg ( filepath , outfile , font_files ) : tree = _embed_font_to_svg ( filepath , font_files ) tree . write ( outfile , encoding = 'utf-8' , pretty_print = True )

Write ttf and otf font content from font_files in the svg file in filepath and write the result in outfile .


def _check_inputs(self):
        ''' make some basic checks on the inputs to make sure they are valid'''
        try:
            _ = self._inputs[0]
        except TypeError:
            raise RuntimeError(
                "inputs should be iterable but found type='{0}', value="
                "'{1}'".format(type(self._inputs), str(self._inputs)))
        from melody.inputs import Input
        for check_input in self._inputs:
            if not isinstance(check_input, Input):
                raise RuntimeError(
                    "input should be a subclass of the Input class but "
                    "found type='{0}', value='{1}'".format(type(check_input),
                                                           str(check_input)))


def _check_function(self):
        ''' make some basic checks on the function to make sure it is valid'''
        # note, callable is valid for Python 2 and Python 3.2 onwards but
        # not inbetween
        if not callable(self._function):
            raise RuntimeError(
                "provided function '{0}' is not callable".
                format(str(self._function)))
        from inspect import getargspec
        arg_info = getargspec(self._function)
        if len(arg_info.args) != 1:
            print str(arg_info)
            raise RuntimeError(
                "provided function should have one argument but found "
                "{0}".format(len(arg_info.args)))


def _recurse(self, inputs, output):
        '''internal recursion routine called by the run method that generates
        all input combinations'''
        if inputs:
            my_input = inputs[0]
            name = my_input.name
            if my_input.state:
                my_options = my_input.options(self.state)
            else:
                my_options = my_input.options
            for option in my_options:
                my_output = list(output)
                my_output.append({name: option})
                self._recurse(inputs[1:], my_output)
        else:
            try:
                valid, result = self._function(output)
            except ValueError:
                raise RuntimeError("function must return 2 values")
            print output, valid, result

def
_recurse
(
self
,
inputs
,
output
)
:
if
inputs
:
my_input
=
inputs
[
0
]
name
=
my_input
.
name
if
my_input
.
state
:
my_options
=
my_input
.
options
(
self
.
state
)
else
:
my_options
=
my_input
.
options
for
option
in
my_options
:
my_output
=
list
(
output
)
my_output
.
append
(
{
name
:
option
}
)
self
.
_recurse
(
inputs
[
1
:
]
,
my_output
)
else
:
try
:
valid
,
result
=
self
.
_function
(
output
)
except
ValueError
:
raise
RuntimeError
(
"function must return 2 values"
)
print
output
,
valid
,
result


def create_input(option, template_name, template_location="template"):

    '''create an input file using jinja2 by filling a template
    with the values from the option variable passed in.'''

    # restructure option list into jinja2 input format
    jinja2_input = {}
    for item in option:
        try:
            jinja2_input.update(item)
        except ValueError:
            raise RuntimeError(
                ("inputs.py, create_input : format of item '{0}' is not "
                 "supported. Expecting a dictionary.".format(str(item))))

    # load the template and fill it with the option variable contents
    import jinja2
    try:
        template_loader = jinja2.FileSystemLoader(searchpath=template_location)
        template_env = jinja2.Environment(loader=template_loader)
        template = template_env.get_template(template_name)
        output_text = template.render(jinja2_input)
    except jinja2.TemplateNotFound:
        raise RuntimeError("template '{0}' not found".format(template_name))
    # return the particular input file as a string
    return output_text

create an input file using jinja2 by filling a template with the values from the option variable passed in .


def _recurse(self, inputs, output, depth, max_depth):
        '''We work out all combinations using this internal recursion method'''
        if depth < max_depth:
            for index, option in enumerate(inputs):
                my_output = list(output)
                my_output.append(option)
                self._recurse(inputs[index + 1:], my_output, depth + 1,
                              max_depth)
        else:
            self._options.append(output)

def _recurse ( self , inputs , output , depth , max_depth ) : if depth < max_depth : for index , option in enumerate ( inputs ) : my_output = list ( output ) my_output . append ( option ) self . _recurse ( inputs [ index + 1 : ] , my_output , depth + 1 , max_depth ) else : self . _options . append ( output )


def to_string(obj):
    '''
    Cast an arbitrary object or sequence to a string type
    '''
    if isinstance(obj, LiteralWrapper):
        val = obj.obj
    elif isinstance(obj, Iterable) and not isinstance(obj, str):
        val = next(obj, None)
    else:
        val = obj
    if val is None:
        yield ''
    elif isinstance(val, str):
        yield val
    elif isinstance(val, node):
        yield strval(val)
    elif isinstance(val, int) or isinstance(val, float):
        yield str(val)
    elif isinstance(item, bool):
        yield 'true' if item else 'false'
    else:
        raise RuntimeError('Unknown type for string conversion: {}'.format(val))


def to_number(obj):
    '''
    Cast an arbitrary object or sequence to a number type
    '''
    if isinstance(obj, LiteralWrapper):
        val = obj.obj
    elif isinstance(obj, Iterable) and not isinstance(obj, str):
        val = next(obj, None)
    else:
        val = obj
    if val is None:
        #FIXME: Should be NaN, not 0
        yield 0
    elif isinstance(val, str):
        yield float(val)
    elif isinstance(val, node):
        yield float(strval(val))
    elif isinstance(val, int) or isinstance(val, float):
        yield val
    else:
        raise RuntimeError('Unknown type for number conversion: {}'.format(val))


def to_boolean(obj):
    '''
    Cast an arbitrary sequence to a boolean type
    '''
    #if hasattr(obj, '__iter__'):
    if isinstance(obj, LiteralWrapper):
        val = obj.obj
    elif isinstance(obj, Iterable) and not isinstance(obj, str):
        val = next(obj, None)
    else:
        val = obj
    if val is None:
        yield False
    elif isinstance(val, bool):
        yield val
    elif isinstance(val, str):
        yield bool(str)
    elif isinstance(val, node):
        yield True
    elif isinstance(val, float) or isinstance(val, int):
        yield bool(val)
    else:
        raise RuntimeError('Unknown type for boolean conversion: {}'.format(val))


def _serialize(xp_ast):
    '''Generate token strings which, when joined together, form a valid
    XPath serialization of the AST.'''

    if hasattr(xp_ast, '_serialize'):
        for tok in xp_ast._serialize():
            yield(tok)
    elif isinstance(xp_ast, str):
        yield(repr(xp_ast))

def _serialize ( xp_ast ) : if hasattr ( xp_ast , '_serialize' ) : for tok in xp_ast . _serialize ( ) : yield ( tok ) elif isinstance ( xp_ast , str ) : yield ( repr ( xp_ast ) )

Generate token strings which when joined together form a valid XPath serialization of the AST .


def change_xml_encoding(filepath, src_enc, dst_enc='utf-8'):
    """ Modify the encoding entry in the XML file.

    Parameters
    ----------
    filepath: str
        Path to the file to be modified.

    src_enc: str
        Encoding that is written in the file

    dst_enc: str
        Encoding to be set in the file.
    """
    enc_attr = "encoding='{}'"
    replace_file_content(filepath, enc_attr.format(src_enc), enc_attr.format(dst_enc), 1)

def change_xml_encoding ( filepath , src_enc , dst_enc = 'utf-8' ) : enc_attr = "encoding='{}'" replace_file_content ( filepath , enc_attr . format ( src_enc ) , enc_attr . format ( dst_enc ) , 1 )

Modify the encoding entry in the XML file .


def save_into_qrcode(text, out_filepath, color='', box_size=10, pixel_size=1850):
    """ Save `text` in a qrcode svg image file.

    Parameters
    ----------
    text: str
        The string to be codified in the QR image.

    out_filepath: str
        Path to the output file

    color: str
        A RGB color expressed in 6 hexadecimal values.

    box_size: scalar
        Size of the QR code boxes.
    """
    try:
        qr = qrcode.QRCode(version=1, error_correction=qrcode.constants.ERROR_CORRECT_L,
                           box_size=box_size, border=0, )
        qr.add_data(text)
        qr.make(fit=True)
    except Exception as exc:
        raise Exception('Error trying to generate QR code '
                        ' from `vcard_string`: {}'.format(text)) from exc
    else:
        img = qr.make_image(image_factory=qrcode.image.svg.SvgPathImage)

    _ = _qrcode_to_file(img, out_filepath)

    if color:
        replace_file_content(out_filepath, 'fill:#000000', 'fill:#{}'.format(color))

def
save_into_qrcode
(
text
,
out_filepath
,
color
=
''
,
box_size
=
10
,
pixel_size
=
1850
)
:
try
:
qr
=
qrcode
.
QRCode
(
version
=
1
,
error_correction
=
qrcode
.
constants
.
ERROR_CORRECT_L
,
box_size
=
box_size
,
border
=
0
,
)
qr
.
add_data
(
text
)
qr
.
make
(
fit
=
True
)
except
Exception
as
exc
:
raise
Exception
(
'Error trying to generate QR code '
' from `vcard_string`: {}'
.
format
(
text
)
)
from
exc
else
:
img
=
qr
.
make_image
(
image_factory
=
qrcode
.
image
.
svg
.
SvgPathImage
)
_
=
_qrcode_to_file
(
img
,
out_filepath
)
if
color
:
replace_file_content
(
out_filepath
,
'fill:#000000'
,
'fill:#{}'
.
format
(
color
)
)


def launch(option):
    '''Set the gromacs input data using the supplied input options, run
    gromacs and extract and return the required outputs.'''

    from melody.inputs import create_input
    _ = create_input(option, template_name="input.mdp")

    # save the input file in the appropriate place and launch gromacs using
    # longbow ...

    # determine if the run was successful
    success = True

    results = None
    if success:
        # extract the required outputs
        results = {"rate": {"value": 35, "units": "ns/day"}, }

    return success, results

def launch ( option ) : from melody . inputs import create_input _ = create_input ( option , template_name = "input.mdp" ) success = True results = None if success : results = { "rate" : { "value" : 35 , "units" : "ns/day" } , } return success , results

Set the gromacs input data using the supplied input options run gromacs and extract and return the required outputs .


def call_command(cmd_name, args_strings):
    """Call CLI command with arguments and returns its return value.

    Parameters
    ----------
    cmd_name: str
        Command name or full path to the binary file.

    arg_strings: str
        Argument strings list.

    Returns
    -------
    return_value
        Command return value.
    """
    if not os.path.isabs(cmd_name):
        cmd_fullpath = which(cmd_name)
    else:
        cmd_fullpath = cmd_name

    try:
        cmd_line = [cmd_fullpath] + args_strings
        log.debug('Calling: `{}`.'.format(' '.join(cmd_line)))
        # retval = subprocess.check_call(cmd_line)
        retval = subprocess.call(' '.join(cmd_line), shell=True)
    except CalledProcessError as ce:
        log.exception(
            "Error calling command with arguments: "
            "{} \n With return code: {}".format(cmd_line, ce.returncode)
        )
        raise
    else:
        return retval

Call CLI command with arguments and returns its return value .


def tex2pdf(tex_file, output_file=None, output_format='pdf'):
    """ Call PDFLatex to convert TeX files to PDF.

    Parameters
    ----------
    tex_file: str
        Path to the input LateX file.

    output_file: str
        Path to the output PDF file.
        If None, will use the same output directory as the tex_file.

    output_format: str
        Output file format. Choices: 'pdf' or 'dvi'. Default: 'pdf'

    Returns
    -------
    return_value
        PDFLatex command call return value.
    """
    if not os.path.exists(tex_file):
        raise IOError('Could not find file {}.'.format(tex_file))

    if output_format != 'pdf' and output_format != 'dvi':
        raise ValueError("Invalid output format given {}. Can only accept 'pdf' or 'dvi'.".format(output_format))

    cmd_name = 'pdflatex'
    check_command(cmd_name)

    args_strings = [cmd_name]
    if output_file is not None:
        args_strings += ['-output-directory="{}" '.format(os.path.abspath(os.path.dirname(output_file)))]

    result_dir = os.path.dirname(output_file) if output_file else os.path.dirname(tex_file)

    args_strings += ['-output-format="{}"'.format(output_format)]
    args_strings += ['"' + tex_file + '"']

    log.debug('Calling command {} with args: {}.'.format(cmd_name, args_strings))
    ret = simple_call(args_strings)

    result_file = os.path.join(result_dir, remove_ext(os.path.basename(tex_file)) + '.' + output_format)
    if os.path.exists(result_file):
        shutil.move(result_file, output_file)
    else:
        raise IOError('Could not find PDFLatex result file.')

    log.debug('Cleaning *.aux and *.log files from folder {}.'.format(result_dir))
    cleanup(result_dir, 'aux')
    cleanup(result_dir, 'log')

    return ret

Call PDFLatex to convert TeX files to PDF .


def options(self, my_psy):
        '''Returns all potential loop fusion options for the psy object
        provided'''
        # compute options dynamically here as they may depend on previous
        # changes to the psy tree
        my_options = []
        invokes = my_psy.invokes.invoke_list
        #print "there are {0} invokes".format(len(invokes))
        if self._dependent_invokes:
            raise RuntimeError(
                "dependent invokes assumes fusion in one invoke might "
                "affect fusion in another invoke. This is not yet "
                "implemented")
        else:
            # treat each invoke separately
            for idx, invoke in enumerate(invokes):
                print "invoke {0}".format(idx)
                # iterate through each outer loop
                for loop in invoke.schedule.loops():
                    if loop.loop_type == "outer":
                        siblings = loop.parent.children
                        my_index = siblings.index(loop)
                        option = []
                        self._recurse(siblings, my_index, option, my_options,
                                      invoke)

        return my_options


def transform(geom, to_sref):
    """Returns a transformed Geometry.

    Arguments:
    geom -- any coercible Geometry value or Envelope
    to_sref -- SpatialReference or EPSG ID as int
    """
    # If we have an envelope, assume it's in the target sref.
    try:
        geom = getattr(geom, 'polygon', Envelope(geom).polygon)
    except (TypeError, ValueError):
        pass
    else:
        geom.AssignSpatialReference(to_sref)
    try:
        geom_sref = geom.GetSpatialReference()
    except AttributeError:
        return transform(Geometry(geom), to_sref)
    if geom_sref is None:
        raise Exception('Cannot transform from unknown spatial reference')
    # Reproject geom if necessary
    if not geom_sref.IsSame(to_sref):
        geom = geom.Clone()
        geom.TransformTo(to_sref)
    return geom