def detail_view(self, request):
        """
        Renders the message view to a response.
        """
        context = {
            'preview': self,
        }

        kwargs = {}
        if self.form_class:
            if request.GET:
                form = self.form_class(data=request.GET)
            else:
                form = self.form_class()

            context['form'] = form
            if not form.is_bound or not form.is_valid():
                return render(request, 'mailviews/previews/detail.html', context)

            kwargs.update(form.get_message_view_kwargs())

        message_view = self.get_message_view(request, **kwargs)

        message = message_view.render_to_message()
        raw = message.message()
        headers = OrderedDict((header, maybe_decode_header(raw[header])) for header in self.headers)

        context.update({
            'message': message,
            'subject': message.subject,
            'body': message.body,
            'headers': headers,
            'raw': raw.as_string(),
        })

        alternatives = getattr(message, 'alternatives', [])
        try:
            html = next(alternative[0] for alternative in alternatives
                if alternative[1] == 'text/html')
            context.update({
                'html': html,
                'escaped_html': b64encode(html.encode('utf-8')),
            })
        except StopIteration:
            pass

        return render(request, self.template_name, context)

def detail_view ( self , request ) : context = { 'preview' : self , } kwargs = { } if self . form_class : if request . GET : form = self . form_class ( data = request . GET ) else : form = self . form_class ( ) context [ 'form' ] = form if not form . is_bound or not form . is_valid ( ) : return render ( request , 'mailviews/previews/detail.html' , context ) kwargs . update ( form . get_message_view_kwargs ( ) ) message_view = self . get_message_view ( request , ** kwargs ) message = message_view . render_to_message ( ) raw = message . message ( ) headers = OrderedDict ( ( header , maybe_decode_header ( raw [ header ] ) ) for header in self . headers ) context . update ( { 'message' : message , 'subject' : message . subject , 'body' : message . body , 'headers' : headers , 'raw' : raw . as_string ( ) , } ) alternatives = getattr ( message , 'alternatives' , [ ] ) try : html = next ( alternative [ 0 ] for alternative in alternatives if alternative [ 1 ] == 'text/html' ) context . update ( { 'html' : html , 'escaped_html' : b64encode ( html . encode ( 'utf-8' ) ) , } ) except StopIteration : pass return render ( request , self . template_name , context )

Renders the message view to a response .


def split_docstring(value):
    """
    Splits the docstring of the given value into it's summary and body.

    :returns: a 2-tuple of the format ``(summary, body)``
    """
    docstring = textwrap.dedent(getattr(value, '__doc__', ''))
    if not docstring:
        return None

    pieces = docstring.strip().split('\n\n', 1)
    try:
        body = pieces[1]
    except IndexError:
        body = None

    return Docstring(pieces[0], body)

def split_docstring ( value ) : docstring = textwrap . dedent ( getattr ( value , '__doc__' , '' ) ) if not docstring : return None pieces = docstring . strip ( ) . split ( '\n\n' , 1 ) try : body = pieces [ 1 ] except IndexError : body = None return Docstring ( pieces [ 0 ] , body )

Splits the docstring of the given value into it s summary and body .


def render_to_message(self, extra_context=None, **kwargs):
        """
        Renders and returns an unsent message with the provided context.

        Any extra keyword arguments passed will be passed through as keyword
        arguments to the message constructor.

        :param extra_context: Any additional context to use when rendering the
            templated content.
        :type extra_context: :class:`dict`
        :returns: A message instance.
        :rtype: :attr:`.message_class`
        """
        if extra_context is None:
            extra_context = {}

        # Ensure our custom headers are added to the underlying message class.
        kwargs.setdefault('headers', {}).update(self.headers)

        context = self.get_context_data(**extra_context)
        return self.message_class(
            subject=self.render_subject(context),
            body=self.render_body(context),
            **kwargs)

def render_to_message ( self , extra_context = None , ** kwargs ) : if extra_context is None : extra_context = { } kwargs . setdefault ( 'headers' , { } ) . update ( self . headers ) context = self . get_context_data ( ** extra_context ) return self . message_class ( subject = self . render_subject ( context ) , body = self . render_body ( context ) , ** kwargs )

Renders and returns an unsent message with the provided context .


def send(self, extra_context=None, **kwargs):
        """
        Renders and sends an email message.

        All keyword arguments other than ``extra_context`` are passed through
        as keyword arguments when constructing a new :attr:`message_class`
        instance for this message.

        This method exists primarily for convenience, and the proper
        rendering of your message should not depend on the behavior of this
        method. To alter how a message is created, override
        :meth:``render_to_message`` instead, since that should always be
        called, even if a message is not sent.

        :param extra_context: Any additional context data that will be used
            when rendering this message.
        :type extra_context: :class:`dict`
        """
        message = self.render_to_message(extra_context=extra_context, **kwargs)
        return message.send()

def send ( self , extra_context = None , ** kwargs ) : message = self . render_to_message ( extra_context = extra_context , ** kwargs ) return message . send ( )


def render_subject(self, context):
        """
        Renders the message subject for the given context.

        The context data is automatically unescaped to avoid rendering HTML
        entities in ``text/plain`` content.

        :param context: The context to use when rendering the subject template.
        :type context: :class:`~django.template.Context`
        :returns: A rendered subject.
        :rtype: :class:`str`
        """
        rendered = self.subject_template.render(unescape(context))
        return rendered.strip()

def render_subject ( self , context ) : rendered = self . subject_template . render ( unescape ( context ) ) return rendered . strip ( )

Renders the message subject for the given context .


def render_to_message(self, extra_context=None, *args, **kwargs):
        """
        Renders and returns an unsent message with the given context.

        Any extra keyword arguments passed will be passed through as keyword
        arguments to the message constructor.

        :param extra_context: Any additional context to use when rendering
            templated content.
        :type extra_context: :class:`dict`
        :returns: A message instance.
        :rtype: :attr:`.message_class`
        """
        message = super(TemplatedHTMLEmailMessageView, self)\
            .render_to_message(extra_context, *args, **kwargs)

        if extra_context is None:
            extra_context = {}

        context = self.get_context_data(**extra_context)
        content = self.render_html_body(context)
        message.attach_alternative(content, mimetype='text/html')
        return message

def render_to_message ( self , extra_context = None , * args , ** kwargs ) : message = super ( TemplatedHTMLEmailMessageView , self ) . render_to_message ( extra_context , * args , ** kwargs ) if extra_context is None : extra_context = { } context = self . get_context_data ( ** extra_context ) content = self . render_html_body ( context ) message . attach_alternative ( content , mimetype = 'text/html' ) return message

Renders and returns an unsent message with the given context .


def numeric(_, n):
    """
    NBASE = 1000
    ndigits = total number of base-NBASE digits
    weight = base-NBASE weight of first digit
    sign = 0x0000 if positive, 0x4000 if negative, 0xC000 if nan
    dscale = decimal digits after decimal place
    """
    try:
        nt = n.as_tuple()
    except AttributeError:
        raise TypeError('numeric field requires Decimal value (got %r)' % n)
    digits = []
    if isinstance(nt.exponent, str):
        # NaN, Inf, -Inf
        ndigits = 0
        weight = 0
        sign = 0xC000
        dscale = 0
    else:
        decdigits = list(reversed(nt.digits + (nt.exponent % 4) * (0,)))
        weight = 0
        while decdigits:
            if any(decdigits[:4]):
                break
            weight += 1
            del decdigits[:4]
        while decdigits:
            digits.insert(0, ndig(decdigits[:4]))
            del decdigits[:4]
        ndigits = len(digits)
        weight += nt.exponent // 4 + ndigits - 1
        sign = nt.sign * 0x4000
        dscale = -min(0, nt.exponent)
    data = [ndigits, weight, sign, dscale] + digits
    return ('ihhHH%dH' % ndigits, [2 * len(data)] + data)

NBASE = 1000 ndigits = total number of base - NBASE digits weight = base - NBASE weight of first digit sign = 0x0000 if positive 0x4000 if negative 0xC000 if nan dscale = decimal digits after decimal place


def execute_from_command_line(argv=None):
    """
    A simple method that runs a ManagementUtility.
    """
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--monitors-dir', default=MONITORS_DIR)
    parser.add_argument('--alerts-dir', default=ALERTS_DIR)
    parser.add_argument('--config', default=SMA_INI_FILE)

    parser.add_argument('--warning', help='set logging to warning', action='store_const', dest='loglevel',
                        const=logging.WARNING, default=logging.INFO)
    parser.add_argument('--quiet', help='set logging to ERROR', action='store_const', dest='loglevel',
                        const=logging.ERROR, default=logging.INFO)
    parser.add_argument('--debug', help='set logging to DEBUG',
                        action='store_const', dest='loglevel',
                        const=logging.DEBUG, default=logging.INFO)
    parser.add_argument('--verbose', help='set logging to COMM',
                        action='store_const', dest='loglevel',
                        const=5, default=logging.INFO)

    parser.sub = parser.add_subparsers()

    parse_service = parser.sub.add_parser('service', help='Run SMA as service (daemon).')
    parse_service.set_defaults(which='service')

    parse_oneshot = parser.sub.add_parser('one-shot', help='Run SMA once and exit')
    parse_oneshot.set_defaults(which='one-shot')

    parse_alerts = parser.sub.add_parser('alerts', help='Alerts options.')
    parse_alerts.set_defaults(which='alerts')
    parse_alerts.add_argument('--test', help = 'Test alert', action='store_true')
    parse_alerts.add_argument('alert_section', nargs='?', help='Alert section to see')

    parse_results = parser.sub.add_parser('results', help='Monitors results')
    parse_results.set_defaults(which='results')

    parser.set_default_subparser('one-shot')
    args = parser.parse_args(argv[1:])

    create_logger('sma', args.loglevel)

    if not getattr(args, 'which', None) or args.which == 'one-shot':
        sma = SMA(args.monitors_dir, args.alerts_dir, args.config)
        sma.evaluate_and_alert()
    elif args.which == 'service':
        sma = SMAService(args.monitors_dir, args.alerts_dir, args.config)
        sma.start()
    elif args.which == 'alerts' and args.test:
        sma = SMA(args.monitors_dir, args.alerts_dir, args.config)
        sma.alerts.test()
    elif args.which == 'results':
        print(SMA(args.monitors_dir, args.alerts_dir, args.config).results)

def
execute_from_command_line
(
argv
=
)
:
parser
=
argparse
.
ArgumentParser
(
description
=
__doc__
)
parser
.
add_argument
(
'--monitors-dir'
,
default
=
MONITORS_DIR
)
parser
.
add_argument
(
'--alerts-dir'
,
default
=
ALERTS_DIR
)
parser
.
add_argument
(
'--config'
,
default
=
SMA_INI_FILE
)
parser
.
add_argument
(
'--warning'
,
help
=
'set logging to warning'
,
action
=
'store_const'
,
dest
=
'loglevel'
,
const
=
logging
.
WARNING
,
default
=
logging
.
INFO
)
parser
.
add_argument
(
'--quiet'
,
help
=
'set logging to ERROR'
,
action
=
'store_const'
,
dest
=
'loglevel'
,
const
=
logging
.
ERROR
,
default
=
logging
.
INFO
)
parser
.
add_argument
(
'--debug'
,
help
=
'set logging to DEBUG'
,
action
=
'store_const'
,
dest
=
'loglevel'
,
const
=
logging
.
DEBUG
,
default
=
logging
.
INFO
)
parser
.
add_argument
(
'--verbose'
,
help
=
'set logging to COMM'
,
action
=
'store_const'
,
dest
=
'loglevel'
,
const
=
5
,
default
=
logging
.
INFO
)
parser
.
sub
=
parser
.
add_subparsers
(
)
parse_service
=
parser
.
sub
.
add_parser
(
'service'
,
help
=
'Run SMA as service (daemon).'
)
parse_service
.
set_defaults
(
which
=
'service'
)
parse_oneshot
=
parser
.
sub
.
add_parser
(
'one-shot'
,
help
=
'Run SMA once and exit'
)
parse_oneshot
.
set_defaults
(
which
=
'one-shot'
)
parse_alerts
=
parser
.
sub
.
add_parser
(
'alerts'
,
help
=
'Alerts options.'
)
parse_alerts
.
set_defaults
(
which
=
'alerts'
)
parse_alerts
.
add_argument
(
'--test'
,
help
=
'Test alert'
,
action
=
'store_true'
)
parse_alerts
.
add_argument
(
'alert_section'
,
nargs
=
'?'
,
help
=
'Alert section to see'
)
parse_results
=
parser
.
sub
.
add_parser
(
'results'
,
help
=
'Monitors results'
)
parse_results
.
set_defaults
(
which
=
'results'
)
parser
.
set_default_subparser
(
'one-shot'
)
args
=
parser
.
parse_args
(
argv
[
1
:
]
)
create_logger
(
'sma'
,
args
.
loglevel
)
if
not
getattr
(
args
,
'which'
,
)
or
args
.
which
==
'one-shot'
:
sma
=
SMA
(
args
.
monitors_dir
,
args
.
alerts_dir
,
args
.
config
)
sma
.
evaluate_and_alert
(
)
elif
args
.
which
==
'service'
:
sma
=
SMAService
(
args
.
monitors_dir
,
args
.
alerts_dir
,
args
.
config
)
sma
.
start
(
)
elif
args
.
which
==
'alerts'
and
args
.
test
:
sma
=
SMA
(
args
.
monitors_dir
,
args
.
alerts_dir
,
args
.
config
)
sma
.
alerts
.
test
(
)
elif
args
.
which
==
'results'
:
print
(
SMA
(
args
.
monitors_dir
,
args
.
alerts_dir
,
args
.
config
)
.
results
)

A simple method that runs a ManagementUtility .


def _getCallingContext():
    """
    Utility function for the RedisLogRecord.

    Returns the module, function, and lineno of the function
    that called the logger.

    We look way up in the stack.  The stack at this point is:
    [0] logger.py _getCallingContext (hey, that's me!)
    [1] logger.py __init__
    [2] logger.py makeRecord
    [3] _log
    [4] <logging method>
    [5] caller of logging method
    """
    frames = inspect.stack()

    if len(frames) > 4:
        context = frames[5]
    else:
        context = frames[0]

    modname = context[1]
    lineno = context[2]

    if context[3]:
        funcname = context[3]
    else:
        funcname = ""

    # python docs say you don't want references to
    # frames lying around.  Bad things can happen.
    del context
    del frames

    return modname, funcname, lineno

def _getCallingContext ( ) : frames = inspect . stack ( ) if len ( frames ) > 4 : context = frames [ 5 ] else : context = frames [ 0 ] modname = context [ 1 ] lineno = context [ 2 ] if context [ 3 ] : funcname = context [ 3 ] else : funcname = "" del context del frames return modname , funcname , lineno

Utility function for the RedisLogRecord .


def format(self, record):
        """
        JSON-encode a record for serializing through redis.

        Convert date to iso format, and stringify any exceptions.
        """
        data = record._raw.copy()

        # serialize the datetime date as utc string
        data['time'] = data['time'].isoformat()

        # stringify exception data
        if data.get('traceback'):
            data['traceback'] = self.formatException(data['traceback'])

        return json.dumps(data)

def format ( self , record ) : data = record . _raw . copy ( ) data [ 'time' ] = data [ 'time' ] . isoformat ( ) if data . get ( 'traceback' ) : data [ 'traceback' ] = self . formatException ( data [ 'traceback' ] ) return json . dumps ( data )

JSON - encode a record for serializing through redis .


def emit(self, record):
        """
        Publish record to redis logging channel
        """
        try:
            self.redis_client.publish(self.channel, self.format(record))
        except redis.RedisError:
            pass

def emit ( self , record ) : try : self . redis_client . publish ( self . channel , self . format ( record ) ) except redis . RedisError : pass


def emit(self, record):
        """
        Publish record to redis logging list
        """
        try:
            if self.max_messages:
                p = self.redis_client.pipeline()
                p.rpush(self.key, self.format(record))
                p.ltrim(self.key, -self.max_messages, -1)
                p.execute()
            else:
                self.redis_client.rpush(self.key, self.format(record))
        except redis.RedisError:
            pass

def emit ( self , record ) : try : if self . max_messages : p = self . redis_client . pipeline ( ) p . rpush ( self . key , self . format ( record ) ) p . ltrim ( self . key , - self . max_messages , - 1 ) p . execute ( ) else : self . redis_client . rpush ( self . key , self . format ( record ) ) except redis . RedisError : pass


def require_template_debug(f):
    """Decorated function is a no-op if TEMPLATE_DEBUG is False"""
    def _(*args, **kwargs):
        TEMPLATE_DEBUG = getattr(settings, 'TEMPLATE_DEBUG', False)
        return f(*args, **kwargs) if TEMPLATE_DEBUG else ''
    return _

def require_template_debug ( f ) : def _ ( * args , ** kwargs ) : TEMPLATE_DEBUG = getattr ( settings , 'TEMPLATE_DEBUG' , False ) return f ( * args , ** kwargs ) if TEMPLATE_DEBUG else '' return _

Decorated function is a no - op if TEMPLATE_DEBUG is False


def _display_details(var_data):
    """
    Given a dictionary of variable attribute data from get_details display the
    data in the terminal.
    """
    meta_keys = (key for key in list(var_data.keys())
                 if key.startswith('META_'))
    for key in meta_keys:
        display_key = key[5:].capitalize()
        pprint('{0}: {1}'.format(display_key, var_data.pop(key)))
    pprint(var_data)

def
_display_details
(
var_data
)
:
meta_keys
=
(
key
for
key
in
list
(
var_data
.
keys
(
)
)
if
key
.
startswith
(
'META_'
)
)
for
key
in
meta_keys
:
display_key
=
key
[
5
:
]
.
capitalize
(
)
pprint
(
'{0}: {1}'
.
format
(
display_key
,
var_data
.
pop
(
key
)
)
)
pprint
(
var_data
)

Given a dictionary of variable attribute data from get_details display the data in the terminal .


def set_trace(context):
    """
    Start a pdb set_trace inside of the template with the context available as
    'context'. Uses ipdb if available.
    """
    try:
        import ipdb as pdb
    except ImportError:
        import pdb
        print("For best results, pip install ipdb.")
    print("Variables that are available in the current context:")
    render = lambda s: template.Template(s).render(context)
    availables = get_variables(context)
    pprint(availables)
    print('Type `availables` to show this list.')
    print('Type <variable_name> to access one.')
    print('Use render("template string") to test template rendering')
    # Cram context variables into the local scope
    for var in availables:
        locals()[var] = context[var]
    pdb.set_trace()
    return ''

Start a pdb set_trace inside of the template with the context available as context . Uses ipdb if available .


def pydevd(context):
    """
    Start a pydev settrace
    """
    global pdevd_not_available
    if pdevd_not_available:
        return ''
    try:
        import pydevd
    except ImportError:
        pdevd_not_available = True
        return ''
    render = lambda s: template.Template(s).render(context)
    availables = get_variables(context)
    for var in availables:
        locals()[var] = context[var]
    #catch the case where no client is listening
    try:
        pydevd.settrace()
    except socket.error:
        pdevd_not_available = True
    return ''

def pydevd ( context ) : global pdevd_not_available if pdevd_not_available : return '' try : import pydevd except ImportError : pdevd_not_available = True return '' render = lambda s : template . Template ( s ) . render ( context ) availables = get_variables ( context ) for var in availables : locals ( ) [ var ] = context [ var ] try : pydevd . settrace ( ) except socket . error : pdevd_not_available = True return ''


def _flatten(iterable):
    """
    Given an iterable with nested iterables, generate a flat iterable
    """
    for i in iterable:
        if isinstance(i, Iterable) and not isinstance(i, string_types):
            for sub_i in _flatten(i):
                yield sub_i
        else:
            yield i

def _flatten ( iterable ) : for i in iterable : if isinstance ( i , Iterable ) and not isinstance ( i , string_types ) : for sub_i in _flatten ( i ) : yield sub_i else : yield i

Given an iterable with nested iterables generate a flat iterable


def _get_detail_value(var, attr):
    """
    Given a variable and one of its attributes that are available inside of
    a template, return its 'method' if it is a callable, its class name if it
    is a model manager, otherwise return its value
    """
    value = getattr(var, attr)
    # Rename common Django class names
    kls = getattr(getattr(value, '__class__', ''), '__name__', '')
    if kls in ('ManyRelatedManager', 'RelatedManager', 'EmptyManager'):
        return kls
    if callable(value):
        return 'routine'
    return value

def _get_detail_value ( var , attr ) : value = getattr ( var , attr ) kls = getattr ( getattr ( value , '__class__' , '' ) , '__name__' , '' ) if kls in ( 'ManyRelatedManager' , 'RelatedManager' , 'EmptyManager' ) : return kls if callable ( value ) : return 'routine' return value

Given a variable and one of its attributes that are available inside of a template return its method if it is a callable its class name if it is a model manager otherwise return its value


def get_attributes(var):
    """
    Given a varaible, return the list of attributes that are available inside
    of a template
    """
    is_valid = partial(is_valid_in_template, var)
    return list(filter(is_valid, dir(var)))

def get_attributes ( var ) : is_valid = partial ( is_valid_in_template , var ) return list ( filter ( is_valid , dir ( var ) ) )

Given a varaible return the list of attributes that are available inside of a template


def is_valid_in_template(var, attr):
    """
    Given a variable and one of its attributes, determine if the attribute is
    accessible inside of a Django template and return True or False accordingly
    """
    # Remove private variables or methods
    if attr.startswith('_'):
        return False
    # Remove any attributes that raise an acception when read
    try:
        value = getattr(var, attr)
    except:
        return False
    if isroutine(value):
        # Remove any routines that are flagged with 'alters_data'
        if getattr(value, 'alters_data', False):
            return False
        else:
            # Remove any routines that require arguments
            try:
                argspec = getargspec(value)
                num_args = len(argspec.args) if argspec.args else 0
                num_defaults = len(argspec.defaults) if argspec.defaults else 0
                if num_args - num_defaults > 1:
                    return False
            except TypeError:
                # C extension callables are routines, but getargspec fails with
                # a TypeError when these are passed.
                pass
    return True

def is_valid_in_template ( var , attr ) : if attr . startswith ( '_' ) : return False try : value = getattr ( var , attr ) except : return False if isroutine ( value ) : if getattr ( value , 'alters_data' , False ) : return False else : try : argspec = getargspec ( value ) num_args = len ( argspec . args ) if argspec . args else 0 num_defaults = len ( argspec . defaults ) if argspec . defaults else 0 if num_args - num_defaults > 1 : return False except TypeError : pass return True

Given a variable and one of its attributes determine if the attribute is accessible inside of a Django template and return True or False accordingly


def parse_log_messages(self, text):
        """Will parse git log messages in the 'short' format"""
        regex = r"commit ([0-9a-f]+)\nAuthor: (.*?)\n\n(.*?)(?:\n\n|$)"
        messages = re.findall(regex, text, re.DOTALL)

        parsed = []
        for commit, author, message in messages:
            parsed.append((
                commit[:10],
                re.sub(r"\s*<.*?>", "", author), # Remove email address if present
                message.strip()
            ))
        return parsed

Will parse git log messages in the short format


def determine_paths(self, package_name=None, create_package_dir=False, dry_run=False):
        """Determine paths automatically and a little intelligently"""

        # Give preference to the environment variable here as it will not
        # derefrence sym links
        self.project_dir = Path(os.getenv('PWD') or os.getcwd())

        # Try and work out the project name
        distribution = self.get_distribution()
        if distribution:
            # Get name from setup.py
            self.project_name = distribution.get_name()
        else:
            # ...failing that, use the current directory name
            self.project_name = self.project_dir.name

        # Descend into the 'src' directory to find the package
        # if necessary
        if os.path.isdir(self.project_dir / "src"):
            package_search_dir = self.project_dir / "src"
        else:
            package_search_dir = self.project_dir

        created_package_dir = False
        if not package_name:
            # Lets try and work out the package_name from the project_name
            package_name = self.project_name.replace("-", "_")

            # Now do some fuzzy matching
            def get_matches(name):
                possibles = [n for n in os.listdir(package_search_dir) if os.path.isdir(package_search_dir / n)]
                return difflib.get_close_matches(name, possibles, n=1, cutoff=0.8)

            close = get_matches(package_name)

            # If no matches, try removing the first part of the package name
            # (e.g. django-guardian becomes guardian)
            if not close and "_" in package_name:
                short_package_name = "_".join(package_name.split("_")[1:])
                close = get_matches(short_package_name)

            if not close:
                if create_package_dir:
                    package_dir = package_search_dir / package_name
                    # Gets set to true even during dry run
                    created_package_dir = True
                    if not dry_run:
                        print("Creating package directory at %s" % package_dir)
                        os.mkdir(package_dir)
                    else:
                        print("Would have created package directory at %s" % package_dir)
                else:
                    raise CommandError("Could not guess the package name. Specify it using --name.")
            else:
                package_name = close[0]

        self.package_name = package_name
        self.package_dir = package_search_dir / package_name

        if not os.path.exists(self.package_dir) and not created_package_dir:
            raise CommandError("Package directory did not exist at %s. Perhaps specify it using --name" % self.package_dir)


def check_integrity(sakefile, settings):
    """
    Checks the format of the sakefile dictionary
    to ensure it conforms to specification

    Args:
        A dictionary that is the parsed Sakefile (from sake.py)
        The setting dictionary (for print functions)
    Returns:
        True if the Sakefile is conformant
        False if not
    """
    sprint = settings["sprint"]
    error = settings["error"]
    sprint("Call to check_integrity issued", level="verbose")
    if not sakefile:
        error("Sakefile is empty")
        return False
    # checking for duplicate targets
    if len(sakefile.keys()) != len(set(sakefile.keys())):
        error("Sakefile contains duplicate targets")
        return False
    for target in sakefile:
        if target == "all":
            if not check_target_integrity(target, sakefile["all"], all=True):
                error("Failed to accept target 'all'")
                return False
            continue
        if "formula" not in sakefile[target]:
            if not check_target_integrity(target, sakefile[target],
                                          meta=True):
                errmes = "Failed to accept meta-target '{}'".format(target)
                error(errmes)
                return False
            for atom_target in sakefile[target]:
                if atom_target == "help":
                    continue
                if not check_target_integrity(atom_target,
                                              sakefile[target][atom_target],
                                              parent=target):
                    errmes = "Failed to accept target '{}'\n".format(
                                                                atom_target)
                    error(errmes)
                    return False
            continue
        if not check_target_integrity(target, sakefile[target]):
            errmes = "Failed to accept target '{}'\n".format(target)
            error(errmes)
            return False
    return True

def
check_integrity
(
sakefile
,
settings
)
:
sprint
=
settings
[
"sprint"
]
error
=
settings
[
"error"
]
sprint
(
"Call to check_integrity issued"
,
level
=
"verbose"
)
if
not
sakefile
:
error
(
"Sakefile is empty"
)
return
False
if
len
(
sakefile
.
keys
(
)
)
!=
len
(
set
(
sakefile
.
keys
(
)
)
)
:
error
(
"Sakefile contains duplicate targets"
)
return
False
for
target
in
sakefile
:
if
target
==
"all"
:
if
not
check_target_integrity
(
target
,
sakefile
[
"all"
]
,
all
=
True
)
:
error
(
"Failed to accept target 'all'"
)
return
False
continue
if
"formula"
not
in
sakefile
[
target
]
:
if
not
check_target_integrity
(
target
,
sakefile
[
target
]
,
meta
=
True
)
:
errmes
=
"Failed to accept meta-target '{}'"
.
format
(
target
)
error
(
errmes
)
return
False
for
atom_target
in
sakefile
[
target
]
:
if
atom_target
==
"help"
:
continue
if
not
check_target_integrity
(
atom_target
,
sakefile
[
target
]
[
atom_target
]
,
parent
=
target
)
:
errmes
=
"Failed to accept target '{}'\n"
.
format
(
atom_target
)
error
(
errmes
)
return
False
continue
if
not
check_target_integrity
(
target
,
sakefile
[
target
]
)
:
errmes
=
"Failed to accept target '{}'\n"
.
format
(
target
)
error
(
errmes
)
return
False
return
True


def check_shastore_version(from_store, settings):
    """
    This function gives us the option to emit errors or warnings
    after sake upgrades
    """
    sprint = settings["sprint"]
    error = settings["error"]

    sprint("checking .shastore version for potential incompatibilities",
           level="verbose")
    if not from_store or 'sake version' not in from_store:
        errmes = ["Since you've used this project last, a new version of ",
                  "sake was installed that introduced backwards incompatible",
                  " changes. Run 'sake clean', and rebuild before continuing\n"]
        errmes = " ".join(errmes)
        error(errmes)
        sys.exit(1)


def get_sha(a_file, settings=None):
    """
    Returns sha1 hash of the file supplied as an argument
    """
    if settings:
        error = settings["error"]
    else:
        error = ERROR_FN
    try:
        BLOCKSIZE = 65536
        hasher = hashlib.sha1()
        with io.open(a_file, "rb") as fh:
            buf = fh.read(BLOCKSIZE)
            while len(buf) > 0:
                hasher.update(buf)
                buf = fh.read(BLOCKSIZE)
        the_hash = hasher.hexdigest()
    except IOError:
        errmes = "File '{}' could not be read! Exiting!".format(a_file)
        error(errmes)
        sys.exit(1)
    except:
        errmes = "Unspecified error returning sha1 hash. Exiting!"
        error(errmes)
        sys.exit(1)
    return the_hash


def write_shas_to_shastore(sha_dict):
    """
    Writes a sha1 dictionary stored in memory to
    the .shastore file
    """
    if sys.version_info[0] < 3:
        fn_open = open
    else:
        fn_open = io.open
    with fn_open(".shastore", "w") as fh:
        fh.write("---\n")
        fh.write('sake version: {}\n'.format(constants.VERSION))
        if sha_dict:
            fh.write(yaml.dump(sha_dict))
        fh.write("...")

def
write_shas_to_shastore
(
sha_dict
)
:
if
sys
.
version_info
[
0
]
<
3
:
fn_open
=
open
else
:
fn_open
=
io
.
open
with
fn_open
(
".shastore"
,
"w"
)
as
fh
:
fh
.
write
(
"---\n"
)
fh
.
write
(
'sake version: {}\n'
.
format
(
constants
.
VERSION
)
)
if
sha_dict
:
fh
.
write
(
yaml
.
dump
(
sha_dict
)
)
fh
.
write
(
"..."
)

Writes a sha1 dictionary stored in memory to the . shastore file


def take_shas_of_all_files(G, settings):
    """
    Takes sha1 hash of all dependencies and outputs of all targets

    Args:
        The graph we are going to build
        The settings dictionary

    Returns:
        A dictionary where the keys are the filenames and the
        value is the sha1 hash
    """
    global ERROR_FN
    sprint = settings["sprint"]
    error = settings["error"]
    ERROR_FN = error
    sha_dict = {}
    all_files = []
    for target in G.nodes(data=True):
        sprint("About to take shas of files in target '{}'".format(target[0]),
               level="verbose")
        if 'dependencies' in target[1]:
            sprint("It has dependencies", level="verbose")
            deplist = []
            for dep in target[1]['dependencies']:
                glist = glob.glob(dep)
                if glist:
                    for oneglob in glist:
                        deplist.append(oneglob)
                else:
                    deplist.append(dep)
            target[1]['dependencies'] = list(deplist)
            for dep in target[1]['dependencies']:
                sprint("  - {}".format(dep), level="verbose")
                all_files.append(dep)
        if 'output' in target[1]:
            sprint("It has outputs", level="verbose")
            for out in acts.get_all_outputs(target[1]):
                sprint("  - {}".format(out), level="verbose")
                all_files.append(out)
    if len(all_files):
        sha_dict['files'] = {}
        # check if files exist and de-dupe
        extant_files = []
        for item in all_files:
            if item not in extant_files and os.path.isfile(item):
                extant_files.append(item)
        pool = Pool()
        results = pool.map(get_sha, extant_files)
        pool.close()
        pool.join()
        for fn, sha in zip(extant_files, results):
            sha_dict['files'][fn] = {'sha': sha}
        return sha_dict
    sprint("No dependencies", level="verbose")


def run_commands(commands, settings):
    """
    Runs the commands supplied as an argument
    It will exit the program if the commands return a
    non-zero code

    Args:
        the commands to run
        The settings dictionary
    """
    sprint = settings["sprint"]
    quiet = settings["quiet"]
    error = settings["error"]
    enhanced_errors = True
    the_shell = None
    if settings["no_enhanced_errors"]:
        enhanced_errors = False
    if "shell" in settings:
        the_shell = settings["shell"]
    windows_p = sys.platform == "win32"

    STDOUT = None
    STDERR = None
    if quiet:
        STDOUT = PIPE
        STDERR = PIPE

    commands = commands.rstrip()
    sprint("About to run commands '{}'".format(commands), level="verbose")
    if not quiet:
        sprint(commands)

    if the_shell:
        tmp = shlex.split(the_shell)
        the_shell = tmp[0]
        tmp = tmp[1:]
        if enhanced_errors and not windows_p:
            tmp.append("-e")
        tmp.append(commands)
        commands = tmp
    else:
        if enhanced_errors and not windows_p:
            commands = ["-e", commands]

    p = Popen(commands, shell=True, stdout=STDOUT, stderr=STDERR,
              executable=the_shell)
    out, err = p.communicate()
    if p.returncode:
        if quiet:
            error(err.decode(locale.getpreferredencoding()))
        error("Command failed to run")
        sys.exit(1)

def
run_commands
(
commands
,
settings
)
:
sprint
=
settings
[
"sprint"
]
quiet
=
settings
[
"quiet"
]
error
=
settings
[
"error"
]
enhanced_errors
=
True
the_shell
=
if
settings
[
"no_enhanced_errors"
]
:
enhanced_errors
=
False
if
"shell"
in
settings
:
the_shell
=
settings
[
"shell"
]
windows_p
=
sys
.
platform
==
"win32"
STDOUT
=
STDERR
=
if
quiet
:
STDOUT
=
PIPE
STDERR
=
PIPE
commands
=
commands
.
rstrip
(
)
sprint
(
"About to run commands '{}'"
.
format
(
commands
)
,
level
=
"verbose"
)
if
not
quiet
:
sprint
(
commands
)
if
the_shell
:
tmp
=
shlex
.
split
(
the_shell
)
the_shell
=
tmp
[
0
]
tmp
=
tmp
[
1
:
]
if
enhanced_errors
and
not
windows_p
:
tmp
.
append
(
"-e"
)
tmp
.
append
(
commands
)
commands
=
tmp
else
:
if
enhanced_errors
and
not
windows_p
:
commands
=
[
"-e"
,
commands
]
p
=
Popen
(
commands
,
shell
=
True
,
stdout
=
STDOUT
,
stderr
=
STDERR
,
executable
=
the_shell
)
out
,
err
=
p
.
communicate
(
)
if
p
.
returncode
:
if
quiet
:
error
(
err
.
decode
(
locale
.
getpreferredencoding
(
)
)
)
error
(
"Command failed to run"
)
sys
.
exit
(
1
)

Runs the commands supplied as an argument It will exit the program if the commands return a non - zero code


def get_the_node_dict(G, name):
    """
    Helper function that returns the node data
    of the node with the name supplied
    """
    for node in G.nodes(data=True):
        if node[0] == name:
            return node[1]

def get_the_node_dict ( G , name ) : for node in G . nodes ( data = True ) : if node [ 0 ] == name : return node [ 1 ]


def get_direct_ancestors(G, list_of_nodes):
    """
    Returns a list of nodes that are the parents
    from all of the nodes given as an argument.
    This is for use in the parallel topo sort
    """
    parents = []
    for item in list_of_nodes:
        anc = G.predecessors(item)
        for one in anc:
            parents.append(one)
    return parents

def get_direct_ancestors ( G , list_of_nodes ) : parents = [ ] for item in list_of_nodes : anc = G . predecessors ( item ) for one in anc : parents . append ( one ) return parents

Returns a list of nodes that are the parents from all of the nodes given as an argument . This is for use in the parallel topo sort


def get_sinks(G):
    """
    A sink is a node with no children.
    This means that this is the end of the line,
    and it should be run last in topo sort. This
    returns a list of all sinks in a graph
    """
    sinks = []
    for node in G:
        if not len(list(G.successors(node))):
            sinks.append(node)
    return sinks

def get_sinks ( G ) : sinks = [ ] for node in G : if not len ( list ( G . successors ( node ) ) ) : sinks . append ( node ) return sinks

A sink is a node with no children . This means that this is the end of the line and it should be run last in topo sort . This returns a list of all sinks in a graph


def get_levels(G):
    """
    For the parallel topo sort to work, the targets have
    to be executed in layers such that there is no
    dependency relationship between any nodes in a layer.
    What is returned is a list of lists representing all
    the layers, or levels
    """
    levels = []
    ends = get_sinks(G)
    levels.append(ends)
    while get_direct_ancestors(G, ends):
        ends = get_direct_ancestors(G, ends)
        levels.append(ends)
    levels.reverse()
    return levels

def get_levels ( G ) : levels = [ ] ends = get_sinks ( G ) levels . append ( ends ) while get_direct_ancestors ( G , ends ) : ends = get_direct_ancestors ( G , ends ) levels . append ( ends ) levels . reverse ( ) return levels

For the parallel topo sort to work the targets have to be executed in layers such that there is no dependency relationship between any nodes in a layer . What is returned is a list of lists representing all the layers or levels


def merge_from_store_and_in_mems(from_store, in_mem_shas, dont_update_shas_of):
    """
    If we don't merge the shas from the sha store and if we build a
    subgraph, the .shastore will only contain the shas of the files
    from the subgraph and the rest of the graph will have to be
    rebuilt
    """
    if not from_store:
        for item in dont_update_shas_of:
            if item in in_mem_shas['files']:
                del in_mem_shas['files'][item]
        return in_mem_shas
    for key in from_store['files']:
        if key not in in_mem_shas['files'] and key not in dont_update_shas_of:
            in_mem_shas['files'][key] = from_store['files'][key]
    for item in dont_update_shas_of:
        if item in in_mem_shas['files']:
            del in_mem_shas['files'][item]
    return in_mem_shas

def merge_from_store_and_in_mems ( from_store , in_mem_shas , dont_update_shas_of ) : if not from_store : for item in dont_update_shas_of : if item in in_mem_shas [ 'files' ] : del in_mem_shas [ 'files' ] [ item ] return in_mem_shas for key in from_store [ 'files' ] : if key not in in_mem_shas [ 'files' ] and key not in dont_update_shas_of : in_mem_shas [ 'files' ] [ key ] = from_store [ 'files' ] [ key ] for item in dont_update_shas_of : if item in in_mem_shas [ 'files' ] : del in_mem_shas [ 'files' ] [ item ] return in_mem_shas

If we don t merge the shas from the sha store and if we build a subgraph the . shastore will only contain the shas of the files from the subgraph and the rest of the graph will have to be rebuilt


def find_standard_sakefile(settings):
    """Returns the filename of the appropriate sakefile"""
    error = settings["error"]
    if settings["customsake"]:
        custom = settings["customsake"]
        if not os.path.isfile(custom):
            error("Specified sakefile '{}' doesn't exist", custom)
            sys.exit(1)
        return custom
    # no custom specified, going over defaults in order
    for name in ["Sakefile", "Sakefile.yaml", "Sakefile.yml"]:
        if os.path.isfile(name):
            return name
    error("Error: there is no Sakefile to read")
    sys.exit(1)

def
find_standard_sakefile
(
settings
)
:
error
=
settings
[
"error"
]
if
settings
[
"customsake"
]
:
custom
=
settings
[
"customsake"
]
if
not
os
.
path
.
isfile
(
custom
)
:
error
(
"Specified sakefile '{}' doesn't exist"
,
custom
)
sys
.
exit
(
1
)
return
custom
for
name
in
[
"Sakefile"
,
"Sakefile.yaml"
,
"Sakefile.yml"
]
:
if
os
.
path
.
isfile
(
name
)
:
return
name
error
(
"Error: there is no Sakefile to read"
)
sys
.
exit
(
1
)


def get_ties(G):
    """
    If you specify a target that shares a dependency with another target,
    both targets need to be updated. This is because running one will resolve
    the sha mismatch and sake will think that the other one doesn't have to
    run. This is called a "tie". This function will find such ties.
    """
    # we are going to make a dictionary whose keys are every dependency
    # and whose values are a list of all targets that use that dependency.
    # after making the dictionary, values whose length is above one will
    # be called "ties"
    ties = []
    dep_dict = {}
    for node in G.nodes(data=True):
        if 'dependencies' in node[1]:
            for item in node[1]['dependencies']:
                if item not in dep_dict:
                    dep_dict[item] = []
                dep_dict[item].append(node[0])
    for item in dep_dict:
        if len(list(set(dep_dict[item]))) > 1:
            ties.append(list(set(dep_dict[item])))
    return ties

def get_ties ( G ) : ties = [ ] dep_dict = { } for node in G . nodes ( data = True ) : if 'dependencies' in node [ 1 ] : for item in node [ 1 ] [ 'dependencies' ] : if item not in dep_dict : dep_dict [ item ] = [ ] dep_dict [ item ] . append ( node [ 0 ] ) for item in dep_dict : if len ( list ( set ( dep_dict [ item ] ) ) ) > 1 : ties . append ( list ( set ( dep_dict [ item ] ) ) ) return ties

If you specify a target that shares a dependency with another target both targets need to be updated . This is because running one will resolve the sha mismatch and sake will think that the other one doesn t have to run . This is called a tie . This function will find such ties .


def get_tied_targets(original_targets, the_ties):
    """
    This function gets called when a target is specified to ensure
    that all 'tied' targets also get included in the subgraph to
    be built
    """
    my_ties = []
    for original_target in original_targets:
        for item in the_ties:
            if original_target in item:
                for thing in item:
                    my_ties.append(thing)
    my_ties = list(set(my_ties))
    if my_ties:
        ties_message = ""
        ties_message += "The following targets share dependencies and must be run together:"
        for item in sorted(my_ties):
            ties_message += "\n  - {}".format(item)
        return list(set(my_ties+original_targets)), ties_message
    return original_targets, ""

This function gets called when a target is specified to ensure that all tied targets also get included in the subgraph to be built


def construct_graph(sakefile, settings):
    """
    Takes the sakefile dictionary and builds a NetworkX graph

    Args:
        A dictionary that is the parsed Sakefile (from sake.py)
        The settings dictionary

    Returns:
        A NetworkX graph
    """
    verbose = settings["verbose"]
    sprint = settings["sprint"]
    G = nx.DiGraph()
    sprint("Going to construct Graph", level="verbose")
    for target in sakefile:
        if target == "all":
            # we don't want this node
            continue
        if "formula" not in sakefile[target]:
            # that means this is a meta target
            for atomtarget in sakefile[target]:
                if atomtarget == "help":
                    continue
                sprint("Adding '{}'".format(atomtarget), level="verbose")
                data_dict = sakefile[target][atomtarget]
                data_dict["parent"] = target
                G.add_node(atomtarget, **data_dict)
        else:
            sprint("Adding '{}'".format(target), level="verbose")
            G.add_node(target, **sakefile[target])
    sprint("Nodes are built\nBuilding connections", level="verbose")
    for node in G.nodes(data=True):
        sprint("checking node {} for dependencies".format(node[0]),
               level="verbose")
        # normalize all paths in output
        for k, v in node[1].items():
            if v is None: node[1][k] = []
        if "output" in node[1]:
            for index, out in enumerate(node[1]['output']):
                node[1]['output'][index] = clean_path(node[1]['output'][index])
        if "dependencies" not in node[1]:
            continue
        sprint("it has dependencies", level="verbose")
        connects = []
        # normalize all paths in dependencies
        for index, dep in enumerate(node[1]['dependencies']):
            dep = os.path.normpath(dep)
            shrt = "dependencies"
            node[1]['dependencies'][index] = clean_path(node[1][shrt][index])
    for node in G.nodes(data=True):
        connects = []
        if "dependencies" not in node[1]:
            continue
        for dep in node[1]['dependencies']:
            matches = check_for_dep_in_outputs(dep, verbose, G)
            if not matches:
                continue
            for match in matches:
                sprint("Appending {} to matches".format(match), level="verbose")
                connects.append(match)
        if connects:
            for connect in connects:
                G.add_edge(connect, node[0])
    return G

def
construct_graph
(
sakefile
,
settings
)
:
verbose
=
settings
[
"verbose"
]
sprint
=
settings
[
"sprint"
]
G
=
nx
.
DiGraph
(
)
sprint
(
"Going to construct Graph"
,
level
=
"verbose"
)
for
target
in
sakefile
:
if
target
==
"all"
:
continue
if
"formula"
not
in
sakefile
[
target
]
:
for
atomtarget
in
sakefile
[
target
]
:
if
atomtarget
==
"help"
:
continue
sprint
(
"Adding '{}'"
.
format
(
atomtarget
)
,
level
=
"verbose"
)
data_dict
=
sakefile
[
target
]
[
atomtarget
]
data_dict
[
"parent"
]
=
target
G
.
add_node
(
atomtarget
,
**
data_dict
)
else
:
sprint
(
"Adding '{}'"
.
format
(
target
)
,
level
=
"verbose"
)
G
.
add_node
(
target
,
**
sakefile
[
target
]
)
sprint
(
"Nodes are built\nBuilding connections"
,
level
=
"verbose"
)
for
node
in
G
.
nodes
(
data
=
True
)
:
sprint
(
"checking node {} for dependencies"
.
format
(
node
[
0
]
)
,
level
=
"verbose"
)
for
k
,
v
in
node
[
1
]
.
items
(
)
:
if
v
is
:
node
[
1
]
[
k
]
=
[
]
if
"output"
in
node
[
1
]
:
for
index
,
out
in
enumerate
(
node
[
1
]
[
'output'
]
)
:
node
[
1
]
[
'output'
]
[
index
]
=
clean_path
(
node
[
1
]
[
'output'
]
[
index
]
)
if
"dependencies"
not
in
node
[
1
]
:
continue
sprint
(
"it has dependencies"
,
level
=
"verbose"
)
connects
=
[
]
for
index
,
dep
in
enumerate
(
node
[
1
]
[
'dependencies'
]
)
:
dep
=
os
.
path
.
normpath
(
dep
)
shrt
=
"dependencies"
node
[
1
]
[
'dependencies'
]
[
index
]
=
clean_path
(
node
[
1
]
[
shrt
]
[
index
]
)
for
node
in
G
.
nodes
(
data
=
True
)
:
connects
=
[
]
if
"dependencies"
not
in
node
[
1
]
:
continue
for
dep
in
node
[
1
]
[
'dependencies'
]
:
matches
=
check_for_dep_in_outputs
(
dep
,
verbose
,
G
)
if
not
matches
:
continue
for
match
in
matches
:
sprint
(
"Appending {} to matches"
.
format
(
match
)
,
level
=
"verbose"
)
connects
.
append
(
match
)
if
connects
:
for
connect
in
connects
:
G
.
add_edge
(
connect
,
node
[
0
]
)
return
G


def clean_all(G, settings):
    """
    Removes all the output files from all targets. Takes
    the graph as the only argument

    Args:
        The networkx graph object
        The settings dictionary

    Returns:
        0 if successful
        1 if removing even one file failed
    """
    quiet = settings["quiet"]
    recon = settings["recon"]
    sprint = settings["sprint"]
    error = settings["error"]
    all_outputs = []
    for node in G.nodes(data=True):
        if "output" in node[1]:
            for item in get_all_outputs(node[1]):
                all_outputs.append(item)
    all_outputs.append(".shastore")
    retcode = 0
    for item in sorted(all_outputs):
        if os.path.isfile(item):
            if recon:
                sprint("Would remove file: {}".format(item))
                continue
            sprint("Attempting to remove file '{}'", level="verbose")
            try:
                os.remove(item)
                sprint("Removed file", level="verbose")
            except:
                errmes = "Error: file '{}' failed to be removed"
                error(errmes.format(item))
                retcode = 1
    if not retcode and not recon:
        sprint("All clean", color=True)
    return retcode

def
clean_all
(
G
,
settings
)
:
quiet
=
settings
[
"quiet"
]
recon
=
settings
[
"recon"
]
sprint
=
settings
[
"sprint"
]
error
=
settings
[
"error"
]
all_outputs
=
[
]
for
node
in
G
.
nodes
(
data
=
True
)
:
if
"output"
in
node
[
1
]
:
for
item
in
get_all_outputs
(
node
[
1
]
)
:
all_outputs
.
append
(
item
)
all_outputs
.
append
(
".shastore"
)
retcode
=
0
for
item
in
sorted
(
all_outputs
)
:
if
os
.
path
.
isfile
(
item
)
:
if
recon
:
sprint
(
"Would remove file: {}"
.
format
(
item
)
)
continue
sprint
(
"Attempting to remove file '{}'"
,
level
=
"verbose"
)
try
:
os
.
remove
(
item
)
sprint
(
"Removed file"
,
level
=
"verbose"
)
except
:
errmes
=
"Error: file '{}' failed to be removed"
error
(
errmes
.
format
(
item
)
)
retcode
=
1
if
not
retcode
and
not
recon
:
sprint
(
"All clean"
,
color
=
True
)
return
retcode

Removes all the output files from all targets . Takes the graph as the only argument


def write_dot_file(G, filename):
    """
    Writes the graph G in dot file format for graphviz visualization.

    Args:
        a Networkx graph
        A filename to name the dot files
    """
    with io.open(filename, "w") as fh:
        fh.write("strict digraph DependencyDiagram {\n")
        edge_list = G.edges()
        node_list = set(G.nodes())
        if edge_list:
            for edge in sorted(edge_list):
                source, targ = edge
                node_list = node_list - set(source)
                node_list = node_list - set(targ)
                line = '"{}" -> "{}";\n'
                fh.write(line.format(source, targ))
        # draw nodes with no links
        if node_list:
            for node in sorted(node_list):
                line = '"{}"\n'.format(node)
                fh.write(line)
        fh.write("}")

def
write_dot_file
(
G
,
filename
)
:
with
io
.
open
(
filename
,
"w"
)
as
fh
:
fh
.
write
(
"strict digraph DependencyDiagram {\n"
)
edge_list
=
G
.
edges
(
)
node_list
=
set
(
G
.
nodes
(
)
)
if
edge_list
:
for
edge
in
sorted
(
edge_list
)
:
source
,
targ
=
edge
node_list
=
node_list
set
(
source
)
node_list
=
node_list
set
(
targ
)
line
=
'"{}" -> "{}";\n'
fh
.
write
(
line
.
format
(
source
,
targ
)
)
if
node_list
:
for
node
in
sorted
(
node_list
)
:
line
=
'"{}"\n'
.
format
(
node
)
fh
.
write
(
line
)
fh
.
write
(
"}"
)

Writes the graph G in dot file format for graphviz visualization .


def itertable(table):
    """Auxiliary function for iterating over a data table."""
    for item in table:
        res = {
            k.lower(): nfd(v) if isinstance(v, text_type) else v for k, v in item.items()}
        for extra in res.pop('extra', []):
            k, _, v = extra.partition(':')
            res[k.strip()] = v.strip()
        yield res

def itertable ( table ) : for item in table : res = { k . lower ( ) : nfd ( v ) if isinstance ( v , text_type ) else v for k , v in item . items ( ) } for extra in res . pop ( 'extra' , [ ] ) : k , _ , v = extra . partition ( ':' ) res [ k . strip ( ) ] = v . strip ( ) yield res

Auxiliary function for iterating over a data table .


def _make_package(args):  # pragma: no cover
    """Prepare transcriptiondata from the transcription sources."""
    from lingpy.sequence.sound_classes import token2class
    from lingpy.data import Model

    columns = ['LATEX', 'FEATURES', 'SOUND', 'IMAGE', 'COUNT', 'NOTE']
    bipa = TranscriptionSystem('bipa')
    for src, rows in args.repos.iter_sources(type='td'):
        args.log.info('TranscriptionData {0} ...'.format(src['NAME']))
        uritemplate = URITemplate(src['URITEMPLATE']) if src['URITEMPLATE'] else None
        out = [['BIPA_GRAPHEME', 'CLTS_NAME', 'GENERATED', 'EXPLICIT',
                'GRAPHEME', 'URL'] + columns]
        graphemes = set()
        for row in rows:
            if row['GRAPHEME'] in graphemes:
                args.log.warn('skipping duplicate grapheme: {0}'.format(row['GRAPHEME']))
                continue
            graphemes.add(row['GRAPHEME'])
            if not row['BIPA']:
                bipa_sound = bipa[row['GRAPHEME']]
                explicit = ''
            else:
                bipa_sound = bipa[row['BIPA']]
                explicit = '+'
            generated = '+' if bipa_sound.generated else ''
            if is_valid_sound(bipa_sound, bipa):
                bipa_grapheme = bipa_sound.s
                bipa_name = bipa_sound.name
            else:
                bipa_grapheme, bipa_name = '<NA>', '<NA>'
            url = uritemplate.expand(**row) if uritemplate else row.get('URL', '')
            out.append(
                [bipa_grapheme, bipa_name, generated, explicit, row['GRAPHEME'],
                 url] + [
                    row.get(c, '') for c in columns])
        found = len([o for o in out if o[0] != '<NA>'])
        args.log.info('... {0} of {1} graphemes found ({2:.0f}%)'.format(
            found, len(out), found / len(out) * 100))
        with UnicodeWriter(
            pkg_path('transcriptiondata', '{0}.tsv'.format(src['NAME'])), delimiter='\t'
        ) as writer:
            writer.writerows(out)

    count = 0
    with UnicodeWriter(pkg_path('soundclasses', 'lingpy.tsv'), delimiter='\t') as writer:
        writer.writerow(['CLTS_NAME', 'BIPA_GRAPHEME'] + SOUNDCLASS_SYSTEMS)
        for grapheme, sound in sorted(bipa.sounds.items()):
            if not sound.alias:
                writer.writerow(
                    [sound.name, grapheme] + [token2class(
                        grapheme, Model(cls)) for cls in SOUNDCLASS_SYSTEMS])
                count += 1
    args.log.info('SoundClasses: {0} written to file.'.format(count))

Prepare transcriptiondata from the transcription sources .


def is_valid_sound(sound, ts):
    """Check the consistency of a given transcription system conversino"""
    if isinstance(sound, (Marker, UnknownSound)):
        return False
    s1 = ts[sound.name]
    s2 = ts[sound.s]
    return s1.name == s2.name and s1.s == s2.s

def is_valid_sound ( sound , ts ) : if isinstance ( sound , ( Marker , UnknownSound ) ) : return False s1 = ts [ sound . name ] s2 = ts [ sound . s ] return s1 . name == s2 . name and s1 . s == s2 . s


def normalize(self, string):
        """Normalize the string according to normalization list"""
        return ''.join([self._normalize.get(x, x) for x in nfd(string)])

def normalize ( self , string ) : return '' . join ( [ self . _normalize . get ( x , x ) for x in nfd ( string ) ] )


def _from_name(self, string):
        """Parse a sound from its name"""
        components = string.split(' ')
        if frozenset(components) in self.features:
            return self.features[frozenset(components)]
        rest, sound_class = components[:-1], components[-1]
        if sound_class in ['diphthong', 'cluster']:
            if string.startswith('from ') and 'to ' in string:
                extension = {'diphthong': 'vowel', 'cluster': 'consonant'}[sound_class]
                string_ = ' '.join(string.split(' ')[1:-1])
                from_, to_ = string_.split(' to ')
                v1, v2 = frozenset(from_.split(' ') + [extension]), frozenset(
                    to_.split(' ') + [extension])
                if v1 in self.features and v2 in self.features:
                    s1, s2 = (self.features[v1], self.features[v2])
                    if sound_class == 'diphthong':
                        return Diphthong.from_sounds(s1 + s2, s1, s2, self)  # noqa: F405
                    else:
                        return Cluster.from_sounds(s1 + s2, s1, s2, self)  # noqa: F405
                else:
                    # try to generate the sounds if they are not there
                    s1, s2 = self._from_name(from_ + ' ' + extension), self._from_name(
                        to_ + ' ' + extension)
                    if not (isinstance(
                        s1, UnknownSound) or isinstance(s2, UnknownSound)):  # noqa: F405
                        if sound_class == 'diphthong':
                            return Diphthong.from_sounds(  # noqa: F405
                                s1 + s2, s1, s2, self)
                        return Cluster.from_sounds(s1 + s2, s1, s2, self)  # noqa: F405
                    raise ValueError('components could not be found in system')
            else:
                raise ValueError('name string is erroneously encoded')

        if sound_class not in self.sound_classes:
            raise ValueError('no sound class specified')

        args = {self._feature_values.get(comp, '?'): comp for comp in rest}
        if '?' in args:
            raise ValueError('string contains unknown features')
        args['grapheme'] = ''
        args['ts'] = self
        sound = self.sound_classes[sound_class](**args)
        if sound.featureset not in self.features:
            sound.generated = True
            return sound
        return self.features[sound.featureset]


def iteration(self):
        """
        Runs the ipfn algorithm. Automatically detects of working with numpy ndarray or pandas dataframes.
        """

        i = 0
        conv = np.inf
        old_conv = -np.inf
        conv_list = []
        m = self.original

        # If the original data input is in pandas DataFrame format
        if isinstance(self.original, pd.DataFrame):
            ipfn_method = self.ipfn_df
        elif isinstance(self.original, np.ndarray):
            ipfn_method = self.ipfn_np
            self.original = self.original.astype('float64')
        else:
            print('Data input instance not recognized')
            sys.exit(0)
        while ((i <= self.max_itr and conv > self.conv_rate) and
               (i <= self.max_itr and abs(conv - old_conv) > self.rate_tolerance)):
            old_conv = conv
            m, conv = ipfn_method(m, self.aggregates, self.dimensions, self.weight_col)
            conv_list.append(conv)
            i += 1
        converged = 1
        if i <= self.max_itr:
            if not conv > self.conv_rate:
                print('ipfn converged: convergence_rate below threshold')
            elif not abs(conv - old_conv) > self.rate_tolerance:
                print('ipfn converged: convergence_rate not updating or below rate_tolerance')
        else:
            print('Maximum iterations reached')
            converged = 0

        # Handle the verbose
        if self.verbose == 0:
            return m
        elif self.verbose == 1:
            return m, converged
        elif self.verbose == 2:
            return m, converged, pd.DataFrame({'iteration': range(i), 'conv': conv_list}).set_index('iteration')
        else:
            print('wrong verbose input, return None')
            sys.exit(0)

Runs the ipfn algorithm . Automatically detects of working with numpy ndarray or pandas dataframes .


def resolve(self, context, quiet=True):
        """
        Return an object described by the accessor by traversing the attributes
        of context.

        """
        try:
            obj = context
            for level in self.levels:
                if isinstance(obj, dict):
                    obj = obj[level]
                elif isinstance(obj, list) or isinstance(obj, tuple):
                    obj = obj[int(level)]
                else:
                    if callable(getattr(obj, level)):
                        try:
                            obj = getattr(obj, level)()
                        except KeyError:
                            obj = getattr(obj, level)
                    else:
                        # for model field that has choice set
                        # use get_xxx_display to access
                        display = 'get_%s_display' % level
                        obj = getattr(obj, display)() if hasattr(obj, display) else getattr(obj, level)
                if not obj:
                    break
            return obj
        except Exception as e:
            if quiet:
                return ''
            else:
                raise e

def resolve ( self , context , quiet = True ) : try : obj = context for level in self . levels : if isinstance ( obj , dict ) : obj = obj [ level ] elif isinstance ( obj , list ) or isinstance ( obj , tuple ) : obj = obj [ int ( level ) ] else : if callable ( getattr ( obj , level ) ) : try : obj = getattr ( obj , level ) ( ) except KeyError : obj = getattr ( obj , level ) else : display = 'get_%s_display' % level obj = getattr ( obj , display ) ( ) if hasattr ( obj , display ) else getattr ( obj , level ) if not obj : break return obj except Exception as e : if quiet : return '' else : raise e

Return an object described by the accessor by traversing the attributes of context .


def get_days_span(self, month_index):
        """
        Calculate how many days the month spans.
        """
        is_first_month = month_index == 0
        is_last_month = month_index == self.__len__() - 1

        y = int(self.start_date.year + (self.start_date.month + month_index) / 13)
        m = int((self.start_date.month + month_index) % 12 or 12)
        total = calendar.monthrange(y, m)[1]

        if is_first_month and is_last_month:
            return (self.end_date - self.start_date).days + 1
        else:
            if is_first_month:
                return total - self.start_date.day + 1
            elif is_last_month:
                return self.end_date.day
            else:
                return total

def get_days_span ( self , month_index ) : is_first_month = month_index == 0 is_last_month = month_index == self . __len__ ( ) - 1 y = int ( self . start_date . year + ( self . start_date . month + month_index ) / 13 ) m = int ( ( self . start_date . month + month_index ) % 12 or 12 ) total = calendar . monthrange ( y , m ) [ 1 ] if is_first_month and is_last_month : return ( self . end_date - self . start_date ) . days + 1 else : if is_first_month : return total - self . start_date . day + 1 elif is_last_month : return self . end_date . day else : return total

Calculate how many days the month spans .


def _calculate_float(self, byte_array):
        """Returns an IEEE 754 float from an array of 4 bytes

        :param byte_array: Expects an array of 4 bytes

        :type byte_array: array

        :rtype: float
        """
        if len(byte_array) != 4:
            return None

        return struct.unpack('f', struct.pack('4B', *byte_array))[0]

def _calculate_float ( self , byte_array ) : if len ( byte_array ) != 4 : return None return struct . unpack ( 'f' , struct . pack ( '4B' , * byte_array ) ) [ 0 ]


def _calculate_period(self, vals):
        ''' calculate the sampling period in seconds '''
        if len(vals) < 4:
            return None

        if self.firmware['major'] < 16:
            return ((vals[3] << 24) | (vals[2] << 16) | (vals[1] << 8) | vals[0]) / 12e6
        else:
            return self._calculate_float(vals)

def _calculate_period ( self , vals ) : if len ( vals ) < 4 : return None if self . firmware [ 'major' ] < 16 : return ( ( vals [ 3 ] << 24 ) | ( vals [ 2 ] << 16 ) | ( vals [ 1 ] << 8 ) | vals [ 0 ] ) / 12e6 else : return self . _calculate_float ( vals )


def calculate_bin_boundary(self, bb):
        """Calculate the adc value that corresponds to a specific bin boundary diameter in microns.

            :param bb: Bin Boundary in microns

            :type bb: float

            :rtype: int
        """

        return min(enumerate(OPC_LOOKUP), key = lambda x: abs(x[1] - bb))[0]

def calculate_bin_boundary ( self , bb ) : return min ( enumerate ( OPC_LOOKUP ) , key = lambda x : abs ( x [ 1 ] - bb ) ) [ 0 ]

Calculate the adc value that corresponds to a specific bin boundary diameter in microns .


def read_info_string(self):
        """Reads the information string for the OPC

        :rtype: string

        :Example:

        >>> alpha.read_info_string()
        'OPC-N2 FirmwareVer=OPC-018.2....................BD'
        """
        infostring = []

        # Send the command byte and sleep for 9 ms
        self.cnxn.xfer([0x3F])
        sleep(9e-3)

        # Read the info string by sending 60 empty bytes
        for i in range(60):
            resp = self.cnxn.xfer([0x00])[0]
            infostring.append(chr(resp))

        sleep(0.1)

        return ''.join(infostring)

def read_info_string ( self ) : infostring = [ ] self . cnxn . xfer ( [ 0x3F ] ) sleep ( 9e-3 ) for i in range ( 60 ) : resp = self . cnxn . xfer ( [ 0x00 ] ) [ 0 ] infostring . append ( chr ( resp ) ) sleep ( 0.1 ) return '' . join ( infostring )


def ping(self):
        """Checks the connection between the Raspberry Pi and the OPC

        :rtype: Boolean
        """
        b = self.cnxn.xfer([0xCF])[0]           # send the command byte

        sleep(0.1)

        return True if b == 0xF3 else False

def ping ( self ) : b = self . cnxn . xfer ( [ 0xCF ] ) [ 0 ] sleep ( 0.1 ) return True if b == 0xF3 else False


def config(self):
        """Read the configuration variables and returns them as a dictionary

        :rtype: dictionary

        :Example:

        >>> alpha.config()
        {
            'BPD 13': 1.6499,
            'BPD 12': 1.6499,
            'BPD 11': 1.6499,
            'BPD 10': 1.6499,
            'BPD 15': 1.6499,
            'BPD 14': 1.6499,
            'BSVW 15': 1.0,
            ...
        }
        """
        config  = []
        data    = {}

        # Send the command byte and sleep for 10 ms
        self.cnxn.xfer([0x3C])
        sleep(10e-3)

        # Read the config variables by sending 256 empty bytes
        for i in range(256):
            resp = self.cnxn.xfer([0x00])[0]
            config.append(resp)

        # Add the bin bounds to the dictionary of data [bytes 0-29]
        for i in range(0, 15):
            data["Bin Boundary {0}".format(i)] = self._16bit_unsigned(config[2*i], config[2*i + 1])

        # Add the Bin Particle Volumes (BPV) [bytes 32-95]
        for i in range(0, 16):
            data["BPV {0}".format(i)] = self._calculate_float(config[4*i + 32:4*i + 36])

        # Add the Bin Particle Densities (BPD) [bytes 96-159]
        for i in range(0, 16):
            data["BPD {0}".format(i)] = self._calculate_float(config[4*i + 96:4*i + 100])

        # Add the Bin Sample Volume Weight (BSVW) [bytes 160-223]
        for i in range(0, 16):
            data["BSVW {0}".format(i)] = self._calculate_float(config[4*i + 160: 4*i + 164])

        # Add the Gain Scaling Coefficient (GSC) and sample flow rate (SFR)
        data["GSC"] = self._calculate_float(config[224:228])
        data["SFR"] = self._calculate_float(config[228:232])

        # Add laser dac (LDAC) and Fan dac (FanDAC)
        data["LaserDAC"]    = config[232]
        data["FanDAC"]      = config[233]

        # If past firmware 15, add other things
        if self.firmware['major'] > 15.:
            data['TOF_SFR'] = config[234]

        sleep(0.1)

        return data

def
config
(
self
)
:
config
=
[
]
data
=
{
}
self
.
cnxn
.
xfer
(
[
0x3C
]
)
sleep
(
10e-3
)
for
i
in
range
(
256
)
:
resp
=
self
.
cnxn
.
xfer
(
[
0x00
]
)
[
0
]
config
.
append
(
resp
)
for
i
in
range
(
0
,
15
)
:
data
[
"Bin Boundary {0}"
.
format
(
i
)
]
=
self
.
_16bit_unsigned
(
config
[
2
*
i
]
,
config
[
2
*
i
+
1
]
)
for
i
in
range
(
0
,
16
)
:
data
[
"BPV {0}"
.
format
(
i
)
]
=
self
.
_calculate_float
(
config
[
4
*
i
+
32
:
4
*
i
+
36
]
)
for
i
in
range
(
0
,
16
)
:
data
[
"BPD {0}"
.
format
(
i
)
]
=
self
.
_calculate_float
(
config
[
4
*
i
+
96
:
4
*
i
+
100
]
)
for
i
in
range
(
0
,
16
)
:
data
[
"BSVW {0}"
.
format
(
i
)
]
=
self
.
_calculate_float
(
config
[
4
*
i
+
160
:
4
*
i
+
164
]
)
data
[
"GSC"
]
=
self
.
_calculate_float
(
config
[
224
:
228
]
)
data
[
"SFR"
]
=
self
.
_calculate_float
(
config
[
228
:
232
]
)
data
[
"LaserDAC"
]
=
config
[
232
]
data
[
"FanDAC"
]
=
config
[
233
]
if
self
.
firmware
[
'major'
]
>
15.
:
data
[
'TOF_SFR'
]
=
config
[
234
]
sleep
(
0.1
)
return
data


def config2(self):
        """Read the second set of configuration variables and return as a dictionary.

        **NOTE: This method is supported by firmware v18+.**

        :rtype: dictionary

        :Example:

        >>> a.config2()
        {
            'AMFanOnIdle': 0,
            'AMIdleIntervalCount': 0,
            'AMMaxDataArraysInFile': 61798,
            'AMSamplingInterval': 1,
            'AMOnlySavePMData': 0,
            'AMLaserOnIdle': 0
        }
        """
        config  = []
        data    = {}

        # Send the command byte and sleep for 10 ms
        self.cnxn.xfer([0x3D])
        sleep(10e-3)

        # Read the config variables by sending 256 empty bytes
        for i in range(9):
            resp = self.cnxn.xfer([0x00])[0]
            config.append(resp)

        data["AMSamplingInterval"]      = self._16bit_unsigned(config[0], config[1])
        data["AMIdleIntervalCount"]     = self._16bit_unsigned(config[2], config[3])
        data['AMFanOnIdle']             = config[4]
        data['AMLaserOnIdle']           = config[5]
        data['AMMaxDataArraysInFile']   = self._16bit_unsigned(config[6], config[7])
        data['AMOnlySavePMData']        = config[8]

        sleep(0.1)

        return data

def config2 ( self ) : config = [ ] data = { } self . cnxn . xfer ( [ 0x3D ] ) sleep ( 10e-3 ) for i in range ( 9 ) : resp = self . cnxn . xfer ( [ 0x00 ] ) [ 0 ] config . append ( resp ) data [ "AMSamplingInterval" ] = self . _16bit_unsigned ( config [ 0 ] , config [ 1 ] ) data [ "AMIdleIntervalCount" ] = self . _16bit_unsigned ( config [ 2 ] , config [ 3 ] ) data [ 'AMFanOnIdle' ] = config [ 4 ] data [ 'AMLaserOnIdle' ] = config [ 5 ] data [ 'AMMaxDataArraysInFile' ] = self . _16bit_unsigned ( config [ 6 ] , config [ 7 ] ) data [ 'AMOnlySavePMData' ] = config [ 8 ] sleep ( 0.1 ) return data

Read the second set of configuration variables and return as a dictionary .


def set_fan_power(self, power):
        """Set only the Fan power.

        :param power: Fan power value as an integer between 0-255.

        :type power: int

        :rtype: boolean

        :Example:

        >>> alpha.set_fan_power(255)
        True
        """
        # Check to make sure the value is a single byte
        if power > 255:
            raise ValueError("The fan power should be a single byte (0-255).")

        # Send the command byte and wait 10 ms
        a = self.cnxn.xfer([0x42])[0]
        sleep(10e-3)

        # Send the next two bytes
        b = self.cnxn.xfer([0x00])[0]
        c = self.cnxn.xfer([power])[0]

        sleep(0.1)

        return True if a == 0xF3 and b == 0x42 and c == 0x00 else False


def toggle_laser(self, state):
        """Toggle the power state of the laser.

        :param state: Boolean state of the laser

        :type state: boolean

        :rtype: boolean

        :Example:

        >>> alpha.toggle_laser(True)
        True
        """

        # Send the command byte and wait 10 ms
        a = self.cnxn.xfer([0x03])[0]

        sleep(10e-3)

        # If state is true, turn the laser ON, else OFF
        if state:
            b = self.cnxn.xfer([0x02])[0]
        else:
            b = self.cnxn.xfer([0x03])[0]

        sleep(0.1)

        return True if a == 0xF3 and b == 0x03 else False

def toggle_laser ( self , state ) : a = self . cnxn . xfer ( [ 0x03 ] ) [ 0 ] sleep ( 10e-3 ) if state : b = self . cnxn . xfer ( [ 0x02 ] ) [ 0 ] else : b = self . cnxn . xfer ( [ 0x03 ] ) [ 0 ] sleep ( 0.1 ) return True if a == 0xF3 and b == 0x03 else False


def sn(self):
        """Read the Serial Number string. This method is only available on OPC-N2
        firmware versions 18+.

        :rtype: string

        :Example:

        >>> alpha.sn()
        'OPC-N2 123456789'
        """
        string = []

        # Send the command byte and sleep for 9 ms
        self.cnxn.xfer([0x10])
        sleep(9e-3)

        # Read the info string by sending 60 empty bytes
        for i in range(60):
            resp = self.cnxn.xfer([0x00])[0]
            string.append(chr(resp))

        sleep(0.1)

        return ''.join(string)

def sn ( self ) : string = [ ] self . cnxn . xfer ( [ 0x10 ] ) sleep ( 9e-3 ) for i in range ( 60 ) : resp = self . cnxn . xfer ( [ 0x00 ] ) [ 0 ] string . append ( chr ( resp ) ) sleep ( 0.1 ) return '' . join ( string )

Read the Serial Number string . This method is only available on OPC - N2 firmware versions 18 + .


def read_firmware(self):
        """Read the firmware version of the OPC-N2. Firmware v18+ only.

        :rtype: dict

        :Example:

        >>> alpha.read_firmware()
        {
            'major': 18,
            'minor': 2,
            'version': 18.2
        }
        """
        # Send the command byte and sleep for 9 ms
        self.cnxn.xfer([0x12])
        sleep(10e-3)

        self.firmware['major'] = self.cnxn.xfer([0x00])[0]
        self.firmware['minor'] = self.cnxn.xfer([0x00])[0]

        # Build the firmware version
        self.firmware['version'] = float('{}.{}'.format(self.firmware['major'], self.firmware['minor']))

        sleep(0.1)

        return self.firmware

def read_firmware ( self ) : self . cnxn . xfer ( [ 0x12 ] ) sleep ( 10e-3 ) self . firmware [ 'major' ] = self . cnxn . xfer ( [ 0x00 ] ) [ 0 ] self . firmware [ 'minor' ] = self . cnxn . xfer ( [ 0x00 ] ) [ 0 ] self . firmware [ 'version' ] = float ( '{}.{}' . format ( self . firmware [ 'major' ] , self . firmware [ 'minor' ] ) ) sleep ( 0.1 ) return self . firmware

Read the firmware version of the OPC - N2 . Firmware v18 + only .


def pm(self):
        """Read the PM data and reset the histogram

        **NOTE: This method is supported by firmware v18+.**

        :rtype: dictionary

        :Example:

        >>> alpha.pm()
        {
            'PM1': 0.12,
            'PM2.5': 0.24,
            'PM10': 1.42
        }
        """

        resp = []
        data = {}

        # Send the command byte
        self.cnxn.xfer([0x32])

        # Wait 10 ms
        sleep(10e-3)

        # read the histogram
        for i in range(12):
            r = self.cnxn.xfer([0x00])[0]
            resp.append(r)

        # convert to real things and store in dictionary!
        data['PM1']     = self._calculate_float(resp[0:4])
        data['PM2.5']   = self._calculate_float(resp[4:8])
        data['PM10']    = self._calculate_float(resp[8:])

        sleep(0.1)

        return data

def pm ( self ) : resp = [ ] data = { } self . cnxn . xfer ( [ 0x32 ] ) sleep ( 10e-3 ) for i in range ( 12 ) : r = self . cnxn . xfer ( [ 0x00 ] ) [ 0 ] resp . append ( r ) data [ 'PM1' ] = self . _calculate_float ( resp [ 0 : 4 ] ) data [ 'PM2.5' ] = self . _calculate_float ( resp [ 4 : 8 ] ) data [ 'PM10' ] = self . _calculate_float ( resp [ 8 : ] ) sleep ( 0.1 ) return data


def read_gsc_sfr(self):
        """Read the gain-scaling-coefficient and sample flow rate.

        :returns: dictionary containing GSC and SFR
        """
        config  = []
        data    = {}

        # Send the command byte and sleep for 10 ms
        self.cnxn.xfer([0x33])
        sleep(10e-3)

        # Read the config variables by sending 256 empty bytes
        for i in range(8):
            resp = self.cnxn.xfer([0x00])[0]
            config.append(resp)

        data["GSC"] = self._calculate_float(config[0:4])
        data["SFR"] = self._calculate_float(config[4:])

        return data

def read_gsc_sfr ( self ) : config = [ ] data = { } self . cnxn . xfer ( [ 0x33 ] ) sleep ( 10e-3 ) for i in range ( 8 ) : resp = self . cnxn . xfer ( [ 0x00 ] ) [ 0 ] config . append ( resp ) data [ "GSC" ] = self . _calculate_float ( config [ 0 : 4 ] ) data [ "SFR" ] = self . _calculate_float ( config [ 4 : ] ) return data

Read the gain - scaling - coefficient and sample flow rate .


def read_bin_boundaries(self):
        """Return the bin boundaries.

        :returns: dictionary with 17 bin boundaries.
        """
        config  = []
        data    = {}

        # Send the command byte and sleep for 10 ms
        self.cnxn.xfer([0x33])
        sleep(10e-3)

        # Read the config variables by sending 256 empty bytes
        for i in range(30):
            resp = self.cnxn.xfer([0x00])[0]
            config.append(resp)

        # Add the bin bounds to the dictionary of data [bytes 0-29]
        for i in range(0, 14):
            data["Bin Boundary {0}".format(i)] = self._16bit_unsigned(config[2*i], config[2*i + 1])

        return data

def
read_bin_boundaries
(
self
)
:
config
=
[
]
data
=
{
}
self
.
cnxn
.
xfer
(
[
0x33
]
)
sleep
(
10e-3
)
for
i
in
range
(
30
)
:
resp
=
self
.
cnxn
.
xfer
(
[
0x00
]
)
[
0
]
config
.
append
(
resp
)
for
i
in
range
(
0
,
14
)
:
data
[
"Bin Boundary {0}"
.
format
(
i
)
]
=
self
.
_16bit_unsigned
(
config
[
2
*
i
]
,
config
[
2
*
i
+
1
]
)
return
data


def read_bin_particle_density(self):
        """Read the bin particle density

        :returns: float
        """
        config = []

        # Send the command byte and sleep for 10 ms
        self.cnxn.xfer([0x33])
        sleep(10e-3)

        # Read the config variables by sending 256 empty bytes
        for i in range(4):
            resp = self.cnxn.xfer([0x00])[0]
            config.append(resp)

        bpd = self._calculate_float(config)

        return bpd

def read_bin_particle_density ( self ) : config = [ ] self . cnxn . xfer ( [ 0x33 ] ) sleep ( 10e-3 ) for i in range ( 4 ) : resp = self . cnxn . xfer ( [ 0x00 ] ) [ 0 ] config . append ( resp ) bpd = self . _calculate_float ( config ) return bpd


def start(self):
        """
        Starts HDLC controller's threads.
        """

        self.receiver = self.Receiver(
            self.read,
            self.write,
            self.send_lock,
            self.senders,
            self.frames_received,
            callback=self.receive_callback,
            fcs_nack=self.fcs_nack,
        )

        self.receiver.start()

def start ( self ) : self . receiver = self . Receiver ( self . read , self . write , self . send_lock , self . senders , self . frames_received , callback = self . receive_callback , fcs_nack = self . fcs_nack , ) self . receiver . start ( )


def stop(self):
        """
        Stops HDLC controller's threads.
        """

        if self.receiver != None:
            self.receiver.join()

        for s in self.senders.values():
            s.join()

def stop ( self ) : if self . receiver != None : self . receiver . join ( ) for s in self . senders . values ( ) : s . join ( )


def send(self, data):
        """
        Sends a new data frame.

        This method will block until a new room is available for
        a new sender. This limit is determined by the size of the window.
        """

        while len(self.senders) >= self.window:
            pass

        self.senders[self.new_seq_no] = self.Sender(
            self.write,
            self.send_lock,
            data,
            self.new_seq_no,
            timeout=self.sending_timeout,
            callback=self.send_callback,
        )

        self.senders[self.new_seq_no].start()
        self.new_seq_no = (self.new_seq_no + 1) % HDLController.MAX_SEQ_NO

def send ( self , data ) : while len ( self . senders ) >= self . window : pass self . senders [ self . new_seq_no ] = self . Sender ( self . write , self . send_lock , data , self . new_seq_no , timeout = self . sending_timeout , callback = self . send_callback , ) self . senders [ self . new_seq_no ] . start ( ) self . new_seq_no = ( self . new_seq_no + 1 ) % HDLController . MAX_SEQ_NO


def cut(self, by, from_start=True):
        """ Cuts this object from_start to the number requestd
        returns new instance
        """
        s, e = copy(self.start), copy(self.end)
        if from_start:
            e = s + by
        else:
            s = e - by
        return Range(s, e)

def cut ( self , by , from_start = True ) : s , e = copy ( self . start ) , copy ( self . end ) if from_start : e = s + by else : s = e - by return Range ( s , e )


def replace(self, **k):
        """Note returns a new Date obj"""
        if self.date != 'infinity':
            return Date(self.date.replace(**k))
        else:
            return Date('infinity')

def replace ( self , ** k ) : if self . date != 'infinity' : return Date ( self . date . replace ( ** k ) ) else : return Date ( 'infinity' )


def findall(text):
    """Find all the timestrings within a block of text.

    >>> timestring.findall("once upon a time, about 3 weeks ago, there was a boy whom was born on august 15th at 7:20 am. epic.")
    [
     ('3 weeks ago,', <timestring.Date 2014-02-09 00:00:00 4483019280>),
     ('august 15th at 7:20 am', <timestring.Date 2014-08-15 07:20:00 4483019344>)
    ]
    """
    results = TIMESTRING_RE.findall(text)
    dates = []
    for date in results:
        if re.compile('((next|last)\s(\d+|couple(\sof))\s(weeks|months|quarters|years))|(between|from)', re.I).match(date[0]):
            dates.append((date[0].strip(), Range(date[0])))
        else:
            dates.append((date[0].strip(), Date(date[0])))
    return dates

Find all the timestrings within a block of text .


def validate_token(self, request, consumer, token):
        """
        Check the token and raise an `oauth.Error` exception if invalid.
        """
        oauth_server, oauth_request = oauth_provider.utils.initialize_server_request(request)
        oauth_server.verify_request(oauth_request, consumer, token)

def validate_token ( self , request , consumer , token ) : oauth_server , oauth_request = oauth_provider . utils . initialize_server_request ( request ) oauth_server . verify_request ( oauth_request , consumer , token )

Check the token and raise an oauth . Error exception if invalid .


def check_nonce(self, request, oauth_request):
        """
        Checks nonce of request, and return True if valid.
        """
        oauth_nonce = oauth_request['oauth_nonce']
        oauth_timestamp = oauth_request['oauth_timestamp']
        return check_nonce(request, oauth_request, oauth_nonce, oauth_timestamp)

Checks nonce of request and return True if valid .


def deliveries(self):
        """ Get delivery log from Redis"""
        key = make_key(
            event=self.object.event,
            owner_name=self.object.owner.username,
            identifier=self.object.identifier
        )
        return redis.lrange(key, 0, 20)

def deliveries ( self ) : key = make_key ( event = self . object . event , owner_name = self . object . owner . username , identifier = self . object . identifier ) return redis . lrange ( key , 0 , 20 )


def event_choices(events):
    """ Get the possible events from settings """
    if events is None:
        msg = "Please add some events in settings.WEBHOOK_EVENTS."
        raise ImproperlyConfigured(msg)
    try:
        choices = [(x, x) for x in events]
    except TypeError:
        """ Not a valid iterator, so we raise an exception """
        msg = "settings.WEBHOOK_EVENTS must be an iterable object."
        raise ImproperlyConfigured(msg)
    return choices


def worker(wrapped, dkwargs, hash_value=None, *args, **kwargs):
    """
        This is an asynchronous sender callable that uses the Django ORM to store
            webhooks. Redis is used to handle the message queue.

        dkwargs argument requires the following key/values:

            :event: A string representing an event.

        kwargs argument requires the following key/values

            :owner: The user who created/owns the event
    """

    if "event" not in dkwargs:
        msg = "djwebhooks.decorators.redis_hook requires an 'event' argument in the decorator."
        raise TypeError(msg)
    event = dkwargs['event']

    if "owner" not in kwargs:
        msg = "djwebhooks.senders.redis_callable requires an 'owner' argument in the decorated function."
        raise TypeError(msg)
    owner = kwargs['owner']

    if "identifier" not in kwargs:
        msg = "djwebhooks.senders.orm_callable requires an 'identifier' argument in the decorated function."
        raise TypeError(msg)
    identifier = kwargs['identifier']

    senderobj = DjangoRQSenderable(
            wrapped, dkwargs, hash_value, WEBHOOK_ATTEMPTS, *args, **kwargs
    )

    # Add the webhook object just so it's around
    # TODO - error handling if this can't be found
    senderobj.webhook_target = WebhookTarget.objects.get(
        event=event,
        owner=owner,
        identifier=identifier
    )

    # Get the target url and add it
    senderobj.url = senderobj.webhook_target.target_url

    # Get the payload. This overides the senderobj.payload property.
    senderobj.payload = senderobj.get_payload()

    # Get the creator and add it to the payload.
    senderobj.payload['owner'] = getattr(kwargs['owner'], WEBHOOK_OWNER_FIELD)

    # get the event and add it to the payload
    senderobj.payload['event'] = dkwargs['event']

    return senderobj.send()

This is an asynchronous sender callable that uses the Django ORM to store webhooks . Redis is used to handle the message queue .


def send(self, msg):
        """Encodes data to slip protocol and then sends over serial port

        Uses the SlipLib module to convert the message data into SLIP format.
        The message is then sent over the serial port opened with the instance
        of the Faraday class used when invoking send().

        Args:
            msg (bytes): Bytes format message to send over serial port.

        Returns:
            int: Number of bytes transmitted over the serial port.

        """
        # Create a sliplib Driver
        slipDriver = sliplib.Driver()

        # Package data in slip format
        slipData = slipDriver.send(msg)

        # Send data over serial port
        res = self._serialPort.write(slipData)

        # Return number of bytes transmitted over serial port
        return res

def send ( self , msg ) : slipDriver = sliplib . Driver ( ) slipData = slipDriver . send ( msg ) res = self . _serialPort . write ( slipData ) return res


def checkTUN(self):
        """
        Checks the TUN adapter for data and returns any that is found.

        Returns:
            packet: Data read from the TUN adapter
        """
        packet = self._TUN._tun.read(self._TUN._tun.mtu)
        return(packet)

def checkTUN ( self ) : packet = self . _TUN . _tun . read ( self . _TUN . _tun . mtu ) return ( packet )

Checks the TUN adapter for data and returns any that is found .


def monitorTUN(self):
        """
        Monitors the TUN adapter and sends data over serial port.

        Returns:
            ret: Number of bytes sent over serial port
        """
        packet = self.checkTUN()

        if packet:
            try:
                # TODO Do I need to strip off [4:] before sending?
                ret = self._faraday.send(packet)
                return ret

            except AttributeError as error:
                # AttributeError was encounteredthreading.Event()
                print("AttributeError")

def monitorTUN ( self ) : packet = self . checkTUN ( ) if packet : try : ret = self . _faraday . send ( packet ) return ret except AttributeError as error : print ( "AttributeError" )

Monitors the TUN adapter and sends data over serial port .


def checkSerial(self):
        """
        Check the serial port for data to write to the TUN adapter.
        """
        for item in self.rxSerial(self._TUN._tun.mtu):
            # print("about to send: {0}".format(item))
            try:
                self._TUN._tun.write(item)
            except pytun.Error as error:
                print("pytun error writing: {0}".format(item))
                print(error)

def
checkSerial
(
self
)
:
for
item
in
self
.
rxSerial
(
self
.
_TUN
.
_tun
.
mtu
)
:
try
:
self
.
_TUN
.
_tun
.
write
(
item
)
except
pytun
.
Error
as
error
:
print
(
"pytun error writing: {0}"
.
format
(
item
)
)
print
(
error
)

Check the serial port for data to write to the TUN adapter .


def run(self):
        """
        Wrapper function for TUN and serial port monitoring

        Wraps the necessary functions to loop over until self._isRunning
        threading.Event() is set(). This checks for data on the TUN/serial
        interfaces and then sends data over the appropriate interface. This
        function is automatically run when Threading.start() is called on the
        Monitor class.
        """
        while self.isRunning.is_set():
            try:
                try:
                    # self.checkTUN()
                    self.monitorTUN()

                except timeout_decorator.TimeoutError as error:
                    # No data received so just move on
                    pass
                self.checkSerial()
            except KeyboardInterrupt:
                break

def run ( self ) : while self . isRunning . is_set ( ) : try : try : self . monitorTUN ( ) except timeout_decorator . TimeoutError as error : pass self . checkSerial ( ) except KeyboardInterrupt : break


def get_field_settings(self):
        """
        Get the field settings, if the configured setting is a string try
        to get a 'profile' from the global config.
        """
        field_settings = None
        if self.field_settings:
            if isinstance(self.field_settings, six.string_types):
                profiles = settings.CONFIG.get(self.PROFILE_KEY, {})
                field_settings = profiles.get(self.field_settings)
            else:
                field_settings = self.field_settings
        return field_settings

def get_field_settings ( self ) : field_settings = None if self . field_settings : if isinstance ( self . field_settings , six . string_types ) : profiles = settings . CONFIG . get ( self . PROFILE_KEY , { } ) field_settings = profiles . get ( self . field_settings ) else : field_settings = self . field_settings return field_settings

Get the field settings if the configured setting is a string try to get a profile from the global config .


def value_from_datadict(self, *args, **kwargs):
        """
        Pass the submitted value through the sanitizer before returning it.
        """
        value = super(RichTextWidget, self).value_from_datadict(
            *args, **kwargs)
        if value is not None:
            value = self.get_sanitizer()(value)
        return value

def value_from_datadict ( self , * args , ** kwargs ) : value = super ( RichTextWidget , self ) . value_from_datadict ( * args , ** kwargs ) if value is not None : value = self . get_sanitizer ( ) ( value ) return value

Pass the submitted value through the sanitizer before returning it .


def get_sanitizer(self):
        """
        Get the field sanitizer.

        The priority is the first defined in the following order:
        - A sanitizer provided to the widget.
        - Profile (field settings) specific sanitizer, if defined in settings.
        - Global sanitizer defined in settings.
        - Simple no-op sanitizer which just returns the provided value.

        """
        sanitizer = self.sanitizer

        if not sanitizer:
            default_sanitizer = settings.CONFIG.get(self.SANITIZER_KEY)
            field_settings = getattr(self, 'field_settings', None)
            if isinstance(field_settings, six.string_types):
                profiles = settings.CONFIG.get(self.SANITIZER_PROFILES_KEY, {})
                sanitizer = profiles.get(field_settings, default_sanitizer)
            else:
                sanitizer = default_sanitizer

        if isinstance(sanitizer, six.string_types):
            sanitizer = import_string(sanitizer)

        return sanitizer or noop

def get_sanitizer ( self ) : sanitizer = self . sanitizer if not sanitizer : default_sanitizer = settings . CONFIG . get ( self . SANITIZER_KEY ) field_settings = getattr ( self , 'field_settings' , None ) if isinstance ( field_settings , six . string_types ) : profiles = settings . CONFIG . get ( self . SANITIZER_PROFILES_KEY , { } ) sanitizer = profiles . get ( field_settings , default_sanitizer ) else : sanitizer = default_sanitizer if isinstance ( sanitizer , six . string_types ) : sanitizer = import_string ( sanitizer ) return sanitizer or noop


def heappop_max(heap):
    """Maxheap version of a heappop."""
    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
    if heap:
        returnitem = heap[0]
        heap[0] = lastelt
        _siftup_max(heap, 0)
        return returnitem
    return lastelt

def heappop_max ( heap ) : lastelt = heap . pop ( ) if heap : returnitem = heap [ 0 ] heap [ 0 ] = lastelt _siftup_max ( heap , 0 ) return returnitem return lastelt


def heapreplace_max(heap, item):
    """Maxheap version of a heappop followed by a heappush."""
    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
    heap[0] = item
    _siftup_max(heap, 0)
    return returnitem

def heapreplace_max ( heap , item ) : returnitem = heap [ 0 ] heap [ 0 ] = item _siftup_max ( heap , 0 ) return returnitem

Maxheap version of a heappop followed by a heappush .


def heappush_max(heap, item):
    """Push item onto heap, maintaining the heap invariant."""
    heap.append(item)
    _siftdown_max(heap, 0, len(heap) - 1)

def heappush_max ( heap , item ) : heap . append ( item ) _siftdown_max ( heap , 0 , len ( heap ) - 1 )

Push item onto heap maintaining the heap invariant .


def heappushpop_max(heap, item):
    """Fast version of a heappush followed by a heappop."""
    if heap and heap[0] > item:
        # if item >= heap[0], it will be popped immediately after pushed
        item, heap[0] = heap[0], item
        _siftup_max(heap, 0)
    return item

def heappushpop_max ( heap , item ) : if heap and heap [ 0 ] > item : item , heap [ 0 ] = heap [ 0 ] , item _siftup_max ( heap , 0 ) return item

Fast version of a heappush followed by a heappop .


def validate_response(expected_responses):
    """ Decorator to validate responses from QTM """

    def internal_decorator(function):
        @wraps(function)
        async def wrapper(*args, **kwargs):

            response = await function(*args, **kwargs)

            for expected_response in expected_responses:
                if response.startswith(expected_response):
                    return response

            raise QRTCommandException(
                "Expected %s but got %s" % (expected_responses, response)
            )

        return wrapper

    return internal_decorator

def
validate_response
(
expected_responses
)
:
def
internal_decorator
(
function
)
:
@
wraps
(
function
)
async
def
wrapper
(
*
args
,
**
kwargs
)
:
response
=
await
function
(
*
args
,
**
kwargs
)
for
expected_response
in
expected_responses
:
if
response
.
startswith
(
expected_response
)
:
return
response
raise
QRTCommandException
(
"Expected %s but got %s"
%
(
expected_responses
,
response
)
)
return
wrapper
return
internal_decorator