def watch_files(self):
        """watch files for changes, if changed, rebuild blog. this thread
        will quit if the main process ends"""

        try:
            while 1:
                sleep(1)  # check every 1s

                try:
                    files_stat = self.get_files_stat()
                except SystemExit:
                    logger.error("Error occurred, server shut down")
                    self.shutdown_server()

                if self.files_stat != files_stat:
                    logger.info("Changes detected, start rebuilding..")

                    try:
                        generator.re_generate()
                        global _root
                        _root = generator.root
                    except SystemExit:  # catch sys.exit, it means fatal error
                        logger.error("Error occurred, server shut down")
                        self.shutdown_server()

                    self.files_stat = files_stat  # update files' stat
        except KeyboardInterrupt:
            # I dont know why, but this exception won't be catched
            # because absolutly each KeyboardInterrupt is catched by
            # the server thread, which will terminate this thread the same time
            logger.info("^C received, shutting down watcher")
            self.shutdown_watcher()

def
watch_files
(
self
)
:
try
:
while
1
:
sleep
(
1
)
try
:
files_stat
=
self
.
get_files_stat
(
)
except
SystemExit
:
logger
.
error
(
"Error occurred, server shut down"
)
self
.
shutdown_server
(
)
if
self
.
files_stat
!=
files_stat
:
logger
.
info
(
"Changes detected, start rebuilding.."
)
try
:
generator
.
re_generate
(
)
global
_root
_root
=
generator
.
root
except
SystemExit
:
logger
.
error
(
"Error occurred, server shut down"
)
self
.
shutdown_server
(
)
self
.
files_stat
=
files_stat
except
KeyboardInterrupt
:
logger
.
info
(
"^C received, shutting down watcher"
)
self
.
shutdown_watcher
(
)

watch files for changes if changed rebuild blog . this thread will quit if the main process ends


def deploy_blog():
    """Deploy new blog to current directory"""
    logger.info(deploy_blog.__doc__)
    # `rsync -aqu path/to/res/* .`
    call(
        'rsync -aqu ' + join(dirname(__file__), 'res', '*') + ' .',
        shell=True)
    logger.success('Done')
    logger.info('Please edit config.toml to meet your needs')


def using(context, alias):
    '''
    Temporarily update the context to use the BlockContext for the given alias.
    '''

    # An empty alias means look in the current widget set.
    if alias == '':
        yield context
    else:
        try:
            widgets = context.render_context[WIDGET_CONTEXT_KEY]
        except KeyError:
            raise template.TemplateSyntaxError('No widget libraries loaded!')

        try:
            block_set = widgets[alias]
        except KeyError:
            raise template.TemplateSyntaxError('No widget library loaded for alias: %r' % alias)

        context.render_context.push()
        context.render_context[BLOCK_CONTEXT_KEY] = block_set
        context.render_context[WIDGET_CONTEXT_KEY] = widgets

        yield context

        context.render_context.pop()

Temporarily update the context to use the BlockContext for the given alias .


def find_block(context, *names):
    '''
    Find the first matching block in the current block_context
    '''
    block_set = context.render_context[BLOCK_CONTEXT_KEY]
    for name in names:
        block = block_set.get_block(name)
        if block is not None:
            return block

    raise template.TemplateSyntaxError('No widget found for: %r' % (names,))

def
find_block
(
context
,
*
names
)
:
block_set
=
context
.
render_context
[
BLOCK_CONTEXT_KEY
]
for
name
in
names
:
block
=
block_set
.
get_block
(
name
)
if
block
is
not
:
return
block
raise
template
.
TemplateSyntaxError
(
'No widget found for: %r'
%
(
names
,
)
)


def load_widgets(context, **kwargs):
    '''
    Load a series of widget libraries.
    '''
    _soft = kwargs.pop('_soft', False)

    try:
        widgets = context.render_context[WIDGET_CONTEXT_KEY]
    except KeyError:
        widgets = context.render_context[WIDGET_CONTEXT_KEY] = {}

    for alias, template_name in kwargs.items():
        if _soft and alias in widgets:
            continue

        with context.render_context.push({BLOCK_CONTEXT_KEY: BlockContext()}):
            blocks = resolve_blocks(template_name, context)
            widgets[alias] = blocks

    return ''

def load_widgets ( context , ** kwargs ) : _soft = kwargs . pop ( '_soft' , False ) try : widgets = context . render_context [ WIDGET_CONTEXT_KEY ] except KeyError : widgets = context . render_context [ WIDGET_CONTEXT_KEY ] = { } for alias , template_name in kwargs . items ( ) : if _soft and alias in widgets : continue with context . render_context . push ( { BLOCK_CONTEXT_KEY : BlockContext ( ) } ) : blocks = resolve_blocks ( template_name , context ) widgets [ alias ] = blocks return ''


def auto_widget(field):
    '''Return a list of widget names for the provided field.'''
    # Auto-detect
    info = {
        'widget': field.field.widget.__class__.__name__,
        'field': field.field.__class__.__name__,
        'name': field.name,
    }

    return [
        fmt.format(**info)
        for fmt in (
            '{field}_{widget}_{name}',
            '{field}_{name}',
            '{widget}_{name}',
            '{field}_{widget}',
            '{name}',
            '{widget}',
            '{field}',
        )
    ]

def auto_widget ( field ) : info = { 'widget' : field . field . widget . __class__ . __name__ , 'field' : field . field . __class__ . __name__ , 'name' : field . name , } return [ fmt . format ( ** info ) for fmt in ( '{field}_{widget}_{name}' , '{field}_{name}' , '{widget}_{name}' , '{field}_{widget}' , '{name}' , '{widget}' , '{field}' , ) ]

Return a list of widget names for the provided field .


def reuse(context, block_list, **kwargs):
    '''
    Allow reuse of a block within a template.

    {% reuse '_myblock' foo=bar %}

    If passed a list of block names, will use the first that matches:

    {% reuse list_of_block_names .... %}
    '''
    try:
        block_context = context.render_context[BLOCK_CONTEXT_KEY]
    except KeyError:
        block_context = BlockContext()

    if not isinstance(block_list, (list, tuple)):
        block_list = [block_list]

    for block in block_list:
        block = block_context.get_block(block)
        if block:
            break
    else:
        return ''

    with context.push(kwargs):
        return block.render(context)

def reuse ( context , block_list , ** kwargs ) : try : block_context = context . render_context [ BLOCK_CONTEXT_KEY ] except KeyError : block_context = BlockContext ( ) if not isinstance ( block_list , ( list , tuple ) ) : block_list = [ block_list ] for block in block_list : block = block_context . get_block ( block ) if block : break else : return '' with context . push ( kwargs ) : return block . render ( context )

Allow reuse of a block within a template .


def display(self):
        """
        When dealing with optgroups, ensure that the value is properly force_text'd.
        """
        if not self.is_group():
            return self._display
        return ((force_text(k), v) for k, v in self._display)

def display ( self ) : if not self . is_group ( ) : return self . _display return ( ( force_text ( k ) , v ) for k , v in self . _display )

When dealing with optgroups ensure that the value is properly force_text d .


def create_message(self, level, msg_text, extra_tags='', date=None, url=None):
        """
        Message instances are namedtuples of type `Message`.
        The date field is already serialized in datetime.isoformat ECMA-262 format
        """
        if not date:
            now = timezone.now()
        else:
            now = date
        r = now.isoformat()
        if now.microsecond:
            r = r[:23] + r[26:]
        if r.endswith('+00:00'):
            r = r[:-6] + 'Z'

        fingerprint = r + msg_text

        msg_id = hashlib.sha256(fingerprint.encode('ascii', 'ignore')).hexdigest()
        return Message(id=msg_id, message=msg_text, level=level, tags=extra_tags, date=r, url=url)

def create_message ( self , level , msg_text , extra_tags = '' , date = None , url = None ) : if not date : now = timezone . now ( ) else : now = date r = now . isoformat ( ) if now . microsecond : r = r [ : 23 ] + r [ 26 : ] if r . endswith ( '+00:00' ) : r = r [ : - 6 ] + 'Z' fingerprint = r + msg_text msg_id = hashlib . sha256 ( fingerprint . encode ( 'ascii' , 'ignore' ) ) . hexdigest ( ) return Message ( id = msg_id , message = msg_text , level = level , tags = extra_tags , date = r , url = url )

Message instances are namedtuples of type Message . The date field is already serialized in datetime . isoformat ECMA - 262 format


def add_message_for(users, level, message_text, extra_tags='', date=None, url=None, fail_silently=False):
    """
    Send a message to a list of users without passing through `django.contrib.messages`

    :param users: an iterable containing the recipients of the messages
    :param level: message level
    :param message_text: the string containing the message
    :param extra_tags: like the Django api, a string containing extra tags for the message
    :param date: a date, different than the default timezone.now
    :param url: an optional url
    :param fail_silently: not used at the moment
    """
    BackendClass = stored_messages_settings.STORAGE_BACKEND
    backend = BackendClass()
    m = backend.create_message(level, message_text, extra_tags, date, url)
    backend.archive_store(users, m)
    backend.inbox_store(users, m)

def add_message_for ( users , level , message_text , extra_tags = '' , date = None , url = None , fail_silently = False ) : BackendClass = stored_messages_settings . STORAGE_BACKEND backend = BackendClass ( ) m = backend . create_message ( level , message_text , extra_tags , date , url ) backend . archive_store ( users , m ) backend . inbox_store ( users , m )

Send a message to a list of users without passing through django . contrib . messages


def broadcast_message(level, message_text, extra_tags='', date=None, url=None, fail_silently=False):
    """
    Send a message to all users aka broadcast.

    :param level: message level
    :param message_text: the string containing the message
    :param extra_tags: like the Django api, a string containing extra tags for the message
    :param date: a date, different than the default timezone.now
    :param url: an optional url
    :param fail_silently: not used at the moment
    """
    from django.contrib.auth import get_user_model
    users = get_user_model().objects.all()
    add_message_for(users, level, message_text, extra_tags=extra_tags, date=date, url=url, fail_silently=fail_silently)

def broadcast_message ( level , message_text , extra_tags = '' , date = None , url = None , fail_silently = False ) : from django . contrib . auth import get_user_model users = get_user_model ( ) . objects . all ( ) add_message_for ( users , level , message_text , extra_tags = extra_tags , date = date , url = url , fail_silently = fail_silently )

Send a message to all users aka broadcast .


def mark_read(user, message):
    """
    Mark message instance as read for user.
    Returns True if the message was `unread` and thus actually marked as `read` or False in case
    it is already `read` or it does not exist at all.

    :param user: user instance for the recipient
    :param message: a Message instance to mark as read
    """
    BackendClass = stored_messages_settings.STORAGE_BACKEND
    backend = BackendClass()
    backend.inbox_delete(user, message)

def mark_read ( user , message ) : BackendClass = stored_messages_settings . STORAGE_BACKEND backend = BackendClass ( ) backend . inbox_delete ( user , message )

Mark message instance as read for user . Returns True if the message was unread and thus actually marked as read or False in case it is already read or it does not exist at all .


def mark_all_read(user):
    """
    Mark all message instances for a user as read.

    :param user: user instance for the recipient
    """
    BackendClass = stored_messages_settings.STORAGE_BACKEND
    backend = BackendClass()
    backend.inbox_purge(user)

def mark_all_read ( user ) : BackendClass = stored_messages_settings . STORAGE_BACKEND backend = BackendClass ( ) backend . inbox_purge ( user )

Mark all message instances for a user as read .


def stored_messages_archive(context, num_elements=10):
    """
    Renders a list of archived messages for the current user
    """
    if "user" in context:
        user = context["user"]
        if user.is_authenticated():
            qs = MessageArchive.objects.select_related("message").filter(user=user)
            return {
                "messages": qs[:num_elements],
                "count": qs.count(),
            }

def stored_messages_archive ( context , num_elements = 10 ) : if "user" in context : user = context [ "user" ] if user . is_authenticated ( ) : qs = MessageArchive . objects . select_related ( "message" ) . filter ( user = user ) return { "messages" : qs [ : num_elements ] , "count" : qs . count ( ) , }


def _get(self, *args, **kwargs):
        """
        Retrieve unread messages for current user, both from the inbox and
        from other storages
        """
        messages, all_retrieved = super(StorageMixin, self)._get(*args, **kwargs)
        if self.user.is_authenticated():
            inbox_messages = self.backend.inbox_list(self.user)
        else:
            inbox_messages = []

        return messages + inbox_messages, all_retrieved

def _get ( self , * args , ** kwargs ) : messages , all_retrieved = super ( StorageMixin , self ) . _get ( * args , ** kwargs ) if self . user . is_authenticated ( ) : inbox_messages = self . backend . inbox_list ( self . user ) else : inbox_messages = [ ] return messages + inbox_messages , all_retrieved

Retrieve unread messages for current user both from the inbox and from other storages


def add(self, level, message, extra_tags=''):
        """
        If the message level was configured for being stored and request.user
        is not anonymous, save it to the database. Otherwise, let some other
        class handle the message.

        Notice: controls like checking the message is not empty and the level
        is above the filter need to be performed here, but it could happen
        they'll be performed again later if the message does not need to be
        stored.
        """
        if not message:
            return
        # Check that the message level is not less than the recording level.
        level = int(level)
        if level < self.level:
            return
        # Check if the message doesn't have a level that needs to be persisted
        if level not in stored_messages_settings.STORE_LEVELS or self.user.is_anonymous():
            return super(StorageMixin, self).add(level, message, extra_tags)

        self.added_new = True
        m = self.backend.create_message(level, message, extra_tags)
        self.backend.archive_store([self.user], m)
        self._queued_messages.append(m)

def add ( self , level , message , extra_tags = '' ) : if not message : return level = int ( level ) if level < self . level : return if level not in stored_messages_settings . STORE_LEVELS or self . user . is_anonymous ( ) : return super ( StorageMixin , self ) . add ( level , message , extra_tags ) self . added_new = True m = self . backend . create_message ( level , message , extra_tags ) self . backend . archive_store ( [ self . user ] , m ) self . _queued_messages . append ( m )

If the message level was configured for being stored and request . user is not anonymous save it to the database . Otherwise let some other class handle the message .


def _store(self, messages, response, *args, **kwargs):
        """
        persistent messages are already in the database inside the 'archive',
        so we can say they're already "stored".
        Here we put them in the inbox, or remove from the inbox in case the
        messages were iterated.

        messages contains only new msgs if self.used==True
        else contains both new and unread messages
        """
        contrib_messages = []
        if self.user.is_authenticated():
            if not messages:
                # erase inbox
                self.backend.inbox_purge(self.user)
            else:
                for m in messages:
                    try:
                        self.backend.inbox_store([self.user], m)
                    except MessageTypeNotSupported:
                        contrib_messages.append(m)

        super(StorageMixin, self)._store(contrib_messages, response, *args, **kwargs)

def _store ( self , messages , response , * args , ** kwargs ) : contrib_messages = [ ] if self . user . is_authenticated ( ) : if not messages : self . backend . inbox_purge ( self . user ) else : for m in messages : try : self . backend . inbox_store ( [ self . user ] , m ) except MessageTypeNotSupported : contrib_messages . append ( m ) super ( StorageMixin , self ) . _store ( contrib_messages , response , * args , ** kwargs )

persistent messages are already in the database inside the archive so we can say they re already stored . Here we put them in the inbox or remove from the inbox in case the messages were iterated .


def _prepare_messages(self, messages):
        """
        Like the base class method, prepares a list of messages for storage
        but avoid to do this for `models.Message` instances.
        """
        for message in messages:
            if not self.backend.can_handle(message):
                message._prepare()

def _prepare_messages ( self , messages ) : for message in messages : if not self . backend . can_handle ( message ) : message . _prepare ( )

Like the base class method prepares a list of messages for storage but avoid to do this for models . Message instances .


def jocker(test_options=None):
    """Main entry point for script."""
    version = ver_check()
    options = test_options or docopt(__doc__, version=version)
    _set_global_verbosity_level(options.get('--verbose'))
    jocker_lgr.debug(options)
    jocker_run(options)

def jocker ( test_options = None ) : version = ver_check ( ) options = test_options or docopt ( __doc__ , version = version ) _set_global_verbosity_level ( options . get ( '--verbose' ) ) jocker_lgr . debug ( options ) jocker_run ( options )


def init(base_level=DEFAULT_BASE_LOGGING_LEVEL,
         verbose_level=DEFAULT_VERBOSE_LOGGING_LEVEL,
         logging_config=None):
    """initializes a base logger

    you can use this to init a logger in any of your files.
    this will use config.py's LOGGER param and logging.dictConfig to configure
    the logger for you.

    :param int|logging.LEVEL base_level: desired base logging level
    :param int|logging.LEVEL verbose_level: desired verbose logging level
    :param dict logging_dict: dictConfig based configuration.
     used to override the default configuration from config.py
    :rtype: `python logger`
    """
    if logging_config is None:
        logging_config = {}
    logging_config = logging_config or LOGGER
    # TODO: (IMPRV) only perform file related actions if file handler is
    # TODO: (IMPRV) defined.

    log_file = LOGGER['handlers']['file']['filename']
    log_dir = os.path.dirname(os.path.expanduser(log_file))
    if os.path.isfile(log_dir):
        sys.exit('file {0} exists - log directory cannot be created '
                 'there. please remove the file and try again.'
                 .format(log_dir))
    try:
        if not os.path.exists(log_dir) and not len(log_dir) == 0:
            os.makedirs(log_dir)
        dictconfig.dictConfig(logging_config)
        lgr = logging.getLogger('user')
        lgr.setLevel(base_level)
        return lgr
    except ValueError as e:
        sys.exit('could not initialize logger.'
                 ' verify your logger config'
                 ' and permissions to write to {0} ({1})'
                 .format(log_file, e))


def configure_custom(self, config):
        """Configure an object with a user-supplied factory."""
        c = config.pop('()')
        if not hasattr(c, '__call__') and \
                hasattr(types, 'ClassType') and isinstance(c, types.ClassType):
            c = self.resolve(c)
        props = config.pop('.', None)
        # Check for valid identifiers
        kwargs = dict((k, config[k]) for k in config if valid_ident(k))
        result = c(**kwargs)
        if props:
            for name, value in props.items():
                setattr(result, name, value)
        return result

def configure_custom ( self , config ) : c = config . pop ( '()' ) if not hasattr ( c , '__call__' ) and hasattr ( types , 'ClassType' ) and isinstance ( c , types . ClassType ) : c = self . resolve ( c ) props = config . pop ( '.' , None ) kwargs = dict ( ( k , config [ k ] ) for k in config if valid_ident ( k ) ) result = c ( ** kwargs ) if props : for name , value in props . items ( ) : setattr ( result , name , value ) return result

Configure an object with a user - supplied factory .


def _set_global_verbosity_level(is_verbose_output=False):
    """sets the global verbosity level for console and the jocker_lgr logger.

    :param bool is_verbose_output: should be output be verbose
    """
    global verbose_output
    # TODO: (IMPRV) only raise exceptions in verbose mode
    verbose_output = is_verbose_output
    if verbose_output:
        jocker_lgr.setLevel(logging.DEBUG)
    else:
        jocker_lgr.setLevel(logging.INFO)

def _set_global_verbosity_level ( is_verbose_output = False ) : global verbose_output verbose_output = is_verbose_output if verbose_output : jocker_lgr . setLevel ( logging . DEBUG ) else : jocker_lgr . setLevel ( logging . INFO )

sets the global verbosity level for console and the jocker_lgr logger .


def _import_config(config_file):
    """returns a configuration object

    :param string config_file: path to config file
    """
    # get config file path
    jocker_lgr.debug('config file is: {0}'.format(config_file))
    # append to path for importing
    try:
        jocker_lgr.debug('importing config...')
        with open(config_file, 'r') as c:
            return yaml.safe_load(c.read())
    except IOError as ex:
        jocker_lgr.error(str(ex))
        raise RuntimeError('cannot access config file')
    except yaml.parser.ParserError as ex:
        jocker_lgr.error('invalid yaml file: {0}'.format(ex))
        raise RuntimeError('invalid yaml file')

def
_import_config
(
config_file
)
:
jocker_lgr
.
debug
(
'config file is: {0}'
.
format
(
config_file
)
)
try
:
jocker_lgr
.
debug
(
'importing config...'
)
with
open
(
config_file
,
'r'
)
as
c
:
return
yaml
.
safe_load
(
c
.
read
(
)
)
except
IOError
as
ex
:
jocker_lgr
.
error
(
str
(
ex
)
)
raise
RuntimeError
(
'cannot access config file'
)
except
yaml
.
parser
.
ParserError
as
ex
:
jocker_lgr
.
error
(
'invalid yaml file: {0}'
.
format
(
ex
)
)
raise
RuntimeError
(
'invalid yaml file'
)


def execute(varsfile, templatefile, outputfile=None, configfile=None,
            dryrun=False, build=False, push=False, verbose=False):
    """generates a Dockerfile, builds an image and pushes it to DockerHub

    A `Dockerfile` will be generated by Jinja2 according to the `varsfile`
    imported. If build is true, an image will be generated from the
    `outputfile` which is the generated Dockerfile and committed to the
    image:tag string supplied to `build`.
    If push is true, a build will be triggered and the produced image
    will be pushed to DockerHub upon completion.

    :param string varsfile: path to file with variables.
    :param string templatefile: path to template file to use.
    :param string outputfile: path to output Dockerfile.
    :param string configfile: path to yaml file with docker-py config.
    :param bool dryrun: mock run.
    :param build: False or the image:tag to build to.
    :param push: False or the image:tag to build to. (triggers build)
    :param bool verbose: verbose output.
    """
    if dryrun and (build or push):
        jocker_lgr.error('dryrun requested, cannot build.')
        sys.exit(100)

    _set_global_verbosity_level(verbose)
    j = Jocker(varsfile, templatefile, outputfile, configfile, dryrun,
               build, push)
    formatted_text = j.generate()
    if dryrun:
        g = j.dryrun(formatted_text)
    if build or push:
        j.build_image()
    if push:
        j.push_image()
    if dryrun:
        return g

def
execute
(
varsfile
,
templatefile
,
outputfile
=
,
configfile
=
,
dryrun
=
False
,
build
=
False
,
push
=
False
,
verbose
=
False
)
:
if
dryrun
and
(
build
or
push
)
:
jocker_lgr
.
error
(
'dryrun requested, cannot build.'
)
sys
.
exit
(
100
)
_set_global_verbosity_level
(
verbose
)
j
=
Jocker
(
varsfile
,
templatefile
,
outputfile
,
configfile
,
dryrun
,
build
,
push
)
formatted_text
=
j
.
generate
(
)
if
dryrun
:
g
=
j
.
dryrun
(
formatted_text
)
if
build
or
push
:
j
.
build_image
(
)
if
push
:
j
.
push_image
(
)
if
dryrun
:
return
g

generates a Dockerfile builds an image and pushes it to DockerHub


def _parse_dumb_push_output(self, string):
        """since the push process outputs a single unicode string consisting of
        multiple JSON formatted "status" lines, we need to parse it so that it
        can be read as multiple strings.

        This will receive the string as an input, count curly braces and ignore
        any newlines. When the curly braces stack is 0, it will append the
        entire string it has read up until then to a list and so forth.

        :param string: the string to parse
        :rtype: list of JSON's
        """
        stack = 0
        json_list = []
        tmp_json = ''
        for char in string:
            if not char == '\r' and not char == '\n':
                tmp_json += char
            if char == '{':
                stack += 1
            elif char == '}':
                stack -= 1
            if stack == 0:
                if not len(tmp_json) == 0:
                    json_list.append(tmp_json)
                tmp_json = ''
        return json_list

def _parse_dumb_push_output ( self , string ) : stack = 0 json_list = [ ] tmp_json = '' for char in string : if not char == '\r' and not char == '\n' : tmp_json += char if char == '{' : stack += 1 elif char == '}' : stack -= 1 if stack == 0 : if not len ( tmp_json ) == 0 : json_list . append ( tmp_json ) tmp_json = '' return json_list

since the push process outputs a single unicode string consisting of multiple JSON formatted status lines we need to parse it so that it can be read as multiple strings .


def upload_gif(gif):
    """Uploads an image file to Imgur"""

    client_id = os.environ.get('IMGUR_API_ID')
    client_secret = os.environ.get('IMGUR_API_SECRET')

    if client_id is None or client_secret is None:
        click.echo('Cannot upload - could not find IMGUR_API_ID or IMGUR_API_SECRET environment variables')
        return

    client = ImgurClient(client_id, client_secret)

    click.echo('Uploading file {}'.format(click.format_filename(gif)))

    response = client.upload_from_path(gif)

    click.echo('File uploaded - see your gif at {}'.format(response['link']))


def is_dot(ip):
    """Return true if the IP address is in dotted decimal notation."""
    octets = str(ip).split('.')
    if len(octets) != 4:
        return False
    for i in octets:
        try:
            val = int(i)
        except ValueError:
            return False
        if val > 255 or val < 0:
            return False
    return True

def is_dot ( ip ) : octets = str ( ip ) . split ( '.' ) if len ( octets ) != 4 : return False for i in octets : try : val = int ( i ) except ValueError : return False if val > 255 or val < 0 : return False return True

Return true if the IP address is in dotted decimal notation .


def is_bin(ip):
    """Return true if the IP address is in binary notation."""
    try:
        ip = str(ip)
        if len(ip) != 32:
            return False
        dec = int(ip, 2)
    except (TypeError, ValueError):
        return False
    if dec > 4294967295 or dec < 0:
        return False
    return True

def is_bin ( ip ) : try : ip = str ( ip ) if len ( ip ) != 32 : return False dec = int ( ip , 2 ) except ( TypeError , ValueError ) : return False if dec > 4294967295 or dec < 0 : return False return True

Return true if the IP address is in binary notation .


def is_oct(ip):
    """Return true if the IP address is in octal notation."""
    try:
        dec = int(str(ip), 8)
    except (TypeError, ValueError):
        return False
    if dec > 0o37777777777 or dec < 0:
        return False
    return True

def is_oct ( ip ) : try : dec = int ( str ( ip ) , 8 ) except ( TypeError , ValueError ) : return False if dec > 0o37777777777 or dec < 0 : return False return True

Return true if the IP address is in octal notation .


def is_dec(ip):
    """Return true if the IP address is in decimal notation."""
    try:
        dec = int(str(ip))
    except ValueError:
        return False
    if dec > 4294967295 or dec < 0:
        return False
    return True

def is_dec ( ip ) : try : dec = int ( str ( ip ) ) except ValueError : return False if dec > 4294967295 or dec < 0 : return False return True

Return true if the IP address is in decimal notation .


def _check_nm(nm, notation):
    """Function internally used to check if the given netmask
    is of the specified notation."""
    # Convert to decimal, and check if it's in the list of valid netmasks.
    _NM_CHECK_FUNCT = {
        NM_DOT: _dot_to_dec,
        NM_HEX: _hex_to_dec,
        NM_BIN: _bin_to_dec,
        NM_OCT: _oct_to_dec,
        NM_DEC: _dec_to_dec_long}
    try:
        dec = _NM_CHECK_FUNCT[notation](nm, check=True)
    except ValueError:
        return False
    if dec in _NETMASKS_VALUES:
        return True
    return False

def _check_nm ( nm , notation ) : _NM_CHECK_FUNCT = { NM_DOT : _dot_to_dec , NM_HEX : _hex_to_dec , NM_BIN : _bin_to_dec , NM_OCT : _oct_to_dec , NM_DEC : _dec_to_dec_long } try : dec = _NM_CHECK_FUNCT [ notation ] ( nm , check = True ) except ValueError : return False if dec in _NETMASKS_VALUES : return True return False

Function internally used to check if the given netmask is of the specified notation .


def is_bits_nm(nm):
    """Return true if the netmask is in bits notatation."""
    try:
        bits = int(str(nm))
    except ValueError:
        return False
    if bits > 32 or bits < 0:
        return False
    return True

def is_bits_nm ( nm ) : try : bits = int ( str ( nm ) ) except ValueError : return False if bits > 32 or bits < 0 : return False return True

Return true if the netmask is in bits notatation .


def is_wildcard_nm(nm):
    """Return true if the netmask is in wildcard bits notatation."""
    try:
        dec = 0xFFFFFFFF - _dot_to_dec(nm, check=True)
    except ValueError:
        return False
    if dec in _NETMASKS_VALUES:
        return True
    return False

def is_wildcard_nm ( nm ) : try : dec = 0xFFFFFFFF - _dot_to_dec ( nm , check = True ) except ValueError : return False if dec in _NETMASKS_VALUES : return True return False

Return true if the netmask is in wildcard bits notatation .


def _dot_to_dec(ip, check=True):
    """Dotted decimal notation to decimal conversion."""
    if check and not is_dot(ip):
        raise ValueError('_dot_to_dec: invalid IP: "%s"' % ip)
    octets = str(ip).split('.')
    dec = 0
    dec |= int(octets[0]) << 24
    dec |= int(octets[1]) << 16
    dec |= int(octets[2]) << 8
    dec |= int(octets[3])
    return dec

def
_dot_to_dec
(
ip
,
check
=
True
)
:
if
check
and
not
is_dot
(
ip
)
:
raise
ValueError
(
'_dot_to_dec: invalid IP: "%s"'
%
ip
)
octets
=
str
(
ip
)
.
split
(
'.'
)
dec
=
0
dec
|=
int
(
octets
[
0
]
)
<<
24
dec
|=
int
(
octets
[
1
]
)
<<
16
dec
|=
int
(
octets
[
2
]
)
<<
8
dec
|=
int
(
octets
[
3
]
)
return
dec

Dotted decimal notation to decimal conversion .


def _dec_to_dot(ip):
    """Decimal to dotted decimal notation conversion."""
    first = int((ip >> 24) & 255)
    second = int((ip >> 16) & 255)
    third = int((ip >> 8) & 255)
    fourth = int(ip & 255)
    return '%d.%d.%d.%d' % (first, second, third, fourth)

def _dec_to_dot ( ip ) : first = int ( ( ip >> 24 ) & 255 ) second = int ( ( ip >> 16 ) & 255 ) third = int ( ( ip >> 8 ) & 255 ) fourth = int ( ip & 255 ) return '%d.%d.%d.%d' % ( first , second , third , fourth )

Decimal to dotted decimal notation conversion .


def _hex_to_dec(ip, check=True):
    """Hexadecimal to decimal conversion."""
    if check and not is_hex(ip):
        raise ValueError('_hex_to_dec: invalid IP: "%s"' % ip)
    if isinstance(ip, int):
        ip = hex(ip)
    return int(str(ip), 16)

def
_hex_to_dec
(
ip
,
check
=
True
)
:
if
check
and
not
is_hex
(
ip
)
:
raise
ValueError
(
'_hex_to_dec: invalid IP: "%s"'
%
ip
)
if
isinstance
(
ip
,
int
)
:
ip
=
hex
(
ip
)
return
int
(
str
(
ip
)
,
16
)


def _oct_to_dec(ip, check=True):
    """Octal to decimal conversion."""
    if check and not is_oct(ip):
        raise ValueError('_oct_to_dec: invalid IP: "%s"' % ip)
    if isinstance(ip, int):
        ip = oct(ip)
    return int(str(ip), 8)

def
_oct_to_dec
(
ip
,
check
=
True
)
:
if
check
and
not
is_oct
(
ip
)
:
raise
ValueError
(
'_oct_to_dec: invalid IP: "%s"'
%
ip
)
if
isinstance
(
ip
,
int
)
:
ip
=
oct
(
ip
)
return
int
(
str
(
ip
)
,
8
)


def _bin_to_dec(ip, check=True):
    """Binary to decimal conversion."""
    if check and not is_bin(ip):
        raise ValueError('_bin_to_dec: invalid IP: "%s"' % ip)
    if isinstance(ip, int):
        ip = str(ip)
    return int(str(ip), 2)

def
_bin_to_dec
(
ip
,
check
=
True
)
:
if
check
and
not
is_bin
(
ip
)
:
raise
ValueError
(
'_bin_to_dec: invalid IP: "%s"'
%
ip
)
if
isinstance
(
ip
,
int
)
:
ip
=
str
(
ip
)
return
int
(
str
(
ip
)
,
2
)


def _BYTES_TO_BITS():
    """Generate a table to convert a whole byte to binary.
    This code was taken from the Python Cookbook, 2nd edition - O'Reilly."""
    the_table = 256*[None]
    bits_per_byte = list(range(7, -1, -1))
    for n in range(256):
        l = n
        bits = 8*[None]
        for i in bits_per_byte:
            bits[i] = '01'[n & 1]
            n >>= 1
        the_table[l] = ''.join(bits)
    return the_table

def _BYTES_TO_BITS ( ) : the_table = 256 * [ None ] bits_per_byte = list ( range ( 7 , - 1 , - 1 ) ) for n in range ( 256 ) : l = n bits = 8 * [ None ] for i in bits_per_byte : bits [ i ] = '01' [ n & 1 ] n >>= 1 the_table [ l ] = '' . join ( bits ) return the_table

Generate a table to convert a whole byte to binary . This code was taken from the Python Cookbook 2nd edition - O Reilly .


def _dec_to_bin(ip):
    """Decimal to binary conversion."""
    bits = []
    while ip:
        bits.append(_BYTES_TO_BITS[ip & 255])
        ip >>= 8
    bits.reverse()
    return ''.join(bits) or 32*'0'

def _dec_to_bin ( ip ) : bits = [ ] while ip : bits . append ( _BYTES_TO_BITS [ ip & 255 ] ) ip >>= 8 bits . reverse ( ) return '' . join ( bits ) or 32 * '0'


def _bits_to_dec(nm, check=True):
    """Bits to decimal conversion."""
    if check and not is_bits_nm(nm):
        raise ValueError('_bits_to_dec: invalid netmask: "%s"' % nm)
    bits = int(str(nm))
    return VALID_NETMASKS[bits]

def
_bits_to_dec
(
nm
,
check
=
True
)
:
if
check
and
not
is_bits_nm
(
nm
)
:
raise
ValueError
(
'_bits_to_dec: invalid netmask: "%s"'
%
nm
)
bits
=
int
(
str
(
nm
)
)
return
VALID_NETMASKS
[
bits
]


def _wildcard_to_dec(nm, check=False):
    """Wildcard bits to decimal conversion."""
    if check and not is_wildcard_nm(nm):
        raise ValueError('_wildcard_to_dec: invalid netmask: "%s"' % nm)
    return 0xFFFFFFFF - _dot_to_dec(nm, check=False)


def _detect(ip, _isnm):
    """Function internally used to detect the notation of the
    given IP or netmask."""
    ip = str(ip)
    if len(ip) > 1:
        if ip[0:2] == '0x':
            if _CHECK_FUNCT[IP_HEX][_isnm](ip):
                return IP_HEX
        elif ip[0] == '0':
            if _CHECK_FUNCT[IP_OCT][_isnm](ip):
                return IP_OCT
    if _CHECK_FUNCT[IP_DOT][_isnm](ip):
        return IP_DOT
    elif _isnm and _CHECK_FUNCT[NM_BITS][_isnm](ip):
        return NM_BITS
    elif _CHECK_FUNCT[IP_DEC][_isnm](ip):
        return IP_DEC
    elif _isnm and _CHECK_FUNCT[NM_WILDCARD][_isnm](ip):
        return NM_WILDCARD
    elif _CHECK_FUNCT[IP_BIN][_isnm](ip):
        return IP_BIN
    return IP_UNKNOWN

def _detect ( ip , _isnm ) : ip = str ( ip ) if len ( ip ) > 1 : if ip [ 0 : 2 ] == '0x' : if _CHECK_FUNCT [ IP_HEX ] [ _isnm ] ( ip ) : return IP_HEX elif ip [ 0 ] == '0' : if _CHECK_FUNCT [ IP_OCT ] [ _isnm ] ( ip ) : return IP_OCT if _CHECK_FUNCT [ IP_DOT ] [ _isnm ] ( ip ) : return IP_DOT elif _isnm and _CHECK_FUNCT [ NM_BITS ] [ _isnm ] ( ip ) : return NM_BITS elif _CHECK_FUNCT [ IP_DEC ] [ _isnm ] ( ip ) : return IP_DEC elif _isnm and _CHECK_FUNCT [ NM_WILDCARD ] [ _isnm ] ( ip ) : return NM_WILDCARD elif _CHECK_FUNCT [ IP_BIN ] [ _isnm ] ( ip ) : return IP_BIN return IP_UNKNOWN

Function internally used to detect the notation of the given IP or netmask .


def _convert(ip, notation, inotation, _check, _isnm):
    """Internally used to convert IPs and netmasks to other notations."""
    inotation_orig = inotation
    notation_orig = notation
    inotation = _get_notation(inotation)
    notation = _get_notation(notation)
    if inotation is None:
        raise ValueError('_convert: unknown input notation: "%s"' % inotation_orig)
    if notation is None:
        raise ValueError('_convert: unknown output notation: "%s"' % notation_orig)
    docheck = _check or False
    if inotation == IP_UNKNOWN:
        inotation = _detect(ip, _isnm)
        if inotation == IP_UNKNOWN:
            raise ValueError('_convert: unable to guess input notation or invalid value')
        if _check is None:
            docheck = True
    # We _always_ check this case later.
    if _isnm:
        docheck = False
    dec = 0
    if inotation == IP_DOT:
        dec = _dot_to_dec(ip, docheck)
    elif inotation == IP_HEX:
        dec = _hex_to_dec(ip, docheck)
    elif inotation == IP_BIN:
        dec = _bin_to_dec(ip, docheck)
    elif inotation == IP_OCT:
        dec = _oct_to_dec(ip, docheck)
    elif inotation == IP_DEC:
        dec = _dec_to_dec_long(ip, docheck)
    elif _isnm and inotation == NM_BITS:
        dec = _bits_to_dec(ip, docheck)
    elif _isnm and inotation == NM_WILDCARD:
        dec = _wildcard_to_dec(ip, docheck)
    else:
        raise ValueError('_convert: unknown IP/netmask notation: "%s"' % inotation_orig)
    # Ensure this is a valid netmask.
    if _isnm and dec not in _NETMASKS_VALUES:
        raise ValueError('_convert: invalid netmask: "%s"' % ip)
    if notation == IP_DOT:
        return _dec_to_dot(dec)
    elif notation == IP_HEX:
        return _dec_to_hex(dec)
    elif notation == IP_BIN:
        return _dec_to_bin(dec)
    elif notation == IP_OCT:
        return _dec_to_oct(dec)
    elif notation == IP_DEC:
        return _dec_to_dec_str(dec)
    elif _isnm and notation == NM_BITS:
        return _dec_to_bits(dec)
    elif _isnm and notation == NM_WILDCARD:
        return _dec_to_wildcard(dec)
    else:
        raise ValueError('convert: unknown notation: "%s"' % notation_orig)

Internally used to convert IPs and netmasks to other notations .


def convert(ip, notation=IP_DOT, inotation=IP_UNKNOWN, check=True):
    """Convert among IP address notations.

    Given an IP address, this function returns the address
    in another notation.

    @param ip: the IP address.
    @type ip: integers, strings or object with an appropriate __str()__ method.

    @param notation: the notation of the output (default: IP_DOT).
    @type notation: one of the IP_* constants, or the equivalent strings.

    @param inotation: force the input to be considered in the given notation
                    (default the notation of the input is autodetected).
    @type inotation: one of the IP_* constants, or the equivalent strings.

    @param check: force the notation check on the input.
    @type check: True force the check, False force not to check and None
                do the check only if the inotation is unknown.

    @return: a string representing the IP in the selected notation.

    @raise ValueError: raised when the input is in unknown notation."""
    return _convert(ip, notation, inotation, _check=check, _isnm=False)

def convert ( ip , notation = IP_DOT , inotation = IP_UNKNOWN , check = True ) : return _convert ( ip , notation , inotation , _check = check , _isnm = False )


def convert_nm(nm, notation=IP_DOT, inotation=IP_UNKNOWN, check=True):
    """Convert a netmask to another notation."""
    return _convert(nm, notation, inotation, _check=check, _isnm=True)

def convert_nm ( nm , notation = IP_DOT , inotation = IP_UNKNOWN , check = True ) : return _convert ( nm , notation , inotation , _check = check , _isnm = True )


def _add(self, other):
        """Sum two IP addresses."""
        if isinstance(other, self.__class__):
            sum_ = self._ip_dec + other._ip_dec
        elif isinstance(other, int):
            sum_ = self._ip_dec + other
        else:
            other = self.__class__(other)
            sum_ = self._ip_dec + other._ip_dec
        return sum_

def _add ( self , other ) : if isinstance ( other , self . __class__ ) : sum_ = self . _ip_dec + other . _ip_dec elif isinstance ( other , int ) : sum_ = self . _ip_dec + other else : other = self . __class__ ( other ) sum_ = self . _ip_dec + other . _ip_dec return sum_


def _sub(self, other):
        """Subtract two IP addresses."""
        if isinstance(other, self.__class__):
            sub = self._ip_dec - other._ip_dec
        if isinstance(other, int):
            sub = self._ip_dec - other
        else:
            other = self.__class__(other)
            sub = self._ip_dec - other._ip_dec
        return sub

def _sub ( self , other ) : if isinstance ( other , self . __class__ ) : sub = self . _ip_dec - other . _ip_dec if isinstance ( other , int ) : sub = self . _ip_dec - other else : other = self . __class__ ( other ) sub = self . _ip_dec - other . _ip_dec return sub


def get_bits(self):
        """Return the bits notation of the netmask."""
        return _convert(self._ip, notation=NM_BITS,
                        inotation=IP_DOT, _check=False, _isnm=self._isnm)

def get_bits ( self ) : return _convert ( self . _ip , notation = NM_BITS , inotation = IP_DOT , _check = False , _isnm = self . _isnm )

Return the bits notation of the netmask .


def get_wildcard(self):
        """Return the wildcard bits notation of the netmask."""
        return _convert(self._ip, notation=NM_WILDCARD,
                        inotation=IP_DOT, _check=False, _isnm=self._isnm)

def get_wildcard ( self ) : return _convert ( self . _ip , notation = NM_WILDCARD , inotation = IP_DOT , _check = False , _isnm = self . _isnm )

Return the wildcard bits notation of the netmask .


def set(self, ip, netmask=None):
        """Set the IP address and the netmask."""
        if isinstance(ip, str) and netmask is None:
            ipnm = ip.split('/')
            if len(ipnm) != 2:
                raise ValueError('set: invalid CIDR: "%s"' % ip)
            ip = ipnm[0]
            netmask = ipnm[1]
        if isinstance(ip, IPv4Address):
            self._ip = ip
        else:
            self._ip = IPv4Address(ip)
        if isinstance(netmask, IPv4NetMask):
            self._nm = netmask
        else:
            self._nm = IPv4NetMask(netmask)
        ipl = int(self._ip)
        nml = int(self._nm)
        base_add = ipl & nml
        self._ip_num = 0xFFFFFFFF - 1 - nml
        # NOTE: quite a mess.
        #      This's here to handle /32 (-1) and /31 (0) netmasks.
        if self._ip_num in (-1, 0):
            if self._ip_num == -1:
                self._ip_num = 1
            else:
                self._ip_num = 2
            self._net_ip = None
            self._bc_ip = None
            self._first_ip_dec = base_add
            self._first_ip = IPv4Address(self._first_ip_dec, notation=IP_DEC)
            if self._ip_num == 1:
                last_ip_dec = self._first_ip_dec
            else:
                last_ip_dec = self._first_ip_dec + 1
            self._last_ip = IPv4Address(last_ip_dec, notation=IP_DEC)
            return
        self._net_ip = IPv4Address(base_add, notation=IP_DEC)
        self._bc_ip = IPv4Address(base_add + self._ip_num + 1, notation=IP_DEC)
        self._first_ip_dec = base_add + 1
        self._first_ip = IPv4Address(self._first_ip_dec, notation=IP_DEC)
        self._last_ip = IPv4Address(base_add + self._ip_num, notation=IP_DEC)

def
set
(
self
,
ip
,
netmask
=
)
:
if
isinstance
(
ip
,
str
)
and
netmask
is
:
ipnm
=
ip
.
split
(
'/'
)
if
len
(
ipnm
)
!=
2
:
raise
ValueError
(
'set: invalid CIDR: "%s"'
%
ip
)
ip
=
ipnm
[
0
]
netmask
=
ipnm
[
1
]
if
isinstance
(
ip
,
IPv4Address
)
:
self
.
_ip
=
ip
else
:
self
.
_ip
=
IPv4Address
(
ip
)
if
isinstance
(
netmask
,
IPv4NetMask
)
:
self
.
_nm
=
netmask
else
:
self
.
_nm
=
IPv4NetMask
(
netmask
)
ipl
=
int
(
self
.
_ip
)
nml
=
int
(
self
.
_nm
)
base_add
=
ipl
&
nml
self
.
_ip_num
=
0xFFFFFFFF
1
nml
if
self
.
_ip_num
in
(
1
,
0
)
:
if
self
.
_ip_num
==
1
:
self
.
_ip_num
=
1
else
:
self
.
_ip_num
=
2
self
.
_net_ip
=
self
.
_bc_ip
=
self
.
_first_ip_dec
=
base_add
self
.
_first_ip
=
IPv4Address
(
self
.
_first_ip_dec
,
notation
=
IP_DEC
)
if
self
.
_ip_num
==
1
:
last_ip_dec
=
self
.
_first_ip_dec
else
:
last_ip_dec
=
self
.
_first_ip_dec
+
1
self
.
_last_ip
=
IPv4Address
(
last_ip_dec
,
notation
=
IP_DEC
)
return
self
.
_net_ip
=
IPv4Address
(
base_add
,
notation
=
IP_DEC
)
self
.
_bc_ip
=
IPv4Address
(
base_add
+
self
.
_ip_num
+
1
,
notation
=
IP_DEC
)
self
.
_first_ip_dec
=
base_add
+
1
self
.
_first_ip
=
IPv4Address
(
self
.
_first_ip_dec
,
notation
=
IP_DEC
)
self
.
_last_ip
=
IPv4Address
(
base_add
+
self
.
_ip_num
,
notation
=
IP_DEC
)


def set_ip(self, ip):
        """Change the current IP."""
        self.set(ip=ip, netmask=self._nm)

def set_ip ( self , ip ) : self . set ( ip = ip , netmask = self . _nm )


def set_netmask(self, netmask):
        """Change the current netmask."""
        self.set(ip=self._ip, netmask=netmask)

def set_netmask ( self , netmask ) : self . set ( ip = self . _ip , netmask = netmask )


def is_valid_ip(self, ip):
        """Return true if the given address in amongst the usable addresses,
        or if the given CIDR is contained in this one."""
        if not isinstance(ip, (IPv4Address, CIDR)):
            if str(ip).find('/') == -1:
                ip = IPv4Address(ip)
            else:
                # Support for CIDR strings/objects, an idea of Nicola Novello.
                ip = CIDR(ip)
        if isinstance(ip, IPv4Address):
            if ip < self._first_ip or ip > self._last_ip:
                return False
        elif isinstance(ip, CIDR):
            # NOTE: manage /31 networks; 127.0.0.1/31 is considered to
            #       be included in 127.0.0.1/8.
            if ip._nm._ip_dec == 0xFFFFFFFE \
                    and self._nm._ip_dec != 0xFFFFFFFE:
                compare_to_first = self._net_ip._ip_dec
                compare_to_last = self._bc_ip._ip_dec
            else:
                compare_to_first = self._first_ip._ip_dec
                compare_to_last = self._last_ip._ip_dec
            if ip._first_ip._ip_dec < compare_to_first or \
                    ip._last_ip._ip_dec > compare_to_last:
                return False
        return True

def is_valid_ip ( self , ip ) : if not isinstance ( ip , ( IPv4Address , CIDR ) ) : if str ( ip ) . find ( '/' ) == - 1 : ip = IPv4Address ( ip ) else : ip = CIDR ( ip ) if isinstance ( ip , IPv4Address ) : if ip < self . _first_ip or ip > self . _last_ip : return False elif isinstance ( ip , CIDR ) : if ip . _nm . _ip_dec == 0xFFFFFFFE and self . _nm . _ip_dec != 0xFFFFFFFE : compare_to_first = self . _net_ip . _ip_dec compare_to_last = self . _bc_ip . _ip_dec else : compare_to_first = self . _first_ip . _ip_dec compare_to_last = self . _last_ip . _ip_dec if ip . _first_ip . _ip_dec < compare_to_first or ip . _last_ip . _ip_dec > compare_to_last : return False return True

Return true if the given address in amongst the usable addresses or if the given CIDR is contained in this one .


async def upload_file(self, bucket, file, uploadpath=None, key=None,
                          ContentType=None, **kw):
        """Upload a file to S3 possibly using the multi-part uploader
        Return the key uploaded
        """
        is_filename = False

        if hasattr(file, 'read'):
            if hasattr(file, 'seek'):
                file.seek(0)
            file = file.read()
            size = len(file)
        elif key:
            size = len(file)
        else:
            is_filename = True
            size = os.stat(file).st_size
            key = os.path.basename(file)

        assert key, 'key not available'

        if not ContentType:
            ContentType, _ = mimetypes.guess_type(key)

        if uploadpath:
            if not uploadpath.endswith('/'):
                uploadpath = '%s/' % uploadpath
            key = '%s%s' % (uploadpath, key)

        params = dict(Bucket=bucket, Key=key)

        if not ContentType:
            ContentType = 'application/octet-stream'

        params['ContentType'] = ContentType

        if size > MULTI_PART_SIZE and is_filename:
            resp = await _multipart(self, file, params)
        elif is_filename:
            with open(file, 'rb') as fp:
                params['Body'] = fp.read()
            resp = await self.put_object(**params)
        else:
            params['Body'] = file
            resp = await self.put_object(**params)
        if 'Key' not in resp:
            resp['Key'] = key
        if 'Bucket' not in resp:
            resp['Bucket'] = bucket
        return resp

async
def
upload_file
(
self
,
bucket
,
file
,
uploadpath
=
,
key
=
,
ContentType
=
,
**
kw
)
:
is_filename
=
False
if
hasattr
(
file
,
'read'
)
:
if
hasattr
(
file
,
'seek'
)
:
file
.
seek
(
0
)
file
=
file
.
read
(
)
size
=
len
(
file
)
elif
key
:
size
=
len
(
file
)
else
:
is_filename
=
True
size
=
os
.
stat
(
file
)
.
st_size
key
=
os
.
path
.
basename
(
file
)
assert
key
,
'key not available'
if
not
ContentType
:
ContentType
,
_
=
mimetypes
.
guess_type
(
key
)
if
uploadpath
:
if
not
uploadpath
.
endswith
(
'/'
)
:
uploadpath
=
'%s/'
%
uploadpath
key
=
'%s%s'
%
(
uploadpath
,
key
)
params
=
dict
(
Bucket
=
bucket
,
Key
=
key
)
if
not
ContentType
:
ContentType
=
'application/octet-stream'
params
[
'ContentType'
]
=
ContentType
if
size
>
MULTI_PART_SIZE
and
is_filename
:
resp
=
await
_multipart
(
self
,
file
,
params
)
elif
is_filename
:
with
open
(
file
,
'rb'
)
as
fp
:
params
[
'Body'
]
=
fp
.
read
(
)
resp
=
await
self
.
put_object
(
**
params
)
else
:
params
[
'Body'
]
=
file
resp
=
await
self
.
put_object
(
**
params
)
if
'Key'
not
in
resp
:
resp
[
'Key'
]
=
key
if
'Bucket'
not
in
resp
:
resp
[
'Bucket'
]
=
bucket
return
resp

Upload a file to S3 possibly using the multi - part uploader Return the key uploaded


async def copy_storage_object(self, source_bucket, source_key,
                                  bucket, key):
        """Copy a file from one bucket into another
        """
        info = await self.head_object(Bucket=source_bucket, Key=source_key)
        size = info['ContentLength']

        if size > MULTI_PART_SIZE:
            result = await _multipart_copy(self, source_bucket, source_key,
                                           bucket, key, size)
        else:
            result = await self.copy_object(
                Bucket=bucket, Key=key,
                CopySource=_source_string(source_bucket, source_key)
            )
        return result

async def copy_storage_object ( self , source_bucket , source_key , bucket , key ) : info = await self . head_object ( Bucket = source_bucket , Key = source_key ) size = info [ 'ContentLength' ] if size > MULTI_PART_SIZE : result = await _multipart_copy ( self , source_bucket , source_key , bucket , key , size ) else : result = await self . copy_object ( Bucket = bucket , Key = key , CopySource = _source_string ( source_bucket , source_key ) ) return result


def upload_folder(self, bucket, folder, key=None, skip=None,
                      content_types=None):
        """Recursively upload a ``folder`` into a backet.

        :param bucket: bucket where to upload the folder to
        :param folder: the folder location in the local file system
        :param key: Optional key where the folder is uploaded
        :param skip: Optional list of files to skip
        :param content_types: Optional dictionary mapping suffixes to
            content types
        :return: a coroutine
        """
        uploader = FolderUploader(self, bucket, folder, key, skip,
                                  content_types)
        return uploader.start()

def upload_folder ( self , bucket , folder , key = None , skip = None , content_types = None ) : uploader = FolderUploader ( self , bucket , folder , key , skip , content_types ) return uploader . start ( )

Recursively upload a folder into a backet .


async def _upload_file(self, full_path):
        """Coroutine for uploading a single file
        """
        rel_path = os.path.relpath(full_path, self.folder)
        key = s3_key(os.path.join(self.key, rel_path))
        ct = self.content_types.get(key.split('.')[-1])
        with open(full_path, 'rb') as fp:
            file = fp.read()
        try:
            await self.botocore.upload_file(self.bucket, file, key=key,
                                            ContentType=ct)
        except Exception as exc:
            LOGGER.error('Could not upload "%s": %s', key, exc)
            self.failures[key] = self.all.pop(full_path)
            return
        size = self.all.pop(full_path)
        self.success[key] = size
        self.total_size += size
        percentage = 100*(1 - len(self.all)/self.total_files)
        message = '{0:.0f}% completed - uploaded "{1}" - {2}'.format(
            percentage, key, convert_bytes(size))
        LOGGER.info(message)


async def trigger(self, event, data=None, socket_id=None):
        '''Trigger an ``event`` on this channel
        '''
        json_data = json.dumps(data, cls=self.pusher.encoder)
        query_string = self.signed_query(event, json_data, socket_id)
        signed_path = "%s?%s" % (self.path, query_string)
        pusher = self.pusher
        absolute_url = pusher.get_absolute_path(signed_path)
        response = await pusher.http.post(
            absolute_url, data=json_data,
            headers=[('Content-Type', 'application/json')])
        response.raise_for_status()
        return response.status_code == 202

async def trigger ( self , event , data = None , socket_id = None ) : json_data = json . dumps ( data , cls = self . pusher . encoder ) query_string = self . signed_query ( event , json_data , socket_id ) signed_path = "%s?%s" % ( self . path , query_string ) pusher = self . pusher absolute_url = pusher . get_absolute_path ( signed_path ) response = await pusher . http . post ( absolute_url , data = json_data , headers = [ ( 'Content-Type' , 'application/json' ) ] ) response . raise_for_status ( ) return response . status_code == 202


async def connect(self):
        '''Connect to a Pusher websocket
        '''
        if not self._consumer:
            waiter = self._waiter = asyncio.Future()
            try:
                address = self._websocket_host()
                self.logger.info('Connect to %s', address)
                self._consumer = await self.http.get(address)
                if self._consumer.status_code != 101:
                    raise PusherError("Could not connect to websocket")
            except Exception as exc:
                waiter.set_exception(exc)
                raise
            else:
                await waiter
        return self._consumer

async
def
connect
(
self
)
:
if
not
self
.
_consumer
:
waiter
=
self
.
_waiter
=
asyncio
.
Future
(
)
try
:
address
=
self
.
_websocket_host
(
)
self
.
logger
.
info
(
'Connect to %s'
,
address
)
self
.
_consumer
=
await
self
.
http
.
get
(
address
)
if
self
.
_consumer
.
status_code
!=
101
:
raise
PusherError
(
"Could not connect to websocket"
)
except
Exception
as
exc
:
waiter
.
set_exception
(
exc
)
raise
else
:
await
waiter
return
self
.
_consumer


def on_message(self, websocket, message):
        '''Handle websocket incoming messages
        '''
        waiter = self._waiter
        self._waiter = None
        encoded = json.loads(message)
        event = encoded.get('event')
        channel = encoded.get('channel')
        data = json.loads(encoded.get('data'))
        try:
            if event == PUSHER_ERROR:
                raise PusherError(data['message'], data['code'])
            elif event == PUSHER_CONNECTION:
                self.socket_id = data.get('socket_id')
                self.logger.info('Succesfully connected on socket %s',
                                 self.socket_id)
                waiter.set_result(self.socket_id)
            elif event == PUSHER_SUBSCRIBED:
                self.logger.info('Succesfully subscribed to %s',
                                 encoded.get('channel'))
            elif channel:
                self[channel]._event(event, data)
        except Exception as exc:
            if waiter:
                waiter.set_exception(exc)
            else:
                self.logger.exception('pusher error')

def
on_message
(
self
,
websocket
,
message
)
:
waiter
=
self
.
_waiter
self
.
_waiter
=
encoded
=
json
.
loads
(
message
)
event
=
encoded
.
get
(
'event'
)
channel
=
encoded
.
get
(
'channel'
)
data
=
json
.
loads
(
encoded
.
get
(
'data'
)
)
try
:
if
event
==
PUSHER_ERROR
:
raise
PusherError
(
data
[
'message'
]
,
data
[
'code'
]
)
elif
event
==
PUSHER_CONNECTION
:
self
.
socket_id
=
data
.
get
(
'socket_id'
)
self
.
logger
.
info
(
'Succesfully connected on socket %s'
,
self
.
socket_id
)
waiter
.
set_result
(
self
.
socket_id
)
elif
event
==
PUSHER_SUBSCRIBED
:
self
.
logger
.
info
(
'Succesfully subscribed to %s'
,
encoded
.
get
(
'channel'
)
)
elif
channel
:
self
[
channel
]
.
_event
(
event
,
data
)
except
Exception
as
exc
:
if
waiter
:
waiter
.
set_exception
(
exc
)
else
:
self
.
logger
.
exception
(
'pusher error'
)


def const_equal(str_a, str_b):
    '''Constant time string comparison'''

    if len(str_a) != len(str_b):
        return False

    result = True
    for i in range(len(str_a)):
        result &= (str_a[i] == str_b[i])

    return result

def const_equal ( str_a , str_b ) : if len ( str_a ) != len ( str_b ) : return False result = True for i in range ( len ( str_a ) ) : result &= ( str_a [ i ] == str_b [ i ] ) return result


def decode_html_entities(html):
    """
    Decodes a limited set of HTML entities.
    """
    if not html:
        return html

    for entity, char in six.iteritems(html_entity_map):
        html = html.replace(entity, char)

    return html

def decode_html_entities ( html ) : if not html : return html for entity , char in six . iteritems ( html_entity_map ) : html = html . replace ( entity , char ) return html

Decodes a limited set of HTML entities .


def set_signature_passphrases(self, signature_passphrases):
        '''Set signature passphrases'''
        self.signature_passphrases = self._update_dict(signature_passphrases,
                                                       {}, replace_data=True)

def set_signature_passphrases ( self , signature_passphrases ) : self . signature_passphrases = self . _update_dict ( signature_passphrases , { } , replace_data = True )


def set_encryption_passphrases(self, encryption_passphrases):
        '''Set encryption passphrases'''
        self.encryption_passphrases = self._update_dict(encryption_passphrases,
                                                        {}, replace_data=True)

def set_encryption_passphrases ( self , encryption_passphrases ) : self . encryption_passphrases = self . _update_dict ( encryption_passphrases , { } , replace_data = True )


def set_algorithms(self, signature=None, encryption=None,
                       serialization=None, compression=None):
        '''Set algorithms used for sealing. Defaults can not be overridden.'''

        self.signature_algorithms = \
            self._update_dict(signature, self.DEFAULT_SIGNATURE)

        self.encryption_algorithms = \
            self._update_dict(encryption, self.DEFAULT_ENCRYPTION)

        self.serialization_algorithms = \
            self._update_dict(serialization, self.DEFAULT_SERIALIZATION)

        self.compression_algorithms = \
            self._update_dict(compression, self.DEFAULT_COMPRESSION)

def set_algorithms ( self , signature = None , encryption = None , serialization = None , compression = None ) : self . signature_algorithms = self . _update_dict ( signature , self . DEFAULT_SIGNATURE ) self . encryption_algorithms = self . _update_dict ( encryption , self . DEFAULT_ENCRYPTION ) self . serialization_algorithms = self . _update_dict ( serialization , self . DEFAULT_SERIALIZATION ) self . compression_algorithms = self . _update_dict ( compression , self . DEFAULT_COMPRESSION )

Set algorithms used for sealing . Defaults can not be overridden .


def get_algorithms(self):
        '''Get algorithms used for sealing'''

        return {
            'signature': self.signature_algorithms,
            'encryption': self.encryption_algorithms,
            'serialization': self.serialization_algorithms,
            'compression': self.compression_algorithms,
        }

def get_algorithms ( self ) : return { 'signature' : self . signature_algorithms , 'encryption' : self . encryption_algorithms , 'serialization' : self . serialization_algorithms , 'compression' : self . compression_algorithms , }


def _set_options(self, options):
        '''Private function for setting options used for sealing'''
        if not options:
            return self.options.copy()

        options = options.copy()

        if 'magic' in options:
            self.set_magic(options['magic'])
            del(options['magic'])

        if 'flags' in options:
            flags = options['flags']
            del(options['flags'])
            for key, value in flags.iteritems():
                if not isinstance(value, bool):
                    raise TypeError('Invalid flag type for: %s' % key)
        else:
            flags = self.options['flags']

        if 'info' in options:
            del(options['info'])

        for key, value in options.iteritems():
            if not isinstance(value, int):
                raise TypeError('Invalid option type for: %s' % key)
            if value < 0 or value > 255:
                raise ValueError('Option value out of range for: %s' % key)

        new_options = self.options.copy()
        new_options.update(options)
        new_options['flags'].update(flags)

        return new_options

def
_set_options
(
self
,
options
)
:
if
not
options
:
return
self
.
options
.
copy
(
)
options
=
options
.
copy
(
)
if
'magic'
in
options
:
self
.
set_magic
(
options
[
'magic'
]
)
del
(
options
[
'magic'
]
)
if
'flags'
in
options
:
flags
=
options
[
'flags'
]
del
(
options
[
'flags'
]
)
for
key
,
value
in
flags
.
iteritems
(
)
:
if
not
isinstance
(
value
,
bool
)
:
raise
TypeError
(
'Invalid flag type for: %s'
%
key
)
else
:
flags
=
self
.
options
[
'flags'
]
if
'info'
in
options
:
del
(
options
[
'info'
]
)
for
key
,
value
in
options
.
iteritems
(
)
:
if
not
isinstance
(
value
,
int
)
:
raise
TypeError
(
'Invalid option type for: %s'
%
key
)
if
value
<
0
or
value
>
255
:
raise
ValueError
(
'Option value out of range for: %s'
%
key
)
new_options
=
self
.
options
.
copy
(
)
new_options
.
update
(
options
)
new_options
[
'flags'
]
.
update
(
flags
)
return
new_options


def verify_signature(self, data):
        '''Verify sealed data signature'''

        data = self._remove_magic(data)
        data = urlsafe_nopadding_b64decode(data)
        options = self._read_header(data)
        data = self._add_magic(data)
        self._unsign_data(data, options)

def verify_signature ( self , data ) : data = self . _remove_magic ( data ) data = urlsafe_nopadding_b64decode ( data ) options = self . _read_header ( data ) data = self . _add_magic ( data ) self . _unsign_data ( data , options )


def _encode(self, data, algorithm, key=None):
        '''Encode data with specific algorithm'''

        if algorithm['type'] == 'hmac':
            return data + self._hmac_generate(data, algorithm, key)
        elif algorithm['type'] == 'aes':
            return self._aes_encrypt(data, algorithm, key)
        elif algorithm['type'] == 'no-serialization':
            return data
        elif algorithm['type'] == 'json':
            return json.dumps(data)
        elif algorithm['type'] == 'no-compression':
            return data
        elif algorithm['type'] == 'gzip':
            return self._zlib_compress(data, algorithm)
        else:
            raise Exception('Algorithm not supported: %s' % algorithm['type'])

def
_encode
(
self
,
data
,
algorithm
,
key
=
)
:
if
algorithm
[
'type'
]
==
'hmac'
:
return
data
+
self
.
_hmac_generate
(
data
,
algorithm
,
key
)
elif
algorithm
[
'type'
]
==
'aes'
:
return
self
.
_aes_encrypt
(
data
,
algorithm
,
key
)
elif
algorithm
[
'type'
]
==
'no-serialization'
:
return
data
elif
algorithm
[
'type'
]
==
'json'
:
return
json
.
dumps
(
data
)
elif
algorithm
[
'type'
]
==
'no-compression'
:
return
data
elif
algorithm
[
'type'
]
==
'gzip'
:
return
self
.
_zlib_compress
(
data
,
algorithm
)
else
:
raise
Exception
(
'Algorithm not supported: %s'
%
algorithm
[
'type'
]
)


def _decode(self, data, algorithm, key=None):
        '''Decode data with specific algorithm'''

        if algorithm['type'] == 'hmac':
            verify_signature = data[-algorithm['hash_size']:]
            data = data[:-algorithm['hash_size']]
            signature = self._hmac_generate(data, algorithm, key)
            if not const_equal(verify_signature, signature):
                raise Exception('Invalid signature')
            return data
        elif algorithm['type'] == 'aes':
            return self._aes_decrypt(data, algorithm, key)
        elif algorithm['type'] == 'no-serialization':
            return data
        elif algorithm['type'] == 'json':
            return json.loads(data)
        elif algorithm['type'] == 'no-compression':
            return data
        elif algorithm['type'] == 'gzip':
            return self._zlib_decompress(data, algorithm)
        else:
            raise Exception('Algorithm not supported: %s' % algorithm['type'])

def
_decode
(
self
,
data
,
algorithm
,
key
=
)
:
if
algorithm
[
'type'
]
==
'hmac'
:
verify_signature
=
data
[
algorithm
[
'hash_size'
]
:
]
data
=
data
[
:
algorithm
[
'hash_size'
]
]
signature
=
self
.
_hmac_generate
(
data
,
algorithm
,
key
)
if
not
const_equal
(
verify_signature
,
signature
)
:
raise
Exception
(
'Invalid signature'
)
return
data
elif
algorithm
[
'type'
]
==
'aes'
:
return
self
.
_aes_decrypt
(
data
,
algorithm
,
key
)
elif
algorithm
[
'type'
]
==
'no-serialization'
:
return
data
elif
algorithm
[
'type'
]
==
'json'
:
return
json
.
loads
(
data
)
elif
algorithm
[
'type'
]
==
'no-compression'
:
return
data
elif
algorithm
[
'type'
]
==
'gzip'
:
return
self
.
_zlib_decompress
(
data
,
algorithm
)
else
:
raise
Exception
(
'Algorithm not supported: %s'
%
algorithm
[
'type'
]
)


def _sign_data(self, data, options):
        '''Add signature to data'''

        if options['signature_algorithm_id'] not in self.signature_algorithms:
            raise Exception('Unknown signature algorithm id: %d'
                            % options['signature_algorithm_id'])

        signature_algorithm = \
            self.signature_algorithms[options['signature_algorithm_id']]

        algorithm = self._get_algorithm_info(signature_algorithm)

        key_salt = get_random_bytes(algorithm['salt_size'])
        key = self._generate_key(options['signature_passphrase_id'],
                            self.signature_passphrases, key_salt, algorithm)

        data = self._encode(data, algorithm, key)

        return data + key_salt

def
_sign_data
(
self
,
data
,
options
)
:
if
options
[
'signature_algorithm_id'
]
not
in
self
.
signature_algorithms
:
raise
Exception
(
'Unknown signature algorithm id: %d'
%
options
[
'signature_algorithm_id'
]
)
signature_algorithm
=
self
.
signature_algorithms
[
options
[
'signature_algorithm_id'
]
]
algorithm
=
self
.
_get_algorithm_info
(
signature_algorithm
)
key_salt
=
get_random_bytes
(
algorithm
[
'salt_size'
]
)
key
=
self
.
_generate_key
(
options
[
'signature_passphrase_id'
]
,
self
.
signature_passphrases
,
key_salt
,
algorithm
)
data
=
self
.
_encode
(
data
,
algorithm
,
key
)
return
data
+
key_salt


def _unsign_data(self, data, options):
        '''Verify and remove signature'''

        if options['signature_algorithm_id'] not in self.signature_algorithms:
            raise Exception('Unknown signature algorithm id: %d'
                            % options['signature_algorithm_id'])

        signature_algorithm = \
            self.signature_algorithms[options['signature_algorithm_id']]

        algorithm = self._get_algorithm_info(signature_algorithm)

        key_salt = ''
        if algorithm['salt_size']:
            key_salt = data[-algorithm['salt_size']:]
            data = data[:-algorithm['salt_size']]

        key = self._generate_key(options['signature_passphrase_id'],
                            self.signature_passphrases, key_salt, algorithm)

        data = self._decode(data, algorithm, key)

        return data

def
_unsign_data
(
self
,
data
,
options
)
:
if
options
[
'signature_algorithm_id'
]
not
in
self
.
signature_algorithms
:
raise
Exception
(
'Unknown signature algorithm id: %d'
%
options
[
'signature_algorithm_id'
]
)
signature_algorithm
=
self
.
signature_algorithms
[
options
[
'signature_algorithm_id'
]
]
algorithm
=
self
.
_get_algorithm_info
(
signature_algorithm
)
key_salt
=
''
if
algorithm
[
'salt_size'
]
:
key_salt
=
data
[
algorithm
[
'salt_size'
]
:
]
data
=
data
[
:
algorithm
[
'salt_size'
]
]
key
=
self
.
_generate_key
(
options
[
'signature_passphrase_id'
]
,
self
.
signature_passphrases
,
key_salt
,
algorithm
)
data
=
self
.
_decode
(
data
,
algorithm
,
key
)
return
data


def _remove_magic(self, data):
        '''Verify and remove magic'''

        if not self.magic:
            return data

        magic_size = len(self.magic)
        magic = data[:magic_size]
        if magic != self.magic:
            raise Exception('Invalid magic')
        data = data[magic_size:]

        return data

def
_remove_magic
(
self
,
data
)
:
if
not
self
.
magic
:
return
data
magic_size
=
len
(
self
.
magic
)
magic
=
data
[
:
magic_size
]
if
magic
!=
self
.
magic
:
raise
Exception
(
'Invalid magic'
)
data
=
data
[
magic_size
:
]
return
data


def _add_header(self, data, options):
        '''Add header to data'''

        # pylint: disable=W0142

        version_info = self._get_version_info(options['version'])

        flags = options['flags']

        header_flags = dict(
            (i, str(int(j))) for i, j in options['flags'].iteritems())
        header_flags = ''.join(version_info['flags'](**header_flags))
        header_flags = int(header_flags, 2)
        options['flags'] = header_flags

        header = version_info['header']
        header = header(**options)
        header = pack(version_info['header_format'], *header)

        if 'timestamp' in flags and flags['timestamp']:
            timestamp = long(time())
            timestamp = pack(version_info['timestamp_format'], timestamp)
            header = header + timestamp

        return header + data

def _add_header ( self , data , options ) : version_info = self . _get_version_info ( options [ 'version' ] ) flags = options [ 'flags' ] header_flags = dict ( ( i , str ( int ( j ) ) ) for i , j in options [ 'flags' ] . iteritems ( ) ) header_flags = '' . join ( version_info [ 'flags' ] ( ** header_flags ) ) header_flags = int ( header_flags , 2 ) options [ 'flags' ] = header_flags header = version_info [ 'header' ] header = header ( ** options ) header = pack ( version_info [ 'header_format' ] , * header ) if 'timestamp' in flags and flags [ 'timestamp' ] : timestamp = long ( time ( ) ) timestamp = pack ( version_info [ 'timestamp_format' ] , timestamp ) header = header + timestamp return header + data


def _read_header(self, data):
        '''Read header from data'''

        # pylint: disable=W0212

        version = self._read_version(data)
        version_info = self._get_version_info(version)
        header_data = data[:version_info['header_size']]
        header = version_info['header']
        header = header._make(
            unpack(version_info['header_format'], header_data))
        header = dict(header._asdict())

        flags = list("{0:0>8b}".format(header['flags']))
        flags = dict(version_info['flags']._make(flags)._asdict())
        flags = dict((i, bool(int(j))) for i, j in flags.iteritems())
        header['flags'] = flags

        timestamp = None
        if flags['timestamp']:
            ts_start = version_info['header_size']
            ts_end = ts_start + version_info['timestamp_size']
            timestamp_data = data[ts_start:ts_end]
            timestamp = unpack(
                version_info['timestamp_format'], timestamp_data)[0]
        header['info'] = {'timestamp': timestamp}

        return header

def _read_header ( self , data ) : version = self . _read_version ( data ) version_info = self . _get_version_info ( version ) header_data = data [ : version_info [ 'header_size' ] ] header = version_info [ 'header' ] header = header . _make ( unpack ( version_info [ 'header_format' ] , header_data ) ) header = dict ( header . _asdict ( ) ) flags = list ( "{0:0>8b}" . format ( header [ 'flags' ] ) ) flags = dict ( version_info [ 'flags' ] . _make ( flags ) . _asdict ( ) ) flags = dict ( ( i , bool ( int ( j ) ) ) for i , j in flags . iteritems ( ) ) header [ 'flags' ] = flags timestamp = None if flags [ 'timestamp' ] : ts_start = version_info [ 'header_size' ] ts_end = ts_start + version_info [ 'timestamp_size' ] timestamp_data = data [ ts_start : ts_end ] timestamp = unpack ( version_info [ 'timestamp_format' ] , timestamp_data ) [ 0 ] header [ 'info' ] = { 'timestamp' : timestamp } return header


def _remove_header(self, data, options):
        '''Remove header from data'''

        version_info = self._get_version_info(options['version'])
        header_size = version_info['header_size']

        if options['flags']['timestamp']:
            header_size += version_info['timestamp_size']

        data = data[header_size:]

        return data

def _remove_header ( self , data , options ) : version_info = self . _get_version_info ( options [ 'version' ] ) header_size = version_info [ 'header_size' ] if options [ 'flags' ] [ 'timestamp' ] : header_size += version_info [ 'timestamp_size' ] data = data [ header_size : ] return data


def _read_version(self, data):
        '''Read header version from data'''

        version = ord(data[0])
        if version not in self.VERSIONS:
            raise Exception('Version not defined: %d' % version)
        return version

def
_read_version
(
self
,
data
)
:
version
=
ord
(
data
[
0
]
)
if
version
not
in
self
.
VERSIONS
:
raise
Exception
(
'Version not defined: %d'
%
version
)
return
version


def _get_algorithm_info(self, algorithm_info):
        '''Get algorithm info'''

        if algorithm_info['algorithm'] not in self.ALGORITHMS:
            raise Exception('Algorithm not supported: %s'
                            % algorithm_info['algorithm'])

        algorithm = self.ALGORITHMS[algorithm_info['algorithm']]
        algorithm_info.update(algorithm)

        return algorithm_info

def
_get_algorithm_info
(
self
,
algorithm_info
)
:
if
algorithm_info
[
'algorithm'
]
not
in
self
.
ALGORITHMS
:
raise
Exception
(
'Algorithm not supported: %s'
%
algorithm_info
[
'algorithm'
]
)
algorithm
=
self
.
ALGORITHMS
[
algorithm_info
[
'algorithm'
]
]
algorithm_info
.
update
(
algorithm
)
return
algorithm_info


def _generate_key(pass_id, passphrases, salt, algorithm):
        '''Generate and return PBKDF2 key'''

        if pass_id not in passphrases:
            raise Exception('Passphrase not defined for id: %d' % pass_id)

        passphrase = passphrases[pass_id]

        if len(passphrase) < 32:
            raise Exception('Passphrase less than 32 characters long')

        digestmod = EncryptedPickle._get_hashlib(algorithm['pbkdf2_algorithm'])

        encoder = PBKDF2(passphrase, salt,
                         iterations=algorithm['pbkdf2_iterations'],
                         digestmodule=digestmod)

        return encoder.read(algorithm['key_size'])


def _update_dict(data, default_data, replace_data=False):
        '''Update algorithm definition type dictionaries'''

        if not data:
            data = default_data.copy()
            return data

        if not isinstance(data, dict):
            raise TypeError('Value not dict type')
        if len(data) > 255:
            raise ValueError('More than 255 values defined')
        for i in data.keys():
            if not isinstance(i, int):
                raise TypeError('Index not int type')
            if i < 0 or i > 255:
                raise ValueError('Index value out of range')

        if not replace_data:
            data.update(default_data)

        return data

def
_update_dict
(
data
,
default_data
,
replace_data
=
False
)
:
if
not
data
:
data
=
default_data
.
copy
(
)
return
data
if
not
isinstance
(
data
,
dict
)
:
raise
TypeError
(
'Value not dict type'
)
if
len
(
data
)
>
255
:
raise
ValueError
(
'More than 255 values defined'
)
for
i
in
data
.
keys
(
)
:
if
not
isinstance
(
i
,
int
)
:
raise
TypeError
(
'Index not int type'
)
if
i
<
0
or
i
>
255
:
raise
ValueError
(
'Index value out of range'
)
if
not
replace_data
:
data
.
update
(
default_data
)
return
data


def getTableOfContents(self):
        """
        This function populates the internal tableOfContents list with the contents
        of the zip file TOC. If the server does not support ranged requests, this will raise
        and exception. It will also throw an exception if the TOC cannot be found.
        """

        self.directory_size = self.getDirectorySize()
        if self.directory_size > 65536:
            self.directory_size += 2
            self.requestContentDirectory()

        # and find the offset from start of file where it can be found
        directory_start = unpack("i", self.raw_bytes[self.directory_end + 16: self.directory_end + 20])[0]

        # find the data in the raw_bytes
        self.raw_bytes = self.raw_bytes
        current_start = directory_start - self.start
        filestart = 0
        compressedsize = 0
        tableOfContents = []

        try:
            while True:
                # get file name size (n), extra len (m) and comm len (k)
                zip_n = unpack("H", self.raw_bytes[current_start + 28: current_start + 28 + 2])[0]
                zip_m = unpack("H", self.raw_bytes[current_start + 30: current_start + 30 + 2])[0]
                zip_k = unpack("H", self.raw_bytes[current_start + 32: current_start + 32 + 2])[0]

                filename = self.raw_bytes[current_start + 46: current_start + 46 + zip_n]

                # check if this is the index file
                filestart = unpack("I", self.raw_bytes[current_start + 42: current_start + 42 + 4])[0]
                compressedsize = unpack("I", self.raw_bytes[current_start + 20: current_start + 20 + 4])[0]
                uncompressedsize = unpack("I", self.raw_bytes[current_start + 24: current_start + 24 + 4])[0]
                tableItem = {
                    'filename': filename,
                    'compressedsize': compressedsize,
                    'uncompressedsize': uncompressedsize,
                    'filestart': filestart
                }
                tableOfContents.append(tableItem)

                # not this file, move along
                current_start = current_start + 46 + zip_n + zip_m + zip_k
        except:
            pass

        self.tableOfContents = tableOfContents
        return tableOfContents

def getTableOfContents ( self ) : self . directory_size = self . getDirectorySize ( ) if self . directory_size > 65536 : self . directory_size += 2 self . requestContentDirectory ( ) directory_start = unpack ( "i" , self . raw_bytes [ self . directory_end + 16 : self . directory_end + 20 ] ) [ 0 ] self . raw_bytes = self . raw_bytes current_start = directory_start - self . start filestart = 0 compressedsize = 0 tableOfContents = [ ] try : while True : zip_n = unpack ( "H" , self . raw_bytes [ current_start + 28 : current_start + 28 + 2 ] ) [ 0 ] zip_m = unpack ( "H" , self . raw_bytes [ current_start + 30 : current_start + 30 + 2 ] ) [ 0 ] zip_k = unpack ( "H" , self . raw_bytes [ current_start + 32 : current_start + 32 + 2 ] ) [ 0 ] filename = self . raw_bytes [ current_start + 46 : current_start + 46 + zip_n ] filestart = unpack ( "I" , self . raw_bytes [ current_start + 42 : current_start + 42 + 4 ] ) [ 0 ] compressedsize = unpack ( "I" , self . raw_bytes [ current_start + 20 : current_start + 20 + 4 ] ) [ 0 ] uncompressedsize = unpack ( "I" , self . raw_bytes [ current_start + 24 : current_start + 24 + 4 ] ) [ 0 ] tableItem = { 'filename' : filename , 'compressedsize' : compressedsize , 'uncompressedsize' : uncompressedsize , 'filestart' : filestart } tableOfContents . append ( tableItem ) current_start = current_start + 46 + zip_n + zip_m + zip_k except : pass self . tableOfContents = tableOfContents return tableOfContents

This function populates the internal tableOfContents list with the contents of the zip file TOC . If the server does not support ranged requests this will raise and exception . It will also throw an exception if the TOC cannot be found .


def extractFile(self, filename):
        """
        This function will extract a single file from the remote zip without downloading
        the entire zip file. The filename argument should match whatever is in the 'filename'
        key of the tableOfContents.
        """
        files = [x for x in self.tableOfContents if x['filename'] == filename]
        if len(files) == 0:
            raise FileNotFoundException()

        fileRecord = files[0]

        # got here? need to fetch the file size
        metaheadroom = 1024  # should be enough
        request = urllib2.Request(self.zipURI)
        start = fileRecord['filestart']
        end = fileRecord['filestart'] + fileRecord['compressedsize'] + metaheadroom
        request.headers['Range'] = "bytes=%s-%s" % (start, end)
        handle = urllib2.urlopen(request)

        # make sure the response is ranged
        return_range = handle.headers.get('Content-Range')
        if return_range != "bytes %d-%d/%s" % (start, end, self.filesize):
            raise Exception("Ranged requests are not supported for this URI")

        filedata = handle.read()

        # find start of raw file data
        zip_n = unpack("H", filedata[26:28])[0]
        zip_m = unpack("H", filedata[28:30])[0]

        # check compressed size
        has_data_descriptor = bool(unpack("H", filedata[6:8])[0] & 8)
        comp_size = unpack("I", filedata[18:22])[0]
        if comp_size == 0 and has_data_descriptor:
            # assume compressed size in the Central Directory is correct
            comp_size = fileRecord['compressedsize']
        elif comp_size != fileRecord['compressedsize']:
            raise Exception("Something went wrong. Directory and file header disagree of compressed file size")

        raw_zip_data = filedata[30 + zip_n + zip_m: 30 + zip_n + zip_m + comp_size]
        uncompressed_data = ""

        # can't decompress if stored without compression
        compression_method = unpack("H", filedata[8:10])[0]
        if compression_method == 0:
          return raw_zip_data

        dec = zlib.decompressobj(-zlib.MAX_WBITS)
        for chunk in raw_zip_data:
            rv = dec.decompress(chunk)
            if rv:
                uncompressed_data = uncompressed_data + rv

        return uncompressed_data

This function will extract a single file from the remote zip without downloading the entire zip file . The filename argument should match whatever is in the filename key of the tableOfContents .


def do_photometry(self):
        """
        Does photometry and estimates uncertainties by calculating the scatter around a linear fit to the data
        in each orientation. This function is called by other functions and generally the user will not need
        to interact with it directly.
        """

        std_f = np.zeros(4)
        data_save = np.zeros_like(self.postcard)
        self.obs_flux = np.zeros_like(self.reference_flux)

        for i in range(4):
            g = np.where(self.qs == i)[0]
            wh = np.where(self.times[g] > 54947)

            data_save[g] = np.roll(self.postcard[g], int(self.roll_best[i,0]), axis=1)
            data_save[g] = np.roll(data_save[g], int(self.roll_best[i,1]), axis=2)

            self.target_flux_pixels = data_save[:,self.targets == 1]
            self.target_flux = np.sum(self.target_flux_pixels, axis=1)

            self.obs_flux[g] = self.target_flux[g] / self.reference_flux[g]
            self.obs_flux[g] /= np.median(self.obs_flux[g[wh]])

            fitline = np.polyfit(self.times[g][wh], self.obs_flux[g][wh], 1)
            std_f[i] = np.max([np.std(self.obs_flux[g][wh]/(fitline[0]*self.times[g][wh]+fitline[1])), 0.001])

        self.flux_uncert = std_f

def do_photometry ( self ) : std_f = np . zeros ( 4 ) data_save = np . zeros_like ( self . postcard ) self . obs_flux = np . zeros_like ( self . reference_flux ) for i in range ( 4 ) : g = np . where ( self . qs == i ) [ 0 ] wh = np . where ( self . times [ g ] > 54947 ) data_save [ g ] = np . roll ( self . postcard [ g ] , int ( self . roll_best [ i , 0 ] ) , axis = 1 ) data_save [ g ] = np . roll ( data_save [ g ] , int ( self . roll_best [ i , 1 ] ) , axis = 2 ) self . target_flux_pixels = data_save [ : , self . targets == 1 ] self . target_flux = np . sum ( self . target_flux_pixels , axis = 1 ) self . obs_flux [ g ] = self . target_flux [ g ] / self . reference_flux [ g ] self . obs_flux [ g ] /= np . median ( self . obs_flux [ g [ wh ] ] ) fitline = np . polyfit ( self . times [ g ] [ wh ] , self . obs_flux [ g ] [ wh ] , 1 ) std_f [ i ] = np . max ( [ np . std ( self . obs_flux [ g ] [ wh ] / ( fitline [ 0 ] * self . times [ g ] [ wh ] + fitline [ 1 ] ) ) , 0.001 ] ) self . flux_uncert = std_f

Does photometry and estimates uncertainties by calculating the scatter around a linear fit to the data in each orientation . This function is called by other functions and generally the user will not need to interact with it directly .


def generate_panel(self, img):
        """
        Creates the figure shown in ``adjust_aperture`` for visualization purposes. Called by other functions
        and generally not called by the user directly.

        Args:
            img: The data frame to be passed through to be plotted. A cutout of the ``integrated_postcard``

        """
        plt.figure(figsize=(14,6))
        ax = plt.gca()
        fig = plt.gcf()
        plt.subplot(122)

        data_save = np.zeros_like(self.postcard)
        self.roll_best = np.zeros((4,2))

        for i in range(4):
            g = np.where(self.qs == i)[0]
            wh = np.where(self.times[g] > 54947)

            self.roll_best[i] = self.do_rolltest(g, wh)

        self.do_photometry()
        for i in range(4):
            g = np.where(self.qs == i)[0]
            plt.errorbar(self.times[g], self.obs_flux[g], yerr=self.flux_uncert[i], fmt=fmt[i])

        plt.xlabel('Time', fontsize=20)
        plt.ylabel('Relative Flux', fontsize=20)

        plt.subplot(121)
        implot = plt.imshow(img, interpolation='nearest', cmap='gray', vmin=98000*52, vmax=104000*52)
        cid = fig.canvas.mpl_connect('button_press_event', self.onclick)

        plt.show(block=True)

def
generate_panel
(
self
,
img
)
:
plt
.
figure
(
figsize
=
(
14
,
6
)
)
ax
=
plt
.
gca
(
)
fig
=
plt
.
gcf
(
)
plt
.
subplot
(
122
)
data_save
=
np
.
zeros_like
(
self
.
postcard
)
self
.
roll_best
=
np
.
zeros
(
(
4
,
2
)
)
for
i
in
range
(
4
)
:
g
=
np
.
where
(
self
.
qs
==
i
)
[
0
]
wh
=
np
.
where
(
self
.
times
[
g
]
>
54947
)
self
.
roll_best
[
i
]
=
self
.
do_rolltest
(
g
,
wh
)
self
.
do_photometry
(
)
for
i
in
range
(
4
)
:
g
=
np
.
where
(
self
.
qs
==
i
)
[
0
]
plt
.
errorbar
(
self
.
times
[
g
]
,
self
.
obs_flux
[
g
]
,
yerr
=
self
.
flux_uncert
[
i
]
,
fmt
=
fmt
[
i
]
)
plt
.
xlabel
(
'Time'
,
fontsize
=
20
)
plt
.
ylabel
(
'Relative Flux'
,
fontsize
=
20
)
plt
.
subplot
(
121
)
implot
=
plt
.
imshow
(
img
,
interpolation
=
'nearest'
,
cmap
=
'gray'
,
vmin
=
98000
*
52
,
vmax
=
104000
*
52
)
cid
=
fig
.
canvas
.
mpl_connect
(
'button_press_event'
,
self
.
onclick
)
plt
.
show
(
block
=
True
)

Creates the figure shown in adjust_aperture for visualization purposes . Called by other functions and generally not called by the user directly .


def calc_centroids(self):
        """
        Identify the centroid positions for the target star at all epochs. Useful for verifying that there is
        no correlation between flux and position, as might be expected for high proper motion stars.
        """
        self.cm = np.zeros((len(self.postcard), 2))
        for i in range(len(self.postcard)):
            target = self.postcard[i]
            target[self.targets != 1] = 0.0
            self.cm[i] = center_of_mass(target)

def calc_centroids ( self ) : self . cm = np . zeros ( ( len ( self . postcard ) , 2 ) ) for i in range ( len ( self . postcard ) ) : target = self . postcard [ i ] target [ self . targets != 1 ] = 0.0 self . cm [ i ] = center_of_mass ( target )

Identify the centroid positions for the target star at all epochs . Useful for verifying that there is no correlation between flux and position as might be expected for high proper motion stars .


def define_spotsignal(self):
        """
        Identify the "expected" flux value at the time of each observation based on the
        Kepler long-cadence data, to ensure variations observed are not the effects of a single
        large starspot. Only works if the target star was targeted for long or short cadence
        observations during the primary mission.
        """
        client = kplr.API()
        star = client.star(self.kic)

        lcs = star.get_light_curves(short_cadence=False)
        time, flux, ferr, qual = [], [], [], []
        for lc in lcs:
            with lc.open() as f:
                hdu_data = f[1].data
                time.append(hdu_data["time"])
                flux.append(hdu_data["pdcsap_flux"])
                ferr.append(hdu_data["pdcsap_flux_err"])
                qual.append(hdu_data["sap_quality"])
            tout = np.array([])
            fout = np.array([])
            eout = np.array([])
            for i in range(len(flux)):
                t = time[i][qual[i] == 0]
                f = flux[i][qual[i] == 0]
                e = ferr[i][qual[i] == 0]

                t = t[np.isfinite(f)]
                e = e[np.isfinite(f)]
                f = f[np.isfinite(f)]

                e /= np.median(f)
                f /= np.median(f)
                tout = np.append(tout, t[50:]+54833)
                fout = np.append(fout, f[50:])
                eout = np.append(eout, e[50:])

            self.spot_signal = np.zeros(52)

            for i in range(len(self.times)):
                if self.times[i] < 55000:
                    self.spot_signal[i] = 1.0
                else:
                    self.spot_signal[i] = fout[np.abs(self.times[i] - tout) == np.min(np.abs(self.times[i] - tout))]

def define_spotsignal ( self ) : client = kplr . API ( ) star = client . star ( self . kic ) lcs = star . get_light_curves ( short_cadence = False ) time , flux , ferr , qual = [ ] , [ ] , [ ] , [ ] for lc in lcs : with lc . open ( ) as f : hdu_data = f [ 1 ] . data time . append ( hdu_data [ "time" ] ) flux . append ( hdu_data [ "pdcsap_flux" ] ) ferr . append ( hdu_data [ "pdcsap_flux_err" ] ) qual . append ( hdu_data [ "sap_quality" ] ) tout = np . array ( [ ] ) fout = np . array ( [ ] ) eout = np . array ( [ ] ) for i in range ( len ( flux ) ) : t = time [ i ] [ qual [ i ] == 0 ] f = flux [ i ] [ qual [ i ] == 0 ] e = ferr [ i ] [ qual [ i ] == 0 ] t = t [ np . isfinite ( f ) ] e = e [ np . isfinite ( f ) ] f = f [ np . isfinite ( f ) ] e /= np . median ( f ) f /= np . median ( f ) tout = np . append ( tout , t [ 50 : ] + 54833 ) fout = np . append ( fout , f [ 50 : ] ) eout = np . append ( eout , e [ 50 : ] ) self . spot_signal = np . zeros ( 52 ) for i in range ( len ( self . times ) ) : if self . times [ i ] < 55000 : self . spot_signal [ i ] = 1.0 else : self . spot_signal [ i ] = fout [ np . abs ( self . times [ i ] - tout ) == np . min ( np . abs ( self . times [ i ] - tout ) ) ]

Identify the expected flux value at the time of each observation based on the Kepler long - cadence data to ensure variations observed are not the effects of a single large starspot . Only works if the target star was targeted for long or short cadence observations during the primary mission .


def model_uncert(self):
        """
        Estimate the photometric uncertainties on each data point following Equation A.2 of The Paper.
        Based on the kepcal package of Dan Foreman-Mackey.
        """
        Y = self.photometry_array.T
        Y /= np.median(Y, axis=1)[:, None]
        C = np.median(Y, axis=0)

        nstars, nobs = np.shape(Y)

        Z = np.empty((nstars, 4))

        qs = self.qs.astype(int)

        for s in range(4):
            Z[:, s] = np.median((Y / C)[:, qs == s], axis=1)

        resid2 = (Y - Z[:, qs] * C)**2
        z = Z[:, qs]
        trend = z * C[None, :]

        lnS = np.log(np.nanmedian(resid2, axis=0))
        jitter = np.log(0.1*np.nanmedian(np.abs(np.diff(Y, axis=1))))

        cal_ferr = np.sqrt(np.exp(2*(jitter/trend))+z**2*np.exp(lnS)[None, :])

        self.modeled_uncert = cal_ferr
        self.target_uncert = cal_ferr[0]

def model_uncert ( self ) : Y = self . photometry_array . T Y /= np . median ( Y , axis = 1 ) [ : , None ] C = np . median ( Y , axis = 0 ) nstars , nobs = np . shape ( Y ) Z = np . empty ( ( nstars , 4 ) ) qs = self . qs . astype ( int ) for s in range ( 4 ) : Z [ : , s ] = np . median ( ( Y / C ) [ : , qs == s ] , axis = 1 ) resid2 = ( Y - Z [ : , qs ] * C ) ** 2 z = Z [ : , qs ] trend = z * C [ None , : ] lnS = np . log ( np . nanmedian ( resid2 , axis = 0 ) ) jitter = np . log ( 0.1 * np . nanmedian ( np . abs ( np . diff ( Y , axis = 1 ) ) ) ) cal_ferr = np . sqrt ( np . exp ( 2 * ( jitter / trend ) ) + z ** 2 * np . exp ( lnS ) [ None , : ] ) self . modeled_uncert = cal_ferr self . target_uncert = cal_ferr [ 0 ]

Estimate the photometric uncertainties on each data point following Equation A . 2 of The Paper . Based on the kepcal package of Dan Foreman - Mackey .


def _dump_field(self, fd):
        """Dump single field.
        """
        v = {}
        v['label'] = Pbd.LABELS[fd.label]
        v['type'] = fd.type_name if len(fd.type_name) > 0 else Pbd.TYPES[fd.type]
        v['name'] = fd.name
        v['number'] = fd.number
        v['default'] = '[default = {}]'.format(fd.default_value) if len(fd.default_value) > 0 else ''

        f = '{label} {type} {name} = {number} {default};'.format(**v)
        f = ' '.join(f.split())
        self._print(f)

        if len(fd.type_name) > 0:
            self.uses.append(fd.type_name)


def disassemble(self):
        """Disassemble serialized protocol buffers file.
        """
        ser_pb = open(self.input_file, 'rb').read()  # Read serialized pb file

        fd = FileDescriptorProto()
        fd.ParseFromString(ser_pb)
        self.name = fd.name

        self._print('// Reversed by pbd (https://github.com/rsc-dev/pbd)')
        self._print('syntax = "proto2";')
        self._print('')

        if len(fd.package) > 0:
            self._print('package {};'.format(fd.package))
            self.package = fd.package
        else:
            self._print('// Package not defined')

        self._walk(fd)

Disassemble serialized protocol buffers file .


def find_imports(self, pbds):
        """Find all missing imports in list of Pbd instances.
        """
        # List of types used, but not defined
        imports = list(set(self.uses).difference(set(self.defines)))

        # Clumpsy, but enought for now
        for imp in imports:
            for p in pbds:
                if imp in p.defines:
                    self.imports.append(p.name)
                    break

        self.imports = list(set(self.imports))

        for import_file in self.imports:
            self.lines.insert(2, 'import "{}";'.format(import_file))

def
find_imports
(
self
,
pbds
)
:
imports
=
list
(
set
(
self
.
uses
)
.
difference
(
set
(
self
.
defines
)
)
)
for
imp
in
imports
:
for
p
in
pbds
:
if
imp
in
p
.
defines
:
self
.
imports
.
append
(
p
.
name
)
break
self
.
imports
=
list
(
set
(
self
.
imports
)
)
for
import_file
in
self
.
imports
:
self
.
lines
.
insert
(
2
,
'import "{}";'
.
format
(
import_file
)
)

Find all missing imports in list of Pbd instances .


def fasta_dict_to_file(fasta_dict, fasta_file, line_char_limit=None):
    """Write fasta_dict to fasta_file

    :param fasta_dict: returned by fasta_file_to_dict
    :param fasta_file: output file can be a string path or a file object
    :param line_char_limit: None = no limit (default)
    :return: None
    """
    fasta_fp = fasta_file
    if isinstance(fasta_file, str):
        fasta_fp = open(fasta_file, 'wb')

    for key in fasta_dict:
        seq = fasta_dict[key]['seq']
        if line_char_limit:
            seq = '\n'.join([seq[i:i+line_char_limit] for i in range(0, len(seq), line_char_limit)])
        fasta_fp.write(u'{0:s}\n{1:s}\n'.format(fasta_dict[key]['header'], seq))

def fasta_dict_to_file ( fasta_dict , fasta_file , line_char_limit = None ) : fasta_fp = fasta_file if isinstance ( fasta_file , str ) : fasta_fp = open ( fasta_file , 'wb' ) for key in fasta_dict : seq = fasta_dict [ key ] [ 'seq' ] if line_char_limit : seq = '\n' . join ( [ seq [ i : i + line_char_limit ] for i in range ( 0 , len ( seq ) , line_char_limit ) ] ) fasta_fp . write ( u'{0:s}\n{1:s}\n' . format ( fasta_dict [ key ] [ 'header' ] , seq ) )