def running_under_virtualenv():
    """
    Return True if we're running inside a virtualenv, False otherwise.

    """
    if hasattr(sys, 'real_prefix'):
        return True
    elif sys.prefix != getattr(sys, "base_prefix", sys.prefix):
        return True

    return False

def running_under_virtualenv ( ) : if hasattr ( sys , 'real_prefix' ) : return True elif sys . prefix != getattr ( sys , "base_prefix" , sys . prefix ) : return True return False

Return True if we re running inside a virtualenv False otherwise .


def __get_username():
    """ Returns the effective username of the current process. """
    if WINDOWS:
        return getpass.getuser()
    import pwd
    return pwd.getpwuid(os.geteuid()).pw_name

def __get_username ( ) : if WINDOWS : return getpass . getuser ( ) import pwd return pwd . getpwuid ( os . geteuid ( ) ) . pw_name

Returns the effective username of the current process .


def distutils_scheme(dist_name, user=False, home=None, root=None,
                     isolated=False):
    """
    Return a distutils install scheme
    """
    from distutils.dist import Distribution

    scheme = {}

    if isolated:
        extra_dist_args = {"script_args": ["--no-user-cfg"]}
    else:
        extra_dist_args = {}
    dist_args = {'name': dist_name}
    dist_args.update(extra_dist_args)

    d = Distribution(dist_args)
    d.parse_config_files()
    i = d.get_command_obj('install', create=True)
    # NOTE: setting user or home has the side-effect of creating the home dir
    # or user base for installations during finalize_options()
    # ideally, we'd prefer a scheme class that has no side-effects.
    i.user = user or i.user
    i.home = home or i.home
    i.root = root or i.root
    i.finalize_options()
    for key in SCHEME_KEYS:
        scheme[key] = getattr(i, 'install_' + key)

    if i.install_lib is not None:
        # install_lib takes precedence over purelib and platlib
        scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib))

    if running_under_virtualenv():
        scheme['headers'] = os.path.join(
            sys.prefix,
            'include',
            'site',
            'python' + sys.version[:3],
            dist_name,
        )

        if root is not None:
            scheme["headers"] = os.path.join(
                root,
                os.path.abspath(scheme["headers"])[1:],
            )

    return scheme

def distutils_scheme ( dist_name , user = False , home = None , root = None , isolated = False ) : from distutils . dist import Distribution scheme = { } if isolated : extra_dist_args = { "script_args" : [ "--no-user-cfg" ] } else : extra_dist_args = { } dist_args = { 'name' : dist_name } dist_args . update ( extra_dist_args ) d = Distribution ( dist_args ) d . parse_config_files ( ) i = d . get_command_obj ( 'install' , create = True ) i . user = user or i . user i . home = home or i . home i . root = root or i . root i . finalize_options ( ) for key in SCHEME_KEYS : scheme [ key ] = getattr ( i , 'install_' + key ) if i . install_lib is not None : scheme . update ( dict ( purelib = i . install_lib , platlib = i . install_lib ) ) if running_under_virtualenv ( ) : scheme [ 'headers' ] = os . path . join ( sys . prefix , 'include' , 'site' , 'python' + sys . version [ : 3 ] , dist_name , ) if root is not None : scheme [ "headers" ] = os . path . join ( root , os . path . abspath ( scheme [ "headers" ] ) [ 1 : ] , ) return scheme


def parse_cache_control(self, headers):
        """
        Parse the cache control headers returning a dictionary with values
        for the different directives.
        """
        retval = {}

        cc_header = 'cache-control'
        if 'Cache-Control' in headers:
            cc_header = 'Cache-Control'

        if cc_header in headers:
            parts = headers[cc_header].split(',')
            parts_with_args = [
                tuple([x.strip().lower() for x in part.split("=", 1)])
                for part in parts if -1 != part.find("=")
            ]
            parts_wo_args = [
                (name.strip().lower(), 1)
                for name in parts if -1 == name.find("=")
            ]
            retval = dict(parts_with_args + parts_wo_args)
        return retval

def parse_cache_control ( self , headers ) : retval = { } cc_header = 'cache-control' if 'Cache-Control' in headers : cc_header = 'Cache-Control' if cc_header in headers : parts = headers [ cc_header ] . split ( ',' ) parts_with_args = [ tuple ( [ x . strip ( ) . lower ( ) for x in part . split ( "=" , 1 ) ] ) for part in parts if - 1 != part . find ( "=" ) ] parts_wo_args = [ ( name . strip ( ) . lower ( ) , 1 ) for name in parts if - 1 == name . find ( "=" ) ] retval = dict ( parts_with_args + parts_wo_args ) return retval

Parse the cache control headers returning a dictionary with values for the different directives .


def cached_request(self, request):
        """
        Return a cached response if it exists in the cache, otherwise
        return False.
        """
        cache_url = self.cache_url(request.url)
        cc = self.parse_cache_control(request.headers)

        # non-caching states
        no_cache = True if 'no-cache' in cc else False
        if 'max-age' in cc and cc['max-age'] == 0:
            no_cache = True

        # Bail out if no-cache was set
        if no_cache:
            return False

        # It is in the cache, so lets see if it is going to be
        # fresh enough
        resp = self.serializer.loads(request, self.cache.get(cache_url))

        # Check to see if we have a cached object
        if not resp:
            return False

        # If we have a cached 301, return it immediately. We don't
        # need to test our response for other headers b/c it is
        # intrinsically "cacheable" as it is Permanent.
        # See:
        #   https://tools.ietf.org/html/rfc7231#section-6.4.2
        #
        # Client can try to refresh the value by repeating the request
        # with cache busting headers as usual (ie no-cache).
        if resp.status == 301:
            return resp

        headers = CaseInsensitiveDict(resp.headers)
        if not headers or 'date' not in headers:
            # With date or etag, the cached response can never be used
            # and should be deleted.
            if 'etag' not in headers:
                self.cache.delete(cache_url)
            return False

        now = time.time()
        date = calendar.timegm(
            parsedate_tz(headers['date'])
        )
        current_age = max(0, now - date)

        # TODO: There is an assumption that the result will be a
        #       urllib3 response object. This may not be best since we
        #       could probably avoid instantiating or constructing the
        #       response until we know we need it.
        resp_cc = self.parse_cache_control(headers)

        # determine freshness
        freshness_lifetime = 0

        # Check the max-age pragma in the cache control header
        if 'max-age' in resp_cc and resp_cc['max-age'].isdigit():
            freshness_lifetime = int(resp_cc['max-age'])

        # If there isn't a max-age, check for an expires header
        elif 'expires' in headers:
            expires = parsedate_tz(headers['expires'])
            if expires is not None:
                expire_time = calendar.timegm(expires) - date
                freshness_lifetime = max(0, expire_time)

        # determine if we are setting freshness limit in the req
        if 'max-age' in cc:
            try:
                freshness_lifetime = int(cc['max-age'])
            except ValueError:
                freshness_lifetime = 0

        if 'min-fresh' in cc:
            try:
                min_fresh = int(cc['min-fresh'])
            except ValueError:
                min_fresh = 0
            # adjust our current age by our min fresh
            current_age += min_fresh

        # see how fresh we actually are
        fresh = (freshness_lifetime > current_age)

        if fresh:
            return resp

        # we're not fresh. If we don't have an Etag, clear it out
        if 'etag' not in headers:
            self.cache.delete(cache_url)

        # return the original handler
        return False

def cached_request ( self , request ) : cache_url = self . cache_url ( request . url ) cc = self . parse_cache_control ( request . headers ) no_cache = True if 'no-cache' in cc else False if 'max-age' in cc and cc [ 'max-age' ] == 0 : no_cache = True if no_cache : return False resp = self . serializer . loads ( request , self . cache . get ( cache_url ) ) if not resp : return False if resp . status == 301 : return resp headers = CaseInsensitiveDict ( resp . headers ) if not headers or 'date' not in headers : if 'etag' not in headers : self . cache . delete ( cache_url ) return False now = time . time ( ) date = calendar . timegm ( parsedate_tz ( headers [ 'date' ] ) ) current_age = max ( 0 , now - date ) resp_cc = self . parse_cache_control ( headers ) freshness_lifetime = 0 if 'max-age' in resp_cc and resp_cc [ 'max-age' ] . isdigit ( ) : freshness_lifetime = int ( resp_cc [ 'max-age' ] ) elif 'expires' in headers : expires = parsedate_tz ( headers [ 'expires' ] ) if expires is not None : expire_time = calendar . timegm ( expires ) - date freshness_lifetime = max ( 0 , expire_time ) if 'max-age' in cc : try : freshness_lifetime = int ( cc [ 'max-age' ] ) except ValueError : freshness_lifetime = 0 if 'min-fresh' in cc : try : min_fresh = int ( cc [ 'min-fresh' ] ) except ValueError : min_fresh = 0 current_age += min_fresh fresh = ( freshness_lifetime > current_age ) if fresh : return resp if 'etag' not in headers : self . cache . delete ( cache_url ) return False

Return a cached response if it exists in the cache otherwise return False .


def cache_response(self, request, response, body=None):
        """
        Algorithm for caching requests.

        This assumes a requests Response object.
        """
        # From httplib2: Don't cache 206's since we aren't going to
        #                handle byte range requests
        if response.status not in [200, 203, 300, 301]:
            return

        response_headers = CaseInsensitiveDict(response.headers)

        cc_req = self.parse_cache_control(request.headers)
        cc = self.parse_cache_control(response_headers)

        cache_url = self.cache_url(request.url)

        # Delete it from the cache if we happen to have it stored there
        no_store = cc.get('no-store') or cc_req.get('no-store')
        if no_store and self.cache.get(cache_url):
            self.cache.delete(cache_url)

        # If we've been given an etag, then keep the response
        if self.cache_etags and 'etag' in response_headers:
            self.cache.set(
                cache_url,
                self.serializer.dumps(request, response, body=body),
            )

        # Add to the cache any 301s. We do this before looking that
        # the Date headers.
        elif response.status == 301:
            self.cache.set(
                cache_url,
                self.serializer.dumps(request, response)
            )

        # Add to the cache if the response headers demand it. If there
        # is no date header then we can't do anything about expiring
        # the cache.
        elif 'date' in response_headers:
            # cache when there is a max-age > 0
            if cc and cc.get('max-age'):
                if int(cc['max-age']) > 0:
                    self.cache.set(
                        cache_url,
                        self.serializer.dumps(request, response, body=body),
                    )

            # If the request can expire, it means we should cache it
            # in the meantime.
            elif 'expires' in response_headers:
                if response_headers['expires']:
                    self.cache.set(
                        cache_url,
                        self.serializer.dumps(request, response, body=body),
                    )

def cache_response ( self , request , response , body = None ) : if response . status not in [ 200 , 203 , 300 , 301 ] : return response_headers = CaseInsensitiveDict ( response . headers ) cc_req = self . parse_cache_control ( request . headers ) cc = self . parse_cache_control ( response_headers ) cache_url = self . cache_url ( request . url ) no_store = cc . get ( 'no-store' ) or cc_req . get ( 'no-store' ) if no_store and self . cache . get ( cache_url ) : self . cache . delete ( cache_url ) if self . cache_etags and 'etag' in response_headers : self . cache . set ( cache_url , self . serializer . dumps ( request , response , body = body ) , ) elif response . status == 301 : self . cache . set ( cache_url , self . serializer . dumps ( request , response ) ) elif 'date' in response_headers : if cc and cc . get ( 'max-age' ) : if int ( cc [ 'max-age' ] ) > 0 : self . cache . set ( cache_url , self . serializer . dumps ( request , response , body = body ) , ) elif 'expires' in response_headers : if response_headers [ 'expires' ] : self . cache . set ( cache_url , self . serializer . dumps ( request , response , body = body ) , )


def _update_zipimporter_cache(normalized_path, cache, updater=None):
    """
    Update zipimporter cache data for a given normalized path.

    Any sub-path entries are processed as well, i.e. those corresponding to zip
    archives embedded in other zip archives.

    Given updater is a callable taking a cache entry key and the original entry
    (after already removing the entry from the cache), and expected to update
    the entry and possibly return a new one to be inserted in its place.
    Returning None indicates that the entry should not be replaced with a new
    one. If no updater is given, the cache entries are simply removed without
    any additional processing, the same as if the updater simply returned None.

    """
    for p in _collect_zipimporter_cache_entries(normalized_path, cache):
        # N.B. pypy's custom zipimport._zip_directory_cache implementation does
        # not support the complete dict interface:
        # * Does not support item assignment, thus not allowing this function
        #    to be used only for removing existing cache entries.
        #  * Does not support the dict.pop() method, forcing us to use the
        #    get/del patterns instead. For more detailed information see the
        #    following links:
        #      https://bitbucket.org/pypa/setuptools/issue/202/more-robust-zipimporter-cache-invalidation#comment-10495960
        #      https://bitbucket.org/pypy/pypy/src/dd07756a34a41f674c0cacfbc8ae1d4cc9ea2ae4/pypy/module/zipimport/interp_zipimport.py#cl-99
        old_entry = cache[p]
        del cache[p]
        new_entry = updater and updater(p, old_entry)
        if new_entry is not None:
            cache[p] = new_entry

def _update_zipimporter_cache ( normalized_path , cache , updater = None ) : for p in _collect_zipimporter_cache_entries ( normalized_path , cache ) : old_entry = cache [ p ] del cache [ p ] new_entry = updater and updater ( p , old_entry ) if new_entry is not None : cache [ p ] = new_entry

Update zipimporter cache data for a given normalized path .


def _load_template(dev_path):
        """
        There are a couple of template scripts in the package. This
        function loads one of them and prepares it for use.
        """
        # See https://bitbucket.org/pypa/setuptools/issue/134 for info
        # on script file naming and downstream issues with SVR4
        name = 'script.tmpl'
        if dev_path:
            name = name.replace('.tmpl', ' (dev).tmpl')

        raw_bytes = resource_string('setuptools', name)
        return raw_bytes.decode('utf-8')

def
_load_template
(
dev_path
)
:
name
=
'script.tmpl'
if
dev_path
:
name
=
name
.
replace
(
'.tmpl'
,
' (dev).tmpl'
)
raw_bytes
=
resource_string
(
'setuptools'
,
name
)
return
raw_bytes
.
decode
(
'utf-8'
)

There are a couple of template scripts in the package . This function loads one of them and prepares it for use .


def install_site_py(self):
        """Make sure there's a site.py in the target dir, if needed"""

        if self.sitepy_installed:
            return  # already did it, or don't need to

        sitepy = os.path.join(self.install_dir, "site.py")
        source = resource_string("setuptools", "site-patch.py")
        current = ""

        if os.path.exists(sitepy):
            log.debug("Checking existing site.py in %s", self.install_dir)
            f = open(sitepy, 'rb')
            current = f.read()
            # we want str, not bytes
            if PY3:
                current = current.decode()

            f.close()
            if not current.startswith('def __boot():'):
                raise DistutilsError(
                    "%s is not a setuptools-generated site.py; please"
                    " remove it." % sitepy
                )

        if current != source:
            log.info("Creating %s", sitepy)
            if not self.dry_run:
                ensure_directory(sitepy)
                f = open(sitepy, 'wb')
                f.write(source)
                f.close()
            self.byte_compile([sitepy])

        self.sitepy_installed = True

Make sure there s a site . py in the target dir if needed


def save(self):
        """Write changed .pth file back to disk"""
        if not self.dirty:
            return

        data = '\n'.join(map(self.make_relative, self.paths))
        if data:
            log.debug("Saving %s", self.filename)
            data = (
                "import sys; sys.__plen = len(sys.path)\n"
                "%s\n"
                "import sys; new=sys.path[sys.__plen:];"
                " del sys.path[sys.__plen:];"
                " p=getattr(sys,'__egginsert',0); sys.path[p:p]=new;"
                " sys.__egginsert = p+len(new)\n"
            ) % data

            if os.path.islink(self.filename):
                os.unlink(self.filename)
            f = open(self.filename, 'wt')
            f.write(data)
            f.close()

        elif os.path.exists(self.filename):
            log.debug("Deleting empty %s", self.filename)
            os.unlink(self.filename)

        self.dirty = False


def convert(self, value):
        """
        Convert values to an appropriate type. dicts, lists and tuples are
        replaced by their converting alternatives. Strings are checked to
        see if they have a conversion format and are converted if they do.
        """
        if not isinstance(value, ConvertingDict) and isinstance(value, dict):
            value = ConvertingDict(value)
            value.configurator = self
        elif not isinstance(value, ConvertingList) and isinstance(value, list):
            value = ConvertingList(value)
            value.configurator = self
        elif not isinstance(value, ConvertingTuple) and\
                 isinstance(value, tuple):
            value = ConvertingTuple(value)
            value.configurator = self
        elif isinstance(value, six.string_types):  # str for py3k
            m = self.CONVERT_PATTERN.match(value)
            if m:
                d = m.groupdict()
                prefix = d['prefix']
                converter = self.value_converters.get(prefix, None)
                if converter:
                    suffix = d['suffix']
                    converter = getattr(self, converter)
                    value = converter(suffix)
        return value

def convert ( self , value ) : if not isinstance ( value , ConvertingDict ) and isinstance ( value , dict ) : value = ConvertingDict ( value ) value . configurator = self elif not isinstance ( value , ConvertingList ) and isinstance ( value , list ) : value = ConvertingList ( value ) value . configurator = self elif not isinstance ( value , ConvertingTuple ) and isinstance ( value , tuple ) : value = ConvertingTuple ( value ) value . configurator = self elif isinstance ( value , six . string_types ) : m = self . CONVERT_PATTERN . match ( value ) if m : d = m . groupdict ( ) prefix = d [ 'prefix' ] converter = self . value_converters . get ( prefix , None ) if converter : suffix = d [ 'suffix' ] converter = getattr ( self , converter ) value = converter ( suffix ) return value

Convert values to an appropriate type . dicts lists and tuples are replaced by their converting alternatives . Strings are checked to see if they have a conversion format and are converted if they do .


def add_filters(self, filterer, filters):
        """Add filters to a filterer from a list of names."""
        for f in filters:
            try:
                filterer.addFilter(self.config['filters'][f])
            except StandardError as e:
                raise ValueError('Unable to add filter %r: %s' % (f, e))

def
add_filters
(
self
,
filterer
,
filters
)
:
for
f
in
filters
:
try
:
filterer
.
addFilter
(
self
.
config
[
'filters'
]
[
f
]
)
except
StandardError
as
e
:
raise
ValueError
(
'Unable to add filter %r: %s'
%
(
f
,
e
)
)

Add filters to a filterer from a list of names .


def configure_handler(self, config):
        """Configure a handler from a dictionary."""
        formatter = config.pop('formatter', None)
        if formatter:
            try:
                formatter = self.config['formatters'][formatter]
            except StandardError as e:
                raise ValueError('Unable to set formatter '
                                 '%r: %s' % (formatter, e))
        level = config.pop('level', None)
        filters = config.pop('filters', None)
        if '()' in config:
            c = config.pop('()')
            if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType:
                c = self.resolve(c)
            factory = c
        else:
            klass = self.resolve(config.pop('class'))
            # Special case for handler which refers to another handler
            if issubclass(klass, logging.handlers.MemoryHandler) and\
                'target' in config:
                try:
                    config['target'] = self.config['handlers'][config['target']]
                except StandardError as e:
                    raise ValueError('Unable to set target handler '
                                     '%r: %s' % (config['target'], e))
            elif issubclass(klass, logging.handlers.SMTPHandler) and\
                'mailhost' in config:
                config['mailhost'] = self.as_tuple(config['mailhost'])
            elif issubclass(klass, logging.handlers.SysLogHandler) and\
                'address' in config:
                config['address'] = self.as_tuple(config['address'])
            factory = klass
        kwargs = dict((k, config[k]) for k in config if valid_ident(k))
        try:
            result = factory(**kwargs)
        except TypeError as te:
            if "'stream'" not in str(te):
                raise
            # The argument name changed from strm to stream
            # Retry with old name.
            # This is so that code can be used with older Python versions
            #(e.g. by Django)
            kwargs['strm'] = kwargs.pop('stream')
            result = factory(**kwargs)
        if formatter:
            result.setFormatter(formatter)
        if level is not None:
            result.setLevel(_checkLevel(level))
        if filters:
            self.add_filters(result, filters)
        return result

def
configure_handler
(
self
,
config
)
:
formatter
=
config
.
pop
(
'formatter'
,
)
if
formatter
:
try
:
formatter
=
self
.
config
[
'formatters'
]
[
formatter
]
except
StandardError
as
e
:
raise
ValueError
(
'Unable to set formatter '
'%r: %s'
%
(
formatter
,
e
)
)
level
=
config
.
pop
(
'level'
,
)
filters
=
config
.
pop
(
'filters'
,
)
if
'()'
in
config
:
c
=
config
.
pop
(
'()'
)
if
not
hasattr
(
c
,
'__call__'
)
and
hasattr
(
types
,
'ClassType'
)
and
type
(
c
)
!=
types
.
ClassType
:
c
=
self
.
resolve
(
c
)
factory
=
c
else
:
klass
=
self
.
resolve
(
config
.
pop
(
'class'
)
)
if
issubclass
(
klass
,
logging
.
handlers
.
MemoryHandler
)
and
'target'
in
config
:
try
:
config
[
'target'
]
=
self
.
config
[
'handlers'
]
[
config
[
'target'
]
]
except
StandardError
as
e
:
raise
ValueError
(
'Unable to set target handler '
'%r: %s'
%
(
config
[
'target'
]
,
e
)
)
elif
issubclass
(
klass
,
logging
.
handlers
.
SMTPHandler
)
and
'mailhost'
in
config
:
config
[
'mailhost'
]
=
self
.
as_tuple
(
config
[
'mailhost'
]
)
elif
issubclass
(
klass
,
logging
.
handlers
.
SysLogHandler
)
and
'address'
in
config
:
config
[
'address'
]
=
self
.
as_tuple
(
config
[
'address'
]
)
factory
=
klass
kwargs
=
dict
(
(
k
,
config
[
k
]
)
for
k
in
config
if
valid_ident
(
k
)
)
try
:
result
=
factory
(
**
kwargs
)
except
TypeError
as
te
:
if
"'stream'"
not
in
str
(
te
)
:
raise
kwargs
[
'strm'
]
=
kwargs
.
pop
(
'stream'
)
result
=
factory
(
**
kwargs
)
if
formatter
:
result
.
setFormatter
(
formatter
)
if
level
is
not
:
result
.
setLevel
(
_checkLevel
(
level
)
)
if
filters
:
self
.
add_filters
(
result
,
filters
)
return
result


def add_handlers(self, logger, handlers):
        """Add handlers to a logger from a list of names."""
        for h in handlers:
            try:
                logger.addHandler(self.config['handlers'][h])
            except StandardError as e:
                raise ValueError('Unable to add handler %r: %s' % (h, e))

def
add_handlers
(
self
,
logger
,
handlers
)
:
for
h
in
handlers
:
try
:
logger
.
addHandler
(
self
.
config
[
'handlers'
]
[
h
]
)
except
StandardError
as
e
:
raise
ValueError
(
'Unable to add handler %r: %s'
%
(
h
,
e
)
)

Add handlers to a logger from a list of names .


def common_logger_config(self, logger, config, incremental=False):
        """
        Perform configuration which is common to root and non-root loggers.
        """
        level = config.get('level', None)
        if level is not None:
            logger.setLevel(_checkLevel(level))
        if not incremental:
            # Remove any existing handlers
            for h in logger.handlers[:]:
                logger.removeHandler(h)
            handlers = config.get('handlers', None)
            if handlers:
                self.add_handlers(logger, handlers)
            filters = config.get('filters', None)
            if filters:
                self.add_filters(logger, filters)

def common_logger_config ( self , logger , config , incremental = False ) : level = config . get ( 'level' , None ) if level is not None : logger . setLevel ( _checkLevel ( level ) ) if not incremental : for h in logger . handlers [ : ] : logger . removeHandler ( h ) handlers = config . get ( 'handlers' , None ) if handlers : self . add_handlers ( logger , handlers ) filters = config . get ( 'filters' , None ) if filters : self . add_filters ( logger , filters )

Perform configuration which is common to root and non - root loggers .


def _execfile(filename, globals, locals=None):
    """
    Python 3 implementation of execfile.
    """
    mode = 'rb'
    with open(filename, mode) as stream:
        script = stream.read()
    # compile() function in Python 2.6 and 3.1 requires LF line endings.
    if sys.version_info[:2] < (2, 7) or sys.version_info[:2] >= (3, 0) and sys.version_info[:2] < (3, 2):
        script = script.replace(b'\r\n', b'\n')
        script = script.replace(b'\r', b'\n')
    if locals is None:
        locals = globals
    code = compile(script, filename, 'exec')
    exec(code, globals, locals)

def _execfile ( filename , globals , locals = None ) : mode = 'rb' with open ( filename , mode ) as stream : script = stream . read ( ) if sys . version_info [ : 2 ] < ( 2 , 7 ) or sys . version_info [ : 2 ] >= ( 3 , 0 ) and sys . version_info [ : 2 ] < ( 3 , 2 ) : script = script . replace ( b'\r\n' , b'\n' ) script = script . replace ( b'\r' , b'\n' ) if locals is None : locals = globals code = compile ( script , filename , 'exec' ) exec ( code , globals , locals )


def override_temp(replacement):
    """
    Monkey-patch tempfile.tempdir with replacement, ensuring it exists
    """
    if not os.path.isdir(replacement):
        os.makedirs(replacement)

    saved = tempfile.tempdir

    tempfile.tempdir = replacement

    try:
        yield
    finally:
        tempfile.tempdir = saved

def override_temp ( replacement ) : if not os . path . isdir ( replacement ) : os . makedirs ( replacement ) saved = tempfile . tempdir tempfile . tempdir = replacement try : yield finally : tempfile . tempdir = saved

Monkey - patch tempfile . tempdir with replacement ensuring it exists


def get_url_rev(self):
        """
        Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'.
        That's required because although they use SSH they sometimes doesn't
        work with a ssh:// scheme (e.g. Github). But we need a scheme for
        parsing. Hence we remove it again afterwards and return it as a stub.
        """
        if '://' not in self.url:
            assert 'file:' not in self.url
            self.url = self.url.replace('git+', 'git+ssh://')
            url, rev = super(Git, self).get_url_rev()
            url = url.replace('ssh://', '')
        else:
            url, rev = super(Git, self).get_url_rev()

        return url, rev

def get_url_rev ( self ) : if '://' not in self . url : assert 'file:' not in self . url self . url = self . url . replace ( 'git+' , 'git+ssh://' ) url , rev = super ( Git , self ) . get_url_rev ( ) url = url . replace ( 'ssh://' , '' ) else : url , rev = super ( Git , self ) . get_url_rev ( ) return url , rev


def getitem(self, obj, argument):
        """Get an item or attribute of an object but prefer the item."""
        try:
            return obj[argument]
        except (TypeError, LookupError):
            if isinstance(argument, string_types):
                try:
                    attr = str(argument)
                except Exception:
                    pass
                else:
                    try:
                        return getattr(obj, attr)
                    except AttributeError:
                        pass
            return self.undefined(obj=obj, name=argument)

def getitem ( self , obj , argument ) : try : return obj [ argument ] except ( TypeError , LookupError ) : if isinstance ( argument , string_types ) : try : attr = str ( argument ) except Exception : pass else : try : return getattr ( obj , attr ) except AttributeError : pass return self . undefined ( obj = obj , name = argument )

Get an item or attribute of an object but prefer the item .


def _generate(self, source, name, filename, defer_init=False):
        """Internal hook that can be overridden to hook a different generate
        method in.

        .. versionadded:: 2.5
        """
        return generate(source, self, name, filename, defer_init=defer_init)

def _generate ( self , source , name , filename , defer_init = False ) : return generate ( source , self , name , filename , defer_init = defer_init )

Internal hook that can be overridden to hook a different generate method in .


def compile_templates(self, target, extensions=None, filter_func=None,
                          zip='deflated', log_function=None,
                          ignore_errors=True, py_compile=False):
        """Finds all the templates the loader can find, compiles them
        and stores them in `target`.  If `zip` is `None`, instead of in a
        zipfile, the templates will be will be stored in a directory.
        By default a deflate zip algorithm is used, to switch to
        the stored algorithm, `zip` can be set to ``'stored'``.

        `extensions` and `filter_func` are passed to :meth:`list_templates`.
        Each template returned will be compiled to the target folder or
        zipfile.

        By default template compilation errors are ignored.  In case a
        log function is provided, errors are logged.  If you want template
        syntax errors to abort the compilation you can set `ignore_errors`
        to `False` and you will get an exception on syntax errors.

        If `py_compile` is set to `True` .pyc files will be written to the
        target instead of standard .py files.  This flag does not do anything
        on pypy and Python 3 where pyc files are not picked up by itself and
        don't give much benefit.

        .. versionadded:: 2.4
        """
        from jinja2.loaders import ModuleLoader

        if log_function is None:
            log_function = lambda x: None

        if py_compile:
            if not PY2 or PYPY:
                from warnings import warn
                warn(Warning('py_compile has no effect on pypy or Python 3'))
                py_compile = False
            else:
                import imp, marshal
                py_header = imp.get_magic() + \
                    u'\xff\xff\xff\xff'.encode('iso-8859-15')

                # Python 3.3 added a source filesize to the header
                if sys.version_info >= (3, 3):
                    py_header += u'\x00\x00\x00\x00'.encode('iso-8859-15')

        def write_file(filename, data, mode):
            if zip:
                info = ZipInfo(filename)
                info.external_attr = 0o755 << 16
                zip_file.writestr(info, data)
            else:
                f = open(os.path.join(target, filename), mode)
                try:
                    f.write(data)
                finally:
                    f.close()

        if zip is not None:
            from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED, ZIP_STORED
            zip_file = ZipFile(target, 'w', dict(deflated=ZIP_DEFLATED,
                                                 stored=ZIP_STORED)[zip])
            log_function('Compiling into Zip archive "%s"' % target)
        else:
            if not os.path.isdir(target):
                os.makedirs(target)
            log_function('Compiling into folder "%s"' % target)

        try:
            for name in self.list_templates(extensions, filter_func):
                source, filename, _ = self.loader.get_source(self, name)
                try:
                    code = self.compile(source, name, filename, True, True)
                except TemplateSyntaxError as e:
                    if not ignore_errors:
                        raise
                    log_function('Could not compile "%s": %s' % (name, e))
                    continue

                filename = ModuleLoader.get_module_filename(name)

                if py_compile:
                    c = self._compile(code, encode_filename(filename))
                    write_file(filename + 'c', py_header +
                               marshal.dumps(c), 'wb')
                    log_function('Byte-compiled "%s" as %s' %
                                 (name, filename + 'c'))
                else:
                    write_file(filename, code, 'w')
                    log_function('Compiled "%s" as %s' % (name, filename))
        finally:
            if zip:
                zip_file.close()

        log_function('Finished compiling templates')

Finds all the templates the loader can find compiles them and stores them in target . If zip is None instead of in a zipfile the templates will be will be stored in a directory . By default a deflate zip algorithm is used to switch to the stored algorithm zip can be set to stored .


def get_default_cache():
    """Determine the default cache location

    This returns the ``PYTHON_EGG_CACHE`` environment variable, if set.
    Otherwise, on Windows, it returns a "Python-Eggs" subdirectory of the
    "Application Data" directory.  On all other systems, it's "~/.python-eggs".
    """
    try:
        return os.environ['PYTHON_EGG_CACHE']
    except KeyError:
        pass

    if os.name!='nt':
        return os.path.expanduser('~/.python-eggs')

    # XXX this may be locale-specific!
    app_data = 'Application Data'
    app_homes = [
        # best option, should be locale-safe
        (('APPDATA',), None),
        (('USERPROFILE',), app_data),
        (('HOMEDRIVE','HOMEPATH'), app_data),
        (('HOMEPATH',), app_data),
        (('HOME',), None),
        # 95/98/ME
        (('WINDIR',), app_data),
    ]

    for keys, subdir in app_homes:
        dirname = ''
        for key in keys:
            if key in os.environ:
                dirname = os.path.join(dirname, os.environ[key])
            else:
                break
        else:
            if subdir:
                dirname = os.path.join(dirname, subdir)
            return os.path.join(dirname, 'Python-Eggs')
    else:
        raise RuntimeError(
            "Please set the PYTHON_EGG_CACHE enviroment variable"
        )


def find_eggs_in_zip(importer, path_item, only=False):
    """
    Find eggs in zip files; possibly multiple nested eggs.
    """
    if importer.archive.endswith('.whl'):
        # wheels are not supported with this finder
        # they don't have PKG-INFO metadata, and won't ever contain eggs
        return
    metadata = EggMetadata(importer)
    if metadata.has_metadata('PKG-INFO'):
        yield Distribution.from_filename(path_item, metadata=metadata)
    if only:
        # don't yield nested distros
        return
    for subitem in metadata.resource_listdir('/'):
        if subitem.endswith('.egg'):
            subpath = os.path.join(path_item, subitem)
            for dist in find_eggs_in_zip(zipimport.zipimporter(subpath), subpath):
                yield dist

def find_eggs_in_zip ( importer , path_item , only = False ) : if importer . archive . endswith ( '.whl' ) : return metadata = EggMetadata ( importer ) if metadata . has_metadata ( 'PKG-INFO' ) : yield Distribution . from_filename ( path_item , metadata = metadata ) if only : return for subitem in metadata . resource_listdir ( '/' ) : if subitem . endswith ( '.egg' ) : subpath = os . path . join ( path_item , subitem ) for dist in find_eggs_in_zip ( zipimport . zipimporter ( subpath ) , subpath ) : yield dist

Find eggs in zip files ; possibly multiple nested eggs .


def find_on_path(importer, path_item, only=False):
    """Yield distributions accessible on a sys.path directory"""
    path_item = _normalize_cached(path_item)

    if os.path.isdir(path_item) and os.access(path_item, os.R_OK):
        if path_item.lower().endswith('.egg'):
            # unpacked egg
            yield Distribution.from_filename(
                path_item, metadata=PathMetadata(
                    path_item, os.path.join(path_item,'EGG-INFO')
                )
            )
        else:
            # scan for .egg and .egg-info in directory
            for entry in os.listdir(path_item):
                lower = entry.lower()
                if lower.endswith('.egg-info') or lower.endswith('.dist-info'):
                    fullpath = os.path.join(path_item, entry)
                    if os.path.isdir(fullpath):
                        # egg-info directory, allow getting metadata
                        metadata = PathMetadata(path_item, fullpath)
                    else:
                        metadata = FileMetadata(fullpath)
                    yield Distribution.from_location(
                        path_item, entry, metadata, precedence=DEVELOP_DIST
                    )
                elif not only and lower.endswith('.egg'):
                    dists = find_distributions(os.path.join(path_item, entry))
                    for dist in dists:
                        yield dist
                elif not only and lower.endswith('.egg-link'):
                    with open(os.path.join(path_item, entry)) as entry_file:
                        entry_lines = entry_file.readlines()
                    for line in entry_lines:
                        if not line.strip():
                            continue
                        path = os.path.join(path_item, line.rstrip())
                        dists = find_distributions(path)
                        for item in dists:
                            yield item
                        break

def find_on_path ( importer , path_item , only = False ) : path_item = _normalize_cached ( path_item ) if os . path . isdir ( path_item ) and os . access ( path_item , os . R_OK ) : if path_item . lower ( ) . endswith ( '.egg' ) : yield Distribution . from_filename ( path_item , metadata = PathMetadata ( path_item , os . path . join ( path_item , 'EGG-INFO' ) ) ) else : for entry in os . listdir ( path_item ) : lower = entry . lower ( ) if lower . endswith ( '.egg-info' ) or lower . endswith ( '.dist-info' ) : fullpath = os . path . join ( path_item , entry ) if os . path . isdir ( fullpath ) : metadata = PathMetadata ( path_item , fullpath ) else : metadata = FileMetadata ( fullpath ) yield Distribution . from_location ( path_item , entry , metadata , precedence = DEVELOP_DIST ) elif not only and lower . endswith ( '.egg' ) : dists = find_distributions ( os . path . join ( path_item , entry ) ) for dist in dists : yield dist elif not only and lower . endswith ( '.egg-link' ) : with open ( os . path . join ( path_item , entry ) ) as entry_file : entry_lines = entry_file . readlines ( ) for line in entry_lines : if not line . strip ( ) : continue path = os . path . join ( path_item , line . rstrip ( ) ) dists = find_distributions ( path ) for item in dists : yield item break

Yield distributions accessible on a sys . path directory


def declare_namespace(packageName):
    """Declare that package 'packageName' is a namespace package"""

    imp.acquire_lock()
    try:
        if packageName in _namespace_packages:
            return

        path, parent = sys.path, None
        if '.' in packageName:
            parent = '.'.join(packageName.split('.')[:-1])
            declare_namespace(parent)
            if parent not in _namespace_packages:
                __import__(parent)
            try:
                path = sys.modules[parent].__path__
            except AttributeError:
                raise TypeError("Not a package:", parent)

        # Track what packages are namespaces, so when new path items are added,
        # they can be updated
        _namespace_packages.setdefault(parent,[]).append(packageName)
        _namespace_packages.setdefault(packageName,[])

        for path_item in path:
            # Ensure all the parent's path items are reflected in the child,
            # if they apply
            _handle_ns(packageName, path_item)

    finally:
        imp.release_lock()

def
declare_namespace
(
packageName
)
:
imp
.
acquire_lock
(
)
try
:
if
packageName
in
_namespace_packages
:
return
path
,
parent
=
sys
.
path
,
if
'.'
in
packageName
:
parent
=
'.'
.
join
(
packageName
.
split
(
'.'
)
[
:
1
]
)
declare_namespace
(
parent
)
if
parent
not
in
_namespace_packages
:
__import__
(
parent
)
try
:
path
=
sys
.
modules
[
parent
]
.
__path__
except
AttributeError
:
raise
TypeError
(
"Not a package:"
,
parent
)
_namespace_packages
.
setdefault
(
parent
,
[
]
)
.
append
(
packageName
)
_namespace_packages
.
setdefault
(
packageName
,
[
]
)
for
path_item
in
path
:
_handle_ns
(
packageName
,
path_item
)
finally
:
imp
.
release_lock
(
)

Declare that package packageName is a namespace package


def _get_mro(cls):
    """Get an mro for a type or classic class"""
    if not isinstance(cls, type):
        class cls(cls, object): pass
        return cls.__mro__[1:]
    return cls.__mro__

def _get_mro ( cls ) : if not isinstance ( cls , type ) : class cls ( cls , object ) : pass return cls . __mro__ [ 1 : ] return cls . __mro__


def _find_adapter(registry, ob):
    """Return an adapter factory for `ob` from `registry`"""
    for t in _get_mro(getattr(ob, '__class__', type(ob))):
        if t in registry:
            return registry[t]

def _find_adapter ( registry , ob ) : for t in _get_mro ( getattr ( ob , '__class__' , type ( ob ) ) ) : if t in registry : return registry [ t ]

Return an adapter factory for ob from registry


def ensure_directory(path):
    """Ensure that the parent directory of `path` exists"""
    dirname = os.path.dirname(path)
    if not os.path.isdir(dirname):
        os.makedirs(dirname)

def ensure_directory ( path ) : dirname = os . path . dirname ( path ) if not os . path . isdir ( dirname ) : os . makedirs ( dirname )

Ensure that the parent directory of path exists


def iter_entry_points(self, group, name=None):
        """Yield entry point objects from `group` matching `name`

        If `name` is None, yields all entry points in `group` from all
        distributions in the working set, otherwise only ones matching
        both `group` and `name` are yielded (in distribution order).
        """
        for dist in self:
            entries = dist.get_entry_map(group)
            if name is None:
                for ep in entries.values():
                    yield ep
            elif name in entries:
                yield entries[name]

def iter_entry_points ( self , group , name = None ) : for dist in self : entries = dist . get_entry_map ( group ) if name is None : for ep in entries . values ( ) : yield ep elif name in entries : yield entries [ name ]

Yield entry point objects from group matching name


def can_add(self, dist):
        """Is distribution `dist` acceptable for this environment?

        The distribution must match the platform and python version
        requirements specified when this environment was created, or False
        is returned.
        """
        return (self.python is None or dist.py_version is None
            or dist.py_version==self.python) \
            and compatible_platforms(dist.platform, self.platform)

def can_add ( self , dist ) : return ( self . python is None or dist . py_version is None or dist . py_version == self . python ) and compatible_platforms ( dist . platform , self . platform )

Is distribution dist acceptable for this environment?


def best_match(self, req, working_set, installer=None):
        """Find distribution best matching `req` and usable on `working_set`

        This calls the ``find(req)`` method of the `working_set` to see if a
        suitable distribution is already active.  (This may raise
        ``VersionConflict`` if an unsuitable version of the project is already
        active in the specified `working_set`.)  If a suitable distribution
        isn't active, this method returns the newest distribution in the
        environment that meets the ``Requirement`` in `req`.  If no suitable
        distribution is found, and `installer` is supplied, then the result of
        calling the environment's ``obtain(req, installer)`` method will be
        returned.
        """
        dist = working_set.find(req)
        if dist is not None:
            return dist
        for dist in self[req.key]:
            if dist in req:
                return dist
        # try to download/install
        return self.obtain(req, installer)

def best_match ( self , req , working_set , installer = None ) : dist = working_set . find ( req ) if dist is not None : return dist for dist in self [ req . key ] : if dist in req : return dist return self . obtain ( req , installer )

Find distribution best matching req and usable on working_set


def evaluate_marker(cls, text, extra=None):
        """
        Evaluate a PEP 426 environment marker on CPython 2.4+.
        Return a boolean indicating the marker result in this environment.
        Raise SyntaxError if marker is invalid.

        This implementation uses the 'parser' module, which is not implemented
        on
        Jython and has been superseded by the 'ast' module in Python 2.6 and
        later.
        """
        return cls.interpret(parser.expr(text).totuple(1)[1])

def evaluate_marker ( cls , text , extra = None ) : return cls . interpret ( parser . expr ( text ) . totuple ( 1 ) [ 1 ] )

Evaluate a PEP 426 environment marker on CPython 2 . 4 + . Return a boolean indicating the marker result in this environment . Raise SyntaxError if marker is invalid .


def _markerlib_evaluate(cls, text):
        """
        Evaluate a PEP 426 environment marker using markerlib.
        Return a boolean indicating the marker result in this environment.
        Raise SyntaxError if marker is invalid.
        """
        from pip._vendor import _markerlib
        # markerlib implements Metadata 1.2 (PEP 345) environment markers.
        # Translate the variables to Metadata 2.0 (PEP 426).
        env = _markerlib.default_environment()
        for key in env.keys():
            new_key = key.replace('.', '_')
            env[new_key] = env.pop(key)
        try:
            result = _markerlib.interpret(text, env)
        except NameError as e:
            raise SyntaxError(e.args[0])
        return result

def _markerlib_evaluate ( cls , text ) : from pip . _vendor import _markerlib env = _markerlib . default_environment ( ) for key in env . keys ( ) : new_key = key . replace ( '.' , '_' ) env [ new_key ] = env . pop ( key ) try : result = _markerlib . interpret ( text , env ) except NameError as e : raise SyntaxError ( e . args [ 0 ] ) return result

Evaluate a PEP 426 environment marker using markerlib . Return a boolean indicating the marker result in this environment . Raise SyntaxError if marker is invalid .


def format(self, record):
        """
        Calls the standard formatter, but will indent all of the log messages
        by our current indentation level.
        """
        formatted = logging.Formatter.format(self, record)
        formatted = "".join([
            (" " * get_indentation()) + line
            for line in formatted.splitlines(True)
        ])
        return formatted

def
format
(
self
,
record
)
:
formatted
=
logging
.
Formatter
.
format
(
self
,
record
)
formatted
=
""
.
join
(
[
(
" "
*
get_indentation
(
)
)
+
line
for
line
in
formatted
.
splitlines
(
True
)
]
)
return
formatted

Calls the standard formatter but will indent all of the log messages by our current indentation level .


def format_currency(number, currency, format=None,
                    locale=LC_NUMERIC, currency_digits=True,
                    format_type='standard', decimal_quantization=True):

    """Return formatted currency value.

    >>> format_currency(1099.98, 'USD', locale='en_US')
    u'$1,099.98'
    >>> format_currency(1099.98, 'USD', locale='es_CO')
    u'US$\\xa01.099,98'
    >>> format_currency(1099.98, 'EUR', locale='de_DE')
    u'1.099,98\\xa0\\u20ac'

    The format can also be specified explicitly.  The currency is
    placed with the '¤' sign.  As the sign gets repeated the format
    expands (¤ being the symbol, ¤¤ is the currency abbreviation and
    ¤¤¤ is the full name of the currency):

    >>> format_currency(1099.98, 'EUR', u'\xa4\xa4 #,##0.00', locale='en_US')
    u'EUR 1,099.98'
    >>> format_currency(1099.98, 'EUR', u'#,##0.00 \xa4\xa4\xa4',
    ...                 locale='en_US')
    u'1,099.98 euros'

    Currencies usually have a specific number of decimal digits. This function
    favours that information over the given format:

    >>> format_currency(1099.98, 'JPY', locale='en_US')
    u'\\xa51,100'
    >>> format_currency(1099.98, 'COP', u'#,##0.00', locale='es_ES')
    u'1.100'

    However, the number of decimal digits can be overriden from the currency
    information, by setting the last parameter to ``False``:

    >>> format_currency(1099.98, 'JPY', locale='en_US', currency_digits=False)
    u'\\xa51,099.98'
    >>> format_currency(1099.98, 'COP', u'#,##0.00', locale='es_ES',
    ...                 currency_digits=False)
    u'1.099,98'

    If a format is not specified the type of currency format to use
    from the locale can be specified:

    >>> format_currency(1099.98, 'EUR', locale='en_US', format_type='standard')
    u'\\u20ac1,099.98'

    When the given currency format type is not available, an exception is
    raised:

    >>> format_currency('1099.98', 'EUR', locale='root', format_type='unknown')
    Traceback (most recent call last):
        ...
    UnknownCurrencyFormatError: "'unknown' is not a known currency format type"

    By default the locale is allowed to truncate and round a high-precision
    number by forcing its format pattern onto the decimal part. You can bypass
    this behavior with the `decimal_quantization` parameter:

    >>> format_currency(1099.9876, 'USD', locale='en_US')
    u'$1,099.99'
    >>> format_currency(1099.9876, 'USD', locale='en_US',
    ...                 decimal_quantization=False)
    u'$1,099.9876'

    :param number: the number to format
    :param currency: the currency code
    :param format: the format string to use
    :param locale: the `Locale` object or locale identifier
    :param currency_digits: use the currency's natural number of decimal digits
    :param format_type: the currency format type to use
    :param decimal_quantization: Truncate and round high-precision numbers to
                                 the format pattern. Defaults to `True`.
    """

    locale = Locale.parse(locale)
    if format:
        pattern = parse_pattern(format)
    else:
        try:
            p = locale.currency_formats[format_type]
            pattern = NumberPattern(
                p.pattern, p.prefix, p.suffix, p.grouping, p.int_prec,
                p.frac_prec, p.exp_prec, p.exp_plus)

        except KeyError:
            raise UnknownCurrencyFormatError(
                "%r is not a known currency format type" % format_type)
    return pattern.apply(
        number, locale, currency=currency, currency_digits=currency_digits,
        decimal_quantization=decimal_quantization)


def parse_pattern(pattern):
    """Parse number format patterns"""
    if isinstance(pattern, NumberPattern):
        return pattern

    def _match_number(pattern):
        rv = number_re.search(pattern)
        if rv is None:
            raise ValueError('Invalid number pattern %r' % pattern)
        return rv.groups()

    pos_pattern = pattern

    # Do we have a negative subpattern?
    if ';' in pattern:
        pos_pattern, neg_pattern = pattern.split(';', 1)
        pos_prefix, number, pos_suffix = _match_number(pos_pattern)
        neg_prefix, _, neg_suffix = _match_number(neg_pattern)
    else:
        pos_prefix, number, pos_suffix = _match_number(pos_pattern)
        neg_prefix = '-' + pos_prefix
        neg_suffix = pos_suffix
    if 'E' in number:
        number, exp = number.split('E', 1)
    else:
        exp = None
    if '@' in number:
        if '.' in number and '0' in number:
            raise ValueError('Significant digit patterns can not contain '
                             '"@" or "0"')
    if '.' in number:
        integer, fraction = number.rsplit('.', 1)
    else:
        integer = number
        fraction = ''

    def parse_precision(p):
        """Calculate the min and max allowed digits"""
        min = max = 0
        for c in p:
            if c in '@0':
                min += 1
                max += 1
            elif c == '#':
                max += 1
            elif c == ',':
                continue
            else:
                break
        return min, max

    int_prec = parse_precision(integer)
    frac_prec = parse_precision(fraction)
    if exp:
        exp_plus = exp.startswith('+')
        exp = exp.lstrip('+')
        exp_prec = parse_precision(exp)
    else:
        exp_plus = None
        exp_prec = None
    grouping = babel.numbers.parse_grouping(integer)
    return NumberPattern(pattern, (pos_prefix, neg_prefix),
                         (pos_suffix, neg_suffix), grouping,
                         int_prec, frac_prec,
                         exp_prec, exp_plus)


def get_decimal_quantum(precision):
    """Return minimal quantum of a number, as defined by precision."""
    assert isinstance(precision, (int, decimal.Decimal))
    return decimal.Decimal(10) ** (-precision)

def get_decimal_quantum ( precision ) : assert isinstance ( precision , ( int , decimal . Decimal ) ) return decimal . Decimal ( 10 ) ** ( - precision )

Return minimal quantum of a number as defined by precision .


def get_decimal_precision(number):
    """Return maximum precision of a decimal instance's fractional part.
    Precision is extracted from the fractional part only.
    """
    # Copied from: https://github.com/mahmoud/boltons/pull/59
    assert isinstance(number, decimal.Decimal)
    decimal_tuple = number.normalize().as_tuple()
    if decimal_tuple.exponent >= 0:
        return 0
    return abs(decimal_tuple.exponent)

def get_decimal_precision ( number ) : assert isinstance ( number , decimal . Decimal ) decimal_tuple = number . normalize ( ) . as_tuple ( ) if decimal_tuple . exponent >= 0 : return 0 return abs ( decimal_tuple . exponent )

Return maximum precision of a decimal instance s fractional part . Precision is extracted from the fractional part only .


def scientific_notation_elements(self, value, locale):
        """ Returns normalized scientific notation components of a value."""
        # Normalize value to only have one lead digit.
        exp = value.adjusted()
        value = value * get_decimal_quantum(exp)
        assert value.adjusted() == 0

        # Shift exponent and value by the minimum number of leading digits
        # imposed by the rendering pattern. And always make that number
        # greater or equal to 1.
        lead_shift = max([1, min(self.int_prec)]) - 1
        exp = exp - lead_shift
        value = value * get_decimal_quantum(-lead_shift)

        # Get exponent sign symbol.
        exp_sign = ''
        if exp < 0:
            exp_sign = babel.numbers.get_minus_sign_symbol(locale)
        elif self.exp_plus:
            exp_sign = babel.numbers.get_plus_sign_symbol(locale)

        # Normalize exponent value now that we have the sign.
        exp = abs(exp)

        return value, exp, exp_sign

def scientific_notation_elements ( self , value , locale ) : exp = value . adjusted ( ) value = value * get_decimal_quantum ( exp ) assert value . adjusted ( ) == 0 lead_shift = max ( [ 1 , min ( self . int_prec ) ] ) - 1 exp = exp - lead_shift value = value * get_decimal_quantum ( - lead_shift ) exp_sign = '' if exp < 0 : exp_sign = babel . numbers . get_minus_sign_symbol ( locale ) elif self . exp_plus : exp_sign = babel . numbers . get_plus_sign_symbol ( locale ) exp = abs ( exp ) return value , exp , exp_sign

Returns normalized scientific notation components of a value .


def total_seconds(td):
    """Python 2.6 compatability"""
    if hasattr(td, 'total_seconds'):
        return td.total_seconds()

    ms = td.microseconds
    secs = (td.seconds + td.days * 24 * 3600)
    return (ms + secs * 10**6) / 10**6

def total_seconds ( td ) : if hasattr ( td , 'total_seconds' ) : return td . total_seconds ( ) ms = td . microseconds secs = ( td . seconds + td . days * 24 * 3600 ) return ( ms + secs * 10 ** 6 ) / 10 ** 6


def parse_requirements(strs):
    """Yield ``Requirement`` objects for each specification in `strs`

    `strs` must be a string, or a (possibly-nested) iterable thereof.
    """
    # create a steppable iterator, so we can handle \-continuations
    lines = iter(yield_lines(strs))

    def scan_list(ITEM, TERMINATOR, line, p, groups, item_name):

        items = []

        while not TERMINATOR(line, p):
            if CONTINUE(line, p):
                try:
                    line = next(lines)
                    p = 0
                except StopIteration:
                    msg = "\\ must not appear on the last nonblank line"
                    raise RequirementParseError(msg)

            match = ITEM(line, p)
            if not match:
                msg = "Expected " + item_name + " in"
                raise RequirementParseError(msg, line, "at", line[p:])

            items.append(match.group(*groups))
            p = match.end()

            match = COMMA(line, p)
            if match:
                # skip the comma
                p = match.end()
            elif not TERMINATOR(line, p):
                msg = "Expected ',' or end-of-list in"
                raise RequirementParseError(msg, line, "at", line[p:])

        match = TERMINATOR(line, p)
        # skip the terminator, if any
        if match:
            p = match.end()
        return line, p, items

    for line in lines:
        match = DISTRO(line)
        if not match:
            raise RequirementParseError("Missing distribution spec", line)
        project_name = match.group(1)
        p = match.end()
        extras = []

        match = OBRACKET(line, p)
        if match:
            p = match.end()
            line, p, extras = scan_list(
                DISTRO, CBRACKET, line, p, (1,), "'extra' name"
            )

        line, p, specs = scan_list(VERSION, LINE_END, line, p, (1, 2),
            "version spec")
        specs = [(op, val) for op, val in specs]
        yield Requirement(project_name, specs, extras)

Yield Requirement objects for each specification in strs


def _get_unpatched(cls):
    """Protect against re-patching the distutils if reloaded

    Also ensures that no other distutils extension monkeypatched the distutils
    first.
    """
    while cls.__module__.startswith('setuptools'):
        cls, = cls.__bases__
    if not cls.__module__.startswith('distutils'):
        raise AssertionError(
            "distutils has already been patched by %r" % cls
        )
    return cls

Protect against re - patching the distutils if reloaded


def check_requirements(dist, attr, value):
    """Verify that install_requires is a valid requirements list"""
    try:
        list(pkg_resources.parse_requirements(value))
    except (TypeError, ValueError) as error:
        tmpl = (
            "{attr!r} must be a string or list of strings "
            "containing valid project/version requirement specifiers; {error}"
        )
        raise DistutilsSetupError(tmpl.format(attr=attr, error=error))


def fetch_build_egg(self, req):
        """Fetch an egg needed for building"""

        try:
            cmd = self._egg_fetcher
            cmd.package_index.to_scan = []
        except AttributeError:
            from setuptools.command.easy_install import easy_install
            dist = self.__class__({'script_args':['easy_install']})
            dist.parse_config_files()
            opts = dist.get_option_dict('easy_install')
            keep = (
                'find_links', 'site_dirs', 'index_url', 'optimize',
                'site_dirs', 'allow_hosts'
            )
            for key in list(opts):
                if key not in keep:
                    del opts[key]   # don't use any other settings
            if self.dependency_links:
                links = self.dependency_links[:]
                if 'find_links' in opts:
                    links = opts['find_links'][1].split() + links
                opts['find_links'] = ('setup', links)
            install_dir = self.get_egg_cache_dir()
            cmd = easy_install(
                dist, args=["x"], install_dir=install_dir, exclude_scripts=True,
                always_copy=False, build_directory=None, editable=False,
                upgrade=False, multi_version=True, no_report=True, user=False
            )
            cmd.ensure_finalized()
            self._egg_fetcher = cmd
        return cmd.easy_install(req)

def fetch_build_egg ( self , req ) : try : cmd = self . _egg_fetcher cmd . package_index . to_scan = [ ] except AttributeError : from setuptools . command . easy_install import easy_install dist = self . __class__ ( { 'script_args' : [ 'easy_install' ] } ) dist . parse_config_files ( ) opts = dist . get_option_dict ( 'easy_install' ) keep = ( 'find_links' , 'site_dirs' , 'index_url' , 'optimize' , 'site_dirs' , 'allow_hosts' ) for key in list ( opts ) : if key not in keep : del opts [ key ] if self . dependency_links : links = self . dependency_links [ : ] if 'find_links' in opts : links = opts [ 'find_links' ] [ 1 ] . split ( ) + links opts [ 'find_links' ] = ( 'setup' , links ) install_dir = self . get_egg_cache_dir ( ) cmd = easy_install ( dist , args = [ "x" ] , install_dir = install_dir , exclude_scripts = True , always_copy = False , build_directory = None , editable = False , upgrade = False , multi_version = True , no_report = True , user = False ) cmd . ensure_finalized ( ) self . _egg_fetcher = cmd return cmd . easy_install ( req )


def do_dice_roll():
	"""
	Roll n-sided dice and return each result and the total
	"""
	options = get_options()
	dice = Dice(options.sides)
	rolls = [dice.roll() for n in range(options.number)]
	for roll in rolls:
		print('rolled', roll)
	if options.number > 1:
		print('total', sum(rolls))

def do_dice_roll ( ) : options = get_options ( ) dice = Dice ( options . sides ) rolls = [ dice . roll ( ) for n in range ( options . number ) ] for roll in rolls : print ( 'rolled' , roll ) if options . number > 1 : print ( 'total' , sum ( rolls ) )

Roll n - sided dice and return each result and the total


def price_converter(obj):
    """Ensures that string prices are converted into Price objects."""
    if isinstance(obj, str):
        obj = PriceClass.parse(obj)
    return obj

def price_converter ( obj ) : if isinstance ( obj , str ) : obj = PriceClass . parse ( obj ) return obj

Ensures that string prices are converted into Price objects .


def price(*args, **kwargs):
    """Price field for attrs.

    See `help(attr.ib)` for full signature.

    Usage:

        >>> from pricing import fields
        ... @attr.s
        ... class Test:
        ...     price: Price = fields.price(default='USD 5.00')
        ...
        ... Test()
        Test(price=USD 5.00)
    """

    kwargs.setdefault('default', 'USD 0.00')
    kwargs.setdefault('converter', price_converter)

    if 'validator' in kwargs:
        validator = kwargs.pop('validator')
        if not isinstance(validator, (tuple, list)):
            validator = [validator]
    else:
        validator = []
    validator.append(instance_of(PriceClass))
    return attr.ib(validator=validator, *args, **kwargs)

def
price
(
*
args
,
**
kwargs
)
:
kwargs
.
setdefault
(
'default'
,
'USD 0.00'
)
kwargs
.
setdefault
(
'converter'
,
price_converter
)
if
'validator'
in
kwargs
:
validator
=
kwargs
.
pop
(
'validator'
)
if
not
isinstance
(
validator
,
(
tuple
,
list
)
)
:
validator
=
[
validator
]
else
:
validator
=
[
]
validator
.
append
(
instance_of
(
PriceClass
)
)
return
attr
.
ib
(
validator
=
validator
,
*
args
,
**
kwargs
)


def validate(self, request):
        """Validate JSON-RPC request.

        :param request: RPC request object
        :type request: dict

        """

        try:
            validate_version(request)
            validate_method(request)
            validate_params(request)
            validate_id(request)
        except (AssertionError, KeyError) as error:
            invalid_request(error)

def validate ( self , request ) : try : validate_version ( request ) validate_method ( request ) validate_params ( request ) validate_id ( request ) except ( AssertionError , KeyError ) as error : invalid_request ( error )


def get_method(self, args):
        """Get request method for service application."""

        try:
            method = self.app[args['method']]
        except KeyError:
            method_not_found(args['id'])
        else:
            return method

def get_method ( self , args ) : try : method = self . app [ args [ 'method' ] ] except KeyError : method_not_found ( args [ 'id' ] ) else : return method

Get request method for service application .


def apply(self, method, args):
        """Apply application method."""

        try:
            params = args['params']
            if isinstance(params, dict):
                result = method(**params)
            else:
                result = method(*params)
        except Exception as error:
            server_error(args['id'], error)
        else:
            return result

def apply ( self , method , args ) : try : params = args [ 'params' ] if isinstance ( params , dict ) : result = method ( ** params ) else : result = method ( * params ) except Exception as error : server_error ( args [ 'id' ] , error ) else : return result


def module(self):
        """The name of the current module if the request was dispatched
        to an actual module.  This is deprecated functionality, use blueprints
        instead.
        """
        from warnings import warn
        warn(DeprecationWarning('modules were deprecated in favor of '
                                'blueprints.  Use request.blueprint '
                                'instead.'), stacklevel=2)
        if self._is_old_module:
            return self.blueprint

def
module
(
self
)
:
from
warnings
import
warn
warn
(
DeprecationWarning
(
'modules were deprecated in favor of '
'blueprints.  Use request.blueprint '
'instead.'
)
,
stacklevel
=
2
)
if
self
.
_is_old_module
:
return
self
.
blueprint

The name of the current module if the request was dispatched to an actual module . This is deprecated functionality use blueprints instead .


def blueprint(self):
        """The name of the current blueprint"""
        if self.url_rule and '.' in self.url_rule.endpoint:
            return self.url_rule.endpoint.rsplit('.', 1)[0]

def blueprint ( self ) : if self . url_rule and '.' in self . url_rule . endpoint : return self . url_rule . endpoint . rsplit ( '.' , 1 ) [ 0 ]


def attach_enctype_error_multidict(request):
    """Since Flask 0.8 we're monkeypatching the files object in case a
    request is detected that does not use multipart form data but the files
    object is accessed.
    """
    oldcls = request.files.__class__
    class newcls(oldcls):
        def __getitem__(self, key):
            try:
                return oldcls.__getitem__(self, key)
            except KeyError as e:
                if key not in request.form:
                    raise
                raise DebugFilesKeyError(request, key)
    newcls.__name__ = oldcls.__name__
    newcls.__module__ = oldcls.__module__
    request.files.__class__ = newcls

def attach_enctype_error_multidict ( request ) : oldcls = request . files . __class__ class newcls ( oldcls ) : def __getitem__ ( self , key ) : try : return oldcls . __getitem__ ( self , key ) except KeyError as e : if key not in request . form : raise raise DebugFilesKeyError ( request , key ) newcls . __name__ = oldcls . __name__ newcls . __module__ = oldcls . __module__ request . files . __class__ = newcls

Since Flask 0 . 8 we re monkeypatching the files object in case a request is detected that does not use multipart form data but the files object is accessed .


def make_abstract_dist(req_to_install):
    """Factory to make an abstract dist object.

    Preconditions: Either an editable req with a source_dir, or satisfied_by or
    a wheel link, or a non-editable req with a source_dir.

    :return: A concrete DistAbstraction.
    """
    if req_to_install.editable:
        return IsSDist(req_to_install)
    elif req_to_install.link and req_to_install.link.is_wheel:
        return IsWheel(req_to_install)
    else:
        return IsSDist(req_to_install)

def make_abstract_dist ( req_to_install ) : if req_to_install . editable : return IsSDist ( req_to_install ) elif req_to_install . link and req_to_install . link . is_wheel : return IsWheel ( req_to_install ) else : return IsSDist ( req_to_install )

Factory to make an abstract dist object .


def add_requirement(self, install_req, parent_req_name=None):
        """Add install_req as a requirement to install.

        :param parent_req_name: The name of the requirement that needed this
            added. The name is used because when multiple unnamed requirements
            resolve to the same name, we could otherwise end up with dependency
            links that point outside the Requirements set. parent_req must
            already be added. Note that None implies that this is a user
            supplied requirement, vs an inferred one.
        :return: Additional requirements to scan. That is either [] if
            the requirement is not applicable, or [install_req] if the
            requirement is applicable and has just been added.
        """
        name = install_req.name
        if not install_req.match_markers():
            logger.warning("Ignoring %s: markers %r don't match your "
                           "environment", install_req.name,
                           install_req.markers)
            return []

        install_req.as_egg = self.as_egg
        install_req.use_user_site = self.use_user_site
        install_req.target_dir = self.target_dir
        install_req.pycompile = self.pycompile
        if not name:
            # url or path requirement w/o an egg fragment
            self.unnamed_requirements.append(install_req)
            return [install_req]
        else:
            if parent_req_name is None and self.has_requirement(name):
                raise InstallationError(
                    'Double requirement given: %s (already in %s, name=%r)'
                    % (install_req, self.get_requirement(name), name))
            if not self.has_requirement(name):
                # Add requirement
                self.requirements[name] = install_req
                # FIXME: what about other normalizations?  E.g., _ vs. -?
                if name.lower() != name:
                    self.requirement_aliases[name.lower()] = name
                result = [install_req]
            else:
                # Canonicalise to the already-added object
                install_req = self.get_requirement(name)
                # No need to scan, this is a duplicate requirement.
                result = []
            if parent_req_name:
                parent_req = self.get_requirement(parent_req_name)
                self._dependencies[parent_req].append(install_req)
            return result

Add install_req as a requirement to install .


def _walk_req_to_install(self, handler):
        """Call handler for all pending reqs.

        :param handler: Handle a single requirement. Should take a requirement
            to install. Can optionally return an iterable of additional
            InstallRequirements to cover.
        """
        # The list() here is to avoid potential mutate-while-iterating bugs.
        discovered_reqs = []
        reqs = itertools.chain(
            list(self.unnamed_requirements), list(self.requirements.values()),
            discovered_reqs)
        for req_to_install in reqs:
            more_reqs = handler(req_to_install)
            if more_reqs:
                discovered_reqs.extend(more_reqs)

def _walk_req_to_install ( self , handler ) : discovered_reqs = [ ] reqs = itertools . chain ( list ( self . unnamed_requirements ) , list ( self . requirements . values ( ) ) , discovered_reqs ) for req_to_install in reqs : more_reqs = handler ( req_to_install ) if more_reqs : discovered_reqs . extend ( more_reqs )


def _check_skip_installed(self, req_to_install, finder):
        """Check if req_to_install should be skipped.

        This will check if the req is installed, and whether we should upgrade
        or reinstall it, taking into account all the relevant user options.

        After calling this req_to_install will only have satisfied_by set to
        None if the req_to_install is to be upgraded/reinstalled etc. Any
        other value will be a dist recording the current thing installed that
        satisfies the requirement.

        Note that for vcs urls and the like we can't assess skipping in this
        routine - we simply identify that we need to pull the thing down,
        then later on it is pulled down and introspected to assess upgrade/
        reinstalls etc.

        :return: A text reason for why it was skipped, or None.
        """
        # Check whether to upgrade/reinstall this req or not.
        req_to_install.check_if_exists()
        if req_to_install.satisfied_by:
            skip_reason = 'satisfied (use --upgrade to upgrade)'
            if self.upgrade:
                best_installed = False
                # For link based requirements we have to pull the
                # tree down and inspect to assess the version #, so
                # its handled way down.
                if not (self.force_reinstall or req_to_install.link):
                    try:
                        finder.find_requirement(req_to_install, self.upgrade)
                    except BestVersionAlreadyInstalled:
                        skip_reason = 'up-to-date'
                        best_installed = True
                    except DistributionNotFound:
                        # No distribution found, so we squash the
                        # error - it will be raised later when we
                        # re-try later to do the install.
                        # Why don't we just raise here?
                        pass

                if not best_installed:
                    # don't uninstall conflict if user install and
                    # conflict is not user install
                    if not (self.use_user_site and not
                            dist_in_usersite(req_to_install.satisfied_by)):
                        req_to_install.conflicts_with = \
                            req_to_install.satisfied_by
                    req_to_install.satisfied_by = None
            return skip_reason
        else:
            return None

def
_check_skip_installed
(
self
,
req_to_install
,
finder
)
:
req_to_install
.
check_if_exists
(
)
if
req_to_install
.
satisfied_by
:
skip_reason
=
'satisfied (use --upgrade to upgrade)'
if
self
.
upgrade
:
best_installed
=
False
if
not
(
self
.
force_reinstall
or
req_to_install
.
link
)
:
try
:
finder
.
find_requirement
(
req_to_install
,
self
.
upgrade
)
except
BestVersionAlreadyInstalled
:
skip_reason
=
'up-to-date'
best_installed
=
True
except
DistributionNotFound
:
pass
if
not
best_installed
:
if
not
(
self
.
use_user_site
and
not
dist_in_usersite
(
req_to_install
.
satisfied_by
)
)
:
req_to_install
.
conflicts_with
=
req_to_install
.
satisfied_by
req_to_install
.
satisfied_by
=
return
skip_reason
else
:
return

Check if req_to_install should be skipped .


def _to_install(self):
        """Create the installation order.

        The installation order is topological - requirements are installed
        before the requiring thing. We break cycles at an arbitrary point,
        and make no other guarantees.
        """
        # The current implementation, which we may change at any point
        # installs the user specified things in the order given, except when
        # dependencies must come earlier to achieve topological order.
        order = []
        ordered_reqs = set()

        def schedule(req):
            if req.satisfied_by or req in ordered_reqs:
                return
            ordered_reqs.add(req)
            for dep in self._dependencies[req]:
                schedule(dep)
            order.append(req)
        for install_req in self.requirements.values():
            schedule(install_req)
        return order

def _to_install ( self ) : order = [ ] ordered_reqs = set ( ) def schedule ( req ) : if req . satisfied_by or req in ordered_reqs : return ordered_reqs . add ( req ) for dep in self . _dependencies [ req ] : schedule ( dep ) order . append ( req ) for install_req in self . requirements . values ( ) : schedule ( install_req ) return order


def _get_all_ns_packages(self):
        """Return sorted list of all package namespaces"""
        nsp = set()
        for pkg in self.distribution.namespace_packages or []:
            pkg = pkg.split('.')
            while pkg:
                nsp.add('.'.join(pkg))
                pkg.pop()
        return sorted(nsp)

def _get_all_ns_packages ( self ) : nsp = set ( ) for pkg in self . distribution . namespace_packages or [ ] : pkg = pkg . split ( '.' ) while pkg : nsp . add ( '.' . join ( pkg ) ) pkg . pop ( ) return sorted ( nsp )


def default(self, obj):
        """
        Convert QuerySet objects to their list counter-parts
        """
        if isinstance(obj, models.Model):
            return self.encode(model_to_dict(obj))
        elif isinstance(obj, models.query.QuerySet):
            return serializers.serialize('json', obj)
        else:
            return super(JsonResponseEncoder, self).default(obj)

def default ( self , obj ) : if isinstance ( obj , models . Model ) : return self . encode ( model_to_dict ( obj ) ) elif isinstance ( obj , models . query . QuerySet ) : return serializers . serialize ( 'json' , obj ) else : return super ( JsonResponseEncoder , self ) . default ( obj )

Convert QuerySet objects to their list counter - parts


def tokenize_annotated(doc, annotation):
    """Tokenize a document and add an annotation attribute to each token
    """
    tokens = tokenize(doc, include_hrefs=False)
    for tok in tokens:
        tok.annotation = annotation
    return tokens

def tokenize_annotated ( doc , annotation ) : tokens = tokenize ( doc , include_hrefs = False ) for tok in tokens : tok . annotation = annotation return tokens


def html_annotate_merge_annotations(tokens_old, tokens_new):
    """Merge the annotations from tokens_old into tokens_new, when the
    tokens in the new document already existed in the old document.
    """
    s = InsensitiveSequenceMatcher(a=tokens_old, b=tokens_new)
    commands = s.get_opcodes()

    for command, i1, i2, j1, j2 in commands:
        if command == 'equal':
            eq_old = tokens_old[i1:i2]
            eq_new = tokens_new[j1:j2]
            copy_annotations(eq_old, eq_new)

def html_annotate_merge_annotations ( tokens_old , tokens_new ) : s = InsensitiveSequenceMatcher ( a = tokens_old , b = tokens_new ) commands = s . get_opcodes ( ) for command , i1 , i2 , j1 , j2 in commands : if command == 'equal' : eq_old = tokens_old [ i1 : i2 ] eq_new = tokens_new [ j1 : j2 ] copy_annotations ( eq_old , eq_new )

Merge the annotations from tokens_old into tokens_new when the tokens in the new document already existed in the old document .


def copy_annotations(src, dest):
    """
    Copy annotations from the tokens listed in src to the tokens in dest
    """
    assert len(src) == len(dest)
    for src_tok, dest_tok in zip(src, dest):
        dest_tok.annotation = src_tok.annotation

def copy_annotations ( src , dest ) : assert len ( src ) == len ( dest ) for src_tok , dest_tok in zip ( src , dest ) : dest_tok . annotation = src_tok . annotation


def compress_tokens(tokens):
    """
    Combine adjacent tokens when there is no HTML between the tokens,
    and they share an annotation
    """
    result = [tokens[0]]
    for tok in tokens[1:]:
        if (not result[-1].post_tags and
            not tok.pre_tags and
            result[-1].annotation == tok.annotation):
            compress_merge_back(result, tok)
        else:
            result.append(tok)
    return result

def compress_tokens ( tokens ) : result = [ tokens [ 0 ] ] for tok in tokens [ 1 : ] : if ( not result [ - 1 ] . post_tags and not tok . pre_tags and result [ - 1 ] . annotation == tok . annotation ) : compress_merge_back ( result , tok ) else : result . append ( tok ) return result

Combine adjacent tokens when there is no HTML between the tokens and they share an annotation


def markup_serialize_tokens(tokens, markup_func):
    """
    Serialize the list of tokens into a list of text chunks, calling
    markup_func around text to add annotations.
    """
    for token in tokens:
        for pre in token.pre_tags:
            yield pre
        html = token.html()
        html = markup_func(html, token.annotation)
        if token.trailing_whitespace:
            html += token.trailing_whitespace
        yield html
        for post in token.post_tags:
            yield post

def markup_serialize_tokens ( tokens , markup_func ) : for token in tokens : for pre in token . pre_tags : yield pre html = token . html ( ) html = markup_func ( html , token . annotation ) if token . trailing_whitespace : html += token . trailing_whitespace yield html for post in token . post_tags : yield post

Serialize the list of tokens into a list of text chunks calling markup_func around text to add annotations .


def expand_tokens(tokens, equal=False):
    """Given a list of tokens, return a generator of the chunks of
    text for the data in the tokens.
    """
    for token in tokens:
        for pre in token.pre_tags:
            yield pre
        if not equal or not token.hide_when_equal:
            if token.trailing_whitespace:
                yield token.html() + token.trailing_whitespace
            else:
                yield token.html()
        for post in token.post_tags:
            yield post

def expand_tokens ( tokens , equal = False ) : for token in tokens : for pre in token . pre_tags : yield pre if not equal or not token . hide_when_equal : if token . trailing_whitespace : yield token . html ( ) + token . trailing_whitespace else : yield token . html ( ) for post in token . post_tags : yield post

Given a list of tokens return a generator of the chunks of text for the data in the tokens .


def locate_unbalanced_end(unbalanced_end, pre_delete, post_delete):
    """ like locate_unbalanced_start, except handling end tags and
    possibly moving the point earlier in the document.  """
    while 1:
        if not unbalanced_end:
            # Success
            break
        finding = unbalanced_end[-1]
        finding_name = finding.split()[0].strip('<>/')
        if not pre_delete:
            break
        next = pre_delete[-1]
        if next is DEL_END or not next.startswith('</'):
            # A word or a start tag
            break
        name = next.split()[0].strip('<>/')
        if name == 'ins' or name == 'del':
            # Can't move into an insert or delete
            break
        if name == finding_name:
            unbalanced_end.pop()
            post_delete.insert(0, pre_delete.pop())
        else:
            # Found a tag that doesn't match
            break

def locate_unbalanced_end ( unbalanced_end , pre_delete , post_delete ) : while 1 : if not unbalanced_end : break finding = unbalanced_end [ - 1 ] finding_name = finding . split ( ) [ 0 ] . strip ( '<>/' ) if not pre_delete : break next = pre_delete [ - 1 ] if next is DEL_END or not next . startswith ( '</' ) : break name = next . split ( ) [ 0 ] . strip ( '<>/' ) if name == 'ins' or name == 'del' : break if name == finding_name : unbalanced_end . pop ( ) post_delete . insert ( 0 , pre_delete . pop ( ) ) else : break

like locate_unbalanced_start except handling end tags and possibly moving the point earlier in the document .


def fixup_chunks(chunks):
    """
    This function takes a list of chunks and produces a list of tokens.
    """
    tag_accum = []
    cur_word = None
    result = []
    for chunk in chunks:
        if isinstance(chunk, tuple):
            if chunk[0] == 'img':
                src = chunk[1]
                tag, trailing_whitespace = split_trailing_whitespace(chunk[2])
                cur_word = tag_token('img', src, html_repr=tag,
                                     pre_tags=tag_accum,
                                     trailing_whitespace=trailing_whitespace)
                tag_accum = []
                result.append(cur_word)

            elif chunk[0] == 'href':
                href = chunk[1]
                cur_word = href_token(href, pre_tags=tag_accum, trailing_whitespace=" ")
                tag_accum = []
                result.append(cur_word)
            continue

        if is_word(chunk):
            chunk, trailing_whitespace = split_trailing_whitespace(chunk)
            cur_word = token(chunk, pre_tags=tag_accum, trailing_whitespace=trailing_whitespace)
            tag_accum = []
            result.append(cur_word)

        elif is_start_tag(chunk):
            tag_accum.append(chunk)

        elif is_end_tag(chunk):
            if tag_accum:
                tag_accum.append(chunk)
            else:
                assert cur_word, (
                    "Weird state, cur_word=%r, result=%r, chunks=%r of %r"
                    % (cur_word, result, chunk, chunks))
                cur_word.post_tags.append(chunk)
        else:
            assert(0)

    if not result:
        return [token('', pre_tags=tag_accum)]
    else:
        result[-1].post_tags.extend(tag_accum)

    return result

This function takes a list of chunks and produces a list of tokens .


def flatten_el(el, include_hrefs, skip_tag=False):
    """ Takes an lxml element el, and generates all the text chunks for
    that tag.  Each start tag is a chunk, each word is a chunk, and each
    end tag is a chunk.

    If skip_tag is true, then the outermost container tag is
    not returned (just its contents)."""
    if not skip_tag:
        if el.tag == 'img':
            yield ('img', el.get('src'), start_tag(el))
        else:
            yield start_tag(el)
    if el.tag in empty_tags and not el.text and not len(el) and not el.tail:
        return
    start_words = split_words(el.text)
    for word in start_words:
        yield html_escape(word)
    for child in el:
        for item in flatten_el(child, include_hrefs=include_hrefs):
            yield item
    if el.tag == 'a' and el.get('href') and include_hrefs:
        yield ('href', el.get('href'))
    if not skip_tag:
        yield end_tag(el)
        end_words = split_words(el.tail)
        for word in end_words:
            yield html_escape(word)

def flatten_el ( el , include_hrefs , skip_tag = False ) : if not skip_tag : if el . tag == 'img' : yield ( 'img' , el . get ( 'src' ) , start_tag ( el ) ) else : yield start_tag ( el ) if el . tag in empty_tags and not el . text and not len ( el ) and not el . tail : return start_words = split_words ( el . text ) for word in start_words : yield html_escape ( word ) for child in el : for item in flatten_el ( child , include_hrefs = include_hrefs ) : yield item if el . tag == 'a' and el . get ( 'href' ) and include_hrefs : yield ( 'href' , el . get ( 'href' ) ) if not skip_tag : yield end_tag ( el ) end_words = split_words ( el . tail ) for word in end_words : yield html_escape ( word )

Takes an lxml element el and generates all the text chunks for that tag . Each start tag is a chunk each word is a chunk and each end tag is a chunk .


def split_words(text):
    """ Splits some text into words. Includes trailing whitespace
    on each word when appropriate.  """
    if not text or not text.strip():
        return []

    words = split_words_re.findall(text)
    return words

def split_words ( text ) : if not text or not text . strip ( ) : return [ ] words = split_words_re . findall ( text ) return words

Splits some text into words . Includes trailing whitespace on each word when appropriate .


def start_tag(el):
    """
    The text representation of the start tag for a tag.
    """
    return '<%s%s>' % (
        el.tag, ''.join([' %s="%s"' % (name, html_escape(value, True))
                         for name, value in el.attrib.items()]))

def
start_tag
(
el
)
:
return
'<%s%s>'
%
(
el
.
tag
,
''
.
join
(
[
' %s="%s"'
%
(
name
,
html_escape
(
value
,
True
)
)
for
name
,
value
in
el
.
attrib
.
items
(
)
]
)
)

The text representation of the start tag for a tag .


def end_tag(el):
    """ The text representation of an end tag for a tag.  Includes
    trailing whitespace when appropriate.  """
    if el.tail and start_whitespace_re.search(el.tail):
        extra = ' '
    else:
        extra = ''
    return '</%s>%s' % (el.tag, extra)

def
end_tag
(
el
)
:
if
el
.
tail
and
start_whitespace_re
.
search
(
el
.
tail
)
:
extra
=
' '
else
:
extra
=
''
return
'</%s>%s'
%
(
el
.
tag
,
extra
)

The text representation of an end tag for a tag . Includes trailing whitespace when appropriate .


def serialize_html_fragment(el, skip_outer=False):
    """ Serialize a single lxml element as HTML.  The serialized form
    includes the elements tail.

    If skip_outer is true, then don't serialize the outermost tag
    """
    assert not isinstance(el, basestring), (
        "You should pass in an element, not a string like %r" % el)
    html = etree.tostring(el, method="html", encoding=_unicode)
    if skip_outer:
        # Get rid of the extra starting tag:
        html = html[html.find('>')+1:]
        # Get rid of the extra end tag:
        html = html[:html.rfind('<')]
        return html.strip()
    else:
        return html

Serialize a single lxml element as HTML . The serialized form includes the elements tail .


def _fixup_ins_del_tags(doc):
    """fixup_ins_del_tags that works on an lxml document in-place
    """
    for tag in ['ins', 'del']:
        for el in doc.xpath('descendant-or-self::%s' % tag):
            if not _contains_block_level_tag(el):
                continue
            _move_el_inside_block(el, tag=tag)
            el.drop_tag()

def _fixup_ins_del_tags ( doc ) : for tag in [ 'ins' , 'del' ] : for el in doc . xpath ( 'descendant-or-self::%s' % tag ) : if not _contains_block_level_tag ( el ) : continue _move_el_inside_block ( el , tag = tag ) el . drop_tag ( )

fixup_ins_del_tags that works on an lxml document in - place


def extract_constant(code, symbol, default=-1):
    """Extract the constant value of 'symbol' from 'code'

    If the name 'symbol' is bound to a constant value by the Python code
    object 'code', return that value.  If 'symbol' is bound to an expression,
    return 'default'.  Otherwise, return 'None'.

    Return value is based on the first assignment to 'symbol'.  'symbol' must
    be a global, or at least a non-"fast" local in the code block.  That is,
    only 'STORE_NAME' and 'STORE_GLOBAL' opcodes are checked, and 'symbol'
    must be present in 'code.co_names'.
    """

    if symbol not in code.co_names:
        # name's not there, can't possibly be an assigment
        return None

    name_idx = list(code.co_names).index(symbol)

    STORE_NAME = 90
    STORE_GLOBAL = 97
    LOAD_CONST = 100

    const = default

    for op, arg in _iter_code(code):

        if op==LOAD_CONST:
            const = code.co_consts[arg]
        elif arg==name_idx and (op==STORE_NAME or op==STORE_GLOBAL):
            return const
        else:
            const = default

def extract_constant ( code , symbol , default = - 1 ) : if symbol not in code . co_names : return None name_idx = list ( code . co_names ) . index ( symbol ) STORE_NAME = 90 STORE_GLOBAL = 97 LOAD_CONST = 100 const = default for op , arg in _iter_code ( code ) : if op == LOAD_CONST : const = code . co_consts [ arg ] elif arg == name_idx and ( op == STORE_NAME or op == STORE_GLOBAL ) : return const else : const = default

Extract the constant value of symbol from code


def cache_url(self, **kwargs):
        """A simplified URL to be used for caching the given query."""
        query = {
            'Operation': self.Operation,
            'Service': "AWSECommerceService",
            'Version': self.Version,
        }
        query.update(kwargs)

        service_domain = SERVICE_DOMAINS[self.Region][0]

        return "http://" + service_domain + "/onca/xml?" + _quote_query(query)

def cache_url ( self , ** kwargs ) : query = { 'Operation' : self . Operation , 'Service' : "AWSECommerceService" , 'Version' : self . Version , } query . update ( kwargs ) service_domain = SERVICE_DOMAINS [ self . Region ] [ 0 ] return "http://" + service_domain + "/onca/xml?" + _quote_query ( query )

A simplified URL to be used for caching the given query .


def autolink(el, link_regexes=_link_regexes,
             avoid_elements=_avoid_elements,
             avoid_hosts=_avoid_hosts,
             avoid_classes=_avoid_classes):
    """
    Turn any URLs into links.

    It will search for links identified by the given regular
    expressions (by default mailto and http(s) links).

    It won't link text in an element in avoid_elements, or an element
    with a class in avoid_classes.  It won't link to anything with a
    host that matches one of the regular expressions in avoid_hosts
    (default localhost and 127.0.0.1).

    If you pass in an element, the element's tail will not be
    substituted, only the contents of the element.
    """
    if el.tag in avoid_elements:
        return
    class_name = el.get('class')
    if class_name:
        class_name = class_name.split()
        for match_class in avoid_classes:
            if match_class in class_name:
                return
    for child in list(el):
        autolink(child, link_regexes=link_regexes,
                 avoid_elements=avoid_elements,
                 avoid_hosts=avoid_hosts,
                 avoid_classes=avoid_classes)
        if child.tail:
            text, tail_children = _link_text(
                child.tail, link_regexes, avoid_hosts, factory=el.makeelement)
            if tail_children:
                child.tail = text
                index = el.index(child)
                el[index+1:index+1] = tail_children
    if el.text:
        text, pre_children = _link_text(
            el.text, link_regexes, avoid_hosts, factory=el.makeelement)
        if pre_children:
            el.text = text
            el[:0] = pre_children

def autolink ( el , link_regexes = _link_regexes , avoid_elements = _avoid_elements , avoid_hosts = _avoid_hosts , avoid_classes = _avoid_classes ) : if el . tag in avoid_elements : return class_name = el . get ( 'class' ) if class_name : class_name = class_name . split ( ) for match_class in avoid_classes : if match_class in class_name : return for child in list ( el ) : autolink ( child , link_regexes = link_regexes , avoid_elements = avoid_elements , avoid_hosts = avoid_hosts , avoid_classes = avoid_classes ) if child . tail : text , tail_children = _link_text ( child . tail , link_regexes , avoid_hosts , factory = el . makeelement ) if tail_children : child . tail = text index = el . index ( child ) el [ index + 1 : index + 1 ] = tail_children if el . text : text , pre_children = _link_text ( el . text , link_regexes , avoid_hosts , factory = el . makeelement ) if pre_children : el . text = text el [ : 0 ] = pre_children


def kill_conditional_comments(self, doc):
        """
        IE conditional comments basically embed HTML that the parser
        doesn't normally see.  We can't allow anything like that, so
        we'll kill any comments that could be conditional.
        """
        bad = []
        self._kill_elements(
            doc, lambda el: _conditional_comment_re.search(el.text),
            etree.Comment)

def kill_conditional_comments ( self , doc ) : bad = [ ] self . _kill_elements ( doc , lambda el : _conditional_comment_re . search ( el . text ) , etree . Comment )

IE conditional comments basically embed HTML that the parser doesn t normally see . We can t allow anything like that so we ll kill any comments that could be conditional .


def document_fromstring(html, guess_charset=True, parser=None):
    """Parse a whole document into a string."""
    if not isinstance(html, _strings):
        raise TypeError('string required')

    if parser is None:
        parser = html_parser

    return parser.parse(html, useChardet=guess_charset).getroot()

def
document_fromstring
(
html
,
guess_charset
=
True
,
parser
=
)
:
if
not
isinstance
(
html
,
_strings
)
:
raise
TypeError
(
'string required'
)
if
parser
is
:
parser
=
html_parser
return
parser
.
parse
(
html
,
useChardet
=
guess_charset
)
.
getroot
(
)