def _rectify_products(self, product=None):
        """ puts products in a normalized format """
        products = self.product if product is None else product
        if products is None:
            return None
        if not isinstance(products, (list, tuple)):
            products = [products]
        return products

def _rectify_products ( self , product = None ) : products = self . product if product is None else product if products is None : return None if not isinstance ( products , ( list , tuple ) ) : products = [ products ] return products


def _product_file_hash(self, product=None):
        """
        Get the hash of the each product file
        """
        if self.hasher is None:
            return None
        else:
            products = self._rectify_products(product)
            product_file_hash = [
                util_hash.hash_file(p, hasher=self.hasher, base='hex')
                for p in products
            ]
            return product_file_hash

def _product_file_hash ( self , product = None ) : if self . hasher is None : return None else : products = self . _rectify_products ( product ) product_file_hash = [ util_hash . hash_file ( p , hasher = self . hasher , base = 'hex' ) for p in products ] return product_file_hash


def expired(self, cfgstr=None, product=None):
        """
        Check to see if a previously existing stamp is still valid and if the
        expected result of that computation still exists.

        Args:
            cfgstr (str, optional): override the default cfgstr if specified
            product (PathLike or Sequence[PathLike], optional): override the
                default product if specified
        """
        products = self._rectify_products(product)
        certificate = self._get_certificate(cfgstr=cfgstr)
        if certificate is None:
            # We dont have a certificate, so we are expired
            is_expired = True
        elif products is None:
            # We dont have a product to check, so assume not expired
            is_expired = False
        elif not all(map(os.path.exists, products)):
            # We are expired if the expected product does not exist
            is_expired = True
        else:
            # We are expired if the hash of the existing product data
            # does not match the expected hash in the certificate
            product_file_hash = self._product_file_hash(products)
            certificate_hash = certificate.get('product_file_hash', None)
            is_expired = product_file_hash != certificate_hash
        return is_expired

def expired ( self , cfgstr = None , product = None ) : products = self . _rectify_products ( product ) certificate = self . _get_certificate ( cfgstr = cfgstr ) if certificate is None : is_expired = True elif products is None : is_expired = False elif not all ( map ( os . path . exists , products ) ) : is_expired = True else : product_file_hash = self . _product_file_hash ( products ) certificate_hash = certificate . get ( 'product_file_hash' , None ) is_expired = product_file_hash != certificate_hash return is_expired

Check to see if a previously existing stamp is still valid and if the expected result of that computation still exists .


def renew(self, cfgstr=None, product=None):
        """
        Recertify that the product has been recomputed by writing a new
        certificate to disk.
        """
        products = self._rectify_products(product)
        certificate = {
            'timestamp': util_time.timestamp(),
            'product': products,
        }
        if products is not None:
            if not all(map(os.path.exists, products)):
                raise IOError(
                    'The stamped product must exist: {}'.format(products))
            certificate['product_file_hash'] = self._product_file_hash(products)
        self.cacher.save(certificate, cfgstr=cfgstr)
        return certificate

def
renew
(
self
,
cfgstr
=
,
product
=
)
:
products
=
self
.
_rectify_products
(
product
)
certificate
=
{
'timestamp'
:
util_time
.
timestamp
(
)
,
'product'
:
products
,
}
if
products
is
not
:
if
not
all
(
map
(
os
.
path
.
exists
,
products
)
)
:
raise
IOError
(
'The stamped product must exist: {}'
.
format
(
products
)
)
certificate
[
'product_file_hash'
]
=
self
.
_product_file_hash
(
products
)
self
.
cacher
.
save
(
certificate
,
cfgstr
=
cfgstr
)
return
certificate

Recertify that the product has been recomputed by writing a new certificate to disk .


def isatty(self):  # nocover
        """
        Returns true of the redirect is a terminal.

        Notes:
            Needed for IPython.embed to work properly when this class is used
            to override stdout / stderr.
        """
        return (self.redirect is not None and
                hasattr(self.redirect, 'isatty') and self.redirect.isatty())

def isatty ( self ) : return ( self . redirect is not None and hasattr ( self . redirect , 'isatty' ) and self . redirect . isatty ( ) )

Returns true of the redirect is a terminal .


def encoding(self):
        """
        Gets the encoding of the `redirect` IO object

        Doctest:
            >>> redirect = io.StringIO()
            >>> assert TeeStringIO(redirect).encoding is None
            >>> assert TeeStringIO(None).encoding is None
            >>> assert TeeStringIO(sys.stdout).encoding is sys.stdout.encoding
            >>> redirect = io.TextIOWrapper(io.StringIO())
            >>> assert TeeStringIO(redirect).encoding is redirect.encoding
        """
        if self.redirect is not None:
            return self.redirect.encoding
        else:
            return super(TeeStringIO, self).encoding

def encoding ( self ) : if self . redirect is not None : return self . redirect . encoding else : return super ( TeeStringIO , self ) . encoding

Gets the encoding of the redirect IO object


def write(self, msg):
        """
        Write to this and the redirected stream
        """
        if self.redirect is not None:
            self.redirect.write(msg)
        if six.PY2:
            from xdoctest.utils.util_str import ensure_unicode
            msg = ensure_unicode(msg)
        super(TeeStringIO, self).write(msg)

def write ( self , msg ) : if self . redirect is not None : self . redirect . write ( msg ) if six . PY2 : from xdoctest . utils . util_str import ensure_unicode msg = ensure_unicode ( msg ) super ( TeeStringIO , self ) . write ( msg )


def flush(self):  # nocover
        """
        Flush to this and the redirected stream
        """
        if self.redirect is not None:
            self.redirect.flush()
        super(TeeStringIO, self).flush()

def flush ( self ) : if self . redirect is not None : self . redirect . flush ( ) super ( TeeStringIO , self ) . flush ( )


def log_part(self):
        """ Log what has been captured so far """
        self.cap_stdout.seek(self._pos)
        text = self.cap_stdout.read()
        self._pos = self.cap_stdout.tell()
        self.parts.append(text)
        self.text = text

def log_part ( self ) : self . cap_stdout . seek ( self . _pos ) text = self . cap_stdout . read ( ) self . _pos = self . cap_stdout . tell ( ) self . parts . append ( text ) self . text = text


def platform_data_dir():
    """
    Returns path for user-specific data files

    Returns:
        PathLike : path to the data dir used by the current operating system
    """
    if LINUX:  # nocover
        dpath_ = os.environ.get('XDG_DATA_HOME', '~/.local/share')
    elif DARWIN:  # nocover
        dpath_  = '~/Library/Application Support'
    elif WIN32:  # nocover
        dpath_ = os.environ.get('APPDATA', '~/AppData/Roaming')
    else:  # nocover
        raise '~/AppData/Local'
    dpath = normpath(expanduser(dpath_))
    return dpath

def
platform_data_dir
(
)
:
if
LINUX
:
dpath_
=
os
.
environ
.
get
(
'XDG_DATA_HOME'
,
'~/.local/share'
)
elif
DARWIN
:
dpath_
=
'~/Library/Application Support'
elif
WIN32
:
dpath_
=
os
.
environ
.
get
(
'APPDATA'
,
'~/AppData/Roaming'
)
else
:
raise
'~/AppData/Local'
dpath
=
normpath
(
expanduser
(
dpath_
)
)
return
dpath

Returns path for user - specific data files


def platform_config_dir():
    """
    Returns a directory which should be writable for any application
    This should be used for persistent configuration files.

    Returns:
        PathLike : path to the cahce dir used by the current operating system
    """
    if LINUX:  # nocover
        dpath_ = os.environ.get('XDG_CONFIG_HOME', '~/.config')
    elif DARWIN:  # nocover
        dpath_  = '~/Library/Application Support'
    elif WIN32:  # nocover
        dpath_ = os.environ.get('APPDATA', '~/AppData/Roaming')
    else:  # nocover
        raise NotImplementedError('Unknown Platform  %r' % (sys.platform,))
    dpath = normpath(expanduser(dpath_))
    return dpath

def
platform_config_dir
(
)
:
if
LINUX
:
dpath_
=
os
.
environ
.
get
(
'XDG_CONFIG_HOME'
,
'~/.config'
)
elif
DARWIN
:
dpath_
=
'~/Library/Application Support'
elif
WIN32
:
dpath_
=
os
.
environ
.
get
(
'APPDATA'
,
'~/AppData/Roaming'
)
else
:
raise
NotImplementedError
(
'Unknown Platform  %r'
%
(
sys
.
platform
,
)
)
dpath
=
normpath
(
expanduser
(
dpath_
)
)
return
dpath

Returns a directory which should be writable for any application This should be used for persistent configuration files .


def ensure_app_data_dir(appname, *args):
    """
    Calls `get_app_data_dir` but ensures the directory exists.

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    SeeAlso:
        get_app_data_dir

    Example:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_data_dir('ubelt')
        >>> assert exists(dpath)
    """
    from ubelt import util_path
    dpath = get_app_data_dir(appname, *args)
    util_path.ensuredir(dpath)
    return dpath

def ensure_app_data_dir ( appname , * args ) : from ubelt import util_path dpath = get_app_data_dir ( appname , * args ) util_path . ensuredir ( dpath ) return dpath

Calls get_app_data_dir but ensures the directory exists .


def ensure_app_config_dir(appname, *args):
    """
    Calls `get_app_config_dir` but ensures the directory exists.

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    SeeAlso:
        get_app_config_dir

    Example:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_config_dir('ubelt')
        >>> assert exists(dpath)
    """
    from ubelt import util_path
    dpath = get_app_config_dir(appname, *args)
    util_path.ensuredir(dpath)
    return dpath

def ensure_app_config_dir ( appname , * args ) : from ubelt import util_path dpath = get_app_config_dir ( appname , * args ) util_path . ensuredir ( dpath ) return dpath

Calls get_app_config_dir but ensures the directory exists .


def ensure_app_cache_dir(appname, *args):
    """
    Calls `get_app_cache_dir` but ensures the directory exists.

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    SeeAlso:
        get_app_cache_dir

    Example:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_cache_dir('ubelt')
        >>> assert exists(dpath)
    """
    from ubelt import util_path
    dpath = get_app_cache_dir(appname, *args)
    util_path.ensuredir(dpath)
    return dpath

def ensure_app_cache_dir ( appname , * args ) : from ubelt import util_path dpath = get_app_cache_dir ( appname , * args ) util_path . ensuredir ( dpath ) return dpath

Calls get_app_cache_dir but ensures the directory exists .


def startfile(fpath, verbose=True):  # nocover
    """
    Uses default program defined by the system to open a file.
    This is done via `os.startfile` on windows, `open` on mac, and `xdg-open`
    on linux.

    Args:
        fpath (PathLike): a file to open using the program associated with the
            files extension type.
        verbose (int): verbosity

    References:
        http://stackoverflow.com/questions/2692873/quote-posix

    DisableExample:
        >>> # This test interacts with a GUI frontend, not sure how to test.
        >>> import ubelt as ub
        >>> base = ub.ensure_app_cache_dir('ubelt')
        >>> fpath1 = join(base, 'test_open.txt')
        >>> ub.touch(fpath1)
        >>> proc = ub.startfile(fpath1)
    """
    from ubelt import util_cmd
    if verbose:
        print('[ubelt] startfile("{}")'.format(fpath))
    fpath = normpath(fpath)
    if not exists(fpath):
        raise Exception('Cannot start nonexistant file: %r' % fpath)
    if not WIN32:
        import pipes
        fpath = pipes.quote(fpath)
    if LINUX:
        info = util_cmd.cmd(('xdg-open', fpath), detach=True, verbose=verbose)
    elif DARWIN:
        info = util_cmd.cmd(('open', fpath), detach=True, verbose=verbose)
    elif WIN32:
        os.startfile(fpath)
        info = None
    else:
        raise RuntimeError('Unknown Platform')
    if info is not None:
        if not info['proc']:
            raise Exception('startfile failed')

def
startfile
(
fpath
,
verbose
=
True
)
:
from
ubelt
import
util_cmd
if
verbose
:
print
(
'[ubelt] startfile("{}")'
.
format
(
fpath
)
)
fpath
=
normpath
(
fpath
)
if
not
exists
(
fpath
)
:
raise
Exception
(
'Cannot start nonexistant file: %r'
%
fpath
)
if
not
WIN32
:
import
pipes
fpath
=
pipes
.
quote
(
fpath
)
if
LINUX
:
info
=
util_cmd
.
cmd
(
(
'xdg-open'
,
fpath
)
,
detach
=
True
,
verbose
=
verbose
)
elif
DARWIN
:
info
=
util_cmd
.
cmd
(
(
'open'
,
fpath
)
,
detach
=
True
,
verbose
=
verbose
)
elif
WIN32
:
os
.
startfile
(
fpath
)
info
=
else
:
raise
RuntimeError
(
'Unknown Platform'
)
if
info
is
not
:
if
not
info
[
'proc'
]
:
raise
Exception
(
'startfile failed'
)

Uses default program defined by the system to open a file . This is done via os . startfile on windows open on mac and xdg - open on linux .


def find_exe(name, multi=False, path=None):
    """
    Locate a command.

    Search your local filesystem for an executable and return the first
    matching file with executable permission.

    Args:
        name (str): globstr of matching filename

        multi (bool): if True return all matches instead of just the first.
            Defaults to False.

        path (str or Iterable[PathLike]): overrides the system PATH variable.

    Returns:
        PathLike or List[PathLike] or None: returns matching executable(s).

    SeeAlso:
        shutil.which - which is available in Python 3.3+.

    Notes:
        This is essentially the `which` UNIX command

    References:
        https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
        https://docs.python.org/dev/library/shutil.html#shutil.which

    Example:
        >>> find_exe('ls')
        >>> find_exe('ping')
        >>> assert find_exe('which') == find_exe(find_exe('which'))
        >>> find_exe('which', multi=True)
        >>> find_exe('ping', multi=True)
        >>> find_exe('cmake', multi=True)
        >>> find_exe('nvcc', multi=True)
        >>> find_exe('noexist', multi=True)

    Example:
        >>> assert not find_exe('noexist', multi=False)
        >>> assert find_exe('ping', multi=False)
        >>> assert not find_exe('noexist', multi=True)
        >>> assert find_exe('ping', multi=True)

    Benchmark:
        >>> # xdoctest: +IGNORE_WANT
        >>> import ubelt as ub
        >>> import shutil
        >>> for timer in ub.Timerit(100, bestof=10, label='ub.find_exe'):
        >>>     ub.find_exe('which')
        >>> for timer in ub.Timerit(100, bestof=10, label='shutil.which'):
        >>>     shutil.which('which')
        Timed best=58.71 µs, mean=59.64 ± 0.96 µs for ub.find_exe
        Timed best=72.75 µs, mean=73.07 ± 0.22 µs for shutil.which
    """
    candidates = find_path(name, path=path, exact=True)
    mode = os.X_OK | os.F_OK
    results = (fpath for fpath in candidates
               if os.access(fpath, mode) and not isdir(fpath))
    if not multi:
        for fpath in results:
            return fpath
    else:
        return list(results)

def find_exe ( name , multi = False , path = None ) : candidates = find_path ( name , path = path , exact = True ) mode = os . X_OK | os . F_OK results = ( fpath for fpath in candidates if os . access ( fpath , mode ) and not isdir ( fpath ) ) if not multi : for fpath in results : return fpath else : return list ( results )


def userhome(username=None):
    """
    Returns the user's home directory.
    If `username` is None, this is the directory for the current user.

    Args:
        username (str): name of a user on the system

    Returns:
        PathLike: userhome_dpath: path to the home directory

    Example:
        >>> import getpass
        >>> username = getpass.getuser()
        >>> assert userhome() == expanduser('~')
        >>> assert userhome(username) == expanduser('~')
    """
    if username is None:
        # get home directory for the current user
        if 'HOME' in os.environ:
            userhome_dpath = os.environ['HOME']
        else:  # nocover
            if sys.platform.startswith('win32'):
                # win32 fallback when HOME is not defined
                if 'USERPROFILE' in os.environ:
                    userhome_dpath = os.environ['USERPROFILE']
                elif 'HOMEPATH' in os.environ:
                    drive = os.environ.get('HOMEDRIVE', '')
                    userhome_dpath = join(drive, os.environ['HOMEPATH'])
                else:
                    raise OSError("Cannot determine the user's home directory")
            else:
                # posix fallback when HOME is not defined
                import pwd
                userhome_dpath = pwd.getpwuid(os.getuid()).pw_dir
    else:
        # A specific user directory was requested
        if sys.platform.startswith('win32'):  # nocover
            # get the directory name for the current user
            c_users = dirname(userhome())
            userhome_dpath = join(c_users, username)
            if not exists(userhome_dpath):
                raise KeyError('Unknown user: {}'.format(username))
        else:
            import pwd
            try:
                pwent = pwd.getpwnam(username)
            except KeyError:  # nocover
                raise KeyError('Unknown user: {}'.format(username))
            userhome_dpath = pwent.pw_dir
    return userhome_dpath

Returns the user s home directory . If username is None this is the directory for the current user .


def compressuser(path, home='~'):
    """
    Inverse of `os.path.expanduser`

    Args:
        path (PathLike): path in system file structure
        home (str): symbol used to replace the home path. Defaults to '~', but
            you might want to use '$HOME' or '%USERPROFILE%' instead.

    Returns:
        PathLike: path: shortened path replacing the home directory with a tilde

    CommandLine:
        xdoctest -m ubelt.util_path compressuser

    Example:
        >>> path = expanduser('~')
        >>> assert path != '~'
        >>> assert compressuser(path) == '~'
        >>> assert compressuser(path + '1') == path + '1'
        >>> assert compressuser(path + '/1') == join('~', '1')
        >>> assert compressuser(path + '/1', '$HOME') == join('$HOME', '1')
    """
    path = normpath(path)
    userhome_dpath = userhome()
    if path.startswith(userhome_dpath):
        if len(path) == len(userhome_dpath):
            path = home
        elif path[len(userhome_dpath)] == os.path.sep:
            path = home + path[len(userhome_dpath):]
    return path

def compressuser ( path , home = '~' ) : path = normpath ( path ) userhome_dpath = userhome ( ) if path . startswith ( userhome_dpath ) : if len ( path ) == len ( userhome_dpath ) : path = home elif path [ len ( userhome_dpath ) ] == os . path . sep : path = home + path [ len ( userhome_dpath ) : ] return path


def truepath(path, real=False):
    """
    Normalizes a string representation of a path and does shell-like expansion.

    Args:
        path (PathLike): string representation of a path
        real (bool): if True, all symbolic links are followed. (default: False)

    Returns:
        PathLike : normalized path

    Note:
        This function is similar to the composition of expanduser, expandvars,
        normpath, and (realpath if `real` else abspath). However, on windows
        backslashes are then replaced with forward slashes to offer a
        consistent unix-like experience across platforms.

        On windows expanduser will expand environment variables formatted as
        %name%, whereas on unix, this will not occur.

    CommandLine:
        python -m ubelt.util_path truepath

    Example:
        >>> import ubelt as ub
        >>> assert ub.truepath('~/foo') == join(ub.userhome(), 'foo')
        >>> assert ub.truepath('~/foo') == ub.truepath('~/foo/bar/..')
        >>> assert ub.truepath('~/foo', real=True) == ub.truepath('~/foo')
    """
    path = expanduser(path)
    path = expandvars(path)
    if real:
        path = realpath(path)
    else:
        path = abspath(path)
    path = normpath(path)
    return path

def truepath ( path , real = False ) : path = expanduser ( path ) path = expandvars ( path ) if real : path = realpath ( path ) else : path = abspath ( path ) path = normpath ( path ) return path

Normalizes a string representation of a path and does shell - like expansion .


def ensuredir(dpath, mode=0o1777, verbose=None):
    r"""
    Ensures that directory will exist. Creates new dir with sticky bits by
    default

    Args:
        dpath (PathLike): dir to ensure. Can also be a tuple to send to join
        mode (int): octal mode of directory (default 0o1777)
        verbose (int): verbosity (default 0)

    Returns:
        PathLike: path: the ensured directory

    Notes:
        This function is not thread-safe in Python2

    Example:
        >>> from ubelt.util_platform import *  # NOQA
        >>> import ubelt as ub
        >>> cache_dpath = ub.ensure_app_cache_dir('ubelt')
        >>> dpath = join(cache_dpath, 'ensuredir')
        >>> if exists(dpath):
        ...     os.rmdir(dpath)
        >>> assert not exists(dpath)
        >>> ub.ensuredir(dpath)
        >>> assert exists(dpath)
        >>> os.rmdir(dpath)
    """
    if verbose is None:  # nocover
        verbose = 0
    if isinstance(dpath, (list, tuple)):  # nocover
        dpath = join(*dpath)
    if not exists(dpath):
        if verbose:  # nocover
            print('Ensuring new directory (%r)' % dpath)
        if sys.version_info.major == 2:  # nocover
            os.makedirs(normpath(dpath), mode=mode)
        else:
            os.makedirs(normpath(dpath), mode=mode, exist_ok=True)
    else:
        if verbose:  # nocover
            print('Ensuring existing directory (%r)' % dpath)
    return dpath

def
ensuredir
(
dpath
,
mode
=
0o1777
,
verbose
=
)
:
r
if
verbose
is
:
verbose
=
0
if
isinstance
(
dpath
,
(
list
,
tuple
)
)
:
dpath
=
join
(
*
dpath
)
if
not
exists
(
dpath
)
:
if
verbose
:
print
(
'Ensuring new directory (%r)'
%
dpath
)
if
sys
.
version_info
.
major
==
2
:
os
.
makedirs
(
normpath
(
dpath
)
,
mode
=
mode
)
else
:
os
.
makedirs
(
normpath
(
dpath
)
,
mode
=
mode
,
exist_ok
=
True
)
else
:
if
verbose
:
print
(
'Ensuring existing directory (%r)'
%
dpath
)
return
dpath

r Ensures that directory will exist . Creates new dir with sticky bits by default


def parse_requirements_alt(fname='requirements.txt'):
    """
    pip install requirements-parser
    fname='requirements.txt'
    """
    import requirements
    from os.path import dirname, join, exists
    require_fpath = join(dirname(__file__), fname)
    if exists(require_fpath):
        # Dont use until this handles platform specific dependencies
        with open(require_fpath, 'r') as file:
            requires = list(requirements.parse(file))
        packages = [r.name for r in requires]
        return packages
    return []

def parse_requirements_alt ( fname = 'requirements.txt' ) : import requirements from os . path import dirname , join , exists require_fpath = join ( dirname ( __file__ ) , fname ) if exists ( require_fpath ) : with open ( require_fpath , 'r' ) as file : requires = list ( requirements . parse ( file ) ) packages = [ r . name for r in requires ] return packages return [ ]

pip install requirements - parser fname = requirements . txt


def parse_requirements(fname='requirements.txt'):
    """
    Parse the package dependencies listed in a requirements file but strips
    specific versioning information.

    TODO:
        perhaps use https://github.com/davidfischer/requirements-parser instead

    CommandLine:
        python -c "import setup; print(setup.parse_requirements())"
    """
    from os.path import dirname, join, exists
    import re
    require_fpath = join(dirname(__file__), fname)

    def parse_line(line):
        """
        Parse information from a line in a requirements text file
        """
        info = {}
        if line.startswith('-e '):
            info['package'] = line.split('#egg=')[1]
        else:
            # Remove versioning from the package
            pat = '(' + '|'.join(['>=', '==', '>']) + ')'
            parts = re.split(pat, line, maxsplit=1)
            parts = [p.strip() for p in parts]

            info['package'] = parts[0]
            if len(parts) > 1:
                op, rest = parts[1:]
                if ';' in rest:
                    # Handle platform specific dependencies
                    # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies
                    version, platform_deps = map(str.strip, rest.split(';'))
                    info['platform_deps'] = platform_deps
                else:
                    version = rest  # NOQA
                info['version'] = (op, version)
        return info

    # This breaks on pip install, so check that it exists.
    if exists(require_fpath):
        with open(require_fpath, 'r') as f:
            packages = []
            for line in f.readlines():
                line = line.strip()
                if line and not line.startswith('#'):
                    info = parse_line(line)
                    package = info['package']
                    if not sys.version.startswith('3.4'):
                        # apparently package_deps are broken in 3.4
                        platform_deps = info.get('platform_deps')
                        if platform_deps is not None:
                            package += ';' + platform_deps
                    packages.append(package)
            return packages
    return []

def
parse_requirements
(
fname
=
'requirements.txt'
)
:
from
os
.
path
import
dirname
,
join
,
exists
import
re
require_fpath
=
join
(
dirname
(
__file__
)
,
fname
)
def
parse_line
(
line
)
:
info
=
{
}
if
line
.
startswith
(
'-e '
)
:
info
[
'package'
]
=
line
.
split
(
'#egg='
)
[
1
]
else
:
pat
=
'('
+
'|'
.
join
(
[
'>='
,
'=='
,
'>'
]
)
+
')'
parts
=
re
.
split
(
pat
,
line
,
maxsplit
=
1
)
parts
=
[
p
.
strip
(
)
for
p
in
parts
]
info
[
'package'
]
=
parts
[
0
]
if
len
(
parts
)
>
1
:
op
,
rest
=
parts
[
1
:
]
if
';'
in
rest
:
version
,
platform_deps
=
map
(
str
.
strip
,
rest
.
split
(
';'
)
)
info
[
'platform_deps'
]
=
platform_deps
else
:
version
=
rest
info
[
'version'
]
=
(
op
,
version
)
return
info
if
exists
(
require_fpath
)
:
with
open
(
require_fpath
,
'r'
)
as
f
:
packages
=
[
]
for
line
in
f
.
readlines
(
)
:
line
=
line
.
strip
(
)
if
line
and
not
line
.
startswith
(
'#'
)
:
info
=
parse_line
(
line
)
package
=
info
[
'package'
]
if
not
sys
.
version
.
startswith
(
'3.4'
)
:
platform_deps
=
info
.
get
(
'platform_deps'
)
if
platform_deps
is
not
:
package
+=
';'
+
platform_deps
packages
.
append
(
package
)
return
packages
return
[
]

Parse the package dependencies listed in a requirements file but strips specific versioning information .


def inject_method(self, func, name=None):
    """
    Injects a function into an object instance as a bound method

    The main use case of this function is for monkey patching. While monkey
    patching is sometimes necessary it should generally be avoided. Thus, we
    simply remind the developer that there might be a better way.

    Args:
        self (object): instance to inject a function into
        func (func): the function to inject (must contain an arg for self)
        name (str): name of the method. optional. If not specified the name
            of the function is used.

    Example:
        >>> class Foo(object):
        >>>     def bar(self):
        >>>         return 'bar'
        >>> def baz(self):
        >>>     return 'baz'
        >>> self = Foo()
        >>> assert self.bar() == 'bar'
        >>> assert not hasattr(self, 'baz')
        >>> inject_method(self, baz)
        >>> assert not hasattr(Foo, 'baz'), 'should only change one instance'
        >>> assert self.baz() == 'baz'
        >>> inject_method(self, baz, 'bar')
        >>> assert self.bar() == 'baz'
    """
    # TODO: if func is a bound method we should probably unbind it
    new_method = func.__get__(self, self.__class__)
    if name is None:
        name = func.__name__
    setattr(self, name, new_method)

def inject_method ( self , func , name = None ) : new_method = func . __get__ ( self , self . __class__ ) if name is None : name = func . __name__ setattr ( self , name , new_method )


def touch(fpath, mode=0o666, dir_fd=None, verbose=0, **kwargs):
    """
    change file timestamps

    Works like the touch unix utility

    Args:
        fpath (PathLike): name of the file
        mode (int): file permissions (python3 and unix only)
        dir_fd (file): optional directory file descriptor. If specified, fpath
            is interpreted as relative to this descriptor (python 3 only).
        verbose (int): verbosity
        **kwargs : extra args passed to `os.utime` (python 3 only).

    Returns:
        PathLike: path to the file

    References:
        https://stackoverflow.com/questions/1158076/implement-touch-using-python

    Example:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_cache_dir('ubelt')
        >>> fpath = join(dpath, 'touch_file')
        >>> assert not exists(fpath)
        >>> ub.touch(fpath)
        >>> assert exists(fpath)
        >>> os.unlink(fpath)
    """
    if verbose:
        print('Touching file {}'.format(fpath))
    if six.PY2:  # nocover
        with open(fpath, 'a'):
            os.utime(fpath, None)
    else:
        flags = os.O_CREAT | os.O_APPEND
        with os.fdopen(os.open(fpath, flags=flags, mode=mode, dir_fd=dir_fd)) as f:
            os.utime(f.fileno() if os.utime in os.supports_fd else fpath,
                     dir_fd=None if os.supports_fd else dir_fd, **kwargs)
    return fpath

def
touch
(
fpath
,
mode
=
0o666
,
dir_fd
=
,
verbose
=
0
,
**
kwargs
)
:
if
verbose
:
print
(
'Touching file {}'
.
format
(
fpath
)
)
if
six
.
PY2
:
with
open
(
fpath
,
'a'
)
:
os
.
utime
(
fpath
,
)
else
:
flags
=
os
.
O_CREAT
|
os
.
O_APPEND
with
os
.
fdopen
(
os
.
open
(
fpath
,
flags
=
flags
,
mode
=
mode
,
dir_fd
=
dir_fd
)
)
as
f
:
os
.
utime
(
f
.
fileno
(
)
if
os
.
utime
in
os
.
supports_fd
else
fpath
,
dir_fd
=
if
os
.
supports_fd
else
dir_fd
,
**
kwargs
)
return
fpath


def delete(path, verbose=False):
    """
    Removes a file or recursively removes a directory.
    If a path does not exist, then this is does nothing.

    Args:
        path (PathLike): file or directory to remove
        verbose (bool): if True prints what is being done

    SeeAlso:
        send2trash - A cross-platform Python package for sending files
            to the trash instead of irreversibly deleting them.
            https://github.com/hsoft/send2trash

    Doctest:
        >>> import ubelt as ub
        >>> base = ub.ensure_app_cache_dir('ubelt', 'delete_test')
        >>> dpath1 = ub.ensuredir(join(base, 'dir'))
        >>> ub.ensuredir(join(base, 'dir', 'subdir'))
        >>> ub.touch(join(base, 'dir', 'to_remove1.txt'))
        >>> fpath1 = join(base, 'dir', 'subdir', 'to_remove3.txt')
        >>> fpath2 = join(base, 'dir', 'subdir', 'to_remove2.txt')
        >>> ub.touch(fpath1)
        >>> ub.touch(fpath2)
        >>> assert all(map(exists, (dpath1, fpath1, fpath2)))
        >>> ub.delete(fpath1)
        >>> assert all(map(exists, (dpath1, fpath2)))
        >>> assert not exists(fpath1)
        >>> ub.delete(dpath1)
        >>> assert not any(map(exists, (dpath1, fpath1, fpath2)))

    Doctest:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_cache_dir('ubelt', 'delete_test2')
        >>> dpath1 = ub.ensuredir(join(dpath, 'dir'))
        >>> fpath1 = ub.touch(join(dpath1, 'to_remove.txt'))
        >>> assert exists(fpath1)
        >>> ub.delete(dpath)
        >>> assert not exists(fpath1)
    """
    if not os.path.exists(path):
        # if the file does exists and is not a broken link
        if os.path.islink(path):
            if verbose:  # nocover
                print('Deleting broken link="{}"'.format(path))
            os.unlink(path)
        elif os.path.isdir(path):  # nocover
            # Only on windows will a file be a directory and not exist
            if verbose:
                print('Deleting broken directory link="{}"'.format(path))
            os.rmdir(path)
        elif os.path.isfile(path):  # nocover
            # This is a windows only case
            if verbose:
                print('Deleting broken file link="{}"'.format(path))
            os.unlink(path)
        else:
            if verbose:  # nocover
                print('Not deleting non-existant path="{}"'.format(path))
    else:
        if os.path.islink(path):
            if verbose:  # nocover
                print('Deleting symbolic link="{}"'.format(path))
            os.unlink(path)
        elif os.path.isfile(path):
            if verbose:  # nocover
                print('Deleting file="{}"'.format(path))
            os.unlink(path)
        elif os.path.isdir(path):
            if verbose:  # nocover
                print('Deleting directory="{}"'.format(path))
            if sys.platform.startswith('win32'):  # nocover
                # Workaround bug that prevents shutil from working if
                # the directory contains junctions
                from ubelt import _win32_links
                _win32_links._win32_rmtree(path, verbose=verbose)
            else:
                import shutil
                shutil.rmtree(path)

def
delete
(
path
,
verbose
=
False
)
:
if
not
os
.
path
.
exists
(
path
)
:
if
os
.
path
.
islink
(
path
)
:
if
verbose
:
print
(
'Deleting broken link="{}"'
.
format
(
path
)
)
os
.
unlink
(
path
)
elif
os
.
path
.
isdir
(
path
)
:
if
verbose
:
print
(
'Deleting broken directory link="{}"'
.
format
(
path
)
)
os
.
rmdir
(
path
)
elif
os
.
path
.
isfile
(
path
)
:
if
verbose
:
print
(
'Deleting broken file link="{}"'
.
format
(
path
)
)
os
.
unlink
(
path
)
else
:
if
verbose
:
print
(
'Not deleting non-existant path="{}"'
.
format
(
path
)
)
else
:
if
os
.
path
.
islink
(
path
)
:
if
verbose
:
print
(
'Deleting symbolic link="{}"'
.
format
(
path
)
)
os
.
unlink
(
path
)
elif
os
.
path
.
isfile
(
path
)
:
if
verbose
:
print
(
'Deleting file="{}"'
.
format
(
path
)
)
os
.
unlink
(
path
)
elif
os
.
path
.
isdir
(
path
)
:
if
verbose
:
print
(
'Deleting directory="{}"'
.
format
(
path
)
)
if
sys
.
platform
.
startswith
(
'win32'
)
:
from
ubelt
import
_win32_links
_win32_links
.
_win32_rmtree
(
path
,
verbose
=
verbose
)
else
:
import
shutil
shutil
.
rmtree
(
path
)

Removes a file or recursively removes a directory . If a path does not exist then this is does nothing .


def repr2(data, **kwargs):
    """
    Makes a pretty and easy-to-doctest string representation!

    This is an alternative to repr, and `pprint.pformat` that attempts to be
    both more configurable and generate output that is consistent between
    python versions.

    Notes:
        This function has many keyword arguments that can be used to customize
        the final representation. For convinience some of the more frequently
        used kwargs have short aliases. See `Args` for more details.

    Args:
        data (object): an arbitrary python object
        **kwargs: see `the Kwargs` section

    Kwargs:
        si, stritems, (bool):
            dict/list items use str instead of repr

        strkeys, sk (bool):
            dict keys use str instead of repr

        strvals, sv (bool):
            dict values use str instead of repr

        nl, newlines (int | bool):
            number of top level nestings to place a newline after. If true all
            items are followed by newlines regardless of nesting level.
            Defaults to 1 for lists and True for dicts.

        nobr, nobraces (bool, default=False):
            if True, text will not contain outer braces for containers

        cbr, compact_brace (bool, default=False):
            if True, braces are compactified (i.e. they will not have newlines
            placed directly after them, think java / K&R / 1TBS)

        trailsep, trailing_sep (bool):
            if True, a separator is placed after the last item in a sequence.
            By default this is True if there are any `nl > 0`.

        explicit (bool, default=False):
            changes dict representation from `{k1: v1, ...}` to
            `dict(k1=v1, ...)`.

        precision (int, default=None):
            if specified floats are formatted with this precision

        kvsep (str, default=': '):
            separator between keys and values

        itemsep (str, default=' '):
            separator between items

        sort (bool):
            if True, attempts to sort all unordered collections in the returned
            text. NOTE: currently if True this will sort lists, this may not be
            a correct thing to do, as such the behavior of this arg is subject
            to change.

        suppress_small (bool):
            passed to `numpy.array2string` for ndarrays

        max_line_width (int):
            passed to `numpy.array2string` for ndarrays

        with_dtype (bool):
            only relevant to ndarrays. if True includes the dtype.

    Returns:
        str: outstr: output string

    Notes:
        There are also internal kwargs, which should not be used:
            _return_info (bool):  return information about child context
            _root_info (depth): information about parent context

    CommandLine:
        python -m ubelt.util_format repr2:0
        python -m ubelt.util_format repr2:1

    Example:
        >>> from ubelt.util_format import *
        >>> import ubelt as ub
        >>> dict_ = {
        ...     'custom_types': [slice(0, 1, None), 1/3],
        ...     'nest_dict': {'k1': [1, 2, {3: {4, 5}}],
        ...                   'key2': [1, 2, {3: {4, 5}}],
        ...                   'key3': [1, 2, {3: {4, 5}}],
        ...                   },
        ...     'nest_dict2': {'k': [1, 2, {3: {4, 5}}]},
        ...     'nested_tuples': [tuple([1]), tuple([2, 3]), frozenset([4, 5, 6])],
        ...     'one_tup': tuple([1]),
        ...     'simple_dict': {'spam': 'eggs', 'ham': 'jam'},
        ...     'simple_list': [1, 2, 'red', 'blue'],
        ...     'odict': ub.odict([(1, '1'), (2, '2')]),
        ... }
        >>> result = repr2(dict_, nl=3, precision=2); print(result)
        >>> result = repr2(dict_, nl=2, precision=2); print(result)
        >>> result = repr2(dict_, nl=1, precision=2); print(result)
        >>> result = repr2(dict_, nl=1, precision=2, itemsep='', explicit=True); print(result)
        >>> result = repr2(dict_, nl=1, precision=2, nobr=1, itemsep='', explicit=True); print(result)
        >>> result = repr2(dict_, nl=3, precision=2, cbr=True); print(result)
        >>> result = repr2(dict_, nl=3, precision=2, si=True); print(result)
        >>> result = repr2(dict_, nl=3, sort=True); print(result)
        >>> result = repr2(dict_, nl=3, sort=False, trailing_sep=False); print(result)
        >>> result = repr2(dict_, nl=3, sort=False, trailing_sep=False, nobr=True); print(result)

    Example:
        >>> from ubelt.util_format import *
        >>> def _nest(d, w):
        ...     if d == 0:
        ...         return {}
        ...     else:
        ...         return {'n{}'.format(d): _nest(d - 1, w + 1), 'm{}'.format(d): _nest(d - 1, w + 1)}
        >>> dict_ = _nest(d=4, w=1)
        >>> result = repr2(dict_, nl=6, precision=2, cbr=1)
        >>> print('---')
        >>> print(result)
        >>> result = repr2(dict_, nl=-1, precision=2)
        >>> print('---')
        >>> print(result)
    """
    custom_extensions = kwargs.get('extensions', None)

    _return_info = kwargs.get('_return_info', False)
    kwargs['_root_info'] = _rectify_root_info(kwargs.get('_root_info', None))

    outstr = None
    _leaf_info = None

    if custom_extensions:
        func = custom_extensions.lookup(data)
        if func is not None:
            outstr = func(data, **kwargs)

    if outstr is None:
        if isinstance(data, dict):
            outstr, _leaf_info = _format_dict(data, **kwargs)
        elif isinstance(data, (list, tuple, set, frozenset)):
            outstr, _leaf_info = _format_list(data, **kwargs)

    if outstr is None:
        # check any globally registered functions for special formatters
        func = _FORMATTER_EXTENSIONS.lookup(data)
        if func is not None:
            outstr = func(data, **kwargs)
        else:
            outstr = _format_object(data, **kwargs)

    if _return_info:
        _leaf_info = _rectify_leaf_info(_leaf_info)
        return outstr, _leaf_info
    else:
        return outstr

def repr2 ( data , ** kwargs ) : custom_extensions = kwargs . get ( 'extensions' , None ) _return_info = kwargs . get ( '_return_info' , False ) kwargs [ '_root_info' ] = _rectify_root_info ( kwargs . get ( '_root_info' , None ) ) outstr = None _leaf_info = None if custom_extensions : func = custom_extensions . lookup ( data ) if func is not None : outstr = func ( data , ** kwargs ) if outstr is None : if isinstance ( data , dict ) : outstr , _leaf_info = _format_dict ( data , ** kwargs ) elif isinstance ( data , ( list , tuple , set , frozenset ) ) : outstr , _leaf_info = _format_list ( data , ** kwargs ) if outstr is None : func = _FORMATTER_EXTENSIONS . lookup ( data ) if func is not None : outstr = func ( data , ** kwargs ) else : outstr = _format_object ( data , ** kwargs ) if _return_info : _leaf_info = _rectify_leaf_info ( _leaf_info ) return outstr , _leaf_info else : return outstr

Makes a pretty and easy - to - doctest string representation!


def _join_itemstrs(itemstrs, itemsep, newlines, _leaf_info, nobraces,
                   trailing_sep, compact_brace, lbr, rbr):
    """
    Joins string-ified items with separators newlines and container-braces.
    """
    # positive newlines means start counting from the root
    use_newline = newlines > 0

    # negative countdown values mean start counting from the leafs
    # if compact_brace < 0:
    #     compact_brace = (-compact_brace) >= _leaf_info['max_height']
    if newlines < 0:
        use_newline = (-newlines) < _leaf_info['max_height']

    if use_newline:
        sep = ',\n'
        if nobraces:
            body_str = sep.join(itemstrs)
            if trailing_sep and len(itemstrs) > 0:
                body_str += ','
            retstr = body_str
        else:
            if compact_brace:
                # Why must we modify the indentation below and not here?
                # prefix = ''
                # rest = [ub.indent(s, prefix) for s in itemstrs[1:]]
                # indented = itemstrs[0:1] + rest
                indented = itemstrs
            else:
                import ubelt as ub
                prefix = ' ' * 4
                indented = [ub.indent(s, prefix) for s in itemstrs]

            body_str = sep.join(indented)
            if trailing_sep and len(itemstrs) > 0:
                body_str += ','
            if compact_brace:
                # Why can we modify the indentation here but not above?
                braced_body_str = (lbr + body_str.replace('\n', '\n ') + rbr)
            else:
                braced_body_str = (lbr + '\n' + body_str + '\n' + rbr)
            retstr = braced_body_str
    else:
        sep = ',' + itemsep
        body_str = sep.join(itemstrs)
        if trailing_sep and len(itemstrs) > 0:
            body_str += ','
        retstr  = (lbr + body_str +  rbr)
    return retstr

def
_join_itemstrs
(
itemstrs
,
itemsep
,
newlines
,
_leaf_info
,
nobraces
,
trailing_sep
,
compact_brace
,
lbr
,
rbr
)
:
use_newline
=
newlines
>
0
if
newlines
<
0
:
use_newline
=
(
newlines
)
<
_leaf_info
[
'max_height'
]
if
use_newline
:
sep
=
',\n'
if
nobraces
:
body_str
=
sep
.
join
(
itemstrs
)
if
trailing_sep
and
len
(
itemstrs
)
>
0
:
body_str
+=
','
retstr
=
body_str
else
:
if
compact_brace
:
indented
=
itemstrs
else
:
import
ubelt
as
ub
prefix
=
' '
*
4
indented
=
[
ub
.
indent
(
s
,
prefix
)
for
s
in
itemstrs
]
body_str
=
sep
.
join
(
indented
)
if
trailing_sep
and
len
(
itemstrs
)
>
0
:
body_str
+=
','
if
compact_brace
:
braced_body_str
=
(
lbr
+
body_str
.
replace
(
'\n'
,
'\n '
)
+
rbr
)
else
:
braced_body_str
=
(
lbr
+
'\n'
+
body_str
+
'\n'
+
rbr
)
retstr
=
braced_body_str
else
:
sep
=
','
+
itemsep
body_str
=
sep
.
join
(
itemstrs
)
if
trailing_sep
and
len
(
itemstrs
)
>
0
:
body_str
+=
','
retstr
=
(
lbr
+
body_str
+
rbr
)
return
retstr

Joins string - ified items with separators newlines and container - braces .


def _dict_itemstrs(dict_, **kwargs):
    """
    Create a string representation for each item in a dict.

    Example:
        >>> from ubelt.util_format import *
        >>> dict_ =  {'b': .1, 'l': 'st', 'g': 1.0, 's': 10, 'm': 0.9, 'w': .5}
        >>> kwargs = {'strkeys': True}
        >>> itemstrs, _ = _dict_itemstrs(dict_, **kwargs)
        >>> char_order = [p[0] for p in itemstrs]
        >>> assert char_order == ['b', 'g', 'l', 'm', 's', 'w']
    """
    import ubelt as ub
    explicit = kwargs.get('explicit', False)
    kwargs['explicit'] = _rectify_countdown_or_bool(explicit)
    precision = kwargs.get('precision', None)
    kvsep = kwargs.get('kvsep', ': ')
    if explicit:
        kvsep = '='

    def make_item_str(key, val):
        if explicit or kwargs.get('strkeys', False):
            key_str = six.text_type(key)
        else:
            key_str = repr2(key, precision=precision, newlines=0)

        prefix = key_str + kvsep
        kwargs['_return_info'] = True
        val_str, _leaf_info = repr2(val, **kwargs)

        # If the first line does not end with an open nest char
        # (e.g. for ndarrays), otherwise we need to worry about
        # residual indentation.
        pos = val_str.find('\n')
        first_line = val_str if pos == -1 else val_str[:pos]

        compact_brace = kwargs.get('cbr', kwargs.get('compact_brace', False))

        if compact_brace or not first_line.rstrip().endswith(tuple('([{<')):
            rest = '' if pos == -1 else val_str[pos:]
            val_str = first_line.lstrip() + rest
            if '\n' in prefix:
                # Fix issue with keys that span new lines
                item_str = prefix + val_str
            else:
                item_str = ub.hzcat([prefix, val_str])
        else:
            item_str = prefix + val_str
        return item_str, _leaf_info

    items = list(six.iteritems(dict_))
    _tups = [make_item_str(key, val) for (key, val) in items]
    itemstrs = [t[0] for t in _tups]
    max_height = max([t[1]['max_height'] for t in _tups]) if _tups else 0
    _leaf_info = {
        'max_height': max_height + 1,
    }

    sort = kwargs.get('sort', None)
    if sort is None:
        # Force ordering on unordered dicts
        sort = True
    if isinstance(dict_, collections.OrderedDict):
        # never sort ordered dicts; they are perfect just the way they are!
        sort = False
    if sort:
        itemstrs = _sort_itemstrs(items, itemstrs)
    return itemstrs, _leaf_info

def
_dict_itemstrs
(
dict_
,
**
kwargs
)
:
import
ubelt
as
ub
explicit
=
kwargs
.
get
(
'explicit'
,
False
)
kwargs
[
'explicit'
]
=
_rectify_countdown_or_bool
(
explicit
)
precision
=
kwargs
.
get
(
'precision'
,
)
kvsep
=
kwargs
.
get
(
'kvsep'
,
': '
)
if
explicit
:
kvsep
=
'='
def
make_item_str
(
key
,
val
)
:
if
explicit
or
kwargs
.
get
(
'strkeys'
,
False
)
:
key_str
=
six
.
text_type
(
key
)
else
:
key_str
=
repr2
(
key
,
precision
=
precision
,
newlines
=
0
)
prefix
=
key_str
+
kvsep
kwargs
[
'_return_info'
]
=
True
val_str
,
_leaf_info
=
repr2
(
val
,
**
kwargs
)
pos
=
val_str
.
find
(
'\n'
)
first_line
=
val_str
if
pos
==
1
else
val_str
[
:
pos
]
compact_brace
=
kwargs
.
get
(
'cbr'
,
kwargs
.
get
(
'compact_brace'
,
False
)
)
if
compact_brace
or
not
first_line
.
rstrip
(
)
.
endswith
(
tuple
(
'([{<'
)
)
:
rest
=
''
if
pos
==
1
else
val_str
[
pos
:
]
val_str
=
first_line
.
lstrip
(
)
+
rest
if
'\n'
in
prefix
:
item_str
=
prefix
+
val_str
else
:
item_str
=
ub
.
hzcat
(
[
prefix
,
val_str
]
)
else
:
item_str
=
prefix
+
val_str
return
item_str
,
_leaf_info
items
=
list
(
six
.
iteritems
(
dict_
)
)
_tups
=
[
make_item_str
(
key
,
val
)
for
(
key
,
val
)
in
items
]
itemstrs
=
[
t
[
0
]
for
t
in
_tups
]
max_height
=
max
(
[
t
[
1
]
[
'max_height'
]
for
t
in
_tups
]
)
if
_tups
else
0
_leaf_info
=
{
'max_height'
:
max_height
+
1
,
}
sort
=
kwargs
.
get
(
'sort'
,
)
if
sort
is
:
sort
=
True
if
isinstance
(
dict_
,
collections
.
OrderedDict
)
:
sort
=
False
if
sort
:
itemstrs
=
_sort_itemstrs
(
items
,
itemstrs
)
return
itemstrs
,
_leaf_info

Create a string representation for each item in a dict .


def _list_itemstrs(list_, **kwargs):
    """
    Create a string representation for each item in a list.
    """
    items = list(list_)
    kwargs['_return_info'] = True
    _tups = [repr2(item, **kwargs) for item in items]
    itemstrs = [t[0] for t in _tups]
    max_height = max([t[1]['max_height'] for t in _tups]) if _tups else 0
    _leaf_info = {
        'max_height': max_height + 1,
    }

    sort = kwargs.get('sort', None)
    if sort is None:
        # Force orderings on sets.
        sort = isinstance(list_, (set, frozenset))
    if sort:
        itemstrs = _sort_itemstrs(items, itemstrs)
    return itemstrs, _leaf_info

def _list_itemstrs ( list_ , ** kwargs ) : items = list ( list_ ) kwargs [ '_return_info' ] = True _tups = [ repr2 ( item , ** kwargs ) for item in items ] itemstrs = [ t [ 0 ] for t in _tups ] max_height = max ( [ t [ 1 ] [ 'max_height' ] for t in _tups ] ) if _tups else 0 _leaf_info = { 'max_height' : max_height + 1 , } sort = kwargs . get ( 'sort' , None ) if sort is None : sort = isinstance ( list_ , ( set , frozenset ) ) if sort : itemstrs = _sort_itemstrs ( items , itemstrs ) return itemstrs , _leaf_info

Create a string representation for each item in a list .


def _rectify_countdown_or_bool(count_or_bool):
    """
    used by recursive functions to specify which level to turn a bool on in
    counting down yields True, True, ..., False
    counting up yields False, False, False, ... True

    Args:
        count_or_bool (bool or int): if positive and an integer, it will count
            down, otherwise it will remain the same.

    Returns:
        int or bool: count_or_bool_

    CommandLine:
        python -m utool.util_str --test-_rectify_countdown_or_bool

    Example:
        >>> from ubelt.util_format import _rectify_countdown_or_bool  # NOQA
        >>> count_or_bool = True
        >>> a1 = (_rectify_countdown_or_bool(2))
        >>> a2 = (_rectify_countdown_or_bool(1))
        >>> a3 = (_rectify_countdown_or_bool(0))
        >>> a4 = (_rectify_countdown_or_bool(-1))
        >>> a5 = (_rectify_countdown_or_bool(-2))
        >>> a6 = (_rectify_countdown_or_bool(True))
        >>> a7 = (_rectify_countdown_or_bool(False))
        >>> a8 = (_rectify_countdown_or_bool(None))
        >>> result = [a1, a2, a3, a4, a5, a6, a7, a8]
        >>> print(result)
        [1, 0, 0, -1, -2, True, False, False]
    """
    if count_or_bool is True or count_or_bool is False:
        count_or_bool_ = count_or_bool
    elif isinstance(count_or_bool, int):
        if count_or_bool == 0:
            return 0
        elif count_or_bool > 0:
            count_or_bool_ = count_or_bool - 1
        else:
            # We dont countup negatives anymore
            count_or_bool_ = count_or_bool
    else:
        count_or_bool_ = False
    return count_or_bool_

used by recursive functions to specify which level to turn a bool on in counting down yields True True ... False counting up yields False False False ... True


def register(self, type):
        """
        Registers a custom formatting function with ub.repr2
        """
        def _decorator(func):
            if isinstance(type, tuple):
                for t in type:
                    self.func_registry[t] = func
            else:
                self.func_registry[type] = func
            return func
        return _decorator

def register ( self , type ) : def _decorator ( func ) : if isinstance ( type , tuple ) : for t in type : self . func_registry [ t ] = func else : self . func_registry [ type ] = func return func return _decorator

Registers a custom formatting function with ub . repr2


def lookup(self, data):
        """
        Returns an appropriate function to format `data` if one has been
        registered.
        """
        for func in self.lazy_init:
            func()

        for type, func in self.func_registry.items():
            if isinstance(data, type):
                return func

def lookup ( self , data ) : for func in self . lazy_init : func ( ) for type , func in self . func_registry . items ( ) : if isinstance ( data , type ) : return func

Returns an appropriate function to format data if one has been registered .


def _rectify_hasher(hasher):
    """
    Convert a string-based key into a hasher class

    Notes:
        In terms of speed on 64bit systems, sha1 is the fastest followed by md5
        and sha512. The slowest algorithm is sha256. If xxhash is installed
        the fastest algorithm is xxh64.

    Example:
        >>> assert _rectify_hasher(NoParam) is DEFAULT_HASHER
        >>> assert _rectify_hasher('sha1') is hashlib.sha1
        >>> assert _rectify_hasher('sha256') is hashlib.sha256
        >>> assert _rectify_hasher('sha512') is hashlib.sha512
        >>> assert _rectify_hasher('md5') is hashlib.md5
        >>> assert _rectify_hasher(hashlib.sha1) is hashlib.sha1
        >>> assert _rectify_hasher(hashlib.sha1())().name == 'sha1'
        >>> import pytest
        >>> assert pytest.raises(KeyError, _rectify_hasher, '42')
        >>> #assert pytest.raises(TypeError, _rectify_hasher, object)
        >>> if xxhash:
        >>>     assert _rectify_hasher('xxh64') is xxhash.xxh64
        >>>     assert _rectify_hasher('xxh32') is xxhash.xxh32
    """
    if xxhash is not None:  # pragma: nobranch
        if hasher in {'xxh32', 'xx32', 'xxhash'}:
            return xxhash.xxh32
        if hasher in {'xxh64', 'xx64'}:
            return xxhash.xxh64

    if hasher is NoParam or hasher == 'default':
        hasher = DEFAULT_HASHER
    elif isinstance(hasher, six.string_types):
        if hasher not in hashlib.algorithms_available:
            raise KeyError('unknown hasher: {}'.format(hasher))
        else:
            hasher = getattr(hashlib, hasher)
    elif isinstance(hasher, HASH):
        # by default the result of this function is a class we will make an
        # instance of, if we already have an instance, wrap it in a callable
        # so the external syntax does not need to change.
        return lambda: hasher
    return hasher

def
_rectify_hasher
(
hasher
)
:
if
xxhash
is
not
:
if
hasher
in
{
'xxh32'
,
'xx32'
,
'xxhash'
}
:
return
xxhash
.
xxh32
if
hasher
in
{
'xxh64'
,
'xx64'
}
:
return
xxhash
.
xxh64
if
hasher
is
NoParam
or
hasher
==
'default'
:
hasher
=
DEFAULT_HASHER
elif
isinstance
(
hasher
,
six
.
string_types
)
:
if
hasher
not
in
hashlib
.
algorithms_available
:
raise
KeyError
(
'unknown hasher: {}'
.
format
(
hasher
)
)
else
:
hasher
=
getattr
(
hashlib
,
hasher
)
elif
isinstance
(
hasher
,
HASH
)
:
return
lambda
:
hasher
return
hasher

Convert a string - based key into a hasher class


def _rectify_base(base):
    """
    transforms base shorthand into the full list representation

    Example:
        >>> assert _rectify_base(NoParam) is DEFAULT_ALPHABET
        >>> assert _rectify_base('hex') is _ALPHABET_16
        >>> assert _rectify_base('abc') is _ALPHABET_26
        >>> assert _rectify_base(10) is _ALPHABET_10
        >>> assert _rectify_base(['1', '2']) == ['1', '2']
        >>> import pytest
        >>> assert pytest.raises(TypeError, _rectify_base, 'uselist')
    """
    if base is NoParam or base == 'default':
        return DEFAULT_ALPHABET
    elif base in [26, 'abc', 'alpha']:
        return _ALPHABET_26
    elif base in [16, 'hex']:
        return _ALPHABET_16
    elif base in [10, 'dec']:
        return _ALPHABET_10
    else:
        if not isinstance(base, (list, tuple)):
            raise TypeError(
                'Argument `base` must be a key, list, or tuple; not {}'.format(
                    type(base)))
        return base


def _hashable_sequence(data, types=True):
    r"""
    Extracts the sequence of bytes that would be hashed by hash_data

    Example:
        >>> data = [2, (3, 4)]
        >>> result1 = (b''.join(_hashable_sequence(data, types=False)))
        >>> result2 = (b''.join(_hashable_sequence(data, types=True)))
        >>> assert result1 == b'_[_\x02_,__[_\x03_,_\x04_,__]__]_'
        >>> assert result2 == b'_[_INT\x02_,__[_INT\x03_,_INT\x04_,__]__]_'
    """
    hasher = _HashTracer()
    _update_hasher(hasher, data, types=types)
    return hasher.sequence

def _hashable_sequence ( data , types = True ) : r hasher = _HashTracer ( ) _update_hasher ( hasher , data , types = types ) return hasher . sequence

r Extracts the sequence of bytes that would be hashed by hash_data


def _convert_to_hashable(data, types=True):
    r"""
    Converts `data` into a hashable byte representation if an appropriate
    hashing function is known.

    Args:
        data (object): ordered data with structure
        types (bool): include type prefixes in the hash

    Returns:
        tuple(bytes, bytes): prefix, hashable:
            a prefix hinting the original data type and the byte representation
            of `data`.

    Raises:
        TypeError : if data has no registered hash methods

    Example:
        >>> assert _convert_to_hashable(None) == (b'NULL', b'NONE')
        >>> assert _convert_to_hashable('string') == (b'TXT', b'string')
        >>> assert _convert_to_hashable(1) == (b'INT', b'\x01')
        >>> assert _convert_to_hashable(1.0) == (b'FLT', b'\x01/\x01')
        >>> assert _convert_to_hashable(_intlike[-1](1)) == (b'INT', b'\x01')
    """
    # HANDLE MOST COMMON TYPES FIRST
    if data is None:
        hashable = b'NONE'
        prefix = b'NULL'
    elif isinstance(data, six.binary_type):
        hashable = data
        prefix = b'TXT'
    elif isinstance(data, six.text_type):
        # convert unicode into bytes
        hashable = data.encode('utf-8')
        prefix = b'TXT'
    elif isinstance(data, _intlike):
        # warnings.warn('Hashing ints is slow, numpy is prefered')
        hashable = _int_to_bytes(data)
        # hashable = data.to_bytes(8, byteorder='big')
        prefix = b'INT'
    elif isinstance(data, float):
        a, b = float(data).as_integer_ratio()
        hashable = _int_to_bytes(a) + b'/' +  _int_to_bytes(b)
        prefix = b'FLT'
    else:
        # Then dynamically look up any other type
        hash_func = _HASHABLE_EXTENSIONS.lookup(data)
        prefix, hashable = hash_func(data)
    if types:
        return prefix, hashable
    else:
        return b'', hashable

def _convert_to_hashable ( data , types = True ) : r if data is None : hashable = b'NONE' prefix = b'NULL' elif isinstance ( data , six . binary_type ) : hashable = data prefix = b'TXT' elif isinstance ( data , six . text_type ) : hashable = data . encode ( 'utf-8' ) prefix = b'TXT' elif isinstance ( data , _intlike ) : hashable = _int_to_bytes ( data ) prefix = b'INT' elif isinstance ( data , float ) : a , b = float ( data ) . as_integer_ratio ( ) hashable = _int_to_bytes ( a ) + b'/' + _int_to_bytes ( b ) prefix = b'FLT' else : hash_func = _HASHABLE_EXTENSIONS . lookup ( data ) prefix , hashable = hash_func ( data ) if types : return prefix , hashable else : return b'' , hashable

r Converts data into a hashable byte representation if an appropriate hashing function is known .


def _update_hasher(hasher, data, types=True):
    """
    Converts `data` into a byte representation and calls update on the hasher
    `hashlib.HASH` algorithm.

    Args:
        hasher (HASH): instance of a hashlib algorithm
        data (object): ordered data with structure
        types (bool): include type prefixes in the hash

    Example:
        >>> hasher = hashlib.sha512()
        >>> data = [1, 2, ['a', 2, 'c']]
        >>> _update_hasher(hasher, data)
        >>> print(hasher.hexdigest()[0:8])
        e2c67675

        2ba8d82b
    """
    # Determine if the data should be hashed directly or iterated through
    if isinstance(data, (tuple, list, zip)):
        needs_iteration = True
    else:
        needs_iteration = any(check(data) for check in
                              _HASHABLE_EXTENSIONS.iterable_checks)

    if needs_iteration:
        # Denote that we are hashing over an iterable
        # Multiple structure bytes makes it harder accidently make conflicts
        SEP = b'_,_'
        ITER_PREFIX = b'_[_'
        ITER_SUFFIX = b'_]_'

        iter_ = iter(data)
        hasher.update(ITER_PREFIX)
        # first, try to nest quickly without recursive calls
        # (this works if all data in the sequence is a non-iterable)
        try:
            for item in iter_:
                prefix, hashable = _convert_to_hashable(item, types)
                binary_data = prefix + hashable + SEP
                hasher.update(binary_data)
        except TypeError:
            # need to use recursive calls
            # Update based on current item
            _update_hasher(hasher, item, types)
            for item in iter_:
                # Ensure the items have a spacer between them
                _update_hasher(hasher, item, types)
                hasher.update(SEP)
        hasher.update(ITER_SUFFIX)
    else:
        prefix, hashable = _convert_to_hashable(data, types)
        binary_data = prefix + hashable
        hasher.update(binary_data)

def _update_hasher ( hasher , data , types = True ) : if isinstance ( data , ( tuple , list , zip ) ) : needs_iteration = True else : needs_iteration = any ( check ( data ) for check in _HASHABLE_EXTENSIONS . iterable_checks ) if needs_iteration : SEP = b'_,_' ITER_PREFIX = b'_[_' ITER_SUFFIX = b'_]_' iter_ = iter ( data ) hasher . update ( ITER_PREFIX ) try : for item in iter_ : prefix , hashable = _convert_to_hashable ( item , types ) binary_data = prefix + hashable + SEP hasher . update ( binary_data ) except TypeError : _update_hasher ( hasher , item , types ) for item in iter_ : _update_hasher ( hasher , item , types ) hasher . update ( SEP ) hasher . update ( ITER_SUFFIX ) else : prefix , hashable = _convert_to_hashable ( data , types ) binary_data = prefix + hashable hasher . update ( binary_data )

Converts data into a byte representation and calls update on the hasher hashlib . HASH algorithm .


def _convert_hexstr_base(hexstr, base):
    r"""
    Packs a long hexstr into a shorter length string with a larger base.

    Args:
        hexstr (str): string of hexidecimal symbols to convert
        base (list): symbols of the conversion base

    Example:
        >>> print(_convert_hexstr_base('ffffffff', _ALPHABET_26))
        nxmrlxv
        >>> print(_convert_hexstr_base('0', _ALPHABET_26))
        0
        >>> print(_convert_hexstr_base('-ffffffff', _ALPHABET_26))
        -nxmrlxv
        >>> print(_convert_hexstr_base('aafffff1', _ALPHABET_16))
        aafffff1

    Sympy:
        >>> import sympy as sy
        >>> # Determine the length savings with lossless conversion
        >>> consts = dict(hexbase=16, hexlen=256, baselen=27)
        >>> symbols = sy.symbols('hexbase, hexlen, baselen, newlen')
        >>> haexbase, hexlen, baselen, newlen = symbols
        >>> eqn = sy.Eq(16 ** hexlen,  baselen ** newlen)
        >>> newlen_ans = sy.solve(eqn, newlen)[0].subs(consts).evalf()
        >>> print('newlen_ans = %r' % (newlen_ans,))
        >>> # for a 26 char base we can get 216
        >>> print('Required length for lossless conversion len2 = %r' % (len2,))
        >>> def info(base, len):
        ...     bits = base ** len
        ...     print('base = %r' % (base,))
        ...     print('len = %r' % (len,))
        ...     print('bits = %r' % (bits,))
        >>> info(16, 256)
        >>> info(27, 16)
        >>> info(27, 64)
        >>> info(27, 216)
    """
    if base is _ALPHABET_16:
        # already in hex, no conversion needed
        return hexstr
    baselen = len(base)
    x = int(hexstr, 16)  # first convert to base 16
    if x == 0:
        return '0'
    sign = 1 if x > 0 else -1
    x *= sign
    digits = []
    while x:
        digits.append(base[x % baselen])
        x //= baselen
    if sign < 0:
        digits.append('-')
    digits.reverse()
    newbase_str = ''.join(digits)
    return newbase_str

def _convert_hexstr_base ( hexstr , base ) : r if base is _ALPHABET_16 : return hexstr baselen = len ( base ) x = int ( hexstr , 16 ) if x == 0 : return '0' sign = 1 if x > 0 else - 1 x *= sign digits = [ ] while x : digits . append ( base [ x % baselen ] ) x //= baselen if sign < 0 : digits . append ( '-' ) digits . reverse ( ) newbase_str = '' . join ( digits ) return newbase_str

r Packs a long hexstr into a shorter length string with a larger base .


def _digest_hasher(hasher, hashlen, base):
    """ counterpart to _update_hasher """
    # Get a 128 character hex string
    hex_text = hasher.hexdigest()
    # Shorten length of string (by increasing base)
    base_text = _convert_hexstr_base(hex_text, base)
    # Truncate
    text = base_text[:hashlen]
    return text

def _digest_hasher ( hasher , hashlen , base ) : hex_text = hasher . hexdigest ( ) base_text = _convert_hexstr_base ( hex_text , base ) text = base_text [ : hashlen ] return text


def hash_data(data, hasher=NoParam, base=NoParam, types=False,
              hashlen=NoParam, convert=False):
    """
    Get a unique hash depending on the state of the data.

    Args:
        data (object):
            Any sort of loosely organized data

        hasher (str or HASHER):
            Hash algorithm from hashlib, defaults to `sha512`.

        base (str or List[str]):
            Shorthand key or a list of symbols.  Valid keys are: 'abc', 'hex',
            and 'dec'. Defaults to 'hex'.

        types (bool):
            If True data types are included in the hash, otherwise only the raw
            data is hashed. Defaults to False.

        hashlen (int):
            Maximum number of symbols in the returned hash. If not specified,
            all are returned.  DEPRECATED. Use slice syntax instead.

        convert (bool, optional, default=True):
            if True, try and convert the data to json an the json is hashed
            instead. This can improve runtime in some instances, however the
            hash may differ from the case where convert=False.

    Notes:
        alphabet26 is a pretty nice base, I recommend it.
        However we default to hex because it is standard.
        This means the output of hashdata with base=sha1 will be the same as
        the output of `sha1sum`.

    Returns:
        str: text -  hash string

    Example:
        >>> import ubelt as ub
        >>> print(ub.hash_data([1, 2, (3, '4')], convert=False))
        60b758587f599663931057e6ebdf185a...
        >>> print(ub.hash_data([1, 2, (3, '4')], base='abc',  hasher='sha512')[:32])
        hsrgqvfiuxvvhcdnypivhhthmrolkzej
    """
    if convert and isinstance(data, six.string_types):  # nocover
        try:
            data = json.dumps(data)
        except TypeError as ex:
            # import warnings
            # warnings.warn('Unable to encode input as json due to: {!r}'.format(ex))
            pass

    base = _rectify_base(base)
    hashlen = _rectify_hashlen(hashlen)
    hasher = _rectify_hasher(hasher)()
    # Feed the data into the hasher
    _update_hasher(hasher, data, types=types)
    # Get the hashed representation
    text = _digest_hasher(hasher, hashlen, base)
    return text

def hash_data ( data , hasher = NoParam , base = NoParam , types = False , hashlen = NoParam , convert = False ) : if convert and isinstance ( data , six . string_types ) : try : data = json . dumps ( data ) except TypeError as ex : pass base = _rectify_base ( base ) hashlen = _rectify_hashlen ( hashlen ) hasher = _rectify_hasher ( hasher ) ( ) _update_hasher ( hasher , data , types = types ) text = _digest_hasher ( hasher , hashlen , base ) return text

Get a unique hash depending on the state of the data .


def hash_file(fpath, blocksize=65536, stride=1, hasher=NoParam,
              hashlen=NoParam, base=NoParam):
    """
    Hashes the data in a file on disk.

    Args:
        fpath (PathLike):  file path string

        blocksize (int): 2 ** 16. Affects speed of reading file

        stride (int): strides > 1 skip data to hash, useful for faster
                      hashing, but less accurate, also makes hash dependant on
                      blocksize.

        hasher (HASH): hash algorithm from hashlib, defaults to `sha512`.

        hashlen (int): maximum number of symbols in the returned hash. If
            not specified, all are returned.

        base (list, str): list of symbols or shorthand key. Valid keys are
            'abc', 'hex', and 'dec'. Defaults to 'hex'.

    Notes:
        For better hashes keep stride = 1
        For faster hashes set stride > 1
        blocksize matters when stride > 1

    References:
        http://stackoverflow.com/questions/3431825/md5-checksum-of-a-file
        http://stackoverflow.com/questions/5001893/when-to-use-sha-1-vs-sha-2

    Example:
        >>> import ubelt as ub
        >>> from os.path import join
        >>> fpath = join(ub.ensure_app_cache_dir('ubelt'), 'tmp.txt')
        >>> ub.writeto(fpath, 'foobar')
        >>> print(ub.hash_file(fpath, hasher='sha1', base='hex'))
        8843d7f92416211de9ebb963ff4ce28125932878

    Example:
        >>> import ubelt as ub
        >>> from os.path import join
        >>> fpath = ub.touch(join(ub.ensure_app_cache_dir('ubelt'), 'empty_file'))
        >>> # Test that the output is the same as sha1sum
        >>> if ub.find_exe('sha1sum'):
        >>>     want = ub.cmd(['sha1sum', fpath], verbose=2)['out'].split(' ')[0]
        >>>     got = ub.hash_file(fpath, hasher='sha1')
        >>>     print('want = {!r}'.format(want))
        >>>     print('got = {!r}'.format(got))
        >>>     assert want.endswith(got)
        >>> # Do the same for sha512 sum and md5sum
        >>> if ub.find_exe('sha512sum'):
        >>>     want = ub.cmd(['sha512sum', fpath], verbose=2)['out'].split(' ')[0]
        >>>     got = ub.hash_file(fpath, hasher='sha512')
        >>>     print('want = {!r}'.format(want))
        >>>     print('got = {!r}'.format(got))
        >>>     assert want.endswith(got)
        >>> if ub.find_exe('md5sum'):
        >>>     want = ub.cmd(['md5sum', fpath], verbose=2)['out'].split(' ')[0]
        >>>     got = ub.hash_file(fpath, hasher='md5')
        >>>     print('want = {!r}'.format(want))
        >>>     print('got = {!r}'.format(got))
        >>>     assert want.endswith(got)
    """
    base = _rectify_base(base)
    hashlen = _rectify_hashlen(hashlen)
    hasher = _rectify_hasher(hasher)()
    with open(fpath, 'rb') as file:
        buf = file.read(blocksize)
        if stride > 1:
            # skip blocks when stride is greater than 1
            while len(buf) > 0:
                hasher.update(buf)
                file.seek(blocksize * (stride - 1), 1)
                buf = file.read(blocksize)
        else:
            # otherwise hash the entire file
            while len(buf) > 0:
                hasher.update(buf)
                buf = file.read(blocksize)
    # Get the hashed representation
    text = _digest_hasher(hasher, hashlen, base)
    return text

def hash_file ( fpath , blocksize = 65536 , stride = 1 , hasher = NoParam , hashlen = NoParam , base = NoParam ) : base = _rectify_base ( base ) hashlen = _rectify_hashlen ( hashlen ) hasher = _rectify_hasher ( hasher ) ( ) with open ( fpath , 'rb' ) as file : buf = file . read ( blocksize ) if stride > 1 : while len ( buf ) > 0 : hasher . update ( buf ) file . seek ( blocksize * ( stride - 1 ) , 1 ) buf = file . read ( blocksize ) else : while len ( buf ) > 0 : hasher . update ( buf ) buf = file . read ( blocksize ) text = _digest_hasher ( hasher , hashlen , base ) return text


def register(self, hash_types):
        """
        Registers a function to generate a hash for data of the appropriate
        types. This can be used to register custom classes. Internally this is
        used to define how to hash non-builtin objects like ndarrays and uuids.

        The registered function should return a tuple of bytes. First a small
        prefix hinting at the data type, and second the raw bytes that can be
        hashed.

        Args:
            hash_types (class or tuple of classes):

        Returns:
            func: closure to be used as the decorator

        Example:
            >>> # xdoctest: +SKIP
            >>> # Skip this doctest because we dont want tests to modify
            >>> # the global state.
            >>> import ubelt as ub
            >>> import pytest
            >>> class MyType(object):
            ...     def __init__(self, id):
            ...         self.id = id
            >>> data = MyType(1)
            >>> # Custom types wont work with ub.hash_data by default
            >>> with pytest.raises(TypeError):
            ...     ub.hash_data(data)
            >>> # You can register your functions with ubelt's internal
            >>> # hashable_extension registery.
            >>> @ub.util_hash._HASHABLE_EXTENSIONS.register(MyType)
            >>> def hash_my_type(data):
            ...     return b'mytype', six.b(ub.hash_data(data.id))
            >>> # TODO: allow hash_data to take an new instance of
            >>> # HashableExtensions, so we dont have to modify the global
            >>> # ubelt state when we run tests.
            >>> my_instance = MyType(1)
            >>> ub.hash_data(my_instance)
        """
        # ensure iterable
        if not isinstance(hash_types, (list, tuple)):
            hash_types = [hash_types]
        def _decor_closure(hash_func):
            for hash_type in hash_types:
                key = (hash_type.__module__, hash_type.__name__)
                self.keyed_extensions[key] = (hash_type, hash_func)
            return hash_func
        return _decor_closure

def register ( self , hash_types ) : if not isinstance ( hash_types , ( list , tuple ) ) : hash_types = [ hash_types ] def _decor_closure ( hash_func ) : for hash_type in hash_types : key = ( hash_type . __module__ , hash_type . __name__ ) self . keyed_extensions [ key ] = ( hash_type , hash_func ) return hash_func return _decor_closure

Registers a function to generate a hash for data of the appropriate types . This can be used to register custom classes . Internally this is used to define how to hash non - builtin objects like ndarrays and uuids .


def lookup(self, data):
        """
        Returns an appropriate function to hash `data` if one has been
        registered.

        Raises:
            TypeError : if data has no registered hash methods

        Example:
            >>> import ubelt as ub
            >>> import pytest
            >>> if not ub.modname_to_modpath('numpy'):
            ...     raise pytest.skip('numpy is optional')
            >>> self = HashableExtensions()
            >>> self._register_numpy_extensions()
            >>> self._register_builtin_class_extensions()

            >>> import numpy as np
            >>> data = np.array([1, 2, 3])
            >>> self.lookup(data[0])

            >>> class Foo(object):
            >>>     def __init__(f):
            >>>         f.attr = 1
            >>> data = Foo()
            >>> assert pytest.raises(TypeError, self.lookup, data)

            >>> # If ub.hash_data doesnt support your object,
            >>> # then you can register it.
            >>> @self.register(Foo)
            >>> def _hashfoo(data):
            >>>     return b'FOO', data.attr
            >>> func = self.lookup(data)
            >>> assert func(data)[1] == 1

            >>> data = uuid.uuid4()
            >>> self.lookup(data)
        """
        # Maybe try using functools.singledispatch instead?
        # First try O(1) lookup
        query_hash_type = data.__class__
        key = (query_hash_type.__module__, query_hash_type.__name__)
        try:
            hash_type, hash_func = self.keyed_extensions[key]
        except KeyError:
            raise TypeError('No registered hash func for hashable type=%r' % (
                    query_hash_type))
        return hash_func

Returns an appropriate function to hash data if one has been registered .


def _register_numpy_extensions(self):
        """
        Numpy extensions are builtin
        """
        # system checks
        import numpy as np
        numpy_floating_types = (np.float16, np.float32, np.float64)
        if hasattr(np, 'float128'):  # nocover
            numpy_floating_types = numpy_floating_types + (np.float128,)

        @self.add_iterable_check
        def is_object_ndarray(data):
            # ndarrays of objects cannot be hashed directly.
            return isinstance(data, np.ndarray) and data.dtype.kind == 'O'

        @self.register(np.ndarray)
        def hash_numpy_array(data):
            """
            Example:
                >>> import ubelt as ub
                >>> if not ub.modname_to_modpath('numpy'):
                ...     raise pytest.skip()
                >>> import numpy as np
                >>> data_f32 = np.zeros((3, 3, 3), dtype=np.float64)
                >>> data_i64 = np.zeros((3, 3, 3), dtype=np.int64)
                >>> data_i32 = np.zeros((3, 3, 3), dtype=np.int32)
                >>> hash_f64 = _hashable_sequence(data_f32, types=True)
                >>> hash_i64 = _hashable_sequence(data_i64, types=True)
                >>> hash_i32 = _hashable_sequence(data_i64, types=True)
                >>> assert hash_i64 != hash_f64
                >>> assert hash_i64 != hash_i32
            """
            if data.dtype.kind == 'O':
                msg = 'directly hashing ndarrays with dtype=object is unstable'
                raise TypeError(msg)
            else:
                # tobytes() views the array in 1D (via ravel())
                # encode the shape as well
                header = b''.join(_hashable_sequence((len(data.shape), data.shape)))
                dtype = b''.join(_hashable_sequence(data.dtype.descr))
                hashable = header + dtype + data.tobytes()
            prefix = b'NDARR'
            return prefix, hashable

        @self.register((np.int64, np.int32, np.int16, np.int8) +
                       (np.uint64, np.uint32, np.uint16, np.uint8))
        def _hash_numpy_int(data):
            return _convert_to_hashable(int(data))

        @self.register(numpy_floating_types)
        def _hash_numpy_float(data):
            return _convert_to_hashable(float(data))

        @self.register(np.random.RandomState)
        def _hash_numpy_random_state(data):
            """
            Example:
                >>> import ubelt as ub
                >>> if not ub.modname_to_modpath('numpy'):
                ...     raise pytest.skip()
                >>> import numpy as np
                >>> rng = np.random.RandomState(0)
                >>> _hashable_sequence(rng, types=True)
            """
            hashable = b''.join(_hashable_sequence(data.get_state()))
            prefix = b'RNG'
            return prefix, hashable


def _register_builtin_class_extensions(self):
        """
        Register hashing extensions for a selection of classes included in
        python stdlib.

        Example:
            >>> data = uuid.UUID('7e9d206b-dc02-4240-8bdb-fffe858121d0')
            >>> print(hash_data(data, base='abc', hasher='sha512', types=True)[0:8])
            cryarepd
            >>> data = OrderedDict([('a', 1), ('b', 2), ('c', [1, 2, 3]),
            >>>                     (4, OrderedDict())])
            >>> print(hash_data(data, base='abc', hasher='sha512', types=True)[0:8])
            qjspicvv

            gpxtclct
        """
        @self.register(uuid.UUID)
        def _hash_uuid(data):
            hashable = data.bytes
            prefix = b'UUID'
            return prefix, hashable

        @self.register(OrderedDict)
        def _hash_ordered_dict(data):
            """
            Note, we should not be hashing dicts because they are unordered
            """
            hashable = b''.join(_hashable_sequence(list(data.items())))
            prefix = b'ODICT'
            return prefix, hashable

def _register_builtin_class_extensions ( self ) : @ self . register ( uuid . UUID ) def _hash_uuid ( data ) : hashable = data . bytes prefix = b'UUID' return prefix , hashable @ self . register ( OrderedDict ) def _hash_ordered_dict ( data ) : hashable = b'' . join ( _hashable_sequence ( list ( data . items ( ) ) ) ) prefix = b'ODICT' return prefix , hashable

Register hashing extensions for a selection of classes included in python stdlib .


def _proc_async_iter_stream(proc, stream, buffersize=1):
    """
    Reads output from a process in a separate thread
    """
    from six.moves import queue
    from threading import Thread
    def enqueue_output(proc, stream, stream_queue):
        while proc.poll() is None:
            line = stream.readline()
            # print('ENQUEUE LIVE {!r} {!r}'.format(stream, line))
            stream_queue.put(line)

        for line in _textio_iterlines(stream):
            # print('ENQUEUE FINAL {!r} {!r}'.format(stream, line))
            stream_queue.put(line)

        # print("STREAM IS DONE {!r}".format(stream))
        stream_queue.put(None)  # signal that the stream is finished
        # stream.close()
    stream_queue = queue.Queue(maxsize=buffersize)
    _thread = Thread(target=enqueue_output, args=(proc, stream, stream_queue))
    _thread.daemon = True  # thread dies with the program
    _thread.start()
    return stream_queue

def _proc_async_iter_stream ( proc , stream , buffersize = 1 ) : from six . moves import queue from threading import Thread def enqueue_output ( proc , stream , stream_queue ) : while proc . poll ( ) is None : line = stream . readline ( ) stream_queue . put ( line ) for line in _textio_iterlines ( stream ) : stream_queue . put ( line ) stream_queue . put ( None ) stream_queue = queue . Queue ( maxsize = buffersize ) _thread = Thread ( target = enqueue_output , args = ( proc , stream , stream_queue ) ) _thread . daemon = True _thread . start ( ) return stream_queue


def _tee_output(make_proc, stdout=None, stderr=None, backend='auto'):
    """
    Simultaneously reports and captures stdout and stderr from a process

    subprocess must be created using (stdout=subprocess.PIPE,
    stderr=subprocess.PIPE)
    """
    logged_out = []
    logged_err = []
    if backend == 'auto':
        # backend = 'select' if POSIX else 'thread'
        backend = 'thread'

    if backend == 'select':
        if not POSIX:  # nocover
            raise NotImplementedError('select is only available on posix')
        # the select-based version is stable, but slow
        _proc_iteroutput = _proc_iteroutput_select
    elif backend == 'thread':
        # the thread version is fast, but might run into issues.
        _proc_iteroutput = _proc_iteroutput_thread
    else:
        raise ValueError('backend must be select, thread, or auto')

    proc = make_proc()
    for oline, eline in _proc_iteroutput(proc):
        if oline:
            if stdout:  # pragma: nobranch
                stdout.write(oline)
                stdout.flush()
            logged_out.append(oline)
        if eline:
            if stderr:  # pragma: nobranch
                stderr.write(eline)
                stderr.flush()
            logged_err.append(eline)
    return proc, logged_out, logged_err


def timestamp(method='iso8601'):
    """
    make an iso8601 timestamp

    Args:
        method (str): type of timestamp

    Example:
        >>> stamp = timestamp()
        >>> print('stamp = {!r}'.format(stamp))
        stamp = ...-...-...T...
    """
    if method == 'iso8601':
        # ISO 8601
        # datetime.datetime.utcnow().isoformat()
        # datetime.datetime.now().isoformat()
        # utcnow
        tz_hour = time.timezone // 3600
        utc_offset = str(tz_hour) if tz_hour < 0 else '+' + str(tz_hour)
        stamp = time.strftime('%Y-%m-%dT%H%M%S') + utc_offset
        return stamp
    else:
        raise ValueError('only iso8601 is accepted for now')

def
timestamp
(
method
=
'iso8601'
)
:
if
method
==
'iso8601'
:
tz_hour
=
time
.
timezone
//
3600
utc_offset
=
str
(
tz_hour
)
if
tz_hour
<
0
else
'+'
+
str
(
tz_hour
)
stamp
=
time
.
strftime
(
'%Y-%m-%dT%H%M%S'
)
+
utc_offset
return
stamp
else
:
raise
ValueError
(
'only iso8601 is accepted for now'
)


def import_module_from_path(modpath, index=-1):
    """
    Imports a module via its path

    Args:
        modpath (PathLike): path to the module on disk or within a zipfile.

    Returns:
        module: the imported module

    References:
        https://stackoverflow.com/questions/67631/import-module-given-path

    Notes:
        If the module is part of a package, the package will be imported first.
        These modules may cause problems when reloading via IPython magic

        This can import a module from within a zipfile. To do this modpath
        should specify the path to the zipfile and the path to the module
        within that zipfile separated by a colon or pathsep.
        E.g. `/path/to/archive.zip:mymodule.py`

    Warning:
        It is best to use this with paths that will not conflict with
        previously existing modules.

        If the modpath conflicts with a previously existing module name. And
        the target module does imports of its own relative to this conflicting
        path. In this case, the module that was loaded first will win.

        For example if you try to import '/foo/bar/pkg/mod.py' from the folder
        structure:
          - foo/
            +- bar/
               +- pkg/
                  +  __init__.py
                  |- mod.py
                  |- helper.py

       If there exists another module named `pkg` already in sys.modules
       and mod.py does something like `from . import helper`, Python will
       assume helper belongs to the `pkg` module already in sys.modules.
       This can cause a NameError or worse --- a incorrect helper module.

    Example:
        >>> import xdoctest
        >>> modpath = xdoctest.__file__
        >>> module = import_module_from_path(modpath)
        >>> assert module is xdoctest

    Example:
        >>> # Test importing a module from within a zipfile
        >>> import zipfile
        >>> from xdoctest import utils
        >>> from os.path import join, expanduser
        >>> dpath = expanduser('~/.cache/xdoctest')
        >>> dpath = utils.ensuredir(dpath)
        >>> #dpath = utils.TempDir().ensure()
        >>> # Write to an external module named bar
        >>> external_modpath = join(dpath, 'bar.py')
        >>> open(external_modpath, 'w').write('testvar = 1')
        >>> internal = 'folder/bar.py'
        >>> # Move the external bar module into a zipfile
        >>> zippath = join(dpath, 'myzip.zip')
        >>> with zipfile.ZipFile(zippath, 'w') as myzip:
        >>>     myzip.write(external_modpath, internal)
        >>> # Import the bar module from within the zipfile
        >>> modpath = zippath + ':' + internal
        >>> modpath = zippath + os.path.sep + internal
        >>> module = import_module_from_path(modpath)
        >>> assert module.__name__ == os.path.normpath('folder/bar')
        >>> assert module.testvar == 1

    Doctest:
        >>> import pytest
        >>> with pytest.raises(IOError):
        >>>     import_module_from_path('does-not-exist')
        >>> with pytest.raises(IOError):
        >>>     import_module_from_path('does-not-exist.zip/')
    """
    import os
    if not os.path.exists(modpath):
        import re
        import zipimport
        # We allow (if not prefer or force) the colon to be a path.sep in order
        # to agree with the mod.__name__ attribute that will be produced

        # zip followed by colon or slash
        pat = '(.zip[' + re.escape(os.path.sep) + '/:])'
        parts = re.split(pat, modpath, flags=re.IGNORECASE)
        if len(parts) > 2:
            archivepath = ''.join(parts[:-1])[:-1]
            internal = parts[-1]
            modname = os.path.splitext(internal)[0]
            modname = os.path.normpath(modname)
            if os.path.exists(archivepath):
                zimp_file = zipimport.zipimporter(archivepath)
                module = zimp_file.load_module(modname)
                return module
        raise IOError('modpath={} does not exist'.format(modpath))
    else:
        # the importlib version doesnt work in pytest
        module = _custom_import_modpath(modpath)
        # TODO: use this implementation once pytest fixes importlib
        # module = _pkgutil_import_modpath(modpath)
        return module

def
import_module_from_path
(
modpath
,
index
=
1
)
:
import
os
if
not
os
.
path
.
exists
(
modpath
)
:
import
re
import
zipimport
pat
=
'(.zip['
+
re
.
escape
(
os
.
path
.
sep
)
+
'/:])'
parts
=
re
.
split
(
pat
,
modpath
,
flags
=
re
.
IGNORECASE
)
if
len
(
parts
)
>
2
:
archivepath
=
''
.
join
(
parts
[
:
1
]
)
[
:
1
]
internal
=
parts
[
1
]
modname
=
os
.
path
.
splitext
(
internal
)
[
0
]
modname
=
os
.
path
.
normpath
(
modname
)
if
os
.
path
.
exists
(
archivepath
)
:
zimp_file
=
zipimport
.
zipimporter
(
archivepath
)
module
=
zimp_file
.
load_module
(
modname
)
return
module
raise
IOError
(
'modpath={} does not exist'
.
format
(
modpath
)
)
else
:
module
=
_custom_import_modpath
(
modpath
)
return
module


def _extension_module_tags():
    """
    Returns valid tags an extension module might have
    """
    import sysconfig
    tags = []
    if six.PY2:
        # see also 'SHLIB_EXT'
        multiarch = sysconfig.get_config_var('MULTIARCH')
        if multiarch is not None:
            tags.append(multiarch)
    else:
        # handle PEP 3149 -- ABI version tagged .so files
        # ABI = application binary interface
        tags.append(sysconfig.get_config_var('SOABI'))
        tags.append('abi3')  # not sure why this one is valid but it is
    tags = [t for t in tags if t]
    return tags

def _extension_module_tags ( ) : import sysconfig tags = [ ] if six . PY2 : multiarch = sysconfig . get_config_var ( 'MULTIARCH' ) if multiarch is not None : tags . append ( multiarch ) else : tags . append ( sysconfig . get_config_var ( 'SOABI' ) ) tags . append ( 'abi3' ) tags = [ t for t in tags if t ] return tags


def _syspath_modname_to_modpath(modname, sys_path=None, exclude=None):
    """
    syspath version of modname_to_modpath

    Args:
        modname (str): name of module to find
        sys_path (List[PathLike], default=None):
            if specified overrides `sys.path`
        exclude (List[PathLike], default=None):
            list of directory paths. if specified prevents these directories
            from being searched.

    Notes:
        This is much slower than the pkgutil mechanisms.

    CommandLine:
        python -m xdoctest.static_analysis _syspath_modname_to_modpath

    Example:
        >>> print(_syspath_modname_to_modpath('xdoctest.static_analysis'))
        ...static_analysis.py
        >>> print(_syspath_modname_to_modpath('xdoctest'))
        ...xdoctest
        >>> print(_syspath_modname_to_modpath('_ctypes'))
        ..._ctypes...
        >>> assert _syspath_modname_to_modpath('xdoctest', sys_path=[]) is None
        >>> assert _syspath_modname_to_modpath('xdoctest.static_analysis', sys_path=[]) is None
        >>> assert _syspath_modname_to_modpath('_ctypes', sys_path=[]) is None
        >>> assert _syspath_modname_to_modpath('this', sys_path=[]) is None

    Example:
        >>> # test what happens when the module is not visible in the path
        >>> modname = 'xdoctest.static_analysis'
        >>> modpath = _syspath_modname_to_modpath(modname)
        >>> exclude = [split_modpath(modpath)[0]]
        >>> found = _syspath_modname_to_modpath(modname, exclude=exclude)
        >>> # this only works if installed in dev mode, pypi fails
        >>> assert found is None, 'should not have found {}'.format(found)
    """

    def _isvalid(modpath, base):
        # every directory up to the module, should have an init
        subdir = dirname(modpath)
        while subdir and subdir != base:
            if not exists(join(subdir, '__init__.py')):
                return False
            subdir = dirname(subdir)
        return True

    _fname_we = modname.replace('.', os.path.sep)
    candidate_fnames = [
        _fname_we + '.py',
        # _fname_we + '.pyc',
        # _fname_we + '.pyo',
    ]
    # Add extension library suffixes
    candidate_fnames += [_fname_we + ext for ext in _platform_pylib_exts()]

    if sys_path is None:
        sys_path = sys.path

    # the empty string in sys.path indicates cwd. Change this to a '.'
    candidate_dpaths = ['.' if p == '' else p for p in sys_path]

    if exclude:
        def normalize(p):
            if sys.platform.startswith('win32'):  # nocover
                return realpath(p).lower()
            else:
                return realpath(p)
        # Keep only the paths not in exclude
        real_exclude = {normalize(p) for p in exclude}
        candidate_dpaths = [p for p in candidate_dpaths
                            if normalize(p) not in real_exclude]

    for dpath in candidate_dpaths:
        # Check for directory-based modules (has presidence over files)
        modpath = join(dpath, _fname_we)
        if exists(modpath):
            if isfile(join(modpath, '__init__.py')):
                if _isvalid(modpath, dpath):
                    return modpath

        # If that fails, check for file-based modules
        for fname in candidate_fnames:
            modpath = join(dpath, fname)
            if isfile(modpath):
                if _isvalid(modpath, dpath):
                    return modpath

def _syspath_modname_to_modpath ( modname , sys_path = None , exclude = None ) : def _isvalid ( modpath , base ) : subdir = dirname ( modpath ) while subdir and subdir != base : if not exists ( join ( subdir , '__init__.py' ) ) : return False subdir = dirname ( subdir ) return True _fname_we = modname . replace ( '.' , os . path . sep ) candidate_fnames = [ _fname_we + '.py' , ] candidate_fnames += [ _fname_we + ext for ext in _platform_pylib_exts ( ) ] if sys_path is None : sys_path = sys . path candidate_dpaths = [ '.' if p == '' else p for p in sys_path ] if exclude : def normalize ( p ) : if sys . platform . startswith ( 'win32' ) : return realpath ( p ) . lower ( ) else : return realpath ( p ) real_exclude = { normalize ( p ) for p in exclude } candidate_dpaths = [ p for p in candidate_dpaths if normalize ( p ) not in real_exclude ] for dpath in candidate_dpaths : modpath = join ( dpath , _fname_we ) if exists ( modpath ) : if isfile ( join ( modpath , '__init__.py' ) ) : if _isvalid ( modpath , dpath ) : return modpath for fname in candidate_fnames : modpath = join ( dpath , fname ) if isfile ( modpath ) : if _isvalid ( modpath , dpath ) : return modpath


def modname_to_modpath(modname, hide_init=True, hide_main=False, sys_path=None):
    """
    Finds the path to a python module from its name.

    Determines the path to a python module without directly import it

    Converts the name of a module (__name__) to the path (__file__) where it is
    located without importing the module. Returns None if the module does not
    exist.

    Args:
        modname (str): module filepath
        hide_init (bool): if False, __init__.py will be returned for packages
        hide_main (bool): if False, and hide_init is True, __main__.py will be
            returned for packages, if it exists.
        sys_path (list): if specified overrides `sys.path` (default None)

    Returns:
        str: modpath - path to the module, or None if it doesn't exist

    CommandLine:
        python -m xdoctest.static_analysis modname_to_modpath:0
        pytest  /home/joncrall/code/xdoctest/xdoctest/static_analysis.py::modname_to_modpath:0

    Example:
        >>> modname = 'xdoctest.__main__'
        >>> modpath = modname_to_modpath(modname, hide_main=False)
        >>> assert modpath.endswith('__main__.py')
        >>> modname = 'xdoctest'
        >>> modpath = modname_to_modpath(modname, hide_init=False)
        >>> assert modpath.endswith('__init__.py')
        >>> modpath = basename(modname_to_modpath('_ctypes'))
        >>> assert 'ctypes' in modpath
    """
    modpath = _syspath_modname_to_modpath(modname, sys_path)
    if modpath is None:
        return None

    modpath = normalize_modpath(modpath, hide_init=hide_init,
                                hide_main=hide_main)
    return modpath

def modname_to_modpath ( modname , hide_init = True , hide_main = False , sys_path = None ) : modpath = _syspath_modname_to_modpath ( modname , sys_path ) if modpath is None : return None modpath = normalize_modpath ( modpath , hide_init = hide_init , hide_main = hide_main ) return modpath

Finds the path to a python module from its name .