def create_package(self, path=None, name=None, mode=None):
        """
        Create a tar file package of all the repository files and directories.
        Only files and directories that are tracked in the repository
        are stored in the package tar file.

        **N.B. On some systems packaging requires root permissions.**

        :Parameters:
            #. path (None, string): The real absolute path where to create the
               package. If None, it will be created in the same directory as
               the repository. If '.' or an empty string is passed, the current
               working directory will be used.
            #. name (None, string): The name to give to the package file
               If None, the package directory name will be used with the
               appropriate extension added.
            #. mode (None, string): The writing mode of the tarfile.
               If None, automatically the best compression mode will be chose.
               Available modes are ('w', 'w:', 'w:gz', 'w:bz2')
        """
        # check mode
        assert mode in (None, 'w', 'w:', 'w:gz', 'w:bz2'), 'unkown archive mode %s'%str(mode)
        if mode is None:
            #mode = 'w:bz2'
            mode = 'w:'
        # get root
        if path is None:
            root = os.path.split(self.__path)[0]
        elif path.strip() in ('','.'):
            root = os.getcwd()
        else:
            root = os.path.realpath( os.path.expanduser(path) )
        assert os.path.isdir(root), 'absolute path %s is not a valid directory'%path
        # get name
        if name is None:
            ext = mode.split(":")
            if len(ext) == 2:
                if len(ext[1]):
                    ext = "."+ext[1]
                else:
                    ext = '.tar'
            else:
                ext = '.tar'
            name = os.path.split(self.__path)[1]+ext
        # create tar file
        tarfilePath = os.path.join(root, name)
        try:
            tarHandler = tarfile.TarFile.open(tarfilePath, mode=mode)
        except Exception as e:
            raise Exception("Unable to create package (%s)"%e)
        # walk directory and create empty directories
        for dpath in sorted(list(self.walk_directories_path(recursive=True))):
            t = tarfile.TarInfo( dpath )
            t.type = tarfile.DIRTYPE
            tarHandler.addfile(t)
            tarHandler.add(os.path.join(self.__path,dpath,self.__dirInfo), arcname=self.__dirInfo)
        # walk files and add to tar
        for fpath in self.walk_files_path(recursive=True):
            relaPath, fname = os.path.split(fpath)
            tarHandler.add(os.path.join(self.__path,fpath), arcname=fname)
            tarHandler.add(os.path.join(self.__path,relaPath,self.__fileInfo%fname), arcname=self.__fileInfo%fname)
            tarHandler.add(os.path.join(self.__path,relaPath,self.__fileClass%fname), arcname=self.__fileClass%fname)
        # save repository .pyrepinfo
        tarHandler.add(os.path.join(self.__path,self.__repoFile), arcname=".pyrepinfo")
        # close tar file
        tarHandler.close()

Create a tar file package of all the repository files and directories . Only files and directories that are tracked in the repository are stored in the package tar file .


def rename(self, key: Any, new_key: Any):
        """
        Renames an item in this collection as a transaction.

        Will override if new key name already exists.
        :param key: the current name of the item
        :param new_key: the new name that the item should have
        """
        if new_key == key:
            return

        required_locks = [self._key_locks[key], self._key_locks[new_key]]
        ordered_required_locks = sorted(required_locks, key=lambda x: id(x))
        for lock in ordered_required_locks:
            lock.acquire()

        try:
            if key not in self._data:
                raise KeyError("Attribute to rename \"%s\" does not exist" % key)
            self._data[new_key] = self[key]
            del self._data[key]
        finally:
            for lock in required_locks:
                lock.release()

Renames an item in this collection as a transaction .


def get_text_fingerprint(text, hash_meth, encoding="utf-8"):  # pragma: no cover
    """
    Use default hash method to return hash value of a piece of string
    default setting use 'utf-8' encoding.
    """
    m = hash_meth()
    m.update(text.encode(encoding))
    return m.hexdigest()

def get_text_fingerprint ( text , hash_meth , encoding = "utf-8" ) : m = hash_meth ( ) m . update ( text . encode ( encoding ) ) return m . hexdigest ( )

Use default hash method to return hash value of a piece of string default setting use utf - 8 encoding .


def md5file(abspath, nbytes=0, chunk_size=DEFAULT_CHUNK_SIZE):
    """
    Return md5 hash value of a piece of a file

    Estimate processing time on:

    :param abspath: the absolute path to the file
    :param nbytes: only has first N bytes of the file. if 0 or None,
      hash all file

    CPU = i7-4600U 2.10GHz - 2.70GHz, RAM = 8.00 GB
    1 second can process 0.25GB data

    - 0.59G - 2.43 sec
    - 1.3G - 5.68 sec
    - 1.9G - 7.72 sec
    - 2.5G - 10.32 sec
    - 3.9G - 16.0 sec
    """
    return get_file_fingerprint(abspath, hashlib.md5, nbytes=nbytes, chunk_size=chunk_size)

def md5file ( abspath , nbytes = 0 , chunk_size = DEFAULT_CHUNK_SIZE ) : return get_file_fingerprint ( abspath , hashlib . md5 , nbytes = nbytes , chunk_size = chunk_size )


def sha256file(abspath, nbytes=0, chunk_size=DEFAULT_CHUNK_SIZE):
    """
    Return sha256 hash value of a piece of a file

    Estimate processing time on:

    :param abspath: the absolute path to the file
    :param nbytes: only has first N bytes of the file. if 0 or None,
      hash all file
    """
    return get_file_fingerprint(abspath, hashlib.sha256, nbytes=nbytes, chunk_size=chunk_size)

def sha256file ( abspath , nbytes = 0 , chunk_size = DEFAULT_CHUNK_SIZE ) : return get_file_fingerprint ( abspath , hashlib . sha256 , nbytes = nbytes , chunk_size = chunk_size )


def sha512file(abspath, nbytes=0, chunk_size=DEFAULT_CHUNK_SIZE):
    """
    Return sha512 hash value of a piece of a file

    Estimate processing time on:

    :param abspath: the absolute path to the file
    :param nbytes: only has first N bytes of the file. if 0 or None,
      hash all file
    """
    return get_file_fingerprint(abspath, hashlib.sha512, nbytes=nbytes, chunk_size=chunk_size)

def sha512file ( abspath , nbytes = 0 , chunk_size = DEFAULT_CHUNK_SIZE ) : return get_file_fingerprint ( abspath , hashlib . sha512 , nbytes = nbytes , chunk_size = chunk_size )


def auto_complete_choices(self, case_sensitive=False):
        """
        A command line auto complete similar behavior. Find all item with same
        prefix of this one.

        :param case_sensitive: toggle if it is case sensitive.
        :return: list of :class:`pathlib_mate.pathlib2.Path`.
        """
        self_basename = self.basename
        self_basename_lower = self.basename.lower()
        if case_sensitive:  # pragma: no cover
            def match(basename):
                return basename.startswith(self_basename)
        else:
            def match(basename):
                return basename.lower().startswith(self_basename_lower)

        choices = list()
        if self.is_dir():
            choices.append(self)
            for p in self.sort_by_abspath(self.select(recursive=False)):
                choices.append(p)
        else:
            p_parent = self.parent
            if p_parent.is_dir():
                for p in self.sort_by_abspath(p_parent.select(recursive=False)):
                    if match(p.basename):
                        choices.append(p)
            else:  # pragma: no cover
                raise ValueError("'%s' directory does not exist!" % p_parent)
        return choices

def
auto_complete_choices
(
self
,
case_sensitive
=
False
)
:
self_basename
=
self
.
basename
self_basename_lower
=
self
.
basename
.
lower
(
)
if
case_sensitive
:
def
match
(
basename
)
:
return
basename
.
startswith
(
self_basename
)
else
:
def
match
(
basename
)
:
return
basename
.
lower
(
)
.
startswith
(
self_basename_lower
)
choices
=
list
(
)
if
self
.
is_dir
(
)
:
choices
.
append
(
self
)
for
p
in
self
.
sort_by_abspath
(
self
.
select
(
recursive
=
False
)
)
:
choices
.
append
(
p
)
else
:
p_parent
=
self
.
parent
if
p_parent
.
is_dir
(
)
:
for
p
in
self
.
sort_by_abspath
(
p_parent
.
select
(
recursive
=
False
)
)
:
if
match
(
p
.
basename
)
:
choices
.
append
(
p
)
else
:
raise
ValueError
(
"'%s' directory does not exist!"
%
p_parent
)
return
choices

A command line auto complete similar behavior . Find all item with same prefix of this one .


def print_big_dir(self, top_n=5):
        """
        Print ``top_n`` big dir in this dir.
        """
        self.assert_is_dir_and_exists()

        size_table = sorted(
            [(p, p.dirsize) for p in self.select_dir(recursive=False)],
            key=lambda x: x[1],
            reverse=True,
        )
        for p, size in size_table[:top_n]:
            print("{:<9}    {:<9}".format(repr_data_size(size), p.abspath))

def
print_big_dir
(
self
,
top_n
=
5
)
:
self
.
assert_is_dir_and_exists
(
)
size_table
=
sorted
(
[
(
p
,
p
.
dirsize
)
for
p
in
self
.
select_dir
(
recursive
=
False
)
]
,
key
=
lambda
x
:
x
[
1
]
,
reverse
=
True
,
)
for
p
,
size
in
size_table
[
:
top_n
]
:
print
(
"{:<9}    {:<9}"
.
format
(
repr_data_size
(
size
)
,
p
.
abspath
)
)


def print_big_file(self, top_n=5):
        """
        Print ``top_n`` big file in this dir.
        """
        self.assert_is_dir_and_exists()

        size_table = sorted(
            [(p, p.size) for p in self.select_file(recursive=True)],
            key=lambda x: x[1],
            reverse=True,
        )
        for p, size in size_table[:top_n]:
            print("{:<9}    {:<9}".format(repr_data_size(size), p.abspath))

def
print_big_file
(
self
,
top_n
=
5
)
:
self
.
assert_is_dir_and_exists
(
)
size_table
=
sorted
(
[
(
p
,
p
.
size
)
for
p
in
self
.
select_file
(
recursive
=
True
)
]
,
key
=
lambda
x
:
x
[
1
]
,
reverse
=
True
,
)
for
p
,
size
in
size_table
[
:
top_n
]
:
print
(
"{:<9}    {:<9}"
.
format
(
repr_data_size
(
size
)
,
p
.
abspath
)
)


def print_big_dir_and_big_file(self, top_n=5):
        """Print ``top_n`` big dir and ``top_n`` big file in each dir.
        """
        self.assert_is_dir_and_exists()

        size_table1 = sorted(
            [(p, p.dirsize) for p in self.select_dir(recursive=False)],
            key=lambda x: x[1],
            reverse=True,
        )
        for p1, size1 in size_table1[:top_n]:
            print("{:<9}    {:<9}".format(repr_data_size(size1), p1.abspath))
            size_table2 = sorted(
                [(p, p.size) for p in p1.select_file(recursive=True)],
                key=lambda x: x[1],
                reverse=True,
            )
            for p2, size2 in size_table2[:top_n]:
                print("    {:<9}    {:<9}".format(
                    repr_data_size(size2), p2.abspath))

def
print_big_dir_and_big_file
(
self
,
top_n
=
5
)
:
self
.
assert_is_dir_and_exists
(
)
size_table1
=
sorted
(
[
(
p
,
p
.
dirsize
)
for
p
in
self
.
select_dir
(
recursive
=
False
)
]
,
key
=
lambda
x
:
x
[
1
]
,
reverse
=
True
,
)
for
p1
,
size1
in
size_table1
[
:
top_n
]
:
print
(
"{:<9}    {:<9}"
.
format
(
repr_data_size
(
size1
)
,
p1
.
abspath
)
)
size_table2
=
sorted
(
[
(
p
,
p
.
size
)
for
p
in
p1
.
select_file
(
recursive
=
True
)
]
,
key
=
lambda
x
:
x
[
1
]
,
reverse
=
True
,
)
for
p2
,
size2
in
size_table2
[
:
top_n
]
:
print
(
"    {:<9}    {:<9}"
.
format
(
repr_data_size
(
size2
)
,
p2
.
abspath
)
)

Print top_n big dir and top_n big file in each dir .


def mirror_to(self, dst):  # pragma: no cover
        """
        Create a new folder having exactly same structure with this directory.
        However, all files are just empty file with same file name.

        :param dst: destination directory. The directory can't exists before
        you execute this.

        **中文文档**

        创建一个目录的镜像拷贝, 与拷贝操作不同的是, 文件的副本只是在文件名上
        与原件一致, 但是是空文件, 完全没有内容, 文件大小为0。
        """
        self.assert_is_dir_and_exists()

        src = self.abspath
        dst = os.path.abspath(dst)
        if os.path.exists(dst):  # pragma: no cover
            raise Exception("distination already exist!")

        folder_to_create = list()
        file_to_create = list()

        for current_folder, _, file_list in os.walk(self.abspath):
            current_folder = current_folder.replace(src, dst)
            try:
                os.mkdir(current_folder)
            except:  # pragma: no cover
                pass
            for basename in file_list:
                abspath = os.path.join(current_folder, basename)
                with open(abspath, "wb") as _:
                    pass

def
mirror_to
(
self
,
dst
)
:
self
.
assert_is_dir_and_exists
(
)
src
=
self
.
abspath
dst
=
os
.
path
.
abspath
(
dst
)
if
os
.
path
.
exists
(
dst
)
:
raise
Exception
(
"distination already exist!"
)
folder_to_create
=
list
(
)
file_to_create
=
list
(
)
for
current_folder
,
_
,
file_list
in
os
.
walk
(
self
.
abspath
)
:
current_folder
=
current_folder
.
replace
(
src
,
dst
)
try
:
os
.
mkdir
(
current_folder
)
except
:
pass
for
basename
in
file_list
:
abspath
=
os
.
path
.
join
(
current_folder
,
basename
)
with
open
(
abspath
,
"wb"
)
as
_
:
pass

Create a new folder having exactly same structure with this directory . However all files are just empty file with same file name .


def execute_pyfile(self, py_exe=None):  # pragma: no cover
        """
        Execute every ``.py`` file as main script.

        :param py_exe: str, python command or python executable path.

        **中文文档**

        将目录下的所有Python文件作为主脚本用当前解释器运行。
        """
        import subprocess

        self.assert_is_dir_and_exists()

        if py_exe is None:
            if six.PY2:
                py_exe = "python2"
            elif six.PY3:
                py_exe = "python3"

        for p in self.select_by_ext(".py"):
            subprocess.Popen('%s "%s"' % (py_exe, p.abspath))

def
execute_pyfile
(
self
,
py_exe
=
)
:
import
subprocess
self
.
assert_is_dir_and_exists
(
)
if
py_exe
is
:
if
six
.
PY2
:
py_exe
=
"python2"
elif
six
.
PY3
:
py_exe
=
"python3"
for
p
in
self
.
select_by_ext
(
".py"
)
:
subprocess
.
Popen
(
'%s "%s"'
%
(
py_exe
,
p
.
abspath
)
)

Execute every . py file as main script .


def trail_space(self, filters=lambda p: p.ext == ".py"):  # pragma: no cover
        """
        Trail white space at end of each line for every ``.py`` file.

        **中文文档**

        将目录下的所有被选择的文件中行末的空格删除。
        """
        self.assert_is_dir_and_exists()

        for p in self.select_file(filters):
            try:
                with open(p.abspath, "rb") as f:
                    lines = list()
                    for line in f:
                        lines.append(line.decode("utf-8").rstrip())

                with open(p.abspath, "wb") as f:
                    f.write("\n".join(lines).encode("utf-8"))

            except Exception as e:  # pragma: no cover
                raise e

def trail_space ( self , filters = lambda p : p . ext == ".py" ) : self . assert_is_dir_and_exists ( ) for p in self . select_file ( filters ) : try : with open ( p . abspath , "rb" ) as f : lines = list ( ) for line in f : lines . append ( line . decode ( "utf-8" ) . rstrip ( ) ) with open ( p . abspath , "wb" ) as f : f . write ( "\n" . join ( lines ) . encode ( "utf-8" ) ) except Exception as e : raise e

Trail white space at end of each line for every . py file .


def autopep8(self, **kwargs):  # pragma: no cover
        """
        Auto convert your python code in a directory to pep8 styled code.

        :param kwargs: arguments for ``autopep8.fix_code`` method.

        **中文文档**

        将目录下的所有Python文件用pep8风格格式化。增加其可读性和规范性。
        """
        self.assert_is_dir_and_exists()

        for p in self.select_by_ext(".py"):
            with open(p.abspath, "rb") as f:
                code = f.read().decode("utf-8")

            formatted_code = autopep8.fix_code(code, **kwargs)

            with open(p.abspath, "wb") as f:
                f.write(formatted_code.encode("utf-8"))

def autopep8 ( self , ** kwargs ) : self . assert_is_dir_and_exists ( ) for p in self . select_by_ext ( ".py" ) : with open ( p . abspath , "rb" ) as f : code = f . read ( ) . decode ( "utf-8" ) formatted_code = autopep8 . fix_code ( code , ** kwargs ) with open ( p . abspath , "wb" ) as f : f . write ( formatted_code . encode ( "utf-8" ) )

Auto convert your python code in a directory to pep8 styled code .


def size(self):
        """
        File size in bytes.
        """
        try:
            return self._stat.st_size
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.size

def size ( self ) : try : return self . _stat . st_size except : self . _stat = self . stat ( ) return self . size


def mtime(self):
        """
        Get most recent modify time in timestamp.
        """
        try:
            return self._stat.st_mtime
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.mtime

def mtime ( self ) : try : return self . _stat . st_mtime except : self . _stat = self . stat ( ) return self . mtime

Get most recent modify time in timestamp .


def atime(self):
        """
        Get most recent access time in timestamp.
        """
        try:
            return self._stat.st_atime
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.atime

def atime ( self ) : try : return self . _stat . st_atime except : self . _stat = self . stat ( ) return self . atime

Get most recent access time in timestamp .


def ctime(self):
        """
        Get most recent create time in timestamp.
        """
        try:
            return self._stat.st_ctime
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.ctime

def ctime ( self ) : try : return self . _stat . st_ctime except : self . _stat = self . stat ( ) return self . ctime

Get most recent create time in timestamp .


def unusedoptions(self, sections):
        """Lists options that have not been used to format other values in
        their sections.

        Good for finding out if the user has misspelled any of the options.
        """
        unused = set([])
        for section in _list(sections):
            if not self.has_section(section):
                continue
            options = self.options(section)
            raw_values = [self.get(section, option, raw=True) for option in options]
            for option in options:
                formatter = "%(" + option + ")s"
                for raw_value in raw_values:
                    if formatter in raw_value:
                        break
                else:
                    unused.add(option)
            return list(unused)

def unusedoptions ( self , sections ) : unused = set ( [ ] ) for section in _list ( sections ) : if not self . has_section ( section ) : continue options = self . options ( section ) raw_values = [ self . get ( section , option , raw = True ) for option in options ] for option in options : formatter = "%(" + option + ")s" for raw_value in raw_values : if formatter in raw_value : break else : unused . add ( option ) return list ( unused )

Lists options that have not been used to format other values in their sections . Good for finding out if the user has misspelled any of the options .


def keys(self):
        """List names of options and positional arguments."""
        return self.options.keys() + [p.name for p in self.positional_args]

def keys ( self ) : return self . options . keys ( ) + [ p . name for p in self . positional_args ]

List names of options and positional arguments .


def _add_option(self, option):
        """Add an Option object to the user interface."""
        if option.name in self.options:
            raise ValueError('name already in use')
        if option.abbreviation in self.abbreviations:
            raise ValueError('abbreviation already in use')
        if option.name in [arg.name for arg in self.positional_args]:
            raise ValueError('name already in use by a positional argument')
        self.options[option.name] = option
        if option.abbreviation:
            self.abbreviations[option.abbreviation] = option
        self.option_order.append(option.name)

Add an Option object to the user interface .


def _add_positional_argument(self, posarg):
        """Append a positional argument to the user interface.

        Optional positional arguments must be added after the required ones.
        The user interface can have at most one recurring positional argument,
        and if present, that argument must be the last one.
        """
        if self.positional_args:
            if self.positional_args[-1].recurring:
                raise ValueError("recurring positional arguments must be last")
            if self.positional_args[-1].optional and not posarg.optional:
                raise ValueError("required positional arguments must precede optional ones")
        self.positional_args.append(posarg)

Append a positional argument to the user interface .


def read_docs(self, docsfiles):
        """Read program documentation from a DocParser compatible file.

        docsfiles is a list of paths to potential docsfiles: parse if present.
        A string is taken as a list of one item.
        """
        updates = DocParser()
        for docsfile in _list(docsfiles):
            if os.path.isfile(docsfile):
                updates.parse(docsfile)
        self.docs.update((k, _docs(updates[k], self.docvars)) for k in self.docs if updates.blocks[k])
        for name, text in updates['parameters'].items():
            if name in self:
                self.getparam(name).docs = text[0] % self.docvars
            elif name not in self.ignore:
                raise ValueError("parameter %r does not exist" % name)

def
read_docs
(
self
,
docsfiles
)
:
updates
=
DocParser
(
)
for
docsfile
in
_list
(
docsfiles
)
:
if
os
.
path
.
isfile
(
docsfile
)
:
updates
.
parse
(
docsfile
)
self
.
docs
.
update
(
(
k
,
_docs
(
updates
[
k
]
,
self
.
docvars
)
)
for
k
in
self
.
docs
if
updates
.
blocks
[
k
]
)
for
name
,
text
in
updates
[
'parameters'
]
.
items
(
)
:
if
name
in
self
:
self
.
getparam
(
name
)
.
docs
=
text
[
0
]
%
self
.
docvars
elif
name
not
in
self
.
ignore
:
raise
ValueError
(
"parameter %r does not exist"
%
name
)

Read program documentation from a DocParser compatible file .


def optionhelp(self, indent=0, maxindent=25, width=79):
        """Return user friendly help on program options."""
        def makelabels(option):
            labels = '%*s--%s' % (indent, ' ', option.name)
            if option.abbreviation:
                labels += ', -' + option.abbreviation
            return labels + ': '
        docs = []
        helpindent = _autoindent([makelabels(o) for o in self.options.values()], indent, maxindent)
        for name in self.option_order:
            option = self.options[name]
            labels = makelabels(option)
            helpstring = "%s(%s). %s" % (option.formatname, option.strvalue, option.docs)
            wrapped = self._wrap_labelled(labels, helpstring, helpindent, width)
            docs.extend(wrapped)
        return '\n'.join(docs)

def
optionhelp
(
self
,
indent
=
0
,
maxindent
=
25
,
width
=
79
)
:
def
makelabels
(
option
)
:
labels
=
'%*s--%s'
%
(
indent
,
' '
,
option
.
name
)
if
option
.
abbreviation
:
labels
+=
', -'
+
option
.
abbreviation
return
labels
+
': '
docs
=
[
]
helpindent
=
_autoindent
(
[
makelabels
(
o
)
for
o
in
self
.
options
.
values
(
)
]
,
indent
,
maxindent
)
for
name
in
self
.
option_order
:
option
=
self
.
options
[
name
]
labels
=
makelabels
(
option
)
helpstring
=
"%s(%s). %s"
%
(
option
.
formatname
,
option
.
strvalue
,
option
.
docs
)
wrapped
=
self
.
_wrap_labelled
(
labels
,
helpstring
,
helpindent
,
width
)
docs
.
extend
(
wrapped
)
return
'\n'
.
join
(
docs
)

Return user friendly help on program options .


def posarghelp(self, indent=0, maxindent=25, width=79):
        """Return user friendly help on positional arguments in the program."""
        docs = []
        makelabel = lambda posarg: ' ' * indent + posarg.displayname + ': '
        helpindent = _autoindent([makelabel(p) for p in self.positional_args], indent, maxindent)
        for posarg in self.positional_args:
            label = makelabel(posarg)
            text = posarg.formatname + '. ' + posarg.docs
            wrapped = self._wrap_labelled(label, text, helpindent, width)
            docs.extend(wrapped)
        return '\n'.join(docs)

def
posarghelp
(
self
,
indent
=
0
,
maxindent
=
25
,
width
=
79
)
:
docs
=
[
]
makelabel
=
lambda
posarg
:
' '
*
indent
+
posarg
.
displayname
+
': '
helpindent
=
_autoindent
(
[
makelabel
(
p
)
for
p
in
self
.
positional_args
]
,
indent
,
maxindent
)
for
posarg
in
self
.
positional_args
:
label
=
makelabel
(
posarg
)
text
=
posarg
.
formatname
+
'. '
+
posarg
.
docs
wrapped
=
self
.
_wrap_labelled
(
label
,
text
,
helpindent
,
width
)
docs
.
extend
(
wrapped
)
return
'\n'
.
join
(
docs
)

Return user friendly help on positional arguments in the program .


def strsettings(self, indent=0, maxindent=25, width=0):
        """Return user friendly help on positional arguments.

        indent is the number of spaces preceeding the text on each line.

        The indent of the documentation is dependent on the length of the
        longest label that is shorter than maxindent. A label longer than
        maxindent will be printed on its own line.

        width is maximum allowed page width, use self.width if 0.
        """
        out = []
        makelabel = lambda name: ' ' * indent + name + ': '
        settingsindent = _autoindent([makelabel(s) for s in self.options], indent, maxindent)
        for name in self.option_order:
            option = self.options[name]
            label = makelabel(name)
            settingshelp = "%s(%s): %s" % (option.formatname, option.strvalue, option.location)
            wrapped = self._wrap_labelled(label, settingshelp, settingsindent, width)
            out.extend(wrapped)
        return '\n'.join(out)

def
strsettings
(
self
,
indent
=
0
,
maxindent
=
25
,
width
=
0
)
:
out
=
[
]
makelabel
=
lambda
name
:
' '
*
indent
+
name
+
': '
settingsindent
=
_autoindent
(
[
makelabel
(
s
)
for
s
in
self
.
options
]
,
indent
,
maxindent
)
for
name
in
self
.
option_order
:
option
=
self
.
options
[
name
]
label
=
makelabel
(
name
)
settingshelp
=
"%s(%s): %s"
%
(
option
.
formatname
,
option
.
strvalue
,
option
.
location
)
wrapped
=
self
.
_wrap_labelled
(
label
,
settingshelp
,
settingsindent
,
width
)
out
.
extend
(
wrapped
)
return
'\n'
.
join
(
out
)

Return user friendly help on positional arguments .


def settingshelp(self, width=0):
        """Return a summary of program options, their values and origins.

        width is maximum allowed page width, use self.width if 0.
        """
        out = []
        out.append(self._wrap(self.docs['title'], width=width))
        if self.docs['description']:
            out.append(self._wrap(self.docs['description'], indent=2, width=width))
        out.append('')
        out.append('SETTINGS:')
        out.append(self.strsettings(indent=2, width=width))
        out.append('')
        return '\n'.join(out)

def settingshelp ( self , width = 0 ) : out = [ ] out . append ( self . _wrap ( self . docs [ 'title' ] , width = width ) ) if self . docs [ 'description' ] : out . append ( self . _wrap ( self . docs [ 'description' ] , indent = 2 , width = width ) ) out . append ( '' ) out . append ( 'SETTINGS:' ) out . append ( self . strsettings ( indent = 2 , width = width ) ) out . append ( '' ) return '\n' . join ( out )

Return a summary of program options their values and origins . width is maximum allowed page width use self . width if 0 .


def parse(self, file):
        """Parse text blocks from a file."""
        if isinstance(file, basestring):
            file = open(file)
        line_number = 0
        label = None
        block = self.untagged
        for line in file:
            line_number += 1
            line = line.rstrip('\n')
            if self.tabsize > 0:
                line = line.replace('\t', ' ' * self.tabsize)
            if self.decommenter:
                line = self.decommenter.decomment(line)
                if line is None:
                    continue
            tag = line.split(':', 1)[0].strip()
            # Still in the same block?
            if tag not in self.names:
                if block is None:
                    if line and not line.isspace():
                        raise ParseError(file.name, line, "garbage before first block: %r" % line)
                    continue
                block.addline(line)
                continue
            # Open a new block.
            name = self.names[tag]
            label = line.split(':',1)[1].strip()
            if name in self.labelled_classes:
                if not label:
                    raise ParseError(file.name, line, "missing label for %r block" % name)
                block = self.blocks[name].setdefault(label, self.labelled_classes[name]())
            else:
                if label:
                    msg = "label %r present for unlabelled block %r" % (label, name)
                    raise ParseError(file.name, line_number, msg)
                block = self.blocks[name]
            block.startblock()


def parse(self, argv):
        """Pop, parse and return the first self.nargs items from args.

        if self.nargs > 1 a list of parsed values will be returned.

        Raise BadNumberOfArguments or BadArgument on errors.

        NOTE: argv may be modified in place by this method.
        """
        if len(argv) < self.nargs:
            raise BadNumberOfArguments(self.nargs, len(argv))
        if self.nargs == 1:
            return self.parse_argument(argv.pop(0))
        return [self.parse_argument(argv.pop(0)) for tmp in range(self.nargs)]

def parse ( self , argv ) : if len ( argv ) < self . nargs : raise BadNumberOfArguments ( self . nargs , len ( argv ) ) if self . nargs == 1 : return self . parse_argument ( argv . pop ( 0 ) ) return [ self . parse_argument ( argv . pop ( 0 ) ) for tmp in range ( self . nargs ) ]

Pop parse and return the first self . nargs items from args .


def parsestr(self, argstr):
        """Parse arguments found in settings files.

        Use the values in self.true for True in settings files, or those in
        self.false for False, case insensitive.
        """
        argv = shlex.split(argstr, comments=True)
        if len(argv) != 1:
            raise BadNumberOfArguments(1, len(argv))
        arg = argv[0]
        lower = arg.lower()
        if lower in self.true:
            return True
        if lower in self.false:
            return False
        raise BadArgument(arg, "Allowed values are " + self.allowed + '.')

def
parsestr
(
self
,
argstr
)
:
argv
=
shlex
.
split
(
argstr
,
comments
=
True
)
if
len
(
argv
)
!=
1
:
raise
BadNumberOfArguments
(
1
,
len
(
argv
)
)
arg
=
argv
[
0
]
lower
=
arg
.
lower
(
)
if
lower
in
self
.
true
:
return
True
if
lower
in
self
.
false
:
return
False
raise
BadArgument
(
arg
,
"Allowed values are "
+
self
.
allowed
+
'.'
)

Parse arguments found in settings files . Use the values in self . true for True in settings files or those in self . false for False case insensitive .


def get_separator(self, i):
        """Return the separator that preceding format i, or '' for i == 0."""
        return i and self.separator[min(i - 1, len(self.separator) - 1)] or ''

def get_separator ( self , i ) : return i and self . separator [ min ( i - 1 , len ( self . separator ) - 1 ) ] or ''

Return the separator that preceding format i or for i == 0 .


def authorize_url(self):
        """
        Return a URL to redirect the user to for OAuth authentication.
        """
        auth_url = OAUTH_ROOT + '/authorize'
        params = {
            'client_id': self.client_id,
            'redirect_uri': self.redirect_uri,
        }
        return "{}?{}".format(auth_url, urlencode(params))

def authorize_url ( self ) : auth_url = OAUTH_ROOT + '/authorize' params = { 'client_id' : self . client_id , 'redirect_uri' : self . redirect_uri , } return "{}?{}" . format ( auth_url , urlencode ( params ) )

Return a URL to redirect the user to for OAuth authentication .


def exchange_token(self, code):
        """
        Exchange the authorization code for an access token.
        """
        access_token_url = OAUTH_ROOT + '/access_token'
        params = {
            'client_id': self.client_id,
            'client_secret': self.client_secret,
            'redirect_uri': self.redirect_uri,
            'code': code,
        }
        resp = requests.get(access_token_url, params=params)
        if not resp.ok:
            raise MixcloudOauthError("Could not get access token.")
        return resp.json()['access_token']

def
exchange_token
(
self
,
code
)
:
access_token_url
=
OAUTH_ROOT
+
'/access_token'
params
=
{
'client_id'
:
self
.
client_id
,
'client_secret'
:
self
.
client_secret
,
'redirect_uri'
:
self
.
redirect_uri
,
'code'
:
code
,
}
resp
=
requests
.
get
(
access_token_url
,
params
=
params
)
if
not
resp
.
ok
:
raise
MixcloudOauthError
(
"Could not get access token."
)
return
resp
.
json
(
)
[
'access_token'
]

Exchange the authorization code for an access token .


def acquire(self, *args, **kwargs):
        """ Wraps Lock.acquire """
        with self._stat_lock:
            self._waiting += 1

        self._lock.acquire(*args, **kwargs)

        with self._stat_lock:
            self._locked = True
            self._waiting -= 1

def acquire ( self , * args , ** kwargs ) : with self . _stat_lock : self . _waiting += 1 self . _lock . acquire ( * args , ** kwargs ) with self . _stat_lock : self . _locked = True self . _waiting -= 1


def release(self):
        """ Wraps Lock.release """
        self._lock.release()

        with self._stat_lock:
            self._locked = False
            self._last_released = datetime.now()

def release ( self ) : self . _lock . release ( ) with self . _stat_lock : self . _locked = False self . _last_released = datetime . now ( )


def default_decoder(self, obj):
        """Handle a dict that might contain a wrapped state for a custom type."""
        typename, marshalled_state = self.unwrap_callback(obj)
        if typename is None:
            return obj

        try:
            cls, unmarshaller = self.serializer.unmarshallers[typename]
        except KeyError:
            raise LookupError('no unmarshaller found for type "{}"'.format(typename)) from None

        if cls is not None:
            instance = cls.__new__(cls)
            unmarshaller(instance, marshalled_state)
            return instance
        else:
            return unmarshaller(marshalled_state)

def
default_decoder
(
self
,
obj
)
:
typename
,
marshalled_state
=
self
.
unwrap_callback
(
obj
)
if
typename
is
:
return
obj
try
:
cls
,
unmarshaller
=
self
.
serializer
.
unmarshallers
[
typename
]
except
KeyError
:
raise
LookupError
(
'no unmarshaller found for type "{}"'
.
format
(
typename
)
)
from
if
cls
is
not
:
instance
=
cls
.
__new__
(
cls
)
unmarshaller
(
instance
,
marshalled_state
)
return
instance
else
:
return
unmarshaller
(
marshalled_state
)

Handle a dict that might contain a wrapped state for a custom type .


def wrap_state_dict(self, typename: str, state) -> Dict[str, Any]:
        """
        Wrap the marshalled state in a dictionary.

        The returned dictionary has two keys, corresponding to the ``type_key`` and ``state_key``
        options. The former holds the type name and the latter holds the marshalled state.

        :param typename: registered name of the custom type
        :param state: the marshalled state of the object
        :return: an object serializable by the serializer

        """
        return {self.type_key: typename, self.state_key: state}

def wrap_state_dict ( self , typename : str , state ) -> Dict [ str , Any ] : return { self . type_key : typename , self . state_key : state }

Wrap the marshalled state in a dictionary .


def publish(quiet, dataset_uri):
    """Enable HTTP access to a dataset.

    This only works on datasets in some systems. For example, datasets stored
    in AWS S3 object storage and Microsoft Azure Storage can be published as
    datasets accessible over HTTP. A published dataset is world readable.
    """
    access_uri = http_publish(dataset_uri)
    if not quiet:
        click.secho("Dataset accessible at ", nl=False, fg="green")
    click.secho(access_uri)

def
publish
(
quiet
,
dataset_uri
)
:
access_uri
=
http_publish
(
dataset_uri
)
if
not
quiet
:
click
.
secho
(
"Dataset accessible at "
,
nl
=
False
,
fg
=
"green"
)
click
.
secho
(
access_uri
)


def _prompt_for_values(d):
    """Update the descriptive metadata interactively.

    Uses values entered by the user. Note that the function keeps recursing
    whenever a value is another ``CommentedMap`` or a ``list``. The
    function works as passing dictionaries and lists into a function edits
    the values in place.
    """
    for key, value in d.items():
        if isinstance(value, CommentedMap):
            _prompt_for_values(value)
        elif isinstance(value, list):
            for item in value:
                _prompt_for_values(item)
        else:
            typ = type(value)

            if isinstance(value, ScalarFloat):  # Deal with ruamel.yaml floats.
                typ = float

            new_value = click.prompt(key, type=typ, default=value)
            d[key] = new_value
    return d

def _prompt_for_values ( d ) : for key , value in d . items ( ) : if isinstance ( value , CommentedMap ) : _prompt_for_values ( value ) elif isinstance ( value , list ) : for item in value : _prompt_for_values ( item ) else : typ = type ( value ) if isinstance ( value , ScalarFloat ) : typ = float new_value = click . prompt ( key , type = typ , default = value ) d [ key ] = new_value return d

Update the descriptive metadata interactively .


def create(quiet, name, base_uri, symlink_path):
    """Create a proto dataset."""
    _validate_name(name)

    admin_metadata = dtoolcore.generate_admin_metadata(name)
    parsed_base_uri = dtoolcore.utils.generous_parse_uri(base_uri)

    if parsed_base_uri.scheme == "symlink":
        if symlink_path is None:
            raise click.UsageError("Need to specify symlink path using the -s/--symlink-path option")  # NOQA

    if symlink_path:
        base_uri = dtoolcore.utils.sanitise_uri(
            "symlink:" + parsed_base_uri.path
        )
        parsed_base_uri = dtoolcore.utils.generous_parse_uri(base_uri)

    # Create the dataset.
    proto_dataset = dtoolcore.generate_proto_dataset(
        admin_metadata=admin_metadata,
        base_uri=dtoolcore.utils.urlunparse(parsed_base_uri),
        config_path=CONFIG_PATH)

    # If we are creating a symlink dataset we need to set the symlink_path
    # attribute on the storage broker.
    if symlink_path:
        symlink_abspath = os.path.abspath(symlink_path)
        proto_dataset._storage_broker.symlink_path = symlink_abspath
    try:
        proto_dataset.create()
    except dtoolcore.storagebroker.StorageBrokerOSError as err:
        raise click.UsageError(str(err))

    proto_dataset.put_readme("")

    if quiet:
        click.secho(proto_dataset.uri)
    else:
        # Give the user some feedback and hints on what to do next.
        click.secho("Created proto dataset ", nl=False, fg="green")
        click.secho(proto_dataset.uri)
        click.secho("Next steps: ")

        step = 1

        if parsed_base_uri.scheme != "symlink":
            click.secho("{}. Add raw data, eg:".format(step))
            click.secho(
                "   dtool add item my_file.txt {}".format(proto_dataset.uri),
                fg="cyan")

            if parsed_base_uri.scheme == "file":
                # Find the abspath of the data directory for user feedback.
                data_path = proto_dataset._storage_broker._data_abspath
                click.secho("   Or use your system commands, e.g: ")
                click.secho(
                    "   mv my_data_directory {}/".format(data_path),
                    fg="cyan"
                )
            step = step + 1

        click.secho("{}. Add descriptive metadata, e.g: ".format(step))
        click.secho(
            "   dtool readme interactive {}".format(proto_dataset.uri),
            fg="cyan")
        step = step + 1

        click.secho(
            "{}. Convert the proto dataset into a dataset: ".format(step)
        )
        click.secho("   dtool freeze {}".format(proto_dataset.uri), fg="cyan")


def interactive(proto_dataset_uri):
    """Interactive prompting to populate the readme."""
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri,
        config_path=CONFIG_PATH)

    # Create an CommentedMap representation of the yaml readme template.
    readme_template = _get_readme_template()
    yaml = YAML()
    yaml.explicit_start = True
    yaml.indent(mapping=2, sequence=4, offset=2)
    descriptive_metadata = yaml.load(readme_template)

    descriptive_metadata = _prompt_for_values(descriptive_metadata)

    # Write out the descriptive metadata to the readme file.
    stream = StringIO()

    yaml.dump(descriptive_metadata, stream)

    proto_dataset.put_readme(stream.getvalue())

    click.secho("Updated readme ", fg="green")
    click.secho("To edit the readme using your default editor:")
    click.secho(
        "dtool readme edit {}".format(proto_dataset_uri),
        fg="cyan")

Interactive prompting to populate the readme .


def edit(dataset_uri):
    """Default editor updating of readme content.
    """
    try:
        dataset = dtoolcore.ProtoDataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    except dtoolcore.DtoolCoreTypeError:
        dataset = dtoolcore.DataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    readme_content = dataset.get_readme_content()

    try:
        # Python2 compatibility.
        readme_content = unicode(readme_content, "utf-8")
    except NameError:
        pass

    edited_content = click.edit(readme_content)
    if edited_content is not None:
        _validate_and_put_readme(dataset, edited_content)
        click.secho("Updated readme ", nl=False, fg="green")
    else:
        click.secho("Did not update readme ", nl=False, fg="red")
    click.secho(dataset_uri)

def
edit
(
dataset_uri
)
:
try
:
dataset
=
dtoolcore
.
ProtoDataSet
.
from_uri
(
uri
=
dataset_uri
,
config_path
=
CONFIG_PATH
)
except
dtoolcore
.
DtoolCoreTypeError
:
dataset
=
dtoolcore
.
DataSet
.
from_uri
(
uri
=
dataset_uri
,
config_path
=
CONFIG_PATH
)
readme_content
=
dataset
.
get_readme_content
(
)
try
:
readme_content
=
unicode
(
readme_content
,
"utf-8"
)
except
NameError
:
pass
edited_content
=
click
.
edit
(
readme_content
)
if
edited_content
is
not
:
_validate_and_put_readme
(
dataset
,
edited_content
)
click
.
secho
(
"Updated readme "
,
nl
=
False
,
fg
=
"green"
)
else
:
click
.
secho
(
"Did not update readme "
,
nl
=
False
,
fg
=
"red"
)
click
.
secho
(
dataset_uri
)

Default editor updating of readme content .


def show(dataset_uri):
    """Show the descriptive metadata in the readme."""
    try:
        dataset = dtoolcore.ProtoDataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    except dtoolcore.DtoolCoreTypeError:
        dataset = dtoolcore.DataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    readme_content = dataset.get_readme_content()
    click.secho(readme_content)

def show ( dataset_uri ) : try : dataset = dtoolcore . ProtoDataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH ) except dtoolcore . DtoolCoreTypeError : dataset = dtoolcore . DataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH ) readme_content = dataset . get_readme_content ( ) click . secho ( readme_content )

Show the descriptive metadata in the readme .


def write(proto_dataset_uri, input):
    """Use YAML from a file or stdin to populate the readme.

    To stream content from stdin use "-", e.g.

    echo "desc: my data" | dtool readme write <DS_URI> -
    """
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri
    )
    _validate_and_put_readme(proto_dataset, input.read())

def write ( proto_dataset_uri , input ) : proto_dataset = dtoolcore . ProtoDataSet . from_uri ( uri = proto_dataset_uri ) _validate_and_put_readme ( proto_dataset , input . read ( ) )

Use YAML from a file or stdin to populate the readme .


def item(proto_dataset_uri, input_file, relpath_in_dataset):
    """Add a file to the proto dataset."""
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        proto_dataset_uri,
        config_path=CONFIG_PATH)
    if relpath_in_dataset == "":
        relpath_in_dataset = os.path.basename(input_file)
    proto_dataset.put_item(input_file, relpath_in_dataset)

def item ( proto_dataset_uri , input_file , relpath_in_dataset ) : proto_dataset = dtoolcore . ProtoDataSet . from_uri ( proto_dataset_uri , config_path = CONFIG_PATH ) if relpath_in_dataset == "" : relpath_in_dataset = os . path . basename ( input_file ) proto_dataset . put_item ( input_file , relpath_in_dataset )


def metadata(proto_dataset_uri, relpath_in_dataset, key, value):
    """Add metadata to a file in the proto dataset."""
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri,
        config_path=CONFIG_PATH)
    proto_dataset.add_item_metadata(
        handle=relpath_in_dataset,
        key=key,
        value=value)

def metadata ( proto_dataset_uri , relpath_in_dataset , key , value ) : proto_dataset = dtoolcore . ProtoDataSet . from_uri ( uri = proto_dataset_uri , config_path = CONFIG_PATH ) proto_dataset . add_item_metadata ( handle = relpath_in_dataset , key = key , value = value )

Add metadata to a file in the proto dataset .


def freeze(proto_dataset_uri):
    """Convert a proto dataset into a dataset.

    This step is carried out after all files have been added to the dataset.
    Freezing a dataset finalizes it with a stamp marking it as frozen.
    """
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri,
        config_path=CONFIG_PATH
    )

    num_items = len(list(proto_dataset._identifiers()))
    max_files_limit = int(dtoolcore.utils.get_config_value(
        "DTOOL_MAX_FILES_LIMIT",
        CONFIG_PATH,
        10000
    ))
    assert isinstance(max_files_limit, int)
    if num_items > max_files_limit:
        click.secho(
            "Too many items ({} > {}) in proto dataset".format(
                num_items,
                max_files_limit
            ),
            fg="red"
        )
        click.secho("1. Consider splitting the dataset into smaller datasets")
        click.secho("2. Consider packaging small files using tar")
        click.secho("3. Increase the limit using the DTOOL_MAX_FILES_LIMIT")
        click.secho("   environment variable")
        sys.exit(2)

    handles = [h for h in proto_dataset._storage_broker.iter_item_handles()]
    for h in handles:
        if not valid_handle(h):
            click.secho(
                "Invalid item name: {}".format(h),
                fg="red"
            )
            click.secho("1. Consider renaming the item")
            click.secho("2. Consider removing the item")
            sys.exit(3)

    with click.progressbar(length=len(list(proto_dataset._identifiers())),
                           label="Generating manifest") as progressbar:
        try:
            proto_dataset.freeze(progressbar=progressbar)
        except dtoolcore.storagebroker.DiskStorageBrokerValidationWarning as e:
            click.secho("")
            click.secho(str(e), fg="red", nl=False)
            sys.exit(4)

    click.secho("Dataset frozen ", nl=False, fg="green")
    click.secho(proto_dataset_uri)

Convert a proto dataset into a dataset .


def cp(resume, quiet, dataset_uri, dest_base_uri):
    """Copy a dataset to a different location."""
    _copy(resume, quiet, dataset_uri, dest_base_uri)

def cp ( resume , quiet , dataset_uri , dest_base_uri ) : _copy ( resume , quiet , dataset_uri , dest_base_uri )

Copy a dataset to a different location .


def compress(obj, level=6, return_type="bytes"):
    """Compress anything to bytes or string.

    :params obj:
    :params level:
    :params return_type: if bytes, then return bytes; if str, then return
      base64.b64encode bytes in utf-8 string.
    """
    if isinstance(obj, binary_type):
        b = zlib.compress(obj, level)
    elif isinstance(obj, string_types):
        b = zlib.compress(obj.encode("utf-8"), level)
    else:
        b = zlib.compress(pickle.dumps(obj, protocol=2), level)

    if return_type == "bytes":
        return b
    elif return_type == "str":
        return base64.b64encode(b).decode("utf-8")
    else:
        raise ValueError("'return_type' has to be one of 'bytes', 'str'!")


def find_probable_year_index(self, tokens):
        """
        attempt to deduce if a pre 100 year was lost
         due to padded zeros being taken off
        """
        for index, token in enumerate(self):
            potential_year_tokens = _ymd.find_potential_year_tokens(
                token, tokens)
            if len(potential_year_tokens) == 1 and len(potential_year_tokens[0]) > 2:
                return index

def find_probable_year_index ( self , tokens ) : for index , token in enumerate ( self ) : potential_year_tokens = _ymd . find_potential_year_tokens ( token , tokens ) if len ( potential_year_tokens ) == 1 and len ( potential_year_tokens [ 0 ] ) > 2 : return index


def tzname_in_python2(namefunc):
    """Change unicode output into bytestrings in Python 2

    tzname() API changed in Python 3. It used to return bytes, but was changed
    to unicode strings
    """
    def adjust_encoding(*args, **kwargs):
        name = namefunc(*args, **kwargs)
        if name is not None and not PY3:
            name = name.encode()

        return name

    return adjust_encoding

def tzname_in_python2 ( namefunc ) : def adjust_encoding ( * args , ** kwargs ) : name = namefunc ( * args , ** kwargs ) if name is not None and not PY3 : name = name . encode ( ) return name return adjust_encoding


def _validate_fromutc_inputs(f):
    """
    The CPython version of ``fromutc`` checks that the input is a ``datetime``
    object and that ``self`` is attached as its ``tzinfo``.
    """
    @wraps(f)
    def fromutc(self, dt):
        if not isinstance(dt, datetime):
            raise TypeError("fromutc() requires a datetime argument")
        if dt.tzinfo is not self:
            raise ValueError("dt.tzinfo is not self")

        return f(self, dt)

    return fromutc

The CPython version of fromutc checks that the input is a datetime object and that self is attached as its tzinfo .


def fromutc(self, dt):
        """ Given a datetime in UTC, return local time """
        if not isinstance(dt, datetime):
            raise TypeError("fromutc() requires a datetime argument")

        if dt.tzinfo is not self:
            raise ValueError("dt.tzinfo is not self")

        # Get transitions - if there are none, fixed offset
        transitions = self.transitions(dt.year)
        if transitions is None:
            return dt + self.utcoffset(dt)

        # Get the transition times in UTC
        dston, dstoff = transitions

        dston -= self._std_offset
        dstoff -= self._std_offset

        utc_transitions = (dston, dstoff)
        dt_utc = dt.replace(tzinfo=None)

        isdst = self._naive_isdst(dt_utc, utc_transitions)

        if isdst:
            dt_wall = dt + self._dst_offset
        else:
            dt_wall = dt + self._std_offset

        _fold = int(not isdst and self.is_ambiguous(dt_wall))

        return enfold(dt_wall, fold=_fold)

Given a datetime in UTC return local time


def strip_comment_line_with_symbol(line, start):
    """Strip comments from line string.
    """
    parts = line.split(start)
    counts = [len(findall(r'(?:^|[^"\\]|(?:\\\\|\\")+)(")', part))
              for part in parts]
    total = 0
    for nr, count in enumerate(counts):
        total += count
        if total % 2 == 0:
            return start.join(parts[:nr + 1]).rstrip()
    else:  # pragma: no cover
        return line.rstrip()

def strip_comment_line_with_symbol ( line , start ) : parts = line . split ( start ) counts = [ len ( findall ( r'(?:^|[^"\\]|(?:\\\\|\\")+)(")' , part ) ) for part in parts ] total = 0 for nr , count in enumerate ( counts ) : total += count if total % 2 == 0 : return start . join ( parts [ : nr + 1 ] ) . rstrip ( ) else : return line . rstrip ( )


def strip_comments(string, comment_symbols=frozenset(('#', '//'))):
    """Strip comments from json string.

    :param string: A string containing json with comments started by comment_symbols.
    :param comment_symbols: Iterable of symbols that start a line comment (default # or //).
    :return: The string with the comments removed.
    """
    lines = string.splitlines()
    for k in range(len(lines)):
        for symbol in comment_symbols:
            lines[k] = strip_comment_line_with_symbol(lines[k], start=symbol)
    return '\n'.join(lines)

def strip_comments ( string , comment_symbols = frozenset ( ( '#' , '//' ) ) ) : lines = string . splitlines ( ) for k in range ( len ( lines ) ) : for symbol in comment_symbols : lines [ k ] = strip_comment_line_with_symbol ( lines [ k ] , start = symbol ) return '\n' . join ( lines )


def picknthweekday(year, month, dayofweek, hour, minute, whichweek):
    """ dayofweek == 0 means Sunday, whichweek 5 means last instance """
    first = datetime.datetime(year, month, 1, hour, minute)

    # This will work if dayofweek is ISO weekday (1-7) or Microsoft-style (0-6),
    # Because 7 % 7 = 0
    weekdayone = first.replace(day=((dayofweek - first.isoweekday()) % 7) + 1)
    wd = weekdayone + ((whichweek - 1) * ONEWEEK)
    if (wd.month != month):
        wd -= ONEWEEK

    return wd

def picknthweekday ( year , month , dayofweek , hour , minute , whichweek ) : first = datetime . datetime ( year , month , 1 , hour , minute ) weekdayone = first . replace ( day = ( ( dayofweek - first . isoweekday ( ) ) % 7 ) + 1 ) wd = weekdayone + ( ( whichweek - 1 ) * ONEWEEK ) if ( wd . month != month ) : wd -= ONEWEEK return wd

dayofweek == 0 means Sunday whichweek 5 means last instance


def valuestodict(key):
    """Convert a registry key's values to a dictionary."""
    dout = {}
    size = winreg.QueryInfoKey(key)[1]
    tz_res = None

    for i in range(size):
        key_name, value, dtype = winreg.EnumValue(key, i)
        if dtype == winreg.REG_DWORD or dtype == winreg.REG_DWORD_LITTLE_ENDIAN:
            # If it's a DWORD (32-bit integer), it's stored as unsigned - convert
            # that to a proper signed integer
            if value & (1 << 31):
                value = value - (1 << 32)
        elif dtype == winreg.REG_SZ:
            # If it's a reference to the tzres DLL, load the actual string
            if value.startswith('@tzres'):
                tz_res = tz_res or tzres()
                value = tz_res.name_from_string(value)

            value = value.rstrip('\x00')    # Remove trailing nulls

        dout[key_name] = value

    return dout

def valuestodict ( key ) : dout = { } size = winreg . QueryInfoKey ( key ) [ 1 ] tz_res = None for i in range ( size ) : key_name , value , dtype = winreg . EnumValue ( key , i ) if dtype == winreg . REG_DWORD or dtype == winreg . REG_DWORD_LITTLE_ENDIAN : if value & ( 1 << 31 ) : value = value - ( 1 << 32 ) elif dtype == winreg . REG_SZ : if value . startswith ( '@tzres' ) : tz_res = tz_res or tzres ( ) value = tz_res . name_from_string ( value ) value = value . rstrip ( '\x00' ) dout [ key_name ] = value return dout

Convert a registry key s values to a dictionary .


def name_from_string(self, tzname_str):
        """
        Parse strings as returned from the Windows registry into the time zone
        name as defined in the registry.

        >>> from dateutil.tzwin import tzres
        >>> tzr = tzres()
        >>> print(tzr.name_from_string('@tzres.dll,-251'))
        'Dateline Daylight Time'
        >>> print(tzr.name_from_string('Eastern Standard Time'))
        'Eastern Standard Time'

        :param tzname_str:
            A timezone name string as returned from a Windows registry key.

        :return:
            Returns the localized timezone string from tzres.dll if the string
            is of the form `@tzres.dll,-offset`, else returns the input string.
        """
        if not tzname_str.startswith('@'):
            return tzname_str

        name_splt = tzname_str.split(',-')
        try:
            offset = int(name_splt[1])
        except:
            raise ValueError("Malformed timezone string.")

        return self.load_name(offset)

def
name_from_string
(
self
,
tzname_str
)
:
if
not
tzname_str
.
startswith
(
'@'
)
:
return
tzname_str
name_splt
=
tzname_str
.
split
(
',-'
)
try
:
offset
=
int
(
name_splt
[
1
]
)
except
:
raise
ValueError
(
"Malformed timezone string."
)
return
self
.
load_name
(
offset
)

Parse strings as returned from the Windows registry into the time zone name as defined in the registry .


def gettz(name):
    """
    This retrieves a time zone from the local zoneinfo tarball that is packaged
    with dateutil.

    :param name:
        An IANA-style time zone name, as found in the zoneinfo file.

    :return:
        Returns a :class:`dateutil.tz.tzfile` time zone object.

    .. warning::
        It is generally inadvisable to use this function, and it is only
        provided for API compatibility with earlier versions. This is *not*
        equivalent to ``dateutil.tz.gettz()``, which selects an appropriate
        time zone based on the inputs, favoring system zoneinfo. This is ONLY
        for accessing the dateutil-specific zoneinfo (which may be out of
        date compared to the system zoneinfo).

    .. deprecated:: 2.6
        If you need to use a specific zoneinfofile over the system zoneinfo,
        instantiate a :class:`dateutil.zoneinfo.ZoneInfoFile` object and call
        :func:`dateutil.zoneinfo.ZoneInfoFile.get(name)` instead.

        Use :func:`get_zonefile_instance` to retrieve an instance of the
        dateutil-provided zoneinfo.
    """
    warnings.warn("zoneinfo.gettz() will be removed in future versions, "
                  "to use the dateutil-provided zoneinfo files, instantiate a "
                  "ZoneInfoFile object and use ZoneInfoFile.zones.get() "
                  "instead. See the documentation for details.",
                  DeprecationWarning)

    if len(_CLASS_ZONE_INSTANCE) == 0:
        _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
    return _CLASS_ZONE_INSTANCE[0].zones.get(name)

This retrieves a time zone from the local zoneinfo tarball that is packaged with dateutil .


def gettz_db_metadata():
    """ Get the zonefile metadata

    See `zonefile_metadata`_

    :returns:
        A dictionary with the database metadata

    .. deprecated:: 2.6
        See deprecation warning in :func:`zoneinfo.gettz`. To get metadata,
        query the attribute ``zoneinfo.ZoneInfoFile.metadata``.
    """
    warnings.warn("zoneinfo.gettz_db_metadata() will be removed in future "
                  "versions, to use the dateutil-provided zoneinfo files, "
                  "ZoneInfoFile object and query the 'metadata' attribute "
                  "instead. See the documentation for details.",
                  DeprecationWarning)

    if len(_CLASS_ZONE_INSTANCE) == 0:
        _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
    return _CLASS_ZONE_INSTANCE[0].metadata


def get_config(jid):
    """Get the configuration for the given JID based on XMPP_HTTP_UPLOAD_ACCESS.

    If the JID does not match any rule, ``False`` is returned.
    """

    acls = getattr(settings, 'XMPP_HTTP_UPLOAD_ACCESS', (('.*', False), ))

    for regex, config in acls:
        if isinstance(regex, six.string_types):
            regex = [regex]

        for subex in regex:
            if re.search(subex, jid):
                return config

    return False

def get_config ( jid ) : acls = getattr ( settings , 'XMPP_HTTP_UPLOAD_ACCESS' , ( ( '.*' , False ) , ) ) for regex , config in acls : if isinstance ( regex , six . string_types ) : regex = [ regex ] for subex in regex : if re . search ( subex , jid ) : return config return False

Get the configuration for the given JID based on XMPP_HTTP_UPLOAD_ACCESS .


def datetime_exists(dt, tz=None):
    """
    Given a datetime and a time zone, determine whether or not a given datetime
    would fall in a gap.

    :param dt:
        A :class:`datetime.datetime` (whose time zone will be ignored if ``tz``
        is provided.)

    :param tz:
        A :class:`datetime.tzinfo` with support for the ``fold`` attribute. If
        ``None`` or not provided, the datetime's own time zone will be used.

    :return:
        Returns a boolean value whether or not the "wall time" exists in ``tz``.
    """
    if tz is None:
        if dt.tzinfo is None:
            raise ValueError('Datetime is naive and no time zone provided.')
        tz = dt.tzinfo

    dt = dt.replace(tzinfo=None)

    # This is essentially a test of whether or not the datetime can survive
    # a round trip to UTC.
    dt_rt = dt.replace(tzinfo=tz).astimezone(tzutc()).astimezone(tz)
    dt_rt = dt_rt.replace(tzinfo=None)

    return dt == dt_rt

Given a datetime and a time zone determine whether or not a given datetime would fall in a gap .


def _set_tzdata(self, tzobj):
        """ Set the time zone data of this object from a _tzfile object """
        # Copy the relevant attributes over as private attributes
        for attr in _tzfile.attrs:
            setattr(self, '_' + attr, getattr(tzobj, attr))

def _set_tzdata ( self , tzobj ) : for attr in _tzfile . attrs : setattr ( self , '_' + attr , getattr ( tzobj , attr ) )


def normalized(self):
        """
        Return a version of this object represented entirely using integer
        values for the relative attributes.

        >>> relativedelta(days=1.5, hours=2).normalized()
        relativedelta(days=1, hours=14)

        :return:
            Returns a :class:`dateutil.relativedelta.relativedelta` object.
        """
        # Cascade remainders down (rounding each to roughly nearest
        # microsecond)
        days = int(self.days)

        hours_f = round(self.hours + 24 * (self.days - days), 11)
        hours = int(hours_f)

        minutes_f = round(self.minutes + 60 * (hours_f - hours), 10)
        minutes = int(minutes_f)

        seconds_f = round(self.seconds + 60 * (minutes_f - minutes), 8)
        seconds = int(seconds_f)

        microseconds = round(self.microseconds + 1e6 * (seconds_f - seconds))

        # Constructor carries overflow back up with call to _fix()
        return self.__class__(years=self.years, months=self.months,
                              days=days, hours=hours, minutes=minutes,
                              seconds=seconds, microseconds=microseconds,
                              leapdays=self.leapdays, year=self.year,
                              month=self.month, day=self.day,
                              weekday=self.weekday, hour=self.hour,
                              minute=self.minute, second=self.second,
                              microsecond=self.microsecond)

def normalized ( self ) : days = int ( self . days ) hours_f = round ( self . hours + 24 * ( self . days - days ) , 11 ) hours = int ( hours_f ) minutes_f = round ( self . minutes + 60 * ( hours_f - hours ) , 10 ) minutes = int ( minutes_f ) seconds_f = round ( self . seconds + 60 * ( minutes_f - minutes ) , 8 ) seconds = int ( seconds_f ) microseconds = round ( self . microseconds + 1e6 * ( seconds_f - seconds ) ) return self . __class__ ( years = self . years , months = self . months , days = days , hours = hours , minutes = minutes , seconds = seconds , microseconds = microseconds , leapdays = self . leapdays , year = self . year , month = self . month , day = self . day , weekday = self . weekday , hour = self . hour , minute = self . minute , second = self . second , microsecond = self . microsecond )

Return a version of this object represented entirely using integer values for the relative attributes .


def _hash(secret: bytes, data: bytes, alg: str) -> bytes:
    """
    Create a new HMAC hash.

    :param secret: The secret used when hashing data.
    :type secret: bytes
    :param data: The data to hash.
    :type data: bytes
    :param alg: The algorithm to use when hashing `data`.
    :type alg: str
    :return: New HMAC hash.
    :rtype: bytes
    """
    algorithm = get_algorithm(alg)
    return hmac \
        .new(secret, msg=data, digestmod=algorithm) \
        .digest()

def _hash ( secret : bytes , data : bytes , alg : str ) -> bytes : algorithm = get_algorithm ( alg ) return hmac . new ( secret , msg = data , digestmod = algorithm ) . digest ( )


def decode(secret: Union[str, bytes], token: Union[str, bytes],
           alg: str = default_alg) -> Tuple[dict, dict]:
    """
    Decodes the given token's header and payload and validates the signature.

    :param secret: The secret used to decode the token. Must match the
        secret used when creating the token.
    :type secret: Union[str, bytes]
    :param token: The token to decode.
    :type token: Union[str, bytes]
    :param alg: The algorithm used to decode the token. Must match the
        algorithm used when creating the token.
    :type alg: str
    :return: The decoded header and payload.
    :rtype: Tuple[dict, dict]
    """
    secret = util.to_bytes(secret)
    token = util.to_bytes(token)
    pre_signature, signature_segment = token.rsplit(b'.', 1)
    header_b64, payload_b64 = pre_signature.split(b'.')
    try:
        header_json = util.b64_decode(header_b64)
        header = json.loads(util.from_bytes(header_json))
    except (json.decoder.JSONDecodeError, UnicodeDecodeError, ValueError):
        raise InvalidHeaderError('Invalid header')
    try:
        payload_json = util.b64_decode(payload_b64)
        payload = json.loads(util.from_bytes(payload_json))
    except (json.decoder.JSONDecodeError, UnicodeDecodeError, ValueError):
        raise InvalidPayloadError('Invalid payload')

    if not isinstance(header, dict):
        raise InvalidHeaderError('Invalid header: {}'.format(header))
    if not isinstance(payload, dict):
        raise InvalidPayloadError('Invalid payload: {}'.format(payload))

    signature = util.b64_decode(signature_segment)
    calculated_signature = _hash(secret, pre_signature, alg)

    if not compare_signature(signature, calculated_signature):
        raise InvalidSignatureError('Invalid signature')
    return header, payload

def
decode
(
secret
:
Union
[
str
,
bytes
]
,
token
:
Union
[
str
,
bytes
]
,
alg
:
str
=
default_alg
)
->
Tuple
[
dict
,
dict
]
:
secret
=
util
.
to_bytes
(
secret
)
token
=
util
.
to_bytes
(
token
)
pre_signature
,
signature_segment
=
token
.
rsplit
(
b'.'
,
1
)
header_b64
,
payload_b64
=
pre_signature
.
split
(
b'.'
)
try
:
header_json
=
util
.
b64_decode
(
header_b64
)
header
=
json
.
loads
(
util
.
from_bytes
(
header_json
)
)
except
(
json
.
decoder
.
JSONDecodeError
,
UnicodeDecodeError
,
ValueError
)
:
raise
InvalidHeaderError
(
'Invalid header'
)
try
:
payload_json
=
util
.
b64_decode
(
payload_b64
)
payload
=
json
.
loads
(
util
.
from_bytes
(
payload_json
)
)
except
(
json
.
decoder
.
JSONDecodeError
,
UnicodeDecodeError
,
ValueError
)
:
raise
InvalidPayloadError
(
'Invalid payload'
)
if
not
isinstance
(
header
,
dict
)
:
raise
InvalidHeaderError
(
'Invalid header: {}'
.
format
(
header
)
)
if
not
isinstance
(
payload
,
dict
)
:
raise
InvalidPayloadError
(
'Invalid payload: {}'
.
format
(
payload
)
)
signature
=
util
.
b64_decode
(
signature_segment
)
calculated_signature
=
_hash
(
secret
,
pre_signature
,
alg
)
if
not
compare_signature
(
signature
,
calculated_signature
)
:
raise
InvalidSignatureError
(
'Invalid signature'
)
return
header
,
payload

Decodes the given token s header and payload and validates the signature .


def compare_signature(expected: Union[str, bytes],
                      actual: Union[str, bytes]) -> bool:
    """
    Compares the given signatures.

    :param expected: The expected signature.
    :type expected: Union[str, bytes]
    :param actual: The actual signature.
    :type actual: Union[str, bytes]
    :return: Do the signatures match?
    :rtype: bool
    """
    expected = util.to_bytes(expected)
    actual = util.to_bytes(actual)
    return hmac.compare_digest(expected, actual)

def compare_signature ( expected : Union [ str , bytes ] , actual : Union [ str , bytes ] ) -> bool : expected = util . to_bytes ( expected ) actual = util . to_bytes ( actual ) return hmac . compare_digest ( expected , actual )


def compare_token(expected: Union[str, bytes],
                  actual: Union[str, bytes]) -> bool:
    """
    Compares the given tokens.

    :param expected: The expected token.
    :type expected: Union[str, bytes]
    :param actual: The actual token.
    :type actual: Union[str, bytes]
    :return: Do the tokens match?
    :rtype: bool
    """
    expected = util.to_bytes(expected)
    actual = util.to_bytes(actual)
    _, expected_sig_seg = expected.rsplit(b'.', 1)
    _, actual_sig_seg = actual.rsplit(b'.', 1)
    expected_sig = util.b64_decode(expected_sig_seg)
    actual_sig = util.b64_decode(actual_sig_seg)
    return compare_signature(expected_sig, actual_sig)

def compare_token ( expected : Union [ str , bytes ] , actual : Union [ str , bytes ] ) -> bool : expected = util . to_bytes ( expected ) actual = util . to_bytes ( actual ) _ , expected_sig_seg = expected . rsplit ( b'.' , 1 ) _ , actual_sig_seg = actual . rsplit ( b'.' , 1 ) expected_sig = util . b64_decode ( expected_sig_seg ) actual_sig = util . b64_decode ( actual_sig_seg ) return compare_signature ( expected_sig , actual_sig )


def valid(self, time: int = None) -> bool:
        """
        Is the token valid? This method only checks the timestamps within the
        token and compares them against the current time if none is provided.

        :param time: The timestamp to validate against
        :type time: Union[int, None]
        :return: The validity of the token.
        :rtype: bool
        """
        if time is None:
            epoch = datetime(1970, 1, 1, 0, 0, 0)
            now = datetime.utcnow()
            time = int((now - epoch).total_seconds())
        if isinstance(self.valid_from, int) and time < self.valid_from:
            return False
        if isinstance(self.valid_to, int) and time > self.valid_to:
            return False
        return True

def valid ( self , time : int = None ) -> bool : if time is None : epoch = datetime ( 1970 , 1 , 1 , 0 , 0 , 0 ) now = datetime . utcnow ( ) time = int ( ( now - epoch ) . total_seconds ( ) ) if isinstance ( self . valid_from , int ) and time < self . valid_from : return False if isinstance ( self . valid_to , int ) and time > self . valid_to : return False return True

Is the token valid? This method only checks the timestamps within the token and compares them against the current time if none is provided .


def _pop_claims_from_payload(self):
        """
        Check for registered claims in the payload and move them to the
        registered_claims property, overwriting any extant claims.
        """
        claims_in_payload = [k for k in self.payload.keys() if
                             k in registered_claims.values()]
        for name in claims_in_payload:
            self.registered_claims[name] = self.payload.pop(name)

def _pop_claims_from_payload ( self ) : claims_in_payload = [ k for k in self . payload . keys ( ) if k in registered_claims . values ( ) ] for name in claims_in_payload : self . registered_claims [ name ] = self . payload . pop ( name )

Check for registered claims in the payload and move them to the registered_claims property overwriting any extant claims .


def encode(self) -> str:
        """
        Create a token based on the data held in the class.

        :return: A new token
        :rtype: str
        """
        payload = {}
        payload.update(self.registered_claims)
        payload.update(self.payload)
        return encode(self.secret, payload, self.alg, self.header)

def encode ( self ) -> str : payload = { } payload . update ( self . registered_claims ) payload . update ( self . payload ) return encode ( self . secret , payload , self . alg , self . header )

Create a token based on the data held in the class .


def decode(secret: Union[str, bytes], token: Union[str, bytes],
               alg: str = default_alg) -> 'Jwt':
        """
        Decodes the given token into an instance of `Jwt`.

        :param secret: The secret used to decode the token. Must match the
            secret used when creating the token.
        :type secret: Union[str, bytes]
        :param token: The token to decode.
        :type token: Union[str, bytes]
        :param alg: The algorithm used to decode the token. Must match the
            algorithm used when creating the token.
        :type alg: str
        :return: The decoded token.
        :rtype: `Jwt`
        """
        header, payload = decode(secret, token, alg)
        return Jwt(secret, payload, alg, header)

def decode ( secret : Union [ str , bytes ] , token : Union [ str , bytes ] , alg : str = default_alg ) -> 'Jwt' : header , payload = decode ( secret , token , alg ) return Jwt ( secret , payload , alg , header )

Decodes the given token into an instance of Jwt .


def compare(self, jwt: 'Jwt', compare_dates: bool = False) -> bool:
        """
        Compare against another `Jwt`.

        :param jwt: The token to compare against.
        :type jwt: Jwt
        :param compare_dates: Should the comparision take dates into account?
        :type compare_dates: bool
        :return: Are the two Jwt's the same?
        :rtype: bool
        """
        if self.secret != jwt.secret:
            return False
        if self.payload != jwt.payload:
            return False
        if self.alg != jwt.alg:
            return False
        if self.header != jwt.header:
            return False
        expected_claims = self.registered_claims
        actual_claims = jwt.registered_claims
        if not compare_dates:
            strip = ['exp', 'nbf', 'iat']
            expected_claims = {k: {v if k not in strip else None} for k, v in
                               expected_claims.items()}
            actual_claims = {k: {v if k not in strip else None} for k, v in
                             actual_claims.items()}
        if expected_claims != actual_claims:
            return False
        return True

def compare ( self , jwt : 'Jwt' , compare_dates : bool = False ) -> bool : if self . secret != jwt . secret : return False if self . payload != jwt . payload : return False if self . alg != jwt . alg : return False if self . header != jwt . header : return False expected_claims = self . registered_claims actual_claims = jwt . registered_claims if not compare_dates : strip = [ 'exp' , 'nbf' , 'iat' ] expected_claims = { k : { v if k not in strip else None } for k , v in expected_claims . items ( ) } actual_claims = { k : { v if k not in strip else None } for k , v in actual_claims . items ( ) } if expected_claims != actual_claims : return False return True


def get(self, request, hash, filename):
        """Download a file."""
        if _ws_download is True:
            return HttpResponseForbidden()
        upload = Upload.objects.uploaded().get(hash=hash, name=filename)

        return FileResponse(upload.file, content_type=upload.type)

def get ( self , request , hash , filename ) : if _ws_download is True : return HttpResponseForbidden ( ) upload = Upload . objects . uploaded ( ) . get ( hash = hash , name = filename ) return FileResponse ( upload . file , content_type = upload . type )


def is_compressed_json_file(abspath):
    """Test a file is a valid json file.

    - *.json: uncompressed, utf-8 encode json file
    - *.js: uncompressed, utf-8 encode json file
    - *.gz: compressed, utf-8 encode json file
    """
    abspath = abspath.lower()
    fname, ext = os.path.splitext(abspath)
    if ext in [".json", ".js"]:
        is_compressed = False
    elif ext == ".gz":
        is_compressed = True
    else:
        raise ValueError(
            "'%s' is not a valid json file. "
            "extension has to be '.json' or '.js' for uncompressed, '.gz' "
            "for compressed." % abspath)
    return is_compressed


def dump_set(self, obj, class_name=set_class_name):
        """
        ``set`` dumper.
        """
        return {"$" + class_name: [self._json_convert(item) for item in obj]}

def dump_set ( self , obj , class_name = set_class_name ) : return { "$" + class_name : [ self . _json_convert ( item ) for item in obj ] }


def dump_deque(self, obj, class_name="collections.deque"):
        """
        ``collections.deque`` dumper.
        """
        return {"$" + class_name: [self._json_convert(item) for item in obj]}

def dump_deque ( self , obj , class_name = "collections.deque" ) : return { "$" + class_name : [ self . _json_convert ( item ) for item in obj ] }


def dump_OrderedDict(self, obj, class_name="collections.OrderedDict"):
        """
        ``collections.OrderedDict`` dumper.
        """
        return {
            "$" + class_name: [
                (key, self._json_convert(value)) for key, value in iteritems(obj)
            ]
        }

def dump_OrderedDict ( self , obj , class_name = "collections.OrderedDict" ) : return { "$" + class_name : [ ( key , self . _json_convert ( value ) ) for key , value in iteritems ( obj ) ] }


def dump_nparray(self, obj, class_name=numpy_ndarray_class_name):
        """
        ``numpy.ndarray`` dumper.
        """
        return {"$" + class_name: self._json_convert(obj.tolist())}

def dump_nparray ( self , obj , class_name = numpy_ndarray_class_name ) : return { "$" + class_name : self . _json_convert ( obj . tolist ( ) ) }


def _invalidates_cache(f):
    """
    Decorator for rruleset methods which may invalidate the
    cached length.
    """

    def inner_func(self, *args, **kwargs):
        rv = f(self, *args, **kwargs)
        self._invalidate_cache()
        return rv

    return inner_func

def _invalidates_cache ( f ) : def inner_func ( self , * args , ** kwargs ) : rv = f ( self , * args , ** kwargs ) self . _invalidate_cache ( ) return rv return inner_func

Decorator for rruleset methods which may invalidate the cached length .


def before(self, dt, inc=False):
        """ Returns the last recurrence before the given datetime instance. The
            inc keyword defines what happens if dt is an occurrence. With
            inc=True, if dt itself is an occurrence, it will be returned. """
        if self._cache_complete:
            gen = self._cache
        else:
            gen = self
        last = None
        if inc:
            for i in gen:
                if i > dt:
                    break
                last = i
        else:
            for i in gen:
                if i >= dt:
                    break
                last = i
        return last

def before ( self , dt , inc = False ) : if self . _cache_complete : gen = self . _cache else : gen = self last = None if inc : for i in gen : if i > dt : break last = i else : for i in gen : if i >= dt : break last = i return last

Returns the last recurrence before the given datetime instance . The inc keyword defines what happens if dt is an occurrence . With inc = True if dt itself is an occurrence it will be returned .


def after(self, dt, inc=False):
        """ Returns the first recurrence after the given datetime instance. The
            inc keyword defines what happens if dt is an occurrence. With
            inc=True, if dt itself is an occurrence, it will be returned.  """
        if self._cache_complete:
            gen = self._cache
        else:
            gen = self
        if inc:
            for i in gen:
                if i >= dt:
                    return i
        else:
            for i in gen:
                if i > dt:
                    return i
        return None

def after ( self , dt , inc = False ) : if self . _cache_complete : gen = self . _cache else : gen = self if inc : for i in gen : if i >= dt : return i else : for i in gen : if i > dt : return i return None

Returns the first recurrence after the given datetime instance . The inc keyword defines what happens if dt is an occurrence . With inc = True if dt itself is an occurrence it will be returned .


def xafter(self, dt, count=None, inc=False):
        """
        Generator which yields up to `count` recurrences after the given
        datetime instance, equivalent to `after`.

        :param dt:
            The datetime at which to start generating recurrences.

        :param count:
            The maximum number of recurrences to generate. If `None` (default),
            dates are generated until the recurrence rule is exhausted.

        :param inc:
            If `dt` is an instance of the rule and `inc` is `True`, it is
            included in the output.

        :yields: Yields a sequence of `datetime` objects.
        """

        if self._cache_complete:
            gen = self._cache
        else:
            gen = self

        # Select the comparison function
        if inc:
            def comp(dc, dtc): return dc >= dtc
        else:
            def comp(dc, dtc): return dc > dtc

        # Generate dates
        n = 0
        for d in gen:
            if comp(d, dt):
                if count is not None:
                    n += 1
                    if n > count:
                        break

                yield d

def xafter ( self , dt , count = None , inc = False ) : if self . _cache_complete : gen = self . _cache else : gen = self if inc : def comp ( dc , dtc ) : return dc >= dtc else : def comp ( dc , dtc ) : return dc > dtc n = 0 for d in gen : if comp ( d , dt ) : if count is not None : n += 1 if n > count : break yield d

Generator which yields up to count recurrences after the given datetime instance equivalent to after .


def replace(self, **kwargs):
        """Return new rrule with same attributes except for those attributes given new
           values by whichever keyword arguments are specified."""
        new_kwargs = {"interval": self._interval,
                      "count": self._count,
                      "dtstart": self._dtstart,
                      "freq": self._freq,
                      "until": self._until,
                      "wkst": self._wkst,
                      "cache": False if self._cache is None else True}
        new_kwargs.update(self._original_rule)
        new_kwargs.update(kwargs)
        return rrule(**new_kwargs)

def replace ( self , ** kwargs ) : new_kwargs = { "interval" : self . _interval , "count" : self . _count , "dtstart" : self . _dtstart , "freq" : self . _freq , "until" : self . _until , "wkst" : self . _wkst , "cache" : False if self . _cache is None else True } new_kwargs . update ( self . _original_rule ) new_kwargs . update ( kwargs ) return rrule ( ** new_kwargs )

Return new rrule with same attributes except for those attributes given new values by whichever keyword arguments are specified .


def run_excel_to_html():
    """
    Run the excel_to_html function from the
    command-line.

    Args:
        -p path to file
        -s name of the sheet to convert
        -css classes to apply
        -m attempt to combine merged cells
        -c caption for accessibility
        -su summary for accessibility
        -d details for accessibility

    Example use:

        excel_to_html -p myfile.xlsx -s SheetName -css diablo-python -m true
    """
    # Capture commandline arguments. prog='' argument must
    # match the command name in setup.py entry_points
    parser = argparse.ArgumentParser(prog='excel_to_html')
    parser.add_argument('-p', nargs='?', help='Path to an excel file for conversion.')
    parser.add_argument(
        '-s',
        nargs='?',
        help='The name of a sheet in our excel file. Defaults to "Sheet1".',
    )
    parser.add_argument(
        '-css', nargs='?', help='Space separated css classes to append to the table.'
    )
    parser.add_argument(
        '-m', action='store_true', help='Merge, attempt to combine merged cells.'
    )
    parser.add_argument(
        '-c', nargs='?', help='Caption for creating an accessible table.'
    )
    parser.add_argument(
        '-d',
        nargs='?',
        help='Two strings separated by a | character. The first string \
        is for the html "summary" attribute and the second string is for the html "details" attribute. \
        both values must be provided and nothing more.',
    )
    parser.add_argument(
        '-r', action='store_true', help='Row headers. Does the table have row headers?'
    )

    args = parser.parse_args()
    inputs = {
        'p': args.p,
        's': args.s,
        'css': args.css,
        'm': args.m,
        'c': args.c,
        'd': args.d,
        'r': args.r,
    }

    p = inputs['p']
    s = inputs['s'] if inputs['s'] else 'Sheet1'
    css = inputs['css'] if inputs['css'] else ''
    m = inputs['m'] if inputs['m'] else False
    c = inputs['c'] if inputs['c'] else ''
    d = inputs['d'].split('|') if inputs['d'] else []
    r = inputs['r'] if inputs['r'] else False

    html = fp.excel_to_html(
        p, sheetname=s, css_classes=css, caption=c, details=d, row_headers=r, merge=m
    )

    print(html)

Run the excel_to_html function from the command - line .


def get_inner_template(self, language, template_type, indentation, key, val):
        """
        Gets the requested template for the given language.

        Args:
            language: string, the language of the template to look for.

            template_type: string, 'iterable' or 'singular'.
            An iterable template is needed when the value is an iterable
            and needs more unpacking, e.g. list, tuple. A singular template
            is needed when unpacking is complete and the value is singular,
            e.g. string, int, float.

            indentation: int, the indentation level.

            key: multiple types, the array key.

            val: multiple types, the array values

        Returns:
            string, template formatting for arrays by language.
        """
        #Language specific inner templates
        inner_templates = {'php' : {
                                'iterable' : '%s%s => array \n%s( \n%s%s),\n' % (indentation, key, indentation, val, indentation),
                                'singular' : '%s%s => %s, \n' % (indentation, key, val) },
                           'javascript' : {
                                'iterable' : '%s%s : {\n%s\n%s},\n' % (indentation, key, val, indentation),
                                'singular' : '%s%s: %s,\n' % (indentation, key, val)},
                           'ocaml' : {
                                'iterable' : '%s[| (%s, (\n%s\n%s))|] ;;\n' % (indentation, key, val, indentation),
                                'singular' : '%s(%s, %s);\n' % (indentation, key, val)}}

        return inner_templates[language][template_type]

def
get_inner_template
(
self
,
language
,
template_type
,
indentation
,
key
,
val
)
:
inner_templates
=
{
'php'
:
{
'iterable'
:
'%s%s => array \n%s( \n%s%s),\n'
%
(
indentation
,
key
,
indentation
,
val
,
indentation
)
,
'singular'
:
'%s%s => %s, \n'
%
(
indentation
,
key
,
val
)
}
,
'javascript'
:
{
'iterable'
:
'%s%s : {\n%s\n%s},\n'
%
(
indentation
,
key
,
val
,
indentation
)
,
'singular'
:
'%s%s: %s,\n'
%
(
indentation
,
key
,
val
)
}
,
'ocaml'
:
{
'iterable'
:
'%s[| (%s, (\n%s\n%s))|] ;;\n'
%
(
indentation
,
key
,
val
,
indentation
)
,
'singular'
:
'%s(%s, %s);\n'
%
(
indentation
,
key
,
val
)
}
}
return
inner_templates
[
language
]
[
template_type
]

Gets the requested template for the given language .


def translate_array(self, string, language, level=3, retdata=False):
        """Unserializes a serialized php array and prints it to
        the console as a data structure in the specified language.
        Used to translate or convert a php array into a data structure
        in another language. Currently supports, PHP, Python, Javascript,
        and JSON.

        Args:
            string: a string of serialized php

            language: a string representing the desired output
            format for the array.

            level: integer, indentation level in spaces.
            Defaults to 3.

            retdata: boolean, the method will return the string
            in addition to printing it if set to True. Defaults
            to false.

        Returns:
            None but prints a string to the console if retdata is
            False, otherwise returns a string.
            """
        language = language.lower()
        assert self.is_built_in(language) or language in self.outer_templates, \
            "Sorry, " + language + " is not a supported language."

        # Serialized data converted to a python data structure (list of tuples)
        data = phpserialize.loads(bytes(string, 'utf-8'), array_hook=list, decode_strings=True)

        # If language conversion is supported by python avoid recursion entirely
        # and use a built in library
        if self.is_built_in(language):
            self.get_built_in(language, level, data)
            print(self)
            return self.data_structure if retdata else None

        # The language is not supported. Use recursion to build a data structure.
        def loop_print(iterable, level=3):
            """
            Loops over a python representation of a php array
            (list of tuples) and constructs a representation in another language.
            Translates a php array into another structure.

            Args:
                iterable: list or tuple to unpack.

                level: integer, number of spaces to use for indentation
            """
            retval = ''
            indentation = ' ' * level

            # Base case - variable is not an iterable
            if not self.is_iterable(iterable) or isinstance(iterable, str):
                non_iterable = str(iterable)
                return str(non_iterable)

            # Recursive case
            for item in iterable:
                # If item is a tuple it should be a key, value pair
                if isinstance(item, tuple) and len(item) == 2:
                    # Get the key value pair
                    key = item[0]
                    val = loop_print(item[1], level=level+3)

                    # Translate special values
                    val = self.translate_val(language, val) if language in self.lang_specific_values \
                          and val in self.lang_specific_values[language] else val

                    # Convert keys to their properly formatted strings
                    # Integers are not quoted as array keys
                    key = str(key) if isinstance(key, int) else '\'' + str(key) + '\''

                    # The first item is a key and the second item is an iterable, boolean
                    needs_unpacking = hasattr(item[0],'__iter__') == False \
                                      and hasattr(item[1],'__iter__') == True

                    # The second item is an iterable
                    if needs_unpacking:
                        retval += self.get_inner_template(language, 'iterable', indentation, key, val)
                    # The second item is not an iterable
                    else:
                        # Convert values to their properly formatted strings
                        # Integers and booleans are not quoted as array values
                        val = str(val) if val.isdigit() or val in self.lang_specific_values[language].values() else '\'' + str(val) + '\''

                        retval += self.get_inner_template(language, 'singular', indentation, key, val)

            return retval

        # Execute the recursive call in language specific wrapper template
        self.data_structure = self.outer_templates[language] % (loop_print(data))
        print(self)
        return self.data_structure if retdata else None

def
translate_array
(
self
,
string
,
language
,
level
=
3
,
retdata
=
False
)
:
language
=
language
.
lower
(
)
assert
self
.
is_built_in
(
language
)
or
language
in
self
.
outer_templates
,
"Sorry, "
+
language
+
" is not a supported language."
data
=
phpserialize
.
loads
(
bytes
(
string
,
'utf-8'
)
,
array_hook
=
list
,
decode_strings
=
True
)
if
self
.
is_built_in
(
language
)
:
self
.
get_built_in
(
language
,
level
,
data
)
print
(
self
)
return
self
.
data_structure
if
retdata
else
def
loop_print
(
iterable
,
level
=
3
)
:
retval
=
''
indentation
=
' '
*
level
if
not
self
.
is_iterable
(
iterable
)
or
isinstance
(
iterable
,
str
)
:
non_iterable
=
str
(
iterable
)
return
str
(
non_iterable
)
for
item
in
iterable
:
if
isinstance
(
item
,
tuple
)
and
len
(
item
)
==
2
:
key
=
item
[
0
]
val
=
loop_print
(
item
[
1
]
,
level
=
level
+
3
)
val
=
self
.
translate_val
(
language
,
val
)
if
language
in
self
.
lang_specific_values
and
val
in
self
.
lang_specific_values
[
language
]
else
val
key
=
str
(
key
)
if
isinstance
(
key
,
int
)
else
'\''
+
str
(
key
)
+
'\''
needs_unpacking
=
hasattr
(
item
[
0
]
,
'__iter__'
)
==
False
and
hasattr
(
item
[
1
]
,
'__iter__'
)
==
True
if
needs_unpacking
:
retval
+=
self
.
get_inner_template
(
language
,
'iterable'
,
indentation
,
key
,
val
)
else
:
val
=
str
(
val
)
if
val
.
isdigit
(
)
or
val
in
self
.
lang_specific_values
[
language
]
.
values
(
)
else
'\''
+
str
(
val
)
+
'\''
retval
+=
self
.
get_inner_template
(
language
,
'singular'
,
indentation
,
key
,
val
)
return
retval
self
.
data_structure
=
self
.
outer_templates
[
language
]
%
(
loop_print
(
data
)
)
print
(
self
)
return
self
.
data_structure
if
retdata
else

Unserializes a serialized php array and prints it to the console as a data structure in the specified language . Used to translate or convert a php array into a data structure in another language . Currently supports PHP Python Javascript and JSON .


def get():
    """ Only API function for the config module.

    :return: {dict}     loaded validated configuration.
    """
    config = {}
    try:
        config = _load_config()
    except IOError:
        try:
            _create_default_config()
            config = _load_config()
        except IOError as e:
            raise ConfigError(_FILE_CREATION_ERROR.format(e.args[0]))
    except SyntaxError as e:
        raise ConfigError(_JSON_SYNTAX_ERROR.format(e.args[0]))
    except Exception:
        raise ConfigError(_JSON_SYNTAX_ERROR.format('Yaml syntax error..'))

    try:
        _validate(config)
    except KeyError as e:
        raise ConfigError(_MANDATORY_KEY_ERROR.format(e.args[0]))
    except SyntaxError as e:
        raise ConfigError(_INVALID_KEY_ERROR.format(e.args[0]))
    except ValueError as e:
        raise ConfigError(_INVALID_VALUE_ERROR.format(e.args[0]))

    config['projects-path'] = os.path.expanduser(config['projects-path'])
    _complete_config(config)
    return config

def
get
(
)
:
config
=
{
}
try
:
config
=
_load_config
(
)
except
IOError
:
try
:
_create_default_config
(
)
config
=
_load_config
(
)
except
IOError
as
e
:
raise
ConfigError
(
_FILE_CREATION_ERROR
.
format
(
e
.
args
[
0
]
)
)
except
SyntaxError
as
e
:
raise
ConfigError
(
_JSON_SYNTAX_ERROR
.
format
(
e
.
args
[
0
]
)
)
except
Exception
:
raise
ConfigError
(
_JSON_SYNTAX_ERROR
.
format
(
'Yaml syntax error..'
)
)
try
:
_validate
(
config
)
except
KeyError
as
e
:
raise
ConfigError
(
_MANDATORY_KEY_ERROR
.
format
(
e
.
args
[
0
]
)
)
except
SyntaxError
as
e
:
raise
ConfigError
(
_INVALID_KEY_ERROR
.
format
(
e
.
args
[
0
]
)
)
except
ValueError
as
e
:
raise
ConfigError
(
_INVALID_VALUE_ERROR
.
format
(
e
.
args
[
0
]
)
)
config
[
'projects-path'
]
=
os
.
path
.
expanduser
(
config
[
'projects-path'
]
)
_complete_config
(
config
)
return
config

Only API function for the config module .


def reusable(func):
    """Create a reusable class from a generator function

    Parameters
    ----------
    func: GeneratorCallable[T_yield, T_send, T_return]
        the function to wrap

    Note
    ----
    * the callable must have an inspectable signature
    * If bound to a class, the new reusable generator is callable as a method.
      To opt out of this, add a :func:`staticmethod` decorator above
      this decorator.

    """
    sig = signature(func)
    origin = func
    while hasattr(origin, '__wrapped__'):
        origin = origin.__wrapped__
    return type(
        origin.__name__,
        (ReusableGenerator, ),
        dict([
            ('__doc__',       origin.__doc__),
            ('__module__',    origin.__module__),
            ('__signature__', sig),
            ('__wrapped__',   staticmethod(func)),
        ] + [
            (name, property(compose(itemgetter(name),
                                    attrgetter('_bound_args.arguments'))))
            for name in sig.parameters
        ] + ([
            ('__qualname__',  origin.__qualname__),
        ] if sys.version_info > (3, ) else [])))

def reusable ( func ) : sig = signature ( func ) origin = func while hasattr ( origin , '__wrapped__' ) : origin = origin . __wrapped__ return type ( origin . __name__ , ( ReusableGenerator , ) , dict ( [ ( '__doc__' , origin . __doc__ ) , ( '__module__' , origin . __module__ ) , ( '__signature__' , sig ) , ( '__wrapped__' , staticmethod ( func ) ) , ] + [ ( name , property ( compose ( itemgetter ( name ) , attrgetter ( '_bound_args.arguments' ) ) ) ) for name in sig . parameters ] + ( [ ( '__qualname__' , origin . __qualname__ ) , ] if sys . version_info > ( 3 , ) else [ ] ) ) )


def sendreturn(gen, value):
    """Send an item into a generator expecting a final return value

    Parameters
    ----------
    gen: ~typing.Generator[T_yield, T_send, T_return]
        the generator to send the value to
    value: T_send
        the value to send

    Raises
    ------
    RuntimeError
        if the generator did not return as expected

    Returns
    -------
    T_return
        the generator's return value
    """
    try:
        gen.send(value)
    except StopIteration as e:
        return stopiter_value(e)
    else:
        raise RuntimeError('generator did not return as expected')

def
sendreturn
(
gen
,
value
)
:
try
:
gen
.
send
(
value
)
except
StopIteration
as
e
:
return
stopiter_value
(
e
)
else
:
raise
RuntimeError
(
'generator did not return as expected'
)