def get_from_cache(url: str, cache_dir: Path = None) -> Path:
    """
    Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
    """
    cache_dir.mkdir(parents=True, exist_ok=True)

    filename = re.sub(r'.+/', '', url)
    # get cache path to put the file
    cache_path = cache_dir / filename
    if cache_path.exists():
        return cache_path

    # make HEAD request to check ETag
    response = requests.head(url)

    if response.status_code != 200:
        if "www.dropbox.com" in url:
            # dropbox return code 301, so we ignore this error
            pass
        else:
            raise IOError("HEAD request failed for url {}".format(url))

    # add ETag to filename if it exists
    # etag = response.headers.get("ETag")

    if not cache_path.exists():
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        fd, temp_filename = tempfile.mkstemp()
        logger.info("%s not found in cache, downloading to %s", url, temp_filename)

        # GET file object
        req = requests.get(url, stream=True)
        content_length = req.headers.get('Content-Length')
        total = int(content_length) if content_length is not None else None
        progress = Tqdm.tqdm(unit="B", total=total)
        with open(temp_filename, 'wb') as temp_file:
            for chunk in req.iter_content(chunk_size=1024):
                if chunk: # filter out keep-alive new chunks
                    progress.update(len(chunk))
                    temp_file.write(chunk)

        progress.close()

        logger.info("copying %s to cache at %s", temp_filename, cache_path)
        shutil.copyfile(temp_filename, str(cache_path))
        logger.info("removing temp file %s", temp_filename)
        os.close(fd)
        os.remove(temp_filename)

    return cache_path

Given a URL look for the corresponding dataset in the local cache . If it s not there download it . Then return the path to the cached file .


def fit(self, X, y):
        """Fit CRF according to X, y

        Parameters
        ----------
        X : list of text
            each item is a text
        y: list
           each item is either a label (in multi class problem) or list of
           labels (in multi label problem)
        """
        trainer = pycrfsuite.Trainer(verbose=True)
        for xseq, yseq in zip(X, y):
            trainer.append(xseq, yseq)

        trainer.set_params(self.params)
        if self.filename:
            filename = self.filename
        else:
            filename = 'model.tmp'
        trainer.train(filename)
        tagger = pycrfsuite.Tagger()
        tagger.open(filename)
        self.estimator = tagger

def fit ( self , X , y ) : trainer = pycrfsuite . Trainer ( verbose = True ) for xseq , yseq in zip ( X , y ) : trainer . append ( xseq , yseq ) trainer . set_params ( self . params ) if self . filename : filename = self . filename else : filename = 'model.tmp' trainer . train ( filename ) tagger = pycrfsuite . Tagger ( ) tagger . open ( filename ) self . estimator = tagger


def predict(self, X):
        """Predict class labels for samples in X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Samples.
        """
        if isinstance(X[0], list):
            return [self.estimator.tag(x) for x in X]
        return self.estimator.tag(X)

def predict ( self , X ) : if isinstance ( X [ 0 ] , list ) : return [ self . estimator . tag ( x ) for x in X ] return self . estimator . tag ( X )


def serve(self, port=62000):
        """ Start LanguageBoard web application

        Parameters
        ----------
        port: int
            port to serve web application
        """

        from http.server import HTTPServer, CGIHTTPRequestHandler
        os.chdir(self.log_folder)

        httpd = HTTPServer(('', port), CGIHTTPRequestHandler)
        print("Starting LanguageBoard on port: " + str(httpd.server_port))
        webbrowser.open('http://0.0.0.0:{}'.format(port))
        httpd.serve_forever()

def
serve
(
self
,
port
=
62000
)
:
from
http
.
server
import
HTTPServer
,
CGIHTTPRequestHandler
os
.
chdir
(
self
.
log_folder
)
httpd
=
HTTPServer
(
(
''
,
port
)
,
CGIHTTPRequestHandler
)
print
(
"Starting LanguageBoard on port: "
+
str
(
httpd
.
server_port
)
)
webbrowser
.
open
(
'http://0.0.0.0:{}'
.
format
(
port
)
)
httpd
.
serve_forever
(
)


def predict(self, X):
        """ In order to obtain the most likely label for a list of text

        Parameters
        ----------
        X : list of string
            Raw texts

        Returns
        -------
        C : list of string
            List labels
        """
        x = X
        if not isinstance(X, list):
            x = [X]
        y = self.estimator.predict(x)
        y = [item[0] for item in y]
        y = [self._remove_prefix(label) for label in y]
        if not isinstance(X, list):
            y = y[0]
        return y

def predict ( self , X ) : x = X if not isinstance ( X , list ) : x = [ X ] y = self . estimator . predict ( x ) y = [ item [ 0 ] for item in y ] y = [ self . _remove_prefix ( label ) for label in y ] if not isinstance ( X , list ) : y = y [ 0 ] return y


def fit(self, X, y):
        """Fit KimCNNClassifier according to X, y

        Parameters
        ----------
        X : list of string
            each item is a raw text
        y : list of string
            each item is a label
        """
        ####################
        # Data Loader
        ####################
        word_vector_transformer = WordVectorTransformer(padding='max')
        X = word_vector_transformer.fit_transform(X)
        X = LongTensor(X)
        self.word_vector_transformer = word_vector_transformer

        y_transformer = LabelEncoder()
        y = y_transformer.fit_transform(y)
        y = torch.from_numpy(y)
        self.y_transformer = y_transformer

        dataset = CategorizedDataset(X, y)
        dataloader = DataLoader(dataset,
                                batch_size=self.batch_size,
                                shuffle=True,
                                num_workers=4)

        ####################
        # Model
        ####################
        KERNEL_SIZES = self.kernel_sizes
        NUM_KERNEL = self.num_kernel
        EMBEDDING_DIM = self.embedding_dim

        model = TextCNN(
            vocab_size=word_vector_transformer.get_vocab_size(),
            embedding_dim=EMBEDDING_DIM,
            output_size=len(self.y_transformer.classes_),
            kernel_sizes=KERNEL_SIZES,
            num_kernel=NUM_KERNEL)
        if USE_CUDA:
            model = model.cuda()

        ####################
        # Train
        ####################
        EPOCH = self.epoch
        LR = self.lr

        loss_function = nn.CrossEntropyLoss()
        optimizer = optim.Adam(model.parameters(), lr=LR)

        for epoch in range(EPOCH):
            losses = []
            for i, data in enumerate(dataloader):
                X, y = data
                X, y = Variable(X), Variable(y)

                optimizer.zero_grad()
                model.train()
                output = model(X)

                loss = loss_function(output, y)
                losses.append(loss.data.tolist()[0])
                loss.backward()

                optimizer.step()

                if i % 100 == 0:
                    print("[%d/%d] mean_loss : %0.2f" % (
                        epoch, EPOCH, np.mean(losses)))
                    losses = []
        self.model = model

def
fit
(
self
,
X
,
y
)
:
word_vector_transformer
=
WordVectorTransformer
(
padding
=
'max'
)
X
=
word_vector_transformer
.
fit_transform
(
X
)
X
=
LongTensor
(
X
)
self
.
word_vector_transformer
=
word_vector_transformer
y_transformer
=
LabelEncoder
(
)
y
=
y_transformer
.
fit_transform
(
y
)
y
=
torch
.
from_numpy
(
y
)
self
.
y_transformer
=
y_transformer
dataset
=
CategorizedDataset
(
X
,
y
)
dataloader
=
DataLoader
(
dataset
,
batch_size
=
self
.
batch_size
,
shuffle
=
True
,
num_workers
=
4
)
KERNEL_SIZES
=
self
.
kernel_sizes
NUM_KERNEL
=
self
.
num_kernel
EMBEDDING_DIM
=
self
.
embedding_dim
model
=
TextCNN
(
vocab_size
=
word_vector_transformer
.
get_vocab_size
(
)
,
embedding_dim
=
EMBEDDING_DIM
,
output_size
=
len
(
self
.
y_transformer
.
classes_
)
,
kernel_sizes
=
KERNEL_SIZES
,
num_kernel
=
NUM_KERNEL
)
if
USE_CUDA
:
model
=
model
.
cuda
(
)
EPOCH
=
self
.
epoch
LR
=
self
.
lr
loss_function
=
nn
.
CrossEntropyLoss
(
)
optimizer
=
optim
.
Adam
(
model
.
parameters
(
)
,
lr
=
LR
)
for
epoch
in
range
(
EPOCH
)
:
losses
=
[
]
for
i
,
data
in
enumerate
(
dataloader
)
:
X
,
y
=
data
X
,
y
=
Variable
(
X
)
,
Variable
(
y
)
optimizer
.
zero_grad
(
)
model
.
train
(
)
output
=
model
(
X
)
loss
=
loss_function
(
output
,
y
)
losses
.
append
(
loss
.
data
.
tolist
(
)
[
0
]
)
loss
.
backward
(
)
optimizer
.
step
(
)
if
i
%
100
==
0
:
print
(
"[%d/%d] mean_loss : %0.2f"
%
(
epoch
,
EPOCH
,
np
.
mean
(
losses
)
)
)
losses
=
[
]
self
.
model
=
model


def config_sources(app, environment, cluster, configs_dirs, app_dir,
                   local=False, build=False):
    """Return the config files for an environment & cluster specific app."""
    sources = [
        # Machine-specific
        (configs_dirs, 'hostname'),
        (configs_dirs, 'hostname-local'),
        (configs_dirs, 'hostname-build'),
        # Global
        (configs_dirs, 'common'),
        # Environment + Cluster
        (configs_dirs, 'common-%s' % environment),
        (configs_dirs, 'common-%s-%s' % (environment, cluster)),
        (configs_dirs, 'common-local'),
        (configs_dirs, 'common-build'),
        # Machine-specific overrides
        (configs_dirs, 'common-overrides'),
        # Application-specific
        ([app_dir], '%s-default' % app),
        ([app_dir], '%s-%s' % (app, environment)),
        ([app_dir], '%s-%s-%s' % (app, environment, cluster)),
        (configs_dirs, app),
        (configs_dirs, '%s-%s' % (app, environment)),
        (configs_dirs, '%s-%s-%s' % (app, environment, cluster)),
        ([app_dir], '%s-local' % app),
        ([app_dir], '%s-build' % app),
        (configs_dirs, '%s-local' % app),
        (configs_dirs, '%s-build' % app),
        # Machine-specific application override
        (configs_dirs, '%s-overrides' % app),
    ]

    # Filter out build sources if not requested
    if not build:
        sources = [source for source in sources
                   if not source[1].endswith('-build')]
        # Filter out local sources if not build and not local
        if not local:
            sources = [source for source in sources
                       if not source[1].endswith('-local')]

    return available_sources(sources)

def config_sources ( app , environment , cluster , configs_dirs , app_dir , local = False , build = False ) : sources = [ ( configs_dirs , 'hostname' ) , ( configs_dirs , 'hostname-local' ) , ( configs_dirs , 'hostname-build' ) , ( configs_dirs , 'common' ) , ( configs_dirs , 'common-%s' % environment ) , ( configs_dirs , 'common-%s-%s' % ( environment , cluster ) ) , ( configs_dirs , 'common-local' ) , ( configs_dirs , 'common-build' ) , ( configs_dirs , 'common-overrides' ) , ( [ app_dir ] , '%s-default' % app ) , ( [ app_dir ] , '%s-%s' % ( app , environment ) ) , ( [ app_dir ] , '%s-%s-%s' % ( app , environment , cluster ) ) , ( configs_dirs , app ) , ( configs_dirs , '%s-%s' % ( app , environment ) ) , ( configs_dirs , '%s-%s-%s' % ( app , environment , cluster ) ) , ( [ app_dir ] , '%s-local' % app ) , ( [ app_dir ] , '%s-build' % app ) , ( configs_dirs , '%s-local' % app ) , ( configs_dirs , '%s-build' % app ) , ( configs_dirs , '%s-overrides' % app ) , ] if not build : sources = [ source for source in sources if not source [ 1 ] . endswith ( '-build' ) ] if not local : sources = [ source for source in sources if not source [ 1 ] . endswith ( '-local' ) ] return available_sources ( sources )

Return the config files for an environment & cluster specific app .


def available_sources(sources):
    """Yield the sources that are present."""
    for dirs, name in sources:
        for directory in dirs:
            fn = os.path.join(directory, name) + '.py'
            if os.path.isfile(fn):
                yield fn

def available_sources ( sources ) : for dirs , name in sources : for directory in dirs : fn = os . path . join ( directory , name ) + '.py' if os . path . isfile ( fn ) : yield fn


def smush_config(sources, initial=None):
    """Merge the configuration sources and return the resulting DotDict."""
    if initial is None:
        initial = {}
    config = DotDict(initial)

    for fn in sources:
        log.debug('Merging %s', fn)
        mod = get_config_module(fn)
        config = mod.update(config)
        log.debug('Current config:\n%s', json.dumps(config, indent=4,
                                                    cls=LenientJSONEncoder))
    return config

def
smush_config
(
sources
,
initial
=
)
:
if
initial
is
:
initial
=
{
}
config
=
DotDict
(
initial
)
for
fn
in
sources
:
log
.
debug
(
'Merging %s'
,
fn
)
mod
=
get_config_module
(
fn
)
config
=
mod
.
update
(
config
)
log
.
debug
(
'Current config:\n%s'
,
json
.
dumps
(
config
,
indent
=
4
,
cls
=
LenientJSONEncoder
)
)
return
config

Merge the configuration sources and return the resulting DotDict .


def merge_dicts(d1, d2, _path=None):
    """
    Merge dictionary d2 into d1, overriding entries in d1 with values from d2.

    d1 is mutated.

    _path is for internal, recursive use.
    """
    if _path is None:
        _path = ()
    if isinstance(d1, dict) and isinstance(d2, dict):
        for k, v in d2.items():
            if isinstance(v, MissingValue) and v.name is None:
                v.name = '.'.join(_path + (k,))

            if isinstance(v, DeletedValue):
                d1.pop(k, None)
            elif k not in d1:
                if isinstance(v, dict):
                    d1[k] = merge_dicts({}, v, _path + (k,))
                else:
                    d1[k] = v
            else:
                if isinstance(d1[k], dict) and isinstance(v, dict):
                    d1[k] = merge_dicts(d1[k], v, _path + (k,))
                elif isinstance(d1[k], list) and isinstance(v, list):
                    # Lists are only supported as leaves
                    d1[k] += v
                elif isinstance(d1[k], MissingValue):
                    d1[k] = v
                elif d1[k] is None:
                    d1[k] = v
                elif type(d1[k]) == type(v):
                    d1[k] = v
                else:
                    raise TypeError('Refusing to replace a %s with a %s'
                                    % (type(d1[k]), type(v)))
    else:
        raise TypeError('Cannot merge a %s with a %s' % (type(d1), type(d2)))

    return d1

def
merge_dicts
(
d1
,
d2
,
_path
=
)
:
if
_path
is
:
_path
=
(
)
if
isinstance
(
d1
,
dict
)
and
isinstance
(
d2
,
dict
)
:
for
k
,
v
in
d2
.
items
(
)
:
if
isinstance
(
v
,
MissingValue
)
and
v
.
name
is
:
v
.
name
=
'.'
.
join
(
_path
+
(
k
,
)
)
if
isinstance
(
v
,
DeletedValue
)
:
d1
.
pop
(
k
,
)
elif
k
not
in
d1
:
if
isinstance
(
v
,
dict
)
:
d1
[
k
]
=
merge_dicts
(
{
}
,
v
,
_path
+
(
k
,
)
)
else
:
d1
[
k
]
=
v
else
:
if
isinstance
(
d1
[
k
]
,
dict
)
and
isinstance
(
v
,
dict
)
:
d1
[
k
]
=
merge_dicts
(
d1
[
k
]
,
v
,
_path
+
(
k
,
)
)
elif
isinstance
(
d1
[
k
]
,
list
)
and
isinstance
(
v
,
list
)
:
d1
[
k
]
+=
v
elif
isinstance
(
d1
[
k
]
,
MissingValue
)
:
d1
[
k
]
=
v
elif
d1
[
k
]
is
:
d1
[
k
]
=
v
elif
type
(
d1
[
k
]
)
==
type
(
v
)
:
d1
[
k
]
=
v
else
:
raise
TypeError
(
'Refusing to replace a %s with a %s'
%
(
type
(
d1
[
k
]
)
,
type
(
v
)
)
)
else
:
raise
TypeError
(
'Cannot merge a %s with a %s'
%
(
type
(
d1
)
,
type
(
d2
)
)
)
return
d1

Merge dictionary d2 into d1 overriding entries in d1 with values from d2 .


def filter_dict(unfiltered, filter_keys):
    """Return a subset of a dictionary using the specified keys."""
    filtered = DotDict()
    for k in filter_keys:
        filtered[k] = unfiltered[k]
    return filtered

def filter_dict ( unfiltered , filter_keys ) : filtered = DotDict ( ) for k in filter_keys : filtered [ k ] = unfiltered [ k ] return filtered

Return a subset of a dictionary using the specified keys .


def _convert_item(self, obj):
        """
        Convert obj into a DotDict, or list of DotDict.

        Directly nested lists aren't supported.
        Returns the result
        """
        if isinstance(obj, dict) and not isinstance(obj, DotDict):
            obj = DotDict(obj)
        elif isinstance(obj, list):
            # must mutate and not just reassign, otherwise it will
            # just use original object mutable/immutable
            for i, item in enumerate(obj):
                if isinstance(item, dict) and not isinstance(item, DotDict):
                    obj[i] = DotDict(item)
        return obj

def _convert_item ( self , obj ) : if isinstance ( obj , dict ) and not isinstance ( obj , DotDict ) : obj = DotDict ( obj ) elif isinstance ( obj , list ) : for i , item in enumerate ( obj ) : if isinstance ( item , dict ) and not isinstance ( item , DotDict ) : obj [ i ] = DotDict ( item ) return obj

Convert obj into a DotDict or list of DotDict .


def filter_config(config, deploy_config):
    """Return a config subset using the filter defined in the deploy config."""
    if not os.path.isfile(deploy_config):
        return DotDict()
    config_module = get_config_module(deploy_config)
    return config_module.filter(config)

def filter_config ( config , deploy_config ) : if not os . path . isfile ( deploy_config ) : return DotDict ( ) config_module = get_config_module ( deploy_config ) return config_module . filter ( config )

Return a config subset using the filter defined in the deploy config .


def seeded_auth_token(client, service, seed):
    """Return an auth token based on the client+service+seed tuple."""
    hash_func = hashlib.md5()
    token = ','.join((client, service, seed)).encode('utf-8')
    hash_func.update(token)
    return hash_func.hexdigest()

def seeded_auth_token ( client , service , seed ) : hash_func = hashlib . md5 ( ) token = ',' . join ( ( client , service , seed ) ) . encode ( 'utf-8' ) hash_func . update ( token ) return hash_func . hexdigest ( )

Return an auth token based on the client + service + seed tuple .


def write_config(config, app_dir, filename='configuration.json'):
    """Write configuration to the applicaiton directory."""
    path = os.path.join(app_dir, filename)
    with open(path, 'w') as f:
        json.dump(
            config, f, indent=4, cls=DetectMissingEncoder,
            separators=(',', ': '))

def
write_config
(
config
,
app_dir
,
filename
=
'configuration.json'
)
:
path
=
os
.
path
.
join
(
app_dir
,
filename
)
with
open
(
path
,
'w'
)
as
f
:
json
.
dump
(
config
,
f
,
indent
=
4
,
cls
=
DetectMissingEncoder
,
separators
=
(
','
,
': '
)
)

Write configuration to the applicaiton directory .


def validate_date(date_text):
    """Return True if valid, raise ValueError if not"""
    try:
        if int(date_text) < 0:
            return True
    except ValueError:
        pass

    try:
        datetime.strptime(date_text, '%Y-%m-%d')
        return True
    except ValueError:
        pass

    raise ValueError('Dates must be negative integers or YYYY-MM-DD in the past.')

Return True if valid raise ValueError if not


def get_download_total(rows):
    """Return the total downloads, and the downloads column"""
    headers = rows.pop(0)
    index = headers.index('download_count')
    total_downloads = sum(int(row[index]) for row in rows)

    rows.insert(0, headers)
    return total_downloads, index

def get_download_total ( rows ) : headers = rows . pop ( 0 ) index = headers . index ( 'download_count' ) total_downloads = sum ( int ( row [ index ] ) for row in rows ) rows . insert ( 0 , headers ) return total_downloads , index

Return the total downloads and the downloads column


def add_download_total(rows):
    """Add a final row to rows showing the total downloads"""
    total_row = [""] * len(rows[0])
    total_row[0] = "Total"
    total_downloads, downloads_column = get_download_total(rows)
    total_row[downloads_column] = str(total_downloads)
    rows.append(total_row)

    return rows

def add_download_total ( rows ) : total_row = [ "" ] * len ( rows [ 0 ] ) total_row [ 0 ] = "Total" total_downloads , downloads_column = get_download_total ( rows ) total_row [ downloads_column ] = str ( total_downloads ) rows . append ( total_row ) return rows


def find_and_patch_entry(soup, entry):
    """
    Modify soup so Dash.app can generate TOCs on the fly.
    """
    link = soup.find("a", {"class": "headerlink"}, href="#" + entry.anchor)
    tag = soup.new_tag("a")
    tag["name"] = APPLE_REF_TEMPLATE.format(entry.type, entry.name)
    if link:
        link.parent.insert(0, tag)
        return True
    elif entry.anchor.startswith("module-"):
        soup.h1.parent.insert(0, tag)
        return True
    else:
        return False

def find_and_patch_entry ( soup , entry ) : link = soup . find ( "a" , { "class" : "headerlink" } , href = "#" + entry . anchor ) tag = soup . new_tag ( "a" ) tag [ "name" ] = APPLE_REF_TEMPLATE . format ( entry . type , entry . name ) if link : link . parent . insert ( 0 , tag ) return True elif entry . anchor . startswith ( "module-" ) : soup . h1 . parent . insert ( 0 , tag ) return True else : return False

Modify soup so Dash . app can generate TOCs on the fly .


def inv_entry_to_path(data):
    """
    Determine the path from the intersphinx inventory entry

    Discard the anchors between head and tail to make it
    compatible with situations where extra meta information is encoded.
    """
    path_tuple = data[2].split("#")
    if len(path_tuple) > 1:
        path_str = "#".join((path_tuple[0], path_tuple[-1]))
    else:
        path_str = data[2]
    return path_str

def inv_entry_to_path ( data ) : path_tuple = data [ 2 ] . split ( "#" ) if len ( path_tuple ) > 1 : path_str = "#" . join ( ( path_tuple [ 0 ] , path_tuple [ - 1 ] ) ) else : path_str = data [ 2 ] return path_str


def main(
    source,
    force,
    name,
    quiet,
    verbose,
    destination,
    add_to_dash,
    add_to_global,
    icon,
    index_page,
    enable_js,
    online_redirect_url,
    parser,
):
    """
    Convert docs from SOURCE to Dash.app's docset format.
    """
    try:
        logging.config.dictConfig(
            create_log_config(verbose=verbose, quiet=quiet)
        )
    except ValueError as e:
        click.secho(e.args[0], fg="red")
        raise SystemExit(1)

    if icon:
        icon_data = icon.read()
        if not icon_data.startswith(PNG_HEADER):
            log.error(
                '"{}" is not a valid PNG image.'.format(
                    click.format_filename(icon.name)
                )
            )
            raise SystemExit(1)
    else:
        icon_data = None

    source, dest, name = setup_paths(
        source,
        destination,
        name=name,
        add_to_global=add_to_global,
        force=force,
    )
    if parser is None:
        parser = parsers.get_doctype(source)
        if parser is None:
            log.error(
                '"{}" does not contain a known documentation format.'.format(
                    click.format_filename(source)
                )
            )
            raise SystemExit(errno.EINVAL)
    docset = prepare_docset(
        source, dest, name, index_page, enable_js, online_redirect_url
    )
    doc_parser = parser(doc_path=docset.docs)
    log.info(
        (
            "Converting "
            + click.style("{parser_name}", bold=True)
            + ' docs from "{src}" to "{dst}".'
        ).format(
            parser_name=parser.name,
            src=click.format_filename(source, shorten=True),
            dst=click.format_filename(dest),
        )
    )

    with docset.db_conn:
        log.info("Parsing documentation...")
        toc = patch_anchors(doc_parser, show_progressbar=not quiet)
        for entry in doc_parser.parse():
            docset.db_conn.execute(
                "INSERT INTO searchIndex VALUES (NULL, ?, ?, ?)",
                entry.as_tuple(),
            )
            toc.send(entry)
        count = docset.db_conn.execute(
            "SELECT COUNT(1) FROM searchIndex"
        ).fetchone()[0]
        log.info(
            (
                "Added "
                + click.style("{count:,}", fg="green" if count > 0 else "red")
                + " index entries."
            ).format(count=count)
        )
        toc.close()

    if icon_data:
        add_icon(icon_data, dest)

    if add_to_dash or add_to_global:
        log.info("Adding to Dash.app...")
        os.system('open -a dash "{}"'.format(dest))

Convert docs from SOURCE to Dash . app s docset format .


def create_log_config(verbose, quiet):
    """
    We use logging's levels as an easy-to-use verbosity controller.
    """
    if verbose and quiet:
        raise ValueError(
            "Supplying both --quiet and --verbose makes no sense."
        )
    elif verbose:
        level = logging.DEBUG
    elif quiet:
        level = logging.ERROR
    else:
        level = logging.INFO

    logger_cfg = {"handlers": ["click_handler"], "level": level}

    return {
        "version": 1,
        "formatters": {"click_formatter": {"format": "%(message)s"}},
        "handlers": {
            "click_handler": {
                "level": level,
                "class": "doc2dash.__main__.ClickEchoHandler",
                "formatter": "click_formatter",
            }
        },
        "loggers": {"doc2dash": logger_cfg, "__main__": logger_cfg},
    }

We use logging s levels as an easy - to - use verbosity controller .


def setup_paths(source, destination, name, add_to_global, force):
    """
    Determine source and destination using the options.
    """
    if source[-1] == "/":
        source = source[:-1]
    if not name:
        name = os.path.split(source)[-1]
    elif name.endswith(".docset"):
        name = name.replace(".docset", "")
    if add_to_global:
        destination = DEFAULT_DOCSET_PATH
    dest = os.path.join(destination or "", name + ".docset")
    dst_exists = os.path.lexists(dest)
    if dst_exists and force:
        shutil.rmtree(dest)
    elif dst_exists:
        log.error(
            'Destination path "{}" already exists.'.format(
                click.format_filename(dest)
            )
        )
        raise SystemExit(errno.EEXIST)
    return source, dest, name

def
setup_paths
(
source
,
destination
,
name
,
add_to_global
,
force
)
:
if
source
[
1
]
==
"/"
:
source
=
source
[
:
1
]
if
not
name
:
name
=
os
.
path
.
split
(
source
)
[
1
]
elif
name
.
endswith
(
".docset"
)
:
name
=
name
.
replace
(
".docset"
,
""
)
if
add_to_global
:
destination
=
DEFAULT_DOCSET_PATH
dest
=
os
.
path
.
join
(
destination
or
""
,
name
+
".docset"
)
dst_exists
=
os
.
path
.
lexists
(
dest
)
if
dst_exists
and
force
:
shutil
.
rmtree
(
dest
)
elif
dst_exists
:
log
.
error
(
'Destination path "{}" already exists.'
.
format
(
click
.
format_filename
(
dest
)
)
)
raise
SystemExit
(
errno
.
EEXIST
)
return
source
,
dest
,
name

Determine source and destination using the options .


def prepare_docset(
    source, dest, name, index_page, enable_js, online_redirect_url
):
    """
    Create boilerplate files & directories and copy vanilla docs inside.

    Return a tuple of path to resources and connection to sqlite db.
    """
    resources = os.path.join(dest, "Contents", "Resources")
    docs = os.path.join(resources, "Documents")
    os.makedirs(resources)

    db_conn = sqlite3.connect(os.path.join(resources, "docSet.dsidx"))
    db_conn.row_factory = sqlite3.Row
    db_conn.execute(
        "CREATE TABLE searchIndex(id INTEGER PRIMARY KEY, name TEXT, "
        "type TEXT, path TEXT)"
    )
    db_conn.commit()

    plist_path = os.path.join(dest, "Contents", "Info.plist")
    plist_cfg = {
        "CFBundleIdentifier": name,
        "CFBundleName": name,
        "DocSetPlatformFamily": name.lower(),
        "DashDocSetFamily": "python",
        "isDashDocset": True,
        "isJavaScriptEnabled": enable_js,
    }
    if index_page is not None:
        plist_cfg["dashIndexFilePath"] = index_page
    if online_redirect_url is not None:
        plist_cfg["DashDocSetFallbackURL"] = online_redirect_url

    write_plist(plist_cfg, plist_path)

    shutil.copytree(source, docs)

    return DocSet(path=dest, docs=docs, plist=plist_path, db_conn=db_conn)

Create boilerplate files & directories and copy vanilla docs inside .


def add_icon(icon_data, dest):
    """
    Add icon to docset
    """
    with open(os.path.join(dest, "icon.png"), "wb") as f:
        f.write(icon_data)

def add_icon ( icon_data , dest ) : with open ( os . path . join ( dest , "icon.png" ) , "wb" ) as f : f . write ( icon_data )


def run_cell(self, cell):
        """Run the Cell code using the IPython globals and locals

        Args:
            cell (str): Python code to be executed
        """
        globals = self.ipy_shell.user_global_ns
        locals = self.ipy_shell.user_ns
        globals.update({
            "__ipy_scope__": None,
        })
        try:
            with redirect_stdout(self.stdout):
                self.run(cell, globals, locals)
        except:
            self.code_error = True
            if self.options.debug:
                raise BdbQuit
        finally:
            self.finalize()

def run_cell ( self , cell ) : globals = self . ipy_shell . user_global_ns locals = self . ipy_shell . user_ns globals . update ( { "__ipy_scope__" : None , } ) try : with redirect_stdout ( self . stdout ) : self . run ( cell , globals , locals ) except : self . code_error = True if self . options . debug : raise BdbQuit finally : self . finalize ( )


def filter_dict(d, exclude):
    """Return a new dict with specified keys excluded from the origional dict

    Args:
        d (dict): origional dict
        exclude (list): The keys that are excluded
    """
    ret = {}
    for key, value in d.items():
        if key not in exclude:
            ret.update({key: value})
    return ret

def filter_dict ( d , exclude ) : ret = { } for key , value in d . items ( ) : if key not in exclude : ret . update ( { key : value } ) return ret


def redirect_stdout(new_stdout):
    """Redirect the stdout

    Args:
        new_stdout (io.StringIO): New stdout to use instead
    """
    old_stdout, sys.stdout = sys.stdout, new_stdout
    try:
        yield None
    finally:
        sys.stdout = old_stdout

def redirect_stdout ( new_stdout ) : old_stdout , sys . stdout = sys . stdout , new_stdout try : yield None finally : sys . stdout = old_stdout


def format(obj, options):
    """Return a string representation of the Python object

    Args:
        obj: The Python object
        options: Format options
    """
    formatters = {
        float_types: lambda x: '{:.{}g}'.format(x, options.digits),
    }
    for _types, fmtr in formatters.items():
        if isinstance(obj, _types):
            return fmtr(obj)
    try:
        if six.PY2 and isinstance(obj, six.string_types):
            return str(obj.encode('utf-8'))
        return str(obj)
    except:
        return 'OBJECT'

def format ( obj , options ) : formatters = { float_types : lambda x : '{:.{}g}' . format ( x , options . digits ) , } for _types , fmtr in formatters . items ( ) : if isinstance ( obj , _types ) : return fmtr ( obj ) try : if six . PY2 and isinstance ( obj , six . string_types ) : return str ( obj . encode ( 'utf-8' ) ) return str ( obj ) except : return 'OBJECT'


def get_type_info(obj):
    """Get type information for a Python object

    Args:
        obj: The Python object

    Returns:
        tuple: (object type "catagory", object type name)
    """
    if isinstance(obj, primitive_types):
        return ('primitive', type(obj).__name__)
    if isinstance(obj, sequence_types):
        return ('sequence', type(obj).__name__)
    if isinstance(obj, array_types):
        return ('array', type(obj).__name__)
    if isinstance(obj, key_value_types):
        return ('key-value', type(obj).__name__)
    if isinstance(obj, types.ModuleType):
        return ('module', type(obj).__name__)
    if isinstance(obj, (types.FunctionType, types.MethodType)):
        return ('function', type(obj).__name__)
    if isinstance(obj, type):
        if hasattr(obj, '__dict__'):
            return ('class', obj.__name__)
    if isinstance(type(obj), type):
        if hasattr(obj, '__dict__'):
            cls_name = type(obj).__name__
            if cls_name == 'classobj':
                cls_name = obj.__name__
                return ('class', '{}'.format(cls_name))
            if cls_name == 'instance':
                cls_name = obj.__class__.__name__
            return ('instance', '{} instance'.format(cls_name))

    return ('unknown', type(obj).__name__)

def
get_type_info
(
obj
)
:
if
isinstance
(
obj
,
primitive_types
)
:
return
(
'primitive'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
sequence_types
)
:
return
(
'sequence'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
array_types
)
:
return
(
'array'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
key_value_types
)
:
return
(
'key-value'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
types
.
ModuleType
)
:
return
(
'module'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
(
types
.
FunctionType
,
types
.
MethodType
)
)
:
return
(
'function'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
type
)
:
if
hasattr
(
obj
,
'__dict__'
)
:
return
(
'class'
,
obj
.
__name__
)
if
isinstance
(
type
(
obj
)
,
type
)
:
if
hasattr
(
obj
,
'__dict__'
)
:
cls_name
=
type
(
obj
)
.
__name__
if
cls_name
==
'classobj'
:
cls_name
=
obj
.
__name__
return
(
'class'
,
'{}'
.
format
(
cls_name
)
)
if
cls_name
==
'instance'
:
cls_name
=
obj
.
__class__
.
__name__
return
(
'instance'
,
'{} instance'
.
format
(
cls_name
)
)
return
(
'unknown'
,
type
(
obj
)
.
__name__
)


def spend_key(self):
        """
        Returns private spend key. None if wallet is view-only.

        :rtype: str or None
        """
        key = self._backend.spend_key()
        if key == numbers.EMPTY_KEY:
            return None
        return key

def spend_key ( self ) : key = self . _backend . spend_key ( ) if key == numbers . EMPTY_KEY : return None return key

Returns private spend key . None if wallet is view - only .


def transfer(self, address, amount,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a transfer from the default account. Returns a list of resulting transactions.

        :param address: destination :class:`Address <monero.address.Address>` or subtype
        :param amount: amount to send
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as the destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self.accounts[0].transfer(
                address,
                amount,
                priority=priority,
                payment_id=payment_id,
                unlock_time=unlock_time,
                relay=relay)

def transfer ( self , address , amount , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) : return self . accounts [ 0 ] . transfer ( address , amount , priority = priority , payment_id = payment_id , unlock_time = unlock_time , relay = relay )

Sends a transfer from the default account . Returns a list of resulting transactions .


def transfer_multiple(self, destinations,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a batch of transfers from the default account. Returns a list of resulting
        transactions.

        :param destinations: a list of destination and amount pairs: [(address, amount), ...]
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as a destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self.accounts[0].transfer_multiple(
                destinations,
                priority=priority,
                payment_id=payment_id,
                unlock_time=unlock_time,
                relay=relay)

def transfer_multiple ( self , destinations , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) : return self . accounts [ 0 ] . transfer_multiple ( destinations , priority = priority , payment_id = payment_id , unlock_time = unlock_time , relay = relay )

Sends a batch of transfers from the default account . Returns a list of resulting transactions .


def balance(self, unlocked=False):
        """
        Returns specified balance.

        :param unlocked: if `True`, return the unlocked balance, otherwise return total balance
        :rtype: Decimal
        """
        return self._backend.balances(account=self.index)[1 if unlocked else 0]

def balance ( self , unlocked = False ) : return self . _backend . balances ( account = self . index ) [ 1 if unlocked else 0 ]


def new_address(self, label=None):
        """
        Creates a new address.

        :param label: address label as `str`
        :rtype: :class:`SubAddress <monero.address.SubAddress>`
        """
        return self._backend.new_address(account=self.index, label=label)

def new_address ( self , label = None ) : return self . _backend . new_address ( account = self . index , label = label )


def transfer(self, address, amount,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a transfer. Returns a list of resulting transactions.

        :param address: destination :class:`Address <monero.address.Address>` or subtype
        :param amount: amount to send
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as the destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self._backend.transfer(
            [(address, amount)],
            priority,
            payment_id,
            unlock_time,
            account=self.index,
            relay=relay)

def transfer ( self , address , amount , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) : return self . _backend . transfer ( [ ( address , amount ) ] , priority , payment_id , unlock_time , account = self . index , relay = relay )

Sends a transfer . Returns a list of resulting transactions .


def transfer_multiple(self, destinations,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a batch of transfers. Returns a list of resulting transactions.

        :param destinations: a list of destination and amount pairs:
                    [(:class:`Address <monero.address.Address>`, `Decimal`), ...]
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as the destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self._backend.transfer(
            destinations,
            priority,
            payment_id,
            unlock_time,
            account=self.index,
            relay=relay)

def transfer_multiple ( self , destinations , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) : return self . _backend . transfer ( destinations , priority , payment_id , unlock_time , account = self . index , relay = relay )

Sends a batch of transfers . Returns a list of resulting transactions .


def to_atomic(amount):
    """Convert Monero decimal to atomic integer of piconero."""
    if not isinstance(amount, (Decimal, float) + _integer_types):
        raise ValueError("Amount '{}' doesn't have numeric type. Only Decimal, int, long and "
                "float (not recommended) are accepted as amounts.")
    return int(amount * 10**12)

Convert Monero decimal to atomic integer of piconero .


def address(addr, label=None):
    """Discover the proper class and return instance for a given Monero address.

    :param addr: the address as a string-like object
    :param label: a label for the address (defaults to `None`)

    :rtype: :class:`Address`, :class:`SubAddress` or :class:`IntegratedAddress`
    """
    addr = str(addr)
    if _ADDR_REGEX.match(addr):
        netbyte = bytearray(unhexlify(base58.decode(addr)))[0]
        if netbyte in Address._valid_netbytes:
            return Address(addr, label=label)
        elif netbyte in SubAddress._valid_netbytes:
            return SubAddress(addr, label=label)
        raise ValueError("Invalid address netbyte {nb:x}. Allowed values are: {allowed}".format(
            nb=netbyte,
            allowed=", ".join(map(
                lambda b: '%02x' % b,
                sorted(Address._valid_netbytes + SubAddress._valid_netbytes)))))
    elif _IADDR_REGEX.match(addr):
        return IntegratedAddress(addr)
    raise ValueError("Address must be either 95 or 106 characters long base58-encoded string, "
        "is {addr} ({len} chars length)".format(addr=addr, len=len(addr)))

Discover the proper class and return instance for a given Monero address .


def with_payment_id(self, payment_id=0):
        """Integrates payment id into the address.

        :param payment_id: int, hexadecimal string or :class:`PaymentID <monero.numbers.PaymentID>`
                    (max 64-bit long)

        :rtype: `IntegratedAddress`
        :raises: `TypeError` if the payment id is too long
        """
        payment_id = numbers.PaymentID(payment_id)
        if not payment_id.is_short():
            raise TypeError("Payment ID {0} has more than 64 bits and cannot be integrated".format(payment_id))
        prefix = 54 if self.is_testnet() else 25 if self.is_stagenet() else 19
        data = bytearray([prefix]) + self._decoded[1:65] + struct.pack('>Q', int(payment_id))
        checksum = bytearray(keccak_256(data).digest()[:4])
        return IntegratedAddress(base58.encode(hexlify(data + checksum)))

Integrates payment id into the address .


def encode(cls, hex):
        """Convert hexadecimal string to mnemonic word representation with checksum.
        """
        out = []
        for i in range(len(hex) // 8):
            word = endian_swap(hex[8*i:8*i+8])
            x = int(word, 16)
            w1 = x % cls.n
            w2 = (x // cls.n + w1) % cls.n
            w3 = (x // cls.n // cls.n + w2) % cls.n
            out += [cls.word_list[w1], cls.word_list[w2], cls.word_list[w3]]
        checksum = cls.get_checksum(" ".join(out))
        out.append(checksum)
        return " ".join(out)

def
encode
(
cls
,
hex
)
:
out
=
[
]
for
i
in
range
(
len
(
hex
)
//
8
)
:
word
=
endian_swap
(
hex
[
8
*
i
:
8
*
i
+
8
]
)
x
=
int
(
word
,
16
)
w1
=
x
%
cls
.
n
w2
=
(
x
//
cls
.
n
+
w1
)
%
cls
.
n
w3
=
(
x
//
cls
.
n
//
cls
.
n
+
w2
)
%
cls
.
n
out
+=
[
cls
.
word_list
[
w1
]
,
cls
.
word_list
[
w2
]
,
cls
.
word_list
[
w3
]
]
checksum
=
cls
.
get_checksum
(
" "
.
join
(
out
)
)
out
.
append
(
checksum
)
return
" "
.
join
(
out
)

Convert hexadecimal string to mnemonic word representation with checksum .


def decode(cls, phrase):
        """Calculate hexadecimal representation of the phrase.
        """
        phrase = phrase.split(" ")
        out = ""
        for i in range(len(phrase) // 3):
            word1, word2, word3 = phrase[3*i:3*i+3]
            w1 = cls.word_list.index(word1)
            w2 = cls.word_list.index(word2) % cls.n
            w3 = cls.word_list.index(word3) % cls.n
            x = w1 + cls.n *((w2 - w1) % cls.n) + cls.n * cls.n * ((w3 - w2) % cls.n)
            out += endian_swap("%08x" % x)
        return out

def
decode
(
cls
,
phrase
)
:
phrase
=
phrase
.
split
(
" "
)
out
=
""
for
i
in
range
(
len
(
phrase
)
//
3
)
:
word1
,
word2
,
word3
=
phrase
[
3
*
i
:
3
*
i
+
3
]
w1
=
cls
.
word_list
.
index
(
word1
)
w2
=
cls
.
word_list
.
index
(
word2
)
%
cls
.
n
w3
=
cls
.
word_list
.
index
(
word3
)
%
cls
.
n
x
=
w1
+
cls
.
n
*
(
(
w2
w1
)
%
cls
.
n
)
+
cls
.
n
*
cls
.
n
*
(
(
w3
w2
)
%
cls
.
n
)
out
+=
endian_swap
(
"%08x"
%
x
)
return
out

Calculate hexadecimal representation of the phrase .


def get_checksum(cls, phrase):
        """Given a mnemonic word string, return a string of the computed checksum.

        :rtype: str
        """
        phrase_split = phrase.split(" ")
        if len(phrase_split) < 12:
            raise ValueError("Invalid mnemonic phrase")
        if len(phrase_split) > 13:
            # Standard format
            phrase = phrase_split[:24]
        else:
            # MyMonero format
            phrase = phrase_split[:12]
        wstr = "".join(word[:cls.unique_prefix_length] for word in phrase)
        wstr = bytearray(wstr.encode('utf-8'))
        z = ((crc32(wstr) & 0xffffffff) ^ 0xffffffff ) >> 0
        z2 = ((z ^ 0xffffffff) >> 0) % len(phrase)
        return phrase_split[z2]

def
get_checksum
(
cls
,
phrase
)
:
phrase_split
=
phrase
.
split
(
" "
)
if
len
(
phrase_split
)
<
12
:
raise
ValueError
(
"Invalid mnemonic phrase"
)
if
len
(
phrase_split
)
>
13
:
phrase
=
phrase_split
[
:
24
]
else
:
phrase
=
phrase_split
[
:
12
]
wstr
=
""
.
join
(
word
[
:
cls
.
unique_prefix_length
]
for
word
in
phrase
)
wstr
=
bytearray
(
wstr
.
encode
(
'utf-8'
)
)
z
=
(
(
crc32
(
wstr
)
&
0xffffffff
)
^
0xffffffff
)
>>
0
z2
=
(
(
z
^
0xffffffff
)
>>
0
)
%
len
(
phrase
)
return
phrase_split
[
z2
]

Given a mnemonic word string return a string of the computed checksum .


def one(prompt, *args, **kwargs):
    """Instantiates a picker, registers custom handlers for going back,
    and starts the picker.
    """
    indicator = '‣'
    if sys.version_info < (3, 0):
        indicator = '>'

    def go_back(picker):
        return None, -1

    options, verbose_options = prepare_options(args)
    idx = kwargs.get('idx', 0)

    picker = Picker(verbose_options, title=prompt, indicator=indicator, default_index=idx)
    picker.register_custom_handler(ord('h'), go_back)
    picker.register_custom_handler(curses.KEY_LEFT, go_back)
    with stdout_redirected(sys.stderr):
        option, index = picker.start()
        if index == -1:
            raise QuestionnaireGoBack
        if kwargs.get('return_index', False):
            # `one` was called by a special client, e.g. `many`
            return index
        return options[index]

def one ( prompt , * args , ** kwargs ) : indicator = '‣' if sys . version_info < ( 3 , 0 ) : indicator = '>' def go_back ( picker ) : return None , - 1 options , verbose_options = prepare_options ( args ) idx = kwargs . get ( 'idx' , 0 ) picker = Picker ( verbose_options , title = prompt , indicator = indicator , default_index = idx ) picker . register_custom_handler ( ord ( 'h' ) , go_back ) picker . register_custom_handler ( curses . KEY_LEFT , go_back ) with stdout_redirected ( sys . stderr ) : option , index = picker . start ( ) if index == - 1 : raise QuestionnaireGoBack if kwargs . get ( 'return_index' , False ) : return index return options [ index ]

Instantiates a picker registers custom handlers for going back and starts the picker .


def many(prompt, *args, **kwargs):
    """Calls `pick` in a while loop to allow user to pick many
    options. Returns a list of chosen options.
    """
    def get_options(options, chosen):
        return [options[i] for i, c in enumerate(chosen) if c]

    def get_verbose_options(verbose_options, chosen):
        no, yes = ' ', '✔'
        if sys.version_info < (3, 3):
            no, yes = ' ', '@'
        opts = ['{} {}'.format(yes if c else no, verbose_options[i]) for i, c in enumerate(chosen)]
        return opts + ['{}{}'.format('  ', kwargs.get('done', 'done...'))]

    options, verbose_options = prepare_options(args)
    chosen = [False] * len(options)
    index = kwargs.get('idx', 0)

    default = kwargs.get('default', None)
    if isinstance(default, list):
        for idx in default:
            chosen[idx] = True
    if isinstance(default, int):
        chosen[default] = True

    while True:
        try:
            index = one(prompt, *get_verbose_options(verbose_options, chosen), return_index=True, idx=index)
        except QuestionnaireGoBack:
            if any(chosen):
                raise QuestionnaireGoBack(0)
            else:
                raise QuestionnaireGoBack
        if index == len(options):
            return get_options(options, chosen)
        chosen[index] = not chosen[index]

def
many
(
prompt
,
*
args
,
**
kwargs
)
:
def
get_options
(
options
,
chosen
)
:
return
[
options
[
i
]
for
i
,
c
in
enumerate
(
chosen
)
if
c
]
def
get_verbose_options
(
verbose_options
,
chosen
)
:
no
,
yes
=
' '
,
'✔'
if
sys
.
version_info
<
(
3
,
3
)
:
no
,
yes
=
' '
,
'@'
opts
=
[
'{} {}'
.
format
(
yes
if
c
else
no
,
verbose_options
[
i
]
)
for
i
,
c
in
enumerate
(
chosen
)
]
return
opts
+
[
'{}{}'
.
format
(
'  '
,
kwargs
.
get
(
'done'
,
'done...'
)
)
]
options
,
verbose_options
=
prepare_options
(
args
)
chosen
=
[
False
]
*
len
(
options
)
index
=
kwargs
.
get
(
'idx'
,
0
)
default
=
kwargs
.
get
(
'default'
,
)
if
isinstance
(
default
,
list
)
:
for
idx
in
default
:
chosen
[
idx
]
=
True
if
isinstance
(
default
,
int
)
:
chosen
[
default
]
=
True
while
True
:
try
:
index
=
one
(
prompt
,
*
get_verbose_options
(
verbose_options
,
chosen
)
,
return_index
=
True
,
idx
=
index
)
except
QuestionnaireGoBack
:
if
any
(
chosen
)
:
raise
QuestionnaireGoBack
(
0
)
else
:
raise
QuestionnaireGoBack
if
index
==
len
(
options
)
:
return
get_options
(
options
,
chosen
)
chosen
[
index
]
=
not
chosen
[
index
]

Calls pick in a while loop to allow user to pick many options . Returns a list of chosen options .


def prepare_options(options):
    """Create options and verbose options from strings and non-string iterables in
    `options` array.
    """
    options_, verbose_options = [], []
    for option in options:
        if is_string(option):
            options_.append(option)
            verbose_options.append(option)
        else:
            options_.append(option[0])
            verbose_options.append(option[1])
    return options_, verbose_options

def prepare_options ( options ) : options_ , verbose_options = [ ] , [ ] for option in options : if is_string ( option ) : options_ . append ( option ) verbose_options . append ( option ) else : options_ . append ( option [ 0 ] ) verbose_options . append ( option [ 1 ] ) return options_ , verbose_options

Create options and verbose options from strings and non - string iterables in options array .


def raw(prompt, *args, **kwargs):
    """Calls input to allow user to input an arbitrary string. User can go
    back by entering the `go_back` string. Works in both Python 2 and 3.
    """
    go_back = kwargs.get('go_back', '<')
    type_ = kwargs.get('type', str)
    default = kwargs.get('default', '')
    with stdout_redirected(sys.stderr):
        while True:
            try:
                if kwargs.get('secret', False):
                    answer = getpass.getpass(prompt)
                elif sys.version_info < (3, 0):
                    answer = raw_input(prompt)
                else:
                    answer = input(prompt)

                if not answer:
                    answer = default

                if answer == go_back:
                    raise QuestionnaireGoBack
                return type_(answer)
            except ValueError:
                eprint('\n`{}` is not a valid `{}`\n'.format(answer, type_))

def
raw
(
prompt
,
*
args
,
**
kwargs
)
:
go_back
=
kwargs
.
get
(
'go_back'
,
'<'
)
type_
=
kwargs
.
get
(
'type'
,
str
)
default
=
kwargs
.
get
(
'default'
,
''
)
with
stdout_redirected
(
sys
.
stderr
)
:
while
True
:
try
:
if
kwargs
.
get
(
'secret'
,
False
)
:
answer
=
getpass
.
getpass
(
prompt
)
elif
sys
.
version_info
<
(
3
,
0
)
:
answer
=
raw_input
(
prompt
)
else
:
answer
=
input
(
prompt
)
if
not
answer
:
answer
=
default
if
answer
==
go_back
:
raise
QuestionnaireGoBack
return
type_
(
answer
)
except
ValueError
:
eprint
(
'\n`{}` is not a valid `{}`\n'
.
format
(
answer
,
type_
)
)

Calls input to allow user to input an arbitrary string . User can go back by entering the go_back string . Works in both Python 2 and 3 .


def get_operator(self, op):
        """Assigns function to the operators property of the instance.
        """
        if op in self.OPERATORS:
            return self.OPERATORS.get(op)
        try:
            n_args = len(inspect.getargspec(op)[0])
            if n_args != 2:
                raise TypeError
        except:
            eprint('Error: invalid operator function. Operators must accept two args.')
            raise
        else:
            return op

Assigns function to the operators property of the instance .


def assign_prompter(self, prompter):
        """If you want to change the core prompters registry, you can
        override this method in a Question subclass.
        """
        if is_string(prompter):
            if prompter not in prompters:
                eprint("Error: '{}' is not a core prompter".format(prompter))
                sys.exit()
            self.prompter = prompters[prompter]
        else:
            self.prompter = prompter

def
assign_prompter
(
self
,
prompter
)
:
if
is_string
(
prompter
)
:
if
prompter
not
in
prompters
:
eprint
(
"Error: '{}' is not a core prompter"
.
format
(
prompter
)
)
sys
.
exit
(
)
self
.
prompter
=
prompters
[
prompter
]
else
:
self
.
prompter
=
prompter

If you want to change the core prompters registry you can override this method in a Question subclass .


def add(self, *args, **kwargs):
        """Add a Question instance to the questions dict. Each key points
        to a list of Question instances with that key. Use the `question`
        kwarg to pass a Question instance if you want, or pass in the same
        args you would pass to instantiate a question.
        """
        if 'question' in kwargs and isinstance(kwargs['question'], Question):
            question = kwargs['question']
        else:
            question = Question(*args, **kwargs)
        self.questions.setdefault(question.key, []).append(question)
        return question

def add ( self , * args , ** kwargs ) : if 'question' in kwargs and isinstance ( kwargs [ 'question' ] , Question ) : question = kwargs [ 'question' ] else : question = Question ( * args , ** kwargs ) self . questions . setdefault ( question . key , [ ] ) . append ( question ) return question

Add a Question instance to the questions dict . Each key points to a list of Question instances with that key . Use the question kwarg to pass a Question instance if you want or pass in the same args you would pass to instantiate a question .


def ask(self, error=None):
        """Asks the next question in the questionnaire and returns the answer,
        unless user goes back.
        """
        q = self.next_question
        if q is None:
            return

        try:
            answer = q.prompter(self.get_prompt(q, error), *q.prompter_args, **q.prompter_kwargs)
        except QuestionnaireGoBack as e:
            steps = e.args[0] if e.args else 1
            if steps == 0:
                self.ask()  # user can redo current question even if `can_go_back` is `False`
                return
            self.go_back(steps)
        else:
            if q._validate:
                error = q._validate(answer)
                if error:
                    self.ask(error)
                    return
            if q._transform:
                answer = q._transform(answer)
            self.answers[q.key] = answer
            return answer

def ask ( self , error = None ) : q = self . next_question if q is None : return try : answer = q . prompter ( self . get_prompt ( q , error ) , * q . prompter_args , ** q . prompter_kwargs ) except QuestionnaireGoBack as e : steps = e . args [ 0 ] if e . args else 1 if steps == 0 : self . ask ( ) return self . go_back ( steps ) else : if q . _validate : error = q . _validate ( answer ) if error : self . ask ( error ) return if q . _transform : answer = q . _transform ( answer ) self . answers [ q . key ] = answer return answer

Asks the next question in the questionnaire and returns the answer unless user goes back .


def next_question(self):
        """Returns the next `Question` in the questionnaire, or `None` if there
        are no questions left. Returns first question for whose key there is no
        answer and for which condition is satisfied, or for which there is no
        condition.
        """
        for key, questions in self.questions.items():
            if key in self.answers:
                continue
            for question in questions:
                if self.check_condition(question._condition):
                    return question
        return None

def next_question ( self ) : for key , questions in self . questions . items ( ) : if key in self . answers : continue for question in questions : if self . check_condition ( question . _condition ) : return question return None

Returns the next Question in the questionnaire or None if there are no questions left . Returns first question for whose key there is no answer and for which condition is satisfied or for which there is no condition .


def go_back(self, n=1):
        """Move `n` questions back in the questionnaire by removing the last `n`
        answers.
        """
        if not self.can_go_back:
            return
        N = max(len(self.answers)-abs(n), 0)
        self.answers = OrderedDict(islice(self.answers.items(), N))

def go_back ( self , n = 1 ) : if not self . can_go_back : return N = max ( len ( self . answers ) - abs ( n ) , 0 ) self . answers = OrderedDict ( islice ( self . answers . items ( ) , N ) )

Move n questions back in the questionnaire by removing the last n answers .


def format_answers(self, fmt='obj'):
        """Formats answers depending on `fmt`.
        """
        fmts = ('obj', 'array', 'plain')
        if fmt not in fmts:
            eprint("Error: '{}' not in {}".format(fmt, fmts))
            return

        def stringify(val):
            if type(val) in (list, tuple):
                return ', '.join(str(e) for e in val)
            return val

        if fmt == 'obj':
            return json.dumps(self.answers)
        elif fmt == 'array':
            answers = [[k, v] for k, v in self.answers.items()]
            return json.dumps(answers)
        elif fmt == 'plain':
            answers = '\n'.join('{}: {}'.format(k, stringify(v)) for k, v in self.answers.items())
            return answers

def
format_answers
(
self
,
fmt
=
'obj'
)
:
fmts
=
(
'obj'
,
'array'
,
'plain'
)
if
fmt
not
in
fmts
:
eprint
(
"Error: '{}' not in {}"
.
format
(
fmt
,
fmts
)
)
return
def
stringify
(
val
)
:
if
type
(
val
)
in
(
list
,
tuple
)
:
return
', '
.
join
(
str
(
e
)
for
e
in
val
)
return
val
if
fmt
==
'obj'
:
return
json
.
dumps
(
self
.
answers
)
elif
fmt
==
'array'
:
answers
=
[
[
k
,
v
]
for
k
,
v
in
self
.
answers
.
items
(
)
]
return
json
.
dumps
(
answers
)
elif
fmt
==
'plain'
:
answers
=
'\n'
.
join
(
'{}: {}'
.
format
(
k
,
stringify
(
v
)
)
for
k
,
v
in
self
.
answers
.
items
(
)
)
return
answers


def answer_display(self, s=''):
        """Helper method for displaying the answers so far.
        """
        padding = len(max(self.questions.keys(), key=len)) + 5
        for key in list(self.answers.keys()):
            s += '{:>{}} : {}\n'.format(key, padding, self.answers[key])
        return s

def
answer_display
(
self
,
s
=
''
)
:
padding
=
len
(
max
(
self
.
questions
.
keys
(
)
,
key
=
len
)
)
+
5
for
key
in
list
(
self
.
answers
.
keys
(
)
)
:
s
+=
'{:>{}} : {}\n'
.
format
(
key
,
padding
,
self
.
answers
[
key
]
)
return
s

Helper method for displaying the answers so far .


def add_intent(self, name, lines, reload_cache=False):
        """
        Creates a new intent, optionally checking the cache first

        Args:
            name (str): The associated name of the intent
            lines (list<str>): All the sentences that should activate the intent
            reload_cache: Whether to ignore cached intent if exists
        """
        self.intents.add(name, lines, reload_cache)
        self.padaos.add_intent(name, lines)
        self.must_train = True

def add_intent ( self , name , lines , reload_cache = False ) : self . intents . add ( name , lines , reload_cache ) self . padaos . add_intent ( name , lines ) self . must_train = True

Creates a new intent optionally checking the cache first


def add_entity(self, name, lines, reload_cache=False):
        """
        Adds an entity that matches the given lines.

        Example:
            self.add_intent('weather', ['will it rain on {weekday}?'])
            self.add_entity('{weekday}', ['monday', 'tuesday', 'wednesday'])  # ...

        Args:
            name (str): The name of the entity
            lines (list<str>): Lines of example extracted entities
            reload_cache (bool): Whether to refresh all of cache
        """
        Entity.verify_name(name)
        self.entities.add(Entity.wrap_name(name), lines, reload_cache)
        self.padaos.add_entity(name, lines)
        self.must_train = True

def add_entity ( self , name , lines , reload_cache = False ) : Entity . verify_name ( name ) self . entities . add ( Entity . wrap_name ( name ) , lines , reload_cache ) self . padaos . add_entity ( name , lines ) self . must_train = True

Adds an entity that matches the given lines .


def load_entity(self, name, file_name, reload_cache=False):
        """
       Loads an entity, optionally checking the cache first

       Args:
           name (str): The associated name of the entity
           file_name (str): The location of the entity file
           reload_cache (bool): Whether to refresh all of cache
       """
        Entity.verify_name(name)
        self.entities.load(Entity.wrap_name(name), file_name, reload_cache)
        with open(file_name) as f:
            self.padaos.add_entity(name, f.read().split('\n'))
        self.must_train = True

def load_entity ( self , name , file_name , reload_cache = False ) : Entity . verify_name ( name ) self . entities . load ( Entity . wrap_name ( name ) , file_name , reload_cache ) with open ( file_name ) as f : self . padaos . add_entity ( name , f . read ( ) . split ( '\n' ) ) self . must_train = True

Loads an entity optionally checking the cache first


def load_intent(self, name, file_name, reload_cache=False):
        """
        Loads an intent, optionally checking the cache first

        Args:
            name (str): The associated name of the intent
            file_name (str): The location of the intent file
            reload_cache (bool): Whether to refresh all of cache
        """
        self.intents.load(name, file_name, reload_cache)
        with open(file_name) as f:
            self.padaos.add_intent(name, f.read().split('\n'))
        self.must_train = True

def load_intent ( self , name , file_name , reload_cache = False ) : self . intents . load ( name , file_name , reload_cache ) with open ( file_name ) as f : self . padaos . add_intent ( name , f . read ( ) . split ( '\n' ) ) self . must_train = True

Loads an intent optionally checking the cache first


def remove_intent(self, name):
        """Unload an intent"""
        self.intents.remove(name)
        self.padaos.remove_intent(name)
        self.must_train = True

def remove_intent ( self , name ) : self . intents . remove ( name ) self . padaos . remove_intent ( name ) self . must_train = True


def remove_entity(self, name):
        """Unload an entity"""
        self.entities.remove(name)
        self.padaos.remove_entity(name)

def remove_entity ( self , name ) : self . entities . remove ( name ) self . padaos . remove_entity ( name )


def train(self, debug=True, force=False, single_thread=False, timeout=20):
        """
        Trains all the loaded intents that need to be updated
        If a cache file exists with the same hash as the intent file,
        the intent will not be trained and just loaded from file

        Args:
            debug (bool): Whether to print a message to stdout each time a new intent is trained
            force (bool): Whether to force training if already finished
            single_thread (bool): Whether to force running in a single thread
            timeout (float): Seconds before cancelling training
        Returns:
            bool: True if training succeeded without timeout
        """
        if not self.must_train and not force:
            return
        self.padaos.compile()
        self.train_thread = Thread(target=self._train, kwargs=dict(
            debug=debug,
            single_thread=single_thread,
            timeout=timeout
        ), daemon=True)
        self.train_thread.start()
        self.train_thread.join(timeout)

        self.must_train = False
        return not self.train_thread.is_alive()

def train ( self , debug = True , force = False , single_thread = False , timeout = 20 ) : if not self . must_train and not force : return self . padaos . compile ( ) self . train_thread = Thread ( target = self . _train , kwargs = dict ( debug = debug , single_thread = single_thread , timeout = timeout ) , daemon = True ) self . train_thread . start ( ) self . train_thread . join ( timeout ) self . must_train = False return not self . train_thread . is_alive ( )

Trains all the loaded intents that need to be updated If a cache file exists with the same hash as the intent file the intent will not be trained and just loaded from file


def train_subprocess(self, *args, **kwargs):
        """
        Trains in a subprocess which provides a timeout guarantees everything shuts down properly

        Args:
            See <train>
        Returns:
            bool: True for success, False if timed out
        """
        ret = call([
            sys.executable, '-m', 'padatious', 'train', self.cache_dir,
            '-d', json.dumps(self.serialized_args),
            '-a', json.dumps(args),
            '-k', json.dumps(kwargs),
        ])
        if ret == 2:
            raise TypeError('Invalid train arguments: {} {}'.format(args, kwargs))
        data = self.serialized_args
        self.clear()
        self.apply_training_args(data)
        self.padaos.compile()
        if ret == 0:
            self.must_train = False
            return True
        elif ret == 10:  # timeout
            return False
        else:
            raise ValueError('Training failed and returned code: {}'.format(ret))

def
train_subprocess
(
self
,
*
args
,
**
kwargs
)
:
ret
=
call
(
[
sys
.
executable
,
'-m'
,
'padatious'
,
'train'
,
self
.
cache_dir
,
'-d'
,
json
.
dumps
(
self
.
serialized_args
)
,
'-a'
,
json
.
dumps
(
args
)
,
'-k'
,
json
.
dumps
(
kwargs
)
,
]
)
if
ret
==
2
:
raise
TypeError
(
'Invalid train arguments: {} {}'
.
format
(
args
,
kwargs
)
)
data
=
self
.
serialized_args
self
.
clear
(
)
self
.
apply_training_args
(
data
)
self
.
padaos
.
compile
(
)
if
ret
==
0
:
self
.
must_train
=
False
return
True
elif
ret
==
10
:
return
False
else
:
raise
ValueError
(
'Training failed and returned code: {}'
.
format
(
ret
)
)


def calc_intents(self, query):
        """
        Tests all the intents against the query and returns
        data on how well each one matched against the query

        Args:
            query (str): Input sentence to test against intents
        Returns:
            list<MatchData>: List of intent matches
        See calc_intent() for a description of the returned MatchData
        """
        if self.must_train:
            self.train()
        intents = {} if self.train_thread and self.train_thread.is_alive() else {
            i.name: i for i in self.intents.calc_intents(query, self.entities)
        }
        sent = tokenize(query)
        for perfect_match in self.padaos.calc_intents(query):
            name = perfect_match['name']
            intents[name] = MatchData(name, sent, matches=perfect_match['entities'], conf=1.0)
        return list(intents.values())

def calc_intents ( self , query ) : if self . must_train : self . train ( ) intents = { } if self . train_thread and self . train_thread . is_alive ( ) else { i . name : i for i in self . intents . calc_intents ( query , self . entities ) } sent = tokenize ( query ) for perfect_match in self . padaos . calc_intents ( query ) : name = perfect_match [ 'name' ] intents [ name ] = MatchData ( name , sent , matches = perfect_match [ 'entities' ] , conf = 1.0 ) return list ( intents . values ( ) )


def calc_intent(self, query):
        """
        Tests all the intents against the query and returns
        match data of the best intent

        Args:
            query (str): Input sentence to test against intents
        Returns:
            MatchData: Best intent match
        """
        matches = self.calc_intents(query)
        if len(matches) == 0:
            return MatchData('', '')
        best_match = max(matches, key=lambda x: x.conf)
        best_matches = (match for match in matches if match.conf == best_match.conf)
        return min(best_matches, key=lambda x: sum(map(len, x.matches.values())))

def calc_intent ( self , query ) : matches = self . calc_intents ( query ) if len ( matches ) == 0 : return MatchData ( '' , '' ) best_match = max ( matches , key = lambda x : x . conf ) best_matches = ( match for match in matches if match . conf == best_match . conf ) return min ( best_matches , key = lambda x : sum ( map ( len , x . matches . values ( ) ) ) )


def _train_and_save(obj, cache, data, print_updates):
    """Internal pickleable function used to train objects in another process"""
    obj.train(data)
    if print_updates:
        print('Regenerated ' + obj.name + '.')
    obj.save(cache)

def
_train_and_save
(
obj
,
cache
,
data
,
print_updates
)
:
obj
.
train
(
data
)
if
print_updates
:
print
(
'Regenerated '
+
obj
.
name
+
'.'
)
obj
.
save
(
cache
)


def main(src, pyi_dir, target_dir, incremental, quiet, replace_any, hg, traceback):
    """Re-apply type annotations from .pyi stubs to your codebase."""
    Config.incremental = incremental
    Config.replace_any = replace_any
    returncode = 0
    for src_entry in src:
        for file, error, exc_type, tb in retype_path(
            Path(src_entry),
            pyi_dir=Path(pyi_dir),
            targets=Path(target_dir),
            src_explicitly_given=True,
            quiet=quiet,
            hg=hg,
        ):
            print(f'error: {file}: {error}', file=sys.stderr)
            if traceback:
                print('Traceback (most recent call last):', file=sys.stderr)
                for line in tb:
                    print(line, file=sys.stderr, end='')
                print(f'{exc_type.__name__}: {error}', file=sys.stderr)
            returncode += 1
    if not src and not quiet:
        print('warning: no sources given', file=sys.stderr)

    # According to http://tldp.org/LDP/abs/html/index.html starting with 126
    # we have special returncodes.
    sys.exit(min(returncode, 125))

def
main
(
src
,
pyi_dir
,
target_dir
,
incremental
,
quiet
,
replace_any
,
hg
,
traceback
)
:
Config
.
incremental
=
incremental
Config
.
replace_any
=
replace_any
returncode
=
0
for
src_entry
in
src
:
for
file
,
error
,
exc_type
,
tb
in
retype_path
(
Path
(
src_entry
)
,
pyi_dir
=
Path
(
pyi_dir
)
,
targets
=
Path
(
target_dir
)
,
src_explicitly_given
=
True
,
quiet
=
quiet
,
hg
=
hg
,
)
:
print
(
f'error: {file}: {error}'
,
file
=
sys
.
stderr
)
if
traceback
:
print
(
'Traceback (most recent call last):'
,
file
=
sys
.
stderr
)
for
line
in
tb
:
print
(
line
,
file
=
sys
.
stderr
,
end
=
''
)
print
(
f'{exc_type.__name__}: {error}'
,
file
=
sys
.
stderr
)
returncode
+=
1
if
not
src
and
not
quiet
:
print
(
'warning: no sources given'
,
file
=
sys
.
stderr
)
sys
.
exit
(
min
(
returncode
,
125
)
)

Re - apply type annotations from . pyi stubs to your codebase .


def retype_path(
    src, pyi_dir, targets, *, src_explicitly_given=False, quiet=False, hg=False
):
    """Recursively retype files or directories given. Generate errors."""
    if src.is_dir():
        for child in src.iterdir():
            if child == pyi_dir or child == targets:
                continue
            yield from retype_path(
                child, pyi_dir / src.name, targets / src.name, quiet=quiet, hg=hg,
            )
    elif src.suffix == '.py' or src_explicitly_given:
        try:
            retype_file(src, pyi_dir, targets, quiet=quiet, hg=hg)
        except Exception as e:
            yield (
                src,
                str(e),
                type(e),
                traceback.format_tb(e.__traceback__),
            )

def retype_path ( src , pyi_dir , targets , * , src_explicitly_given = False , quiet = False , hg = False ) : if src . is_dir ( ) : for child in src . iterdir ( ) : if child == pyi_dir or child == targets : continue yield from retype_path ( child , pyi_dir / src . name , targets / src . name , quiet = quiet , hg = hg , ) elif src . suffix == '.py' or src_explicitly_given : try : retype_file ( src , pyi_dir , targets , quiet = quiet , hg = hg ) except Exception as e : yield ( src , str ( e ) , type ( e ) , traceback . format_tb ( e . __traceback__ ) , )

Recursively retype files or directories given . Generate errors .


def retype_file(src, pyi_dir, targets, *, quiet=False, hg=False):
    """Retype `src`, finding types in `pyi_dir`. Save in `targets`.

    The file should remain formatted exactly as it was before, save for:
    - annotations
    - additional imports needed to satisfy annotations
    - additional module-level names needed to satisfy annotations

    Type comments in sources are normalized to type annotations.
    """
    with tokenize.open(src) as src_buffer:
        src_encoding = src_buffer.encoding
        src_node = lib2to3_parse(src_buffer.read())
    try:
        with open((pyi_dir / src.name).with_suffix('.pyi')) as pyi_file:
            pyi_txt = pyi_file.read()
    except FileNotFoundError:
        if not quiet:
            print(
                f'warning: .pyi file for source {src} not found in {pyi_dir}',
                file=sys.stderr,
            )
    else:
        pyi_ast = ast3.parse(pyi_txt)
        assert isinstance(pyi_ast, ast3.Module)
        reapply_all(pyi_ast.body, src_node)
    fix_remaining_type_comments(src_node)
    targets.mkdir(parents=True, exist_ok=True)
    with open(targets / src.name, 'w', encoding=src_encoding) as target_file:
        target_file.write(lib2to3_unparse(src_node, hg=hg))
    return targets / src.name

Retype src finding types in pyi_dir . Save in targets .


def lib2to3_parse(src_txt):
    """Given a string with source, return the lib2to3 Node."""
    grammar = pygram.python_grammar_no_print_statement
    drv = driver.Driver(grammar, pytree.convert)
    if src_txt[-1] != '\n':
        nl = '\r\n' if '\r\n' in src_txt[:1024] else '\n'
        src_txt += nl
    try:
        result = drv.parse_string(src_txt, True)
    except ParseError as pe:
        lineno, column = pe.context[1]
        lines = src_txt.splitlines()
        try:
            faulty_line = lines[lineno - 1]
        except IndexError:
            faulty_line = "<line number missing in source>"
        raise ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}") from None

    if isinstance(result, Leaf):
        result = Node(syms.file_input, [result])

    return result

Given a string with source return the lib2to3 Node .


def lib2to3_unparse(node, *, hg=False):
    """Given a lib2to3 node, return its string representation."""
    code = str(node)
    if hg:
        from retype_hgext import apply_job_security
        code = apply_job_security(code)
    return code

def lib2to3_unparse ( node , * , hg = False ) : code = str ( node ) if hg : from retype_hgext import apply_job_security code = apply_job_security ( code ) return code

Given a lib2to3 node return its string representation .


def reapply_all(ast_node, lib2to3_node):
    """Reapplies the typed_ast node into the lib2to3 tree.

    Also does post-processing. This is done in reverse order to enable placing
    TypeVars and aliases that depend on one another.
    """
    late_processing = reapply(ast_node, lib2to3_node)
    for lazy_func in reversed(late_processing):
        lazy_func()

def reapply_all ( ast_node , lib2to3_node ) : late_processing = reapply ( ast_node , lib2to3_node ) for lazy_func in reversed ( late_processing ) : lazy_func ( )

Reapplies the typed_ast node into the lib2to3 tree .


def fix_remaining_type_comments(node):
    """Converts type comments in `node` to proper annotated assignments."""
    assert node.type == syms.file_input

    last_n = None
    for n in node.post_order():
        if last_n is not None:
            if n.type == token.NEWLINE and is_assignment(last_n):
                fix_variable_annotation_type_comment(n, last_n)
            elif n.type == syms.funcdef and last_n.type == syms.suite:
                fix_signature_annotation_type_comment(n, last_n, offset=1)
            elif n.type == syms.async_funcdef and last_n.type == syms.suite:
                fix_signature_annotation_type_comment(n, last_n, offset=2)
        last_n = n

def fix_remaining_type_comments ( node ) : assert node . type == syms . file_input last_n = None for n in node . post_order ( ) : if last_n is not None : if n . type == token . NEWLINE and is_assignment ( last_n ) : fix_variable_annotation_type_comment ( n , last_n ) elif n . type == syms . funcdef and last_n . type == syms . suite : fix_signature_annotation_type_comment ( n , last_n , offset = 1 ) elif n . type == syms . async_funcdef and last_n . type == syms . suite : fix_signature_annotation_type_comment ( n , last_n , offset = 2 ) last_n = n

Converts type comments in node to proper annotated assignments .


def parse_signature_type_comment(type_comment):
    """Parse the fugly signature type comment into AST nodes.

    Caveats: ASTifying **kwargs is impossible with the current grammar so we
    hack it into unary subtraction (to differentiate from Starred in vararg).

    For example from:
    "(str, int, *int, **Any) -> 'SomeReturnType'"

    To:
    ([ast3.Name, ast.Name, ast3.Name, ast.Name], ast3.Str)
    """
    try:
        result = ast3.parse(type_comment, '<func_type>', 'func_type')
    except SyntaxError:
        raise ValueError(f"invalid function signature type comment: {type_comment!r}")

    assert isinstance(result, ast3.FunctionType)
    if len(result.argtypes) == 1:
        argtypes = result.argtypes[0]
    else:
        argtypes = result.argtypes
    return argtypes, result.returns

Parse the fugly signature type comment into AST nodes .


def parse_type_comment(type_comment):
    """Parse a type comment string into AST nodes."""
    try:
        result = ast3.parse(type_comment, '<type_comment>', 'eval')
    except SyntaxError:
        raise ValueError(f"invalid type comment: {type_comment!r}") from None

    assert isinstance(result, ast3.Expression)
    return result.body

Parse a type comment string into AST nodes .


def copy_arguments_to_annotations(args, type_comment, *, is_method=False):
    """Copies AST nodes from `type_comment` into the ast3.arguments in `args`.

    Does validaation of argument count (allowing for untyped self/cls)
    and type (vararg and kwarg).
    """
    if isinstance(type_comment, ast3.Ellipsis):
        return

    expected = len(args.args)
    if args.vararg:
        expected += 1
    expected += len(args.kwonlyargs)
    if args.kwarg:
        expected += 1
    actual = len(type_comment) if isinstance(type_comment, list) else 1
    if expected != actual:
        if is_method and expected - actual == 1:
            pass  # fine, we're just skipping `self`, `cls`, etc.
        else:
            raise ValueError(
                f"number of arguments in type comment doesn't match; " +
                f"expected {expected}, found {actual}"
            )

    if isinstance(type_comment, list):
        next_value = type_comment.pop
    else:
        # If there's just one value, only one of the loops and ifs below will
        # be populated. We ensure this with the expected/actual length check
        # above.
        _tc = type_comment

        def next_value(index: int = 0) -> ast3.expr:
            return _tc

    for arg in args.args[expected - actual:]:
        ensure_no_annotation(arg.annotation)
        arg.annotation = next_value(0)

    if args.vararg:
        ensure_no_annotation(args.vararg.annotation)
        args.vararg.annotation = next_value(0)

    for arg in args.kwonlyargs:
        ensure_no_annotation(arg.annotation)
        arg.annotation = next_value(0)

    if args.kwarg:
        ensure_no_annotation(args.kwarg.annotation)
        args.kwarg.annotation = next_value(0)

Copies AST nodes from type_comment into the ast3 . arguments in args .


def copy_type_comments_to_annotations(args):
    """Copies argument type comments from the legacy long form to annotations
    in the entire function signature.
    """
    for arg in args.args:
        copy_type_comment_to_annotation(arg)

    if args.vararg:
        copy_type_comment_to_annotation(args.vararg)

    for arg in args.kwonlyargs:
        copy_type_comment_to_annotation(arg)

    if args.kwarg:
        copy_type_comment_to_annotation(args.kwarg)

def copy_type_comments_to_annotations ( args ) : for arg in args . args : copy_type_comment_to_annotation ( arg ) if args . vararg : copy_type_comment_to_annotation ( args . vararg ) for arg in args . kwonlyargs : copy_type_comment_to_annotation ( arg ) if args . kwarg : copy_type_comment_to_annotation ( args . kwarg )

Copies argument type comments from the legacy long form to annotations in the entire function signature .


def maybe_replace_any_if_equal(name, expected, actual):
    """Return the type given in `expected`.

    Raise ValueError if `expected` isn't equal to `actual`.  If --replace-any is
    used, the Any type in `actual` is considered equal.

    The implementation is naively checking if the string representation of
    `actual` is one of "Any", "typing.Any", or "t.Any".  This is done for two
    reasons:

    1. I'm lazy.
    2. We want people to be able to explicitly state that they want Any without it
       being replaced.  This way they can use an alias.
    """
    is_equal = expected == actual
    if not is_equal and Config.replace_any:
        actual_str = minimize_whitespace(str(actual))
        if actual_str and actual_str[0] in {'"', "'"}:
            actual_str = actual_str[1:-1]
        is_equal = actual_str in {'Any', 'typing.Any', 't.Any'}

    if not is_equal:
        expected_annotation = minimize_whitespace(str(expected))
        actual_annotation = minimize_whitespace(str(actual))
        raise ValueError(
            f"incompatible existing {name}. " +
            f"Expected: {expected_annotation!r}, actual: {actual_annotation!r}"
        )

    return expected or actual


def remove_function_signature_type_comment(body):
    """Removes the legacy signature type comment, leaving other comments if any."""
    for node in body.children:
        if node.type == token.INDENT:
            prefix = node.prefix.lstrip()
            if prefix.startswith('# type: '):
                node.prefix = '\n'.join(prefix.split('\n')[1:])
            break

def
remove_function_signature_type_comment
(
body
)
:
for
node
in
body
.
children
:
if
node
.
type
==
token
.
INDENT
:
prefix
=
node
.
prefix
.
lstrip
(
)
if
prefix
.
startswith
(
'# type: '
)
:
node
.
prefix
=
'\n'
.
join
(
prefix
.
split
(
'\n'
)
[
1
:
]
)
break

Removes the legacy signature type comment leaving other comments if any .


def get_offset_and_prefix(body, skip_assignments=False):
    """Returns the offset after which a statement can be inserted to the `body`.

    This offset is calculated to come after all imports, and maybe existing
    (possibly annotated) assignments if `skip_assignments` is True.

    Also returns the indentation prefix that should be applied to the inserted
    node.
    """
    assert body.type in (syms.file_input, syms.suite)

    _offset = 0
    prefix = ''
    for _offset, child in enumerate(body.children):
        if child.type == syms.simple_stmt:
            stmt = child.children[0]
            if stmt.type == syms.expr_stmt:
                expr = stmt.children
                if not skip_assignments:
                    break

                if (
                    len(expr) != 2 or
                    expr[0].type != token.NAME or
                    expr[1].type != syms.annassign or
                    _eq in expr[1].children
                ):
                    break

            elif stmt.type not in (syms.import_name, syms.import_from, token.STRING):
                break

        elif child.type == token.INDENT:
            assert isinstance(child, Leaf)
            prefix = child.value
        elif child.type != token.NEWLINE:
            break

    prefix, child.prefix = child.prefix, prefix
    return _offset, prefix

def get_offset_and_prefix ( body , skip_assignments = False ) : assert body . type in ( syms . file_input , syms . suite ) _offset = 0 prefix = '' for _offset , child in enumerate ( body . children ) : if child . type == syms . simple_stmt : stmt = child . children [ 0 ] if stmt . type == syms . expr_stmt : expr = stmt . children if not skip_assignments : break if ( len ( expr ) != 2 or expr [ 0 ] . type != token . NAME or expr [ 1 ] . type != syms . annassign or _eq in expr [ 1 ] . children ) : break elif stmt . type not in ( syms . import_name , syms . import_from , token . STRING ) : break elif child . type == token . INDENT : assert isinstance ( child , Leaf ) prefix = child . value elif child . type != token . NEWLINE : break prefix , child . prefix = child . prefix , prefix return _offset , prefix

Returns the offset after which a statement can be inserted to the body .


def fix_line_numbers(body):
    r"""Recomputes all line numbers based on the number of \n characters."""
    maxline = 0
    for node in body.pre_order():
        maxline += node.prefix.count('\n')
        if isinstance(node, Leaf):
            node.lineno = maxline
            maxline += str(node.value).count('\n')

def fix_line_numbers ( body ) : r maxline = 0 for node in body . pre_order ( ) : maxline += node . prefix . count ( '\n' ) if isinstance ( node , Leaf ) : node . lineno = maxline maxline += str ( node . value ) . count ( '\n' )

r Recomputes all line numbers based on the number of \ n characters .


def new(n, prefix=None):
    """lib2to3's AST requires unique objects as children."""

    if isinstance(n, Leaf):
        return Leaf(n.type, n.value, prefix=n.prefix if prefix is None else prefix)

    # this is hacky, we assume complex nodes are just being reused once from the
    # original AST.
    n.parent = None
    if prefix is not None:
        n.prefix = prefix
    return n

def new ( n , prefix = None ) : if isinstance ( n , Leaf ) : return Leaf ( n . type , n . value , prefix = n . prefix if prefix is None else prefix ) n . parent = None if prefix is not None : n . prefix = prefix return n

lib2to3 s AST requires unique objects as children .


def _load_info(self):
        '''Get user info for GBDX S3, put into instance vars for convenience.

        Args:
            None.

        Returns:
            Dictionary with S3 access key, S3 secret key, S3 session token,
            user bucket and user prefix (dict).
        '''

        url = '%s/prefix?duration=36000' % self.base_url
        r = self.gbdx_connection.get(url)
        r.raise_for_status()
        return r.json()

def _load_info ( self ) : url = '%s/prefix?duration=36000' % self . base_url r = self . gbdx_connection . get ( url ) r . raise_for_status ( ) return r . json ( )

Get user info for GBDX S3 put into instance vars for convenience .


def histogram_equalize(self, use_bands, **kwargs):
        ''' Equalize and the histogram and normalize value range
            Equalization is on all three bands, not per-band'''
        data = self._read(self[use_bands,...], **kwargs)
        data = np.rollaxis(data.astype(np.float32), 0, 3)
        flattened = data.flatten()
        if 0 in data:
            masked = np.ma.masked_values(data, 0).compressed()
            image_histogram, bin_edges = np.histogram(masked, 256)
        else:
            image_histogram, bin_edges = np.histogram(flattened, 256)
        bins = (bin_edges[:-1] + bin_edges[1:]) / 2.0
        cdf = image_histogram.cumsum()
        cdf = cdf / float(cdf[-1])
        image_equalized = np.interp(flattened, bins, cdf).reshape(data.shape)
        if 'stretch' in kwargs or 'gamma' in kwargs:
            return self._histogram_stretch(image_equalized, **kwargs)
        else:
            return image_equalized

def histogram_equalize ( self , use_bands , ** kwargs ) : data = self . _read ( self [ use_bands , ... ] , ** kwargs ) data = np . rollaxis ( data . astype ( np . float32 ) , 0 , 3 ) flattened = data . flatten ( ) if 0 in data : masked = np . ma . masked_values ( data , 0 ) . compressed ( ) image_histogram , bin_edges = np . histogram ( masked , 256 ) else : image_histogram , bin_edges = np . histogram ( flattened , 256 ) bins = ( bin_edges [ : - 1 ] + bin_edges [ 1 : ] ) / 2.0 cdf = image_histogram . cumsum ( ) cdf = cdf / float ( cdf [ - 1 ] ) image_equalized = np . interp ( flattened , bins , cdf ) . reshape ( data . shape ) if 'stretch' in kwargs or 'gamma' in kwargs : return self . _histogram_stretch ( image_equalized , ** kwargs ) else : return image_equalized

Equalize and the histogram and normalize value range Equalization is on all three bands not per - band


def histogram_match(self, use_bands, blm_source=None, **kwargs):
        ''' Match the histogram to existing imagery '''
        assert has_rio, "To match image histograms please install rio_hist"
        data = self._read(self[use_bands,...], **kwargs)
        data = np.rollaxis(data.astype(np.float32), 0, 3)
        if 0 in data:
            data = np.ma.masked_values(data, 0)
        bounds = self._reproject(box(*self.bounds), from_proj=self.proj, to_proj="EPSG:4326").bounds
        if blm_source == 'browse':
            from gbdxtools.images.browse_image import BrowseImage
            ref = BrowseImage(self.cat_id, bbox=bounds).read()
        else:
            from gbdxtools.images.tms_image import TmsImage
            tms = TmsImage(zoom=self._calc_tms_zoom(self.affine[0]), bbox=bounds, **kwargs)
            ref = np.rollaxis(tms.read(), 0, 3)
        out = np.dstack([rio_match(data[:,:,idx], ref[:,:,idx].astype(np.double)/255.0)
                        for idx in range(data.shape[-1])])
        if 'stretch' in kwargs or 'gamma' in kwargs:
            return self._histogram_stretch(out, **kwargs)
        else:
            return out


def histogram_stretch(self, use_bands, **kwargs):
        ''' entry point for contrast stretching '''
        data = self._read(self[use_bands,...], **kwargs)
        data = np.rollaxis(data.astype(np.float32), 0, 3)
        return self._histogram_stretch(data, **kwargs)

def histogram_stretch ( self , use_bands , ** kwargs ) : data = self . _read ( self [ use_bands , ... ] , ** kwargs ) data = np . rollaxis ( data . astype ( np . float32 ) , 0 , 3 ) return self . _histogram_stretch ( data , ** kwargs )


def ndvi(self, **kwargs):
        """
        Calculates Normalized Difference Vegetation Index using NIR and Red of an image.

        Returns: numpy array with ndvi values
        """
        data = self._read(self[self._ndvi_bands,...]).astype(np.float32)
        return (data[0,:,:] - data[1,:,:]) / (data[0,:,:] + data[1,:,:])

def ndvi ( self , ** kwargs ) : data = self . _read ( self [ self . _ndvi_bands , ... ] ) . astype ( np . float32 ) return ( data [ 0 , : , : ] - data [ 1 , : , : ] ) / ( data [ 0 , : , : ] + data [ 1 , : , : ] )

Calculates Normalized Difference Vegetation Index using NIR and Red of an image .


def ndwi(self):
        """
        Calculates Normalized Difference Water Index using Coastal and NIR2 bands for WV02, WV03.
        For Landsat8 and sentinel2 calculated by using Green and NIR bands.

        Returns: numpy array of ndwi values
        """
        data = self._read(self[self._ndwi_bands,...]).astype(np.float32)
        return (data[1,:,:] - data[0,:,:]) / (data[0,:,:] + data[1,:,:])

def ndwi ( self ) : data = self . _read ( self [ self . _ndwi_bands , ... ] ) . astype ( np . float32 ) return ( data [ 1 , : , : ] - data [ 0 , : , : ] ) / ( data [ 0 , : , : ] + data [ 1 , : , : ] )

Calculates Normalized Difference Water Index using Coastal and NIR2 bands for WV02 WV03 . For Landsat8 and sentinel2 calculated by using Green and NIR bands .


def plot(self, spec="rgb", **kwargs):
        ''' Plot the image with MatplotLib

        Plot sizing includes default borders and spacing. If the image is shown in Jupyter the outside whitespace will be automatically cropped to save size, resulting in a smaller sized image than expected.

        Histogram options:
            * 'equalize': performs histogram equalization on the image.
            * 'minmax': stretch the pixel range to the minimum and maximum input pixel values. Equivalent to stretch=[0,100].
            * 'match': match the histogram to the Maps API imagery. Pass the additional keyword blm_source='browse' to match to the Browse Service (image thumbnail) instead.
            * 'ignore': Skip dynamic range adjustment, in the event the image is already correctly balanced and the values are in the correct range.

        Gamma values greater than 1 will brighten the image midtones, values less than 1 will darken the midtones.

        Plots generated with the histogram options of 'match' and 'equalize' can be combined with the stretch and gamma options. The stretch and gamma adjustments will be applied after the histogram adjustments.

        Args:
            w (float or int): width of plot in inches at 72 dpi, default is 10
            h (float or int): height of plot in inches at 72 dpi, default is 10
            title (str): Title to use on the plot
            fontsize (int): Size of title font, default is 22. Size is measured in points.
            bands (list): bands to use for plotting, such as bands=[4,2,1]. Defaults to the image's natural RGB bands. This option is useful for generating pseudocolor images when passed a list of three bands. If only a single band is provided, a colormapped plot will be generated instead.
            cmap (str): MatPlotLib colormap name to use for single band images. Default is colormap='Grey_R'.
            histogram (str): either 'equalize', 'minmax', 'match', or ignore
            stretch (list): stretch the histogram between two percentile values, default is [2,98]
            gamma (float): adjust image gamma, default is 1.0
        '''

        if self.shape[0] == 1 or ("bands" in kwargs and len(kwargs["bands"]) == 1):
            if "cmap" in kwargs:
                cmap = kwargs["cmap"]
                del kwargs["cmap"]
            else:
                cmap = "Greys_r"
            self._plot(tfm=self._single_band, cmap=cmap, **kwargs)
        else:
            if spec == "rgb" and self._has_token(**kwargs):
                self._plot(tfm=self.rgb, **kwargs)
            else:
                self._plot(tfm=getattr(self, spec), **kwargs)

def plot ( self , spec = "rgb" , ** kwargs ) : if self . shape [ 0 ] == 1 or ( "bands" in kwargs and len ( kwargs [ "bands" ] ) == 1 ) : if "cmap" in kwargs : cmap = kwargs [ "cmap" ] del kwargs [ "cmap" ] else : cmap = "Greys_r" self . _plot ( tfm = self . _single_band , cmap = cmap , ** kwargs ) else : if spec == "rgb" and self . _has_token ( ** kwargs ) : self . _plot ( tfm = self . rgb , ** kwargs ) else : self . _plot ( tfm = getattr ( self , spec ) , ** kwargs )