def Geometry(*args, **kwargs):
    """Returns an ogr.Geometry instance optionally created from a geojson str
    or dict. The spatial reference may also be provided.
    """
    # Look for geojson as a positional or keyword arg.
    arg = kwargs.pop('geojson', None) or len(args) and args[0]
    try:
        srs = kwargs.pop('srs', None) or arg.srs.wkt
    except AttributeError:
        srs = SpatialReference(4326)
    if hasattr(arg, 'keys'):
        geom = ogr.CreateGeometryFromJson(json.dumps(arg))
    elif hasattr(arg, 'startswith'):
        # WKB as hexadecimal string.
        char = arg[0] if arg else ' '
        i = char if isinstance(char, int) else ord(char)
        if i in (0, 1):
            geom = ogr.CreateGeometryFromWkb(arg)
        elif arg.startswith('{'):
            geom = ogr.CreateGeometryFromJson(arg)
        elif arg.startswith('<gml'):
            geom = ogr.CreateGeometryFromGML(arg)
        else:
            raise ValueError('Invalid geometry value: %s' % arg)
    elif hasattr(arg, 'wkb'):
        geom = ogr.CreateGeometryFromWkb(bytes(arg.wkb))
    else:
        geom = ogr.Geometry(*args, **kwargs)
    if geom:
        if not isinstance(srs, SpatialReference):
            srs = SpatialReference(srs)
        geom.AssignSpatialReference(srs)
    return geom

def
Geometry
(
*
args
,
**
kwargs
)
:
arg
=
kwargs
.
pop
(
'geojson'
,
)
or
len
(
args
)
and
args
[
0
]
try
:
srs
=
kwargs
.
pop
(
'srs'
,
)
or
arg
.
srs
.
wkt
except
AttributeError
:
srs
=
SpatialReference
(
4326
)
if
hasattr
(
arg
,
'keys'
)
:
geom
=
ogr
.
CreateGeometryFromJson
(
json
.
dumps
(
arg
)
)
elif
hasattr
(
arg
,
'startswith'
)
:
char
=
arg
[
0
]
if
arg
else
' '
i
=
char
if
isinstance
(
char
,
int
)
else
ord
(
char
)
if
i
in
(
0
,
1
)
:
geom
=
ogr
.
CreateGeometryFromWkb
(
arg
)
elif
arg
.
startswith
(
'{'
)
:
geom
=
ogr
.
CreateGeometryFromJson
(
arg
)
elif
arg
.
startswith
(
'<gml'
)
:
geom
=
ogr
.
CreateGeometryFromGML
(
arg
)
else
:
raise
ValueError
(
'Invalid geometry value: %s'
%
arg
)
elif
hasattr
(
arg
,
'wkb'
)
:
geom
=
ogr
.
CreateGeometryFromWkb
(
bytes
(
arg
.
wkb
)
)
else
:
geom
=
ogr
.
Geometry
(
*
args
,
**
kwargs
)
if
geom
:
if
not
isinstance
(
srs
,
SpatialReference
)
:
srs
=
SpatialReference
(
srs
)
geom
.
AssignSpatialReference
(
srs
)
return
geom

Returns an ogr . Geometry instance optionally created from a geojson str or dict . The spatial reference may also be provided .


def expand(self, other):
        """Expands this envelope by the given Envelope or tuple.

        Arguments:
        other -- Envelope, two-tuple, or four-tuple
        """
        if len(other) == 2:
            other += other
        mid = len(other) // 2
        self.ll = map(min, self.ll, other[:mid])
        self.ur = map(max, self.ur, other[mid:])

def expand ( self , other ) : if len ( other ) == 2 : other += other mid = len ( other ) // 2 self . ll = map ( min , self . ll , other [ : mid ] ) self . ur = map ( max , self . ur , other [ mid : ] )

Expands this envelope by the given Envelope or tuple .


def intersect(self, other):
        """Returns the intersection of this and another Envelope."""
        inter = Envelope(tuple(self))
        if inter.intersects(other):
            mid = len(other) // 2
            inter.ll = map(max, inter.ll, other[:mid])
            inter.ur = map(min, inter.ur, other[mid:])
        else:
            inter.ll = (0, 0)
            inter.ur = (0, 0)
        return inter

def intersect ( self , other ) : inter = Envelope ( tuple ( self ) ) if inter . intersects ( other ) : mid = len ( other ) // 2 inter . ll = map ( max , inter . ll , other [ : mid ] ) inter . ur = map ( min , inter . ur , other [ mid : ] ) else : inter . ll = ( 0 , 0 ) inter . ur = ( 0 , 0 ) return inter

Returns the intersection of this and another Envelope .


def intersects(self, other):
        """Returns true if this envelope intersects another.

        Arguments:
        other -- Envelope or tuple of (minX, minY, maxX, maxY)
        """
        try:
            return (self.min_x <= other.max_x and
                    self.max_x >= other.min_x and
                    self.min_y <= other.max_y and
                    self.max_y >= other.min_y)
        except AttributeError:
            return self.intersects(Envelope(other))

def intersects ( self , other ) : try : return ( self . min_x <= other . max_x and self . max_x >= other . min_x and self . min_y <= other . max_y and self . max_y >= other . min_y ) except AttributeError : return self . intersects ( Envelope ( other ) )

Returns true if this envelope intersects another .


def polygon(self):
        """Returns an OGR Geometry for this envelope."""
        ring = ogr.Geometry(ogr.wkbLinearRing)
        for coord in self.ll, self.lr, self.ur, self.ul, self.ll:
            ring.AddPoint_2D(*coord)
        polyg = ogr.Geometry(ogr.wkbPolygon)
        polyg.AddGeometryDirectly(ring)
        return polyg

def polygon ( self ) : ring = ogr . Geometry ( ogr . wkbLinearRing ) for coord in self . ll , self . lr , self . ur , self . ul , self . ll : ring . AddPoint_2D ( * coord ) polyg = ogr . Geometry ( ogr . wkbPolygon ) polyg . AddGeometryDirectly ( ring ) return polyg

Returns an OGR Geometry for this envelope .


def from_ZNM(cls, Z, N, M, name=''):
        """
        Creates a table from arrays Z, N and M

        Example:
        ________

        >>> Z = [82, 82, 83]
        >>> N = [126, 127, 130]
        >>> M = [-21.34, -18.0, -14.45]
        >>> Table.from_ZNM(Z, N, M, name='Custom Table')
        Z   N
        82  126   -21.34
            127   -18.00
        83  130   -14.45
        Name: Custom Table, dtype: float64
        """
        df = pd.DataFrame.from_dict({'Z': Z, 'N': N, 'M': M}).set_index(['Z', 'N'])['M']
        df.name = name
        return cls(df=df, name=name)

def from_ZNM ( cls , Z , N , M , name = '' ) : df = pd . DataFrame . from_dict ( { 'Z' : Z , 'N' : N , 'M' : M } ) . set_index ( [ 'Z' , 'N' ] ) [ 'M' ] df . name = name return cls ( df = df , name = name )


def to_file(self, path):
        """Export the contents to a file as comma separated values.

        Parameters
        ----------
        path : string
            File path where the data should be saved to

        Example
        -------
        Export the last ten elements of AME2012 to a new file:

        >>> Table('AME2012').tail(10).to_file('last_ten.txt')
        """
        with open(path, 'w') as f:
            f.write('Z   N   M\n')
        self.df.to_csv(path, sep='\t', mode='a')

def
to_file
(
self
,
path
)
:
with
open
(
path
,
'w'
)
as
f
:
f
.
write
(
'Z   N   M\n'
)
self
.
df
.
to_csv
(
path
,
sep
=
'\t'
,
mode
=
'a'
)

Export the contents to a file as comma separated values .


def select(self, condition, name=''):
        """
        Selects nuclei according to a condition on Z,N or M

        Parameters
        ----------
        condition : function,
            Can have one of the signatures f(M), f(Z,N) or f(Z, N, M)
            must return a boolean value
        name: string, optional name for the resulting Table

        Example:
        --------
        Select all nuclei with A > 160:

        >>> A_gt_160 = lambda Z,N: Z + N > 160
        >>> Table('AME2003').select(A_gt_160)
        """
        if condition.func_code.co_argcount == 1:
            idx = [(Z, N) for (Z, N), M in self if condition(M)]
        if condition.func_code.co_argcount == 2:
            idx = [(Z, N) for (Z, N) in self.index if condition(Z, N)]
        if condition.func_code.co_argcount == 3:
            idx = [(Z, N) for (Z, N), M in self if condition(Z, N, M)]
        index = pd.MultiIndex.from_tuples(idx, names=['Z', 'N'])
        return Table(df=self.df.ix[index], name=name)

def select ( self , condition , name = '' ) : if condition . func_code . co_argcount == 1 : idx = [ ( Z , N ) for ( Z , N ) , M in self if condition ( M ) ] if condition . func_code . co_argcount == 2 : idx = [ ( Z , N ) for ( Z , N ) in self . index if condition ( Z , N ) ] if condition . func_code . co_argcount == 3 : idx = [ ( Z , N ) for ( Z , N ) , M in self if condition ( Z , N , M ) ] index = pd . MultiIndex . from_tuples ( idx , names = [ 'Z' , 'N' ] ) return Table ( df = self . df . ix [ index ] , name = name )

Selects nuclei according to a condition on Z N or M


def at(self, nuclei):
        """Return a selection of the Table at positions given by ``nuclei``

        Parameters
        ----------
        nuclei: list of tuples
            A list where each element is tuple of the form (Z,N)

        Example
        -------
        Return binding energies at magic nuclei:

        >>> magic_nuclei = [(20,28), (50,50), (50,82), (82,126)]
        >>> Table('AME2012').binding_energy.at(magic_nuclei)
        Z   N
        20  28      416.014215
        50  50      825.325172
            82     1102.876416
        82  126    1636.486450
        """
        index = pd.MultiIndex.from_tuples(nuclei, names=['Z', 'N'])
        return Table(df=self.df.ix[index], name=self.name)

def at ( self , nuclei ) : index = pd . MultiIndex . from_tuples ( nuclei , names = [ 'Z' , 'N' ] ) return Table ( df = self . df . ix [ index ] , name = self . name )

Return a selection of the Table at positions given by nuclei


def intersection(self, table):
        """
        Select nuclei which also belong to ``table``

        Parameters
        ----------
        table: Table, Table object

        Example:
        ----------
        Table('AME2003').intersection(Table('AME1995'))
        """
        idx = self.df.index & table.df.index
        return Table(df=self.df[idx], name=self.name)

def intersection ( self , table ) : idx = self . df . index & table . df . index return Table ( df = self . df [ idx ] , name = self . name )

Select nuclei which also belong to table


def not_in(self, table):
        """
        Select nuclei not in table

        Parameters
        ----------
        table: Table, Table object from where nuclei should be removed

        Example:
        ----------
        Find the new nuclei in AME2003 with Z,N >= 8:

        >>> Table('AME2003').not_in(Table('AME1995'))[8:,8:].count
        389
        """
        idx = self.df.index - table.df.index
        return Table(df=self.df[idx], name=self.name)

def not_in ( self , table ) : idx = self . df . index - table . df . index return Table ( df = self . df [ idx ] , name = self . name )


def odd_even(self):
        """
        Selects odd-even nuclei from the table
        """
        return self.select(lambda Z, N: (Z % 2) and not(N % 2), name=self.name)

def odd_even ( self ) : return self . select ( lambda Z , N : ( Z % 2 ) and not ( N % 2 ) , name = self . name )

Selects odd - even nuclei from the table


def even_odd(self):
        """
        Selects even-odd nuclei from the table
        """
        return self.select(lambda Z, N: not(Z % 2) and (N % 2), name=self.name)

def even_odd ( self ) : return self . select ( lambda Z , N : not ( Z % 2 ) and ( N % 2 ) , name = self . name )

Selects even - odd nuclei from the table


def even_even(self):
        """
        Selects even-even nuclei from the table
        """
        return self.select(lambda Z, N: not(Z % 2) and not(N % 2), name=self.name)

def even_even ( self ) : return self . select ( lambda Z , N : not ( Z % 2 ) and not ( N % 2 ) , name = self . name )

Selects even - even nuclei from the table


def error(self, relative_to='AME2003'):
        """
        Calculate error difference

        Parameters
        ----------
        relative_to : string,
            a valid mass table name.

        Example:
        ----------
        >>> Table('DUZU').error(relative_to='AME2003')
        """
        df = self.df - Table(relative_to).df
        return Table(df=df)

def error ( self , relative_to = 'AME2003' ) : df = self . df - Table ( relative_to ) . df return Table ( df = df )


def rmse(self, relative_to='AME2003'):
        """Calculate root mean squared error

        Parameters
        ----------
        relative_to : string,
            a valid mass table name.

        Example:
        ----------
        >>> template = '{0:10}|{1:^6.2f}|{2:^6.2f}|{3:^6.2f}'
        >>> print 'Model      ', 'AME95 ', 'AME03 ', 'AME12 '  #  Table header
        ... for name in Table.names:
        ...     print template.format(name, Table(name).rmse(relative_to='AME1995'),
        ...                             Table(name).rmse(relative_to='AME2003'),
        ...                             Table(name).rmse(relative_to='AME2012'))
        Model       AME95  AME03  AME12
        AME2003   | 0.13 | 0.00 | 0.13
        AME2003all| 0.42 | 0.40 | 0.71
        AME2012   | 0.16 | 0.13 | 0.00
        AME2012all| 0.43 | 0.43 | 0.69
        AME1995   | 0.00 | 0.13 | 0.16
        AME1995all| 0.00 | 0.17 | 0.21
        DUZU      | 0.52 | 0.52 | 0.76
        FRDM95    | 0.79 | 0.78 | 0.95
        KTUY05    | 0.78 | 0.77 | 1.03
        ETFSI12   | 0.84 | 0.84 | 1.04
        HFB14     | 0.84 | 0.83 | 1.02
        """

        error = self.error(relative_to=relative_to)
        return math.sqrt((error.df ** 2).mean())

def rmse ( self , relative_to = 'AME2003' ) : error = self . error ( relative_to = relative_to ) return math . sqrt ( ( error . df ** 2 ) . mean ( ) )


def binding_energy(self):
        """
        Return binding energies instead of mass excesses
        """
        M_P = 938.2723
        # MeV
        M_E = 0.5110
        # MeV
        M_N = 939.5656
        # MeV
        AMU = 931.494028
        # MeV
        df = self.Z * (M_P + M_E) + (self.A - self.Z) * M_N - (self.df + self.A * AMU)
        return Table(df=df, name='BE' + '(' + self.name + ')')

def binding_energy ( self ) : M_P = 938.2723 M_E = 0.5110 M_N = 939.5656 AMU = 931.494028 df = self . Z * ( M_P + M_E ) + ( self . A - self . Z ) * M_N - ( self . df + self . A * AMU ) return Table ( df = df , name = 'BE' + '(' + self . name + ')' )


def s2n(self):
        """Return 2 neutron separation energy"""
        M_N = 8.0713171         # neutron mass excess in MeV
        f = lambda parent, daugther: -parent + daugther + 2 * M_N
        return self.derived('s2n', (0, -2), f)

def s2n ( self ) : M_N = 8.0713171 f = lambda parent , daugther : - parent + daugther + 2 * M_N return self . derived ( 's2n' , ( 0 , - 2 ) , f )


def s1n(self):
        """Return 1 neutron separation energy"""
        M_N = 8.0713171         # neutron mass excess in MeV
        f = lambda parent, daugther: -parent + daugther + M_N
        return self.derived('s1n', (0, -1), f)

def s1n ( self ) : M_N = 8.0713171 f = lambda parent , daugther : - parent + daugther + M_N return self . derived ( 's1n' , ( 0 , - 1 ) , f )


def s2p(self):
        """Return 2 proton separation energy"""
        M_P = 7.28897050         # proton mass excess in MeV
        f = lambda parent, daugther: -parent + daugther + 2 * M_P
        return self.derived('s2p', (-2, 0), f)

def s2p ( self ) : M_P = 7.28897050 f = lambda parent , daugther : - parent + daugther + 2 * M_P return self . derived ( 's2p' , ( - 2 , 0 ) , f )


def s1p(self):
        """Return 1 proton separation energy"""
        M_P = 7.28897050         # proton mass excess in MeV
        f = lambda parent, daugther: -parent + daugther + M_P
        return self.derived('s1p', (-1, 0), f)

def s1p ( self ) : M_P = 7.28897050 f = lambda parent , daugther : - parent + daugther + M_P return self . derived ( 's1p' , ( - 1 , 0 ) , f )


def derived(self, name, relative_coords, formula):
        """Helper function for derived quantities"""
        relZ, relN = relative_coords
        daughter_idx = [(x[0] + relZ, x[1] + relN) for x in self.df.index]
        values = formula(self.df.values, self.df.loc[daughter_idx].values)
        return Table(df=pd.Series(values, index=self.df.index, name=name + '(' + self.name + ')'))

def derived ( self , name , relative_coords , formula ) : relZ , relN = relative_coords daughter_idx = [ ( x [ 0 ] + relZ , x [ 1 ] + relN ) for x in self . df . index ] values = formula ( self . df . values , self . df . loc [ daughter_idx ] . values ) return Table ( df = pd . Series ( values , index = self . df . index , name = name + '(' + self . name + ')' ) )


def _uses_db(func, self, *args, **kwargs):
    """ Use as a decorator for operations on the database, to ensure connection setup and
    teardown. Can only be used on methods on objects with a `self.session` attribute.
    """
    if not self.session:
        _logger.debug('Creating new db session')
        self._init_db_session()
    try:
        ret = func(self, *args, **kwargs)
        self.session.commit()
    except:
        self.session.rollback()
        tb = traceback.format_exc()
        _logger.debug(tb)
        raise
    finally:
        _logger.debug('Closing db session')
        self.session.close()
    return ret

def
_uses_db
(
func
,
self
,
*
args
,
**
kwargs
)
:
if
not
self
.
session
:
_logger
.
debug
(
'Creating new db session'
)
self
.
_init_db_session
(
)
try
:
ret
=
func
(
self
,
*
args
,
**
kwargs
)
self
.
session
.
commit
(
)
except
:
self
.
session
.
rollback
(
)
tb
=
traceback
.
format_exc
(
)
_logger
.
debug
(
tb
)
raise
finally
:
_logger
.
debug
(
'Closing db session'
)
self
.
session
.
close
(
)
return
ret

Use as a decorator for operations on the database to ensure connection setup and teardown . Can only be used on methods on objects with a self . session attribute .


def derive_key(self, master_password):
        """ Computes the key from the salt and the master password. """
        encoder = encoding.Encoder(self.charset)

        bytes = ('%s:%s' % (master_password, self.name)).encode('utf8')

        start_time = time.clock()
        # we fix the scrypt parameters in case the defaults change
        digest = scrypt.hash(bytes, self.salt, N=1<<14, r=8, p=1)

        key = encoder.encode(digest, self.key_length)
        derivation_time_in_s = time.clock() - start_time

        _logger.debug('Key derivation took %.2fms', derivation_time_in_s*1000)
        return key

def
derive_key
(
self
,
master_password
)
:
encoder
=
encoding
.
Encoder
(
self
.
charset
)
bytes
=
(
'%s:%s'
%
(
master_password
,
self
.
name
)
)
.
encode
(
'utf8'
)
start_time
=
time
.
clock
(
)
digest
=
scrypt
.
hash
(
bytes
,
self
.
salt
,
N
=
1
<<
14
,
r
=
8
,
p
=
1
)
key
=
encoder
.
encode
(
digest
,
self
.
key_length
)
derivation_time_in_s
=
time
.
clock
(
)
start_time
_logger
.
debug
(
'Key derivation took %.2fms'
,
derivation_time_in_s
*
1000
)
return
key

Computes the key from the salt and the master password .


def bootstrap(self, path_or_uri):
        """ Initialize a database.

        :param database_path: The absolute path to the database to initialize.
        """
        _logger.debug("Bootstrapping new database: %s", path_or_uri)
        self.database_uri = _urify_db(path_or_uri)
        db = sa.create_engine(self.database_uri)
        Base.metadata.create_all(db)

def
bootstrap
(
self
,
path_or_uri
)
:
_logger
.
debug
(
"Bootstrapping new database: %s"
,
path_or_uri
)
self
.
database_uri
=
_urify_db
(
path_or_uri
)
db
=
sa
.
create_engine
(
self
.
database_uri
)
Base
.
metadata
.
create_all
(
db
)


def search(self, query):
        """ Search the database for the given query. Will find partial matches. """
        results = self.session.query(Domain).filter(Domain.name.ilike('%%%s%%' % query)).all()
        return results

def search ( self , query ) : results = self . session . query ( Domain ) . filter ( Domain . name . ilike ( '%%%s%%' % query ) ) . all ( ) return results

Search the database for the given query . Will find partial matches .


def modify_domain(self, domain_name, new_salt=False, username=None):
        """ Modify an existing domain.

        :param domain_name: The name of the domain to modify.
        :param new_salt: Whether to generate a new salt for the domain.
        :param username: If given, change domain username to this value.
        :returns: The modified :class:`Domain <pwm.core.Domain>` object.
        """
        domain = self._get_domain_from_db(domain_name)
        if domain is None:
            raise NoSuchDomainException
        if new_salt:
            _logger.info("Generating new salt..")
            domain.new_salt()
        if username is not None:
            domain.username = username
        return domain

def
modify_domain
(
self
,
domain_name
,
new_salt
=
False
,
username
=
)
:
domain
=
self
.
_get_domain_from_db
(
domain_name
)
if
domain
is
:
raise
NoSuchDomainException
if
new_salt
:
_logger
.
info
(
"Generating new salt.."
)
domain
.
new_salt
(
)
if
username
is
not
:
domain
.
username
=
username
return
domain


def create_domain(self, domain_name, username=None, alphabet=Domain.DEFAULT_ALPHABET,
            length=Domain.DEFAULT_KEY_LENGTH):
        """ Create a new domain entry in the database.

        :param username: The username to associate with this domain.
        :param alphabet: A character set restriction to impose on keys generated for this domain.
        :param length: The length of the generated key, in case of restrictions on the site.
        """
        # Wrap the actual implementation to do some error handling
        try:
            return self._create_domain(domain_name, username, alphabet, length)
        except Exception as ex:
            _logger.warn("Inserting new domain failed: %s", ex)
            raise DuplicateDomainException

def
create_domain
(
self
,
domain_name
,
username
=
,
alphabet
=
Domain
.
DEFAULT_ALPHABET
,
length
=
Domain
.
DEFAULT_KEY_LENGTH
)
:
try
:
return
self
.
_create_domain
(
domain_name
,
username
,
alphabet
,
length
)
except
Exception
as
ex
:
_logger
.
warn
(
"Inserting new domain failed: %s"
,
ex
)
raise
DuplicateDomainException

Create a new domain entry in the database .


def extract_hbs(fileobj, keywords, comment_tags, options):
    """Extract messages from Handlebars templates.

    It returns an iterator yielding tuples in the following form ``(lineno,
    funcname, message, comments)``.

    TODO: Things to improve:
    --- Return comments
    """

    server = get_pipeserver()
    server.sendline(COMMAND+u'PARSE FILE:'+fileobj.name)
    server.expect(RESPONSE+'SENDING OUTPUT')
    server.expect(RESPONSE+'OUTPUT END')
    trans_strings = server.before

    for item in json.loads(trans_strings):
        messages = [item['content']]
        if item['funcname'] == 'ngettext':
            messages.append(item['alt_content'])
        yield item['line_number'],item['funcname'],tuple(messages),[]

def
extract_hbs
(
fileobj
,
keywords
,
comment_tags
,
options
)
:
server
=
get_pipeserver
(
)
server
.
sendline
(
COMMAND
+
u'PARSE FILE:'
+
fileobj
.
name
)
server
.
expect
(
RESPONSE
+
'SENDING OUTPUT'
)
server
.
expect
(
RESPONSE
+
'OUTPUT END'
)
trans_strings
=
server
.
before
for
item
in
json
.
loads
(
trans_strings
)
:
messages
=
[
item
[
'content'
]
]
if
item
[
'funcname'
]
==
'ngettext'
:
messages
.
append
(
item
[
'alt_content'
]
)
yield
item
[
'line_number'
]
,
item
[
'funcname'
]
,
tuple
(
messages
)
,
[
]

Extract messages from Handlebars templates .


def vsiprefix(path):
    """Returns a GDAL virtual filesystem prefixed path.

    Arguments:
    path -- file path as str
    """
    vpath = path.lower()
    scheme = VSI_SCHEMES.get(urlparse(vpath).scheme, '')
    for ext in VSI_TYPES:
        if ext in vpath:
            filesys = VSI_TYPES[ext]
            break
    else:
        filesys = ''
    if filesys and scheme:
        filesys = filesys[:-1]
    return ''.join((filesys, scheme, path))

def vsiprefix ( path ) : vpath = path . lower ( ) scheme = VSI_SCHEMES . get ( urlparse ( vpath ) . scheme , '' ) for ext in VSI_TYPES : if ext in vpath : filesys = VSI_TYPES [ ext ] break else : filesys = '' if filesys and scheme : filesys = filesys [ : - 1 ] return '' . join ( ( filesys , scheme , path ) )

Returns a GDAL virtual filesystem prefixed path .


def srid(self):
        """Returns the EPSG ID as int if it exists."""
        epsg_id = (self.GetAuthorityCode('PROJCS') or
                   self.GetAuthorityCode('GEOGCS'))
        try:
            return int(epsg_id)
        except TypeError:
            return

def srid ( self ) : epsg_id = ( self . GetAuthorityCode ( 'PROJCS' ) or self . GetAuthorityCode ( 'GEOGCS' ) ) try : return int ( epsg_id ) except TypeError : return

Returns the EPSG ID as int if it exists .


def main():
    """ Main entry point for the CLI. """
    args = get_args()
    ret_code = args.target(args)
    _logger.debug('Exiting with code %d', ret_code)
    sys.exit(ret_code)

def
main
(
)
:
args
=
get_args
(
)
ret_code
=
args
.
target
(
args
)
_logger
.
debug
(
'Exiting with code %d'
,
ret_code
)
sys
.
exit
(
ret_code
)


def _init_logging(verbose=False):
    """ Initialize loggers. """
    config = {
        'version': 1,
        'formatters': {
            'console': {
                'format': '* %(message)s',
            }
        },
        'handlers': {
            'console': {
                'class': 'logging.StreamHandler',
                'level': 'DEBUG',
                'formatter': 'console',
                'stream': 'ext://sys.stdout',
            }
        },
        'loggers': {
            'pwm': {
                'level': 'DEBUG' if verbose else 'INFO',
                'handlers': ['console'],
                'propagate': True,
            },
            'requests.packages.urllib3': {
                'level': 'INFO' if verbose else 'WARNING',
                'handlers': ['console'],
                'propagate': True,
            }
        }
    }
    logging.config.dictConfig(config)
    HTTPConnection.debuglevel = 1 if verbose else 0

def
_init_logging
(
verbose
=
False
)
:
config
=
{
'version'
:
1
,
'formatters'
:
{
'console'
:
{
'format'
:
'* %(message)s'
,
}
}
,
'handlers'
:
{
'console'
:
{
'class'
:
'logging.StreamHandler'
,
'level'
:
'DEBUG'
,
'formatter'
:
'console'
,
'stream'
:
'ext://sys.stdout'
,
}
}
,
'loggers'
:
{
'pwm'
:
{
'level'
:
'DEBUG'
if
verbose
else
'INFO'
,
'handlers'
:
[
'console'
]
,
'propagate'
:
True
,
}
,
'requests.packages.urllib3'
:
{
'level'
:
'INFO'
if
verbose
else
'WARNING'
,
'handlers'
:
[
'console'
]
,
'propagate'
:
True
,
}
}
}
logging
.
config
.
dictConfig
(
config
)
HTTPConnection
.
debuglevel
=
1
if
verbose
else
0


def update_file(url, filename):
    """Update the content of a single file."""
    resp = urlopen(url)
    if resp.code != 200:
        raise Exception('GET {} failed.'.format(url))
    with open(_get_package_path(filename), 'w') as fp:
        for l in resp:
            if not l.startswith(b'#'):
                fp.write(l.decode('utf8'))
    print('Updated {}'.format(filename))

def
update_file
(
url
,
filename
)
:
resp
=
urlopen
(
url
)
if
resp
.
code
!=
200
:
raise
Exception
(
'GET {} failed.'
.
format
(
url
)
)
with
open
(
_get_package_path
(
filename
)
,
'w'
)
as
fp
:
for
l
in
resp
:
if
not
l
.
startswith
(
b'#'
)
:
fp
.
write
(
l
.
decode
(
'utf8'
)
)
print
(
'Updated {}'
.
format
(
filename
)
)


def available_drivers():
    """Returns a dictionary of enabled GDAL Driver metadata keyed by the
    'ShortName' attribute.
    """
    drivers = {}
    for i in range(gdal.GetDriverCount()):
        d = gdal.GetDriver(i)
        drivers[d.ShortName] = d.GetMetadata()
    return drivers

def available_drivers ( ) : drivers = { } for i in range ( gdal . GetDriverCount ( ) ) : d = gdal . GetDriver ( i ) drivers [ d . ShortName ] = d . GetMetadata ( ) return drivers

Returns a dictionary of enabled GDAL Driver metadata keyed by the ShortName attribute .


def driver_for_path(path, drivers=None):
    """Returns the gdal.Driver for a path or None based on the file extension.

    Arguments:
    path -- file path as str with a GDAL supported file extension
    """
    ext = (os.path.splitext(path)[1][1:] or path).lower()
    drivers = drivers or ImageDriver.registry if ext else {}
    for name, meta in drivers.items():
        if ext == meta.get('DMD_EXTENSION', '').lower():
            return ImageDriver(name)
    return None

def driver_for_path ( path , drivers = None ) : ext = ( os . path . splitext ( path ) [ 1 ] [ 1 : ] or path ) . lower ( ) drivers = drivers or ImageDriver . registry if ext else { } for name , meta in drivers . items ( ) : if ext == meta . get ( 'DMD_EXTENSION' , '' ) . lower ( ) : return ImageDriver ( name ) return None

Returns the gdal . Driver for a path or None based on the file extension .


def geom_to_array(geom, size, affine):
    """Converts an OGR polygon to a 2D NumPy array.

    Arguments:
    geom -- OGR Geometry
    size -- array size in pixels as a tuple of (width, height)
    affine -- AffineTransform
    """
    driver = ImageDriver('MEM')
    rast = driver.raster(driver.ShortName, size)
    rast.affine = affine
    rast.sref = geom.GetSpatialReference()
    with MemoryLayer.from_records([(1, geom)]) as ml:
        status = gdal.RasterizeLayer(rast.ds, (1,), ml.layer, burn_values=(1,))
    arr = rast.array()
    rast.close()
    return arr

def geom_to_array ( geom , size , affine ) : driver = ImageDriver ( 'MEM' ) rast = driver . raster ( driver . ShortName , size ) rast . affine = affine rast . sref = geom . GetSpatialReference ( ) with MemoryLayer . from_records ( [ ( 1 , geom ) ] ) as ml : status = gdal . RasterizeLayer ( rast . ds , ( 1 , ) , ml . layer , burn_values = ( 1 , ) ) arr = rast . array ( ) rast . close ( ) return arr

Converts an OGR polygon to a 2D NumPy array .


def rasterize(layer, rast):
    """Returns a Raster from layer features.

    Arguments:
    layer -- Layer to rasterize
    rast -- Raster with target affine, size, and sref
    """
    driver = ImageDriver('MEM')
    r2 = driver.raster(driver.ShortName, rast.size)
    r2.affine = rast.affine
    sref = rast.sref
    if not sref.srid:
        sref = SpatialReference(4326)
    r2.sref = sref
    ml = MemoryLayer(sref, layer.GetGeomType())
    ml.load(layer)
    status = gdal.RasterizeLayer(
        r2.ds, (1,), ml.layer, options=['ATTRIBUTE=%s' % ml.id])
    ml.close()
    return r2

def rasterize ( layer , rast ) : driver = ImageDriver ( 'MEM' ) r2 = driver . raster ( driver . ShortName , rast . size ) r2 . affine = rast . affine sref = rast . sref if not sref . srid : sref = SpatialReference ( 4326 ) r2 . sref = sref ml = MemoryLayer ( sref , layer . GetGeomType ( ) ) ml . load ( layer ) status = gdal . RasterizeLayer ( r2 . ds , ( 1 , ) , ml . layer , options = [ 'ATTRIBUTE=%s' % ml . id ] ) ml . close ( ) return r2


def open(path, mode=gdalconst.GA_ReadOnly):
    """Returns a Raster instance.

    Arguments:
    path -- local or remote path as str or file-like object
    Keyword args:
    mode -- gdal constant representing access mode
    """
    path = getattr(path, 'name', path)
    try:
        return Raster(vsiprefix(path), mode)
    except AttributeError:
        try:
            imgdata = path.read()
        except AttributeError:
            raise TypeError('Not a file-like object providing read()')
        else:
            imgio = MemFileIO(delete=False)
            gdal.FileFromMemBuffer(imgio.name, imgdata)
            return Raster(imgio, mode)
    raise ValueError('Failed to open raster from "%r"' % path)


def frombytes(data, size, bandtype=gdal.GDT_Byte):
    """Returns an in-memory raster initialized from a pixel buffer.

    Arguments:
    data -- byte buffer of raw pixel data
    size -- two or three-tuple of (xsize, ysize, bandcount)
    bandtype -- band data type
    """
    r = ImageDriver('MEM').raster('', size, bandtype)
    r.frombytes(data)
    return r

def frombytes ( data , size , bandtype = gdal . GDT_Byte ) : r = ImageDriver ( 'MEM' ) . raster ( '' , size , bandtype ) r . frombytes ( data ) return r

Returns an in - memory raster initialized from a pixel buffer .


def copy(self, source, dest):
        """Returns a copied Raster instance.

        Arguments:
        source -- the source Raster instance or filepath as str
        dest -- destination filepath as str
        """
        if not self.copyable:
            raise IOError('Driver does not support raster copying')
        if not isinstance(source, Raster):
            source = Raster(source)
            should_close = True
        else:
            should_close = False
        if source.name == dest:
            raise ValueError(
                'Input and output are the same location: %s' % source.name)
        settings = driverdict_tolist(self.settings)
        ds = self.CreateCopy(dest, source.ds, self.strictmode,
                             options=settings)
        if should_close:
            source.close()
        return Raster(ds)


def options(self):
        """Returns a dict of driver specific raster creation options.

        See GDAL format docs at http://www.gdal.org/formats_list.html
        """
        if self._options is None:
            try:
                elem = ET.fromstring(
                    self.info.get('DMD_CREATIONOPTIONLIST', ''))
            except ET.ParseError:
                elem = []
            opts = {}
            for child in elem:
                choices = [val.text for val in child]
                if choices:
                    child.attrib.update(choices=choices)
                opts[child.attrib.pop('name')] = child.attrib
            self._options = opts
        return self._options

def options ( self ) : if self . _options is None : try : elem = ET . fromstring ( self . info . get ( 'DMD_CREATIONOPTIONLIST' , '' ) ) except ET . ParseError : elem = [ ] opts = { } for child in elem : choices = [ val . text for val in child ] if choices : child . attrib . update ( choices = choices ) opts [ child . attrib . pop ( 'name' ) ] = child . attrib self . _options = opts return self . _options

Returns a dict of driver specific raster creation options .


def raster(self, path, size, bandtype=gdal.GDT_Byte):
        """Returns a new Raster instance.

        gdal.Driver.Create() does not support all formats.

        Arguments:
        path -- file object or path as str
        size -- two or three-tuple of (xsize, ysize, bandcount)
        bandtype -- GDAL pixel data type
        """
        path = getattr(path, 'name', path)
        try:
            is_multiband = len(size) > 2
            nx, ny, nbands = size if is_multiband else size + (1,)
        except (TypeError, ValueError) as exc:
            exc.args = ('Size must be 2 or 3-item sequence',)
            raise
        if nx < 1 or ny < 1:
            raise ValueError('Invalid raster size %s' % (size,))
        # Do not write to a non-empty file.
        if not self._is_empty(path):
            raise IOError('%s already exists, open with Raster()' % path)
        ds = self.Create(path, nx, ny, nbands, bandtype)
        if not ds:
            raise ValueError(
                'Could not create %s using %s' % (path, str(self)))
        return Raster(ds)

def
raster
(
self
,
path
,
size
,
bandtype
=
gdal
.
GDT_Byte
)
:
path
=
getattr
(
path
,
'name'
,
path
)
try
:
is_multiband
=
len
(
size
)
>
2
nx
,
ny
,
nbands
=
size
if
is_multiband
else
size
+
(
1
,
)
except
(
TypeError
,
ValueError
)
as
exc
:
exc
.
args
=
(
'Size must be 2 or 3-item sequence'
,
)
raise
if
nx
<
1
or
ny
<
1
:
raise
ValueError
(
'Invalid raster size %s'
%
(
size
,
)
)
if
not
self
.
_is_empty
(
path
)
:
raise
IOError
(
'%s already exists, open with Raster()'
%
path
)
ds
=
self
.
Create
(
path
,
nx
,
ny
,
nbands
,
bandtype
)
if
not
ds
:
raise
ValueError
(
'Could not create %s using %s'
%
(
path
,
str
(
self
)
)
)
return
Raster
(
ds
)


def SetGeoTransform(self, affine):
        """Sets the affine transformation.

        Intercepts the gdal.Dataset call to ensure use as a property setter.

        Arguments:
        affine -- AffineTransform or six-tuple of geotransformation values
        """
        if isinstance(affine, collections.Sequence):
            affine = AffineTransform(*affine)
        self._affine = affine
        self.ds.SetGeoTransform(affine)

def SetGeoTransform ( self , affine ) : if isinstance ( affine , collections . Sequence ) : affine = AffineTransform ( * affine ) self . _affine = affine self . ds . SetGeoTransform ( affine )


def array(self, envelope=()):
        """Returns an NDArray, optionally subset by spatial envelope.

        Keyword args:
        envelope -- coordinate extent tuple or Envelope
        """
        args = ()
        if envelope:
            args = self.get_offset(envelope)
        return self.ds.ReadAsArray(*args)

def array ( self , envelope = ( ) ) : args = ( ) if envelope : args = self . get_offset ( envelope ) return self . ds . ReadAsArray ( * args )

Returns an NDArray optionally subset by spatial envelope .


def envelope(self):
        """Returns the minimum bounding rectangle as a tuple of min X, min Y,
        max X, max Y.
        """
        if self._envelope is None:
            origin = self.affine.origin
            ur_x = origin[0] + self.ds.RasterXSize * self.affine.scale[0]
            ll_y = origin[1] + self.ds.RasterYSize * self.affine.scale[1]
            self._envelope = Envelope(origin[0], ll_y, ur_x, origin[1])
        return self._envelope

def envelope ( self ) : if self . _envelope is None : origin = self . affine . origin ur_x = origin [ 0 ] + self . ds . RasterXSize * self . affine . scale [ 0 ] ll_y = origin [ 1 ] + self . ds . RasterYSize * self . affine . scale [ 1 ] self . _envelope = Envelope ( origin [ 0 ] , ll_y , ur_x , origin [ 1 ] ) return self . _envelope

Returns the minimum bounding rectangle as a tuple of min X min Y max X max Y .


def driver(self):
        """Returns the underlying ImageDriver instance."""
        if self._driver is None:
            self._driver = ImageDriver(self.ds.GetDriver())
        return self._driver

def driver ( self ) : if self . _driver is None : self . _driver = ImageDriver ( self . ds . GetDriver ( ) ) return self . _driver

Returns the underlying ImageDriver instance .


def new(self, size=(), affine=None):
        """Derive new Raster instances.

        Keyword args:
        size -- tuple of image size (width, height)
        affine -- AffineTransform or six-tuple of geotransformation values
        """
        size = size or self.size + (len(self),)
        band = self.ds.GetRasterBand(1)
        driver = ImageDriver('MEM')
        rcopy = driver.raster(driver.ShortName, size, band.DataType)
        rcopy.sref = self.GetProjection()
        rcopy.affine = affine or tuple(self.affine)
        colors = band.GetColorTable()
        for outband in rcopy:
            if self.nodata is not None:
                outband.SetNoDataValue(self.nodata)
            if colors:
                outband.SetColorTable(colors)
        return rcopy

def new ( self , size = ( ) , affine = None ) : size = size or self . size + ( len ( self ) , ) band = self . ds . GetRasterBand ( 1 ) driver = ImageDriver ( 'MEM' ) rcopy = driver . raster ( driver . ShortName , size , band . DataType ) rcopy . sref = self . GetProjection ( ) rcopy . affine = affine or tuple ( self . affine ) colors = band . GetColorTable ( ) for outband in rcopy : if self . nodata is not None : outband . SetNoDataValue ( self . nodata ) if colors : outband . SetColorTable ( colors ) return rcopy


def masked_array(self, geometry=None):
        """Returns a MaskedArray using nodata values.

        Keyword args:
        geometry -- any geometry, envelope, or coordinate extent tuple
        """
        if geometry is None:
            return self._masked_array()
        geom = transform(geometry, self.sref)
        env = Envelope.from_geom(geom).intersect(self.envelope)
        arr = self._masked_array(env)
        if geom.GetGeometryType() != ogr.wkbPoint:
            dims = self.get_offset(env)[2:]
            affine = AffineTransform(*tuple(self.affine))
            affine.origin = env.ul
            mask = ~np.ma.make_mask(geom_to_array(geom, dims, affine))
            arr.mask = arr.mask | mask
        return arr

def masked_array ( self , geometry = None ) : if geometry is None : return self . _masked_array ( ) geom = transform ( geometry , self . sref ) env = Envelope . from_geom ( geom ) . intersect ( self . envelope ) arr = self . _masked_array ( env ) if geom . GetGeometryType ( ) != ogr . wkbPoint : dims = self . get_offset ( env ) [ 2 : ] affine = AffineTransform ( * tuple ( self . affine ) ) affine . origin = env . ul mask = ~ np . ma . make_mask ( geom_to_array ( geom , dims , affine ) ) arr . mask = arr . mask | mask return arr

Returns a MaskedArray using nodata values .


def nodata(self):
        """Returns read only property for band nodata value, assuming single
        band rasters for now.
        """
        if self._nodata is None:
            self._nodata = self[0].GetNoDataValue()
        return self._nodata

def nodata ( self ) : if self . _nodata is None : self . _nodata = self [ 0 ] . GetNoDataValue ( ) return self . _nodata

Returns read only property for band nodata value assuming single band rasters for now .


def ReadRaster(self, *args, **kwargs):
        """Returns raster data bytes for partial or full extent.

        Overrides gdal.Dataset.ReadRaster() with the full raster size by
        default.
        """
        args = args or (0, 0, self.ds.RasterXSize, self.ds.RasterYSize)
        return self.ds.ReadRaster(*args, **kwargs)

def ReadRaster ( self , * args , ** kwargs ) : args = args or ( 0 , 0 , self . ds . RasterXSize , self . ds . RasterYSize ) return self . ds . ReadRaster ( * args , ** kwargs )

Returns raster data bytes for partial or full extent .


def resample(self, size, interpolation=gdalconst.GRA_NearestNeighbour):
        """Returns a new instance resampled to provided size.

        Arguments:
        size -- tuple of x,y image dimensions
        """
        # Find the scaling factor for pixel size.
        factors = (size[0] / float(self.RasterXSize),
                   size[1] / float(self.RasterYSize))
        affine = AffineTransform(*tuple(self.affine))
        affine.scale = (affine.scale[0] / factors[0],
                        affine.scale[1] / factors[1])
        dest = self.new(size, affine)
        # Uses self and dest projection when set to None
        gdal.ReprojectImage(self.ds, dest.ds, None, None, interpolation)
        return dest

def resample ( self , size , interpolation = gdalconst . GRA_NearestNeighbour ) : factors = ( size [ 0 ] / float ( self . RasterXSize ) , size [ 1 ] / float ( self . RasterYSize ) ) affine = AffineTransform ( * tuple ( self . affine ) ) affine . scale = ( affine . scale [ 0 ] / factors [ 0 ] , affine . scale [ 1 ] / factors [ 1 ] ) dest = self . new ( size , affine ) gdal . ReprojectImage ( self . ds , dest . ds , None , None , interpolation ) return dest

Returns a new instance resampled to provided size .


def save(self, to, driver=None):
        """Save this instance to the path and format provided.

        Arguments:
        to -- output path as str, file, or MemFileIO instance
        Keyword args:
        driver -- GDAL driver name as string or ImageDriver
        """
        path = getattr(to, 'name', to)
        if not driver and hasattr(path, 'encode'):
            driver = driver_for_path(path, self.driver.filter_copyable())
        elif hasattr(driver, 'encode'):
            driver = ImageDriver(driver)
        if driver is None or not driver.copyable:
            raise ValueError('Copy supporting driver not found for %s' % path)
        driver.copy(self, path).close()

Save this instance to the path and format provided .


def SetProjection(self, sref):
        """Sets the spatial reference.

        Intercepts the gdal.Dataset call to ensure use as a property setter.

        Arguments:
        sref -- SpatialReference or any format supported by the constructor
        """
        if not hasattr(sref, 'ExportToWkt'):
            sref = SpatialReference(sref)
        self._sref = sref
        self.ds.SetProjection(sref.ExportToWkt())

def SetProjection ( self , sref ) : if not hasattr ( sref , 'ExportToWkt' ) : sref = SpatialReference ( sref ) self . _sref = sref self . ds . SetProjection ( sref . ExportToWkt ( ) )


def warp(self, to_sref, dest=None, interpolation=gdalconst.GRA_NearestNeighbour):
        """Returns a new reprojected instance.

        Arguments:
        to_sref -- spatial reference as a proj4 or wkt string, or a
        SpatialReference
        Keyword args:
        dest -- filepath as str
        interpolation -- GDAL interpolation type
        """
        if not hasattr(to_sref, 'ExportToWkt'):
            to_sref = SpatialReference(to_sref)
        dest_wkt = to_sref.ExportToWkt()
        dtype = self[0].DataType
        err_thresh = 0.125
        # Determine new values for destination raster dimensions and
        # geotransform.
        vrt = gdal.AutoCreateWarpedVRT(self.ds, None, dest_wkt,
                                       interpolation, err_thresh)
        if vrt is None:
            raise ValueError('Could not warp %s to %s' % (self, dest_wkt))
        warpsize = (vrt.RasterXSize, vrt.RasterYSize, len(self))
        warptrans = vrt.GetGeoTransform()
        vrt = None
        if dest is None:
            imgio = MemFileIO()
            rwarp = self.driver.raster(imgio, warpsize, dtype)
            imgio.close()
        else:
            rwarp = self.driver.raster(dest, warpsize, dtype)
        rwarp.SetGeoTransform(warptrans)
        rwarp.SetProjection(to_sref)
        if self.nodata is not None:
            for band in rwarp:
                band.SetNoDataValue(self.nodata)
                band = None
        # Uses self and rwarp projection when set to None
        gdal.ReprojectImage(self.ds, rwarp.ds, None, None, interpolation)
        return rwarp

def
warp
(
self
,
to_sref
,
dest
=
,
interpolation
=
gdalconst
.
GRA_NearestNeighbour
)
:
if
not
hasattr
(
to_sref
,
'ExportToWkt'
)
:
to_sref
=
SpatialReference
(
to_sref
)
dest_wkt
=
to_sref
.
ExportToWkt
(
)
dtype
=
self
[
0
]
.
DataType
err_thresh
=
0.125
vrt
=
gdal
.
AutoCreateWarpedVRT
(
self
.
ds
,
,
dest_wkt
,
interpolation
,
err_thresh
)
if
vrt
is
:
raise
ValueError
(
'Could not warp %s to %s'
%
(
self
,
dest_wkt
)
)
warpsize
=
(
vrt
.
RasterXSize
,
vrt
.
RasterYSize
,
len
(
self
)
)
warptrans
=
vrt
.
GetGeoTransform
(
)
vrt
=
if
dest
is
:
imgio
=
MemFileIO
(
)
rwarp
=
self
.
driver
.
raster
(
imgio
,
warpsize
,
dtype
)
imgio
.
close
(
)
else
:
rwarp
=
self
.
driver
.
raster
(
dest
,
warpsize
,
dtype
)
rwarp
.
SetGeoTransform
(
warptrans
)
rwarp
.
SetProjection
(
to_sref
)
if
self
.
nodata
is
not
:
for
band
in
rwarp
:
band
.
SetNoDataValue
(
self
.
nodata
)
band
=
gdal
.
ReprojectImage
(
self
.
ds
,
rwarp
.
ds
,
,
,
interpolation
)
return
rwarp


def calc_chunklen(alph_len):
    '''
    computes the ideal conversion ratio for the given alphabet.
    A ratio is considered ideal when the number of bits in one output
    encoding chunk that don't add up to one input encoding chunk is minimal.
    '''
    binlen, enclen = min([
                          (i, i*8 / math.log(alph_len, 2))
                          for i in range(1, 7)
                         ], key=lambda k: k[1] % 1)

    return binlen, int(enclen)

def calc_chunklen ( alph_len ) : binlen , enclen = min ( [ ( i , i * 8 / math . log ( alph_len , 2 ) ) for i in range ( 1 , 7 ) ] , key = lambda k : k [ 1 ] % 1 ) return binlen , int ( enclen )

computes the ideal conversion ratio for the given alphabet . A ratio is considered ideal when the number of bits in one output encoding chunk that don t add up to one input encoding chunk is minimal .


def lookup_alphabet(charset):
    '''
    retrieves a named charset or treats the input as a custom alphabet and use that
    '''
    if charset in PRESETS:
        return PRESETS[charset]
    if len(charset) < 16:
        _logger.warning('very small alphabet in use, possibly a failed lookup?')
    return charset


def _encode_chunk(self, data, index):
        '''
        gets a chunk from the input data, converts it to a number and
        encodes that number
        '''
        chunk = self._get_chunk(data, index)
        return self._encode_long(self._chunk_to_long(chunk))

def _encode_chunk ( self , data , index ) : chunk = self . _get_chunk ( data , index ) return self . _encode_long ( self . _chunk_to_long ( chunk ) )

gets a chunk from the input data converts it to a number and encodes that number


def _chunk_to_long(self, chunk):
        '''
        parses a chunk of bytes to integer using big-endian representation
        '''
        return sum([
                256**(self.chunklen[0]-1-i) * ord_byte(chunk[i])
                for i in range(self.chunklen[0])
            ])

def _chunk_to_long ( self , chunk ) : return sum ( [ 256 ** ( self . chunklen [ 0 ] - 1 - i ) * ord_byte ( chunk [ i ] ) for i in range ( self . chunklen [ 0 ] ) ] )

parses a chunk of bytes to integer using big - endian representation


def _get_chunk(self, data, index):
        '''
        partition the data into chunks and retrieve the chunk at the given index
        '''
        return data[index*self.chunklen[0]:(index+1)*self.chunklen[0]]

def _get_chunk ( self , data , index ) : return data [ index * self . chunklen [ 0 ] : ( index + 1 ) * self . chunklen [ 0 ] ]


def memoize(func):
    """Cache result of function call."""
    cache = {}

    @wraps(func)
    def inner(filename):
        if filename not in cache:
            cache[filename] = func(filename)
        return cache[filename]
    return inner

def memoize ( func ) : cache = { } @ wraps ( func ) def inner ( filename ) : if filename not in cache : cache [ filename ] = func ( filename ) return cache [ filename ] return inner


def _regexp(filename):
    """Get a list of patterns from a file and make a regular expression."""
    lines = _get_resource_content(filename).decode('utf-8').splitlines()
    return re.compile('|'.join(lines))

def _regexp ( filename ) : lines = _get_resource_content ( filename ) . decode ( 'utf-8' ) . splitlines ( ) return re . compile ( '|' . join ( lines ) )

Get a list of patterns from a file and make a regular expression .


def normalize_date_format(date):
    '''
    Dates can be defined in many ways, but zipline use
    aware datetime objects only. Plus, the software work
    with utc timezone so we convert it.
    '''
    if isinstance(date, int):
        # This is probably epoch time
        date = time.strftime('%Y-%m-%d %H:%M:%S',
                             time.localtime(date))

    # assert isinstance(date, str) or isinstance(date, unicode)
    if isinstance(date, str) or isinstance(date, unicode):
        date = dateutil.parser.parse(date)
    if not date.tzinfo:
        local_tz = pytz.timezone(_detect_timezone())
        local_dt = local_tz.localize(date, is_dst=None)
        # TODO I'm not sure why and when I need to add a date to make it right
        date = local_dt.astimezone(pytz.utc) + pd.datetools.day

    return date

def
normalize_date_format
(
date
)
:
if
isinstance
(
date
,
int
)
:
date
=
time
.
strftime
(
'%Y-%m-%d %H:%M:%S'
,
time
.
localtime
(
date
)
)
if
isinstance
(
date
,
str
)
or
isinstance
(
date
,
unicode
)
:
date
=
dateutil
.
parser
.
parse
(
date
)
if
not
date
.
tzinfo
:
local_tz
=
pytz
.
timezone
(
_detect_timezone
(
)
)
local_dt
=
local_tz
.
localize
(
date
,
is_dst
=
)
date
=
local_dt
.
astimezone
(
pytz
.
utc
)
+
pd
.
datetools
.
day
return
date

Dates can be defined in many ways but zipline use aware datetime objects only . Plus the software work with utc timezone so we convert it .


def _detect_timezone():
    '''
    Get timezone as set by the system
    '''
    default_timezone = 'America/New_York'
    locale_code = locale.getdefaultlocale()
    return default_timezone if not locale_code[0] else \
        str(pytz.country_timezones[locale_code[0][-2:]][0])

def _detect_timezone ( ) : default_timezone = 'America/New_York' locale_code = locale . getdefaultlocale ( ) return default_timezone if not locale_code [ 0 ] else str ( pytz . country_timezones [ locale_code [ 0 ] [ - 2 : ] ] [ 0 ] )


def to_dict(self):
        """
        Returns the model properties as a dict
        """
        result = {}

        for attr, _ in iteritems(self.swagger_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            else:
                result[attr] = value

        return result

def to_dict ( self ) : result = { } for attr , _ in iteritems ( self . swagger_types ) : value = getattr ( self , attr ) if isinstance ( value , list ) : result [ attr ] = list ( map ( lambda x : x . to_dict ( ) if hasattr ( x , "to_dict" ) else x , value ) ) elif hasattr ( value , "to_dict" ) : result [ attr ] = value . to_dict ( ) else : result [ attr ] = value return result


def activate_pdb_hook():
    ''' Catch exceptions with a prompt for post-mortem analyzis'''
    def debug_exception(type_exception, value, tb):
        import pdb
        pdb.post_mortem(tb)

    import sys
    sys.excepthook = debug_exception

def activate_pdb_hook ( ) : def debug_exception ( type_exception , value , tb ) : import pdb pdb . post_mortem ( tb ) import sys sys . excepthook = debug_exception

Catch exceptions with a prompt for post - mortem analyzis


def emphasis(obj, align=True):
    ''' Clearer data printing '''
    if isinstance(obj, dict):
        if align:
            pretty_msg = os.linesep.join(
                ["%25s: %s" % (k, obj[k]) for k in sorted(obj.keys())])
        else:
            pretty_msg = json.dumps(obj, indent=4, sort_keys=True)
    else:
        return obj
    return pretty_msg

def
emphasis
(
obj
,
align
=
True
)
:
if
isinstance
(
obj
,
dict
)
:
if
align
:
pretty_msg
=
os
.
linesep
.
join
(
[
"%25s: %s"
%
(
k
,
obj
[
k
]
)
for
k
in
sorted
(
obj
.
keys
(
)
)
]
)
else
:
pretty_msg
=
json
.
dumps
(
obj
,
indent
=
4
,
sort_keys
=
True
)
else
:
return
obj
return
pretty_msg


async def handle_jobs(job_handler, host, port, *, loop):
    """
    Connects to the remote master and continuously receives calls, executes
    them, then returns a response until interrupted.
    """

    try:

        try:
            reader, writer = await asyncio.open_connection(host, port, loop=loop)
        except OSError:
            logging.error("worker could not connect to server")
            return

        while True:

            try:
                call_encoded = await reader.readuntil(b"\n")
            except (asyncio.IncompleteReadError, ConnectionResetError):
                break
            logging.debug("worker got call")
            call_json = call_encoded.decode("utf-8")
            call = json.loads(call_json)

            response = job_handler(call)

            response_json = json.dumps(response) + "\n"
            response_encoded = response_json.encode("utf-8")
            writer.write(response_encoded)
            logging.debug("worker returned response")

    except KeyboardInterrupt:

        pass

async
def
handle_jobs
(
job_handler
,
host
,
port
,
*
,
loop
)
:
try
:
try
:
reader
,
writer
=
await
asyncio
.
open_connection
(
host
,
port
,
loop
=
loop
)
except
OSError
:
logging
.
error
(
"worker could not connect to server"
)
return
while
True
:
try
:
call_encoded
=
await
reader
.
readuntil
(
b"\n"
)
except
(
asyncio
.
IncompleteReadError
,
ConnectionResetError
)
:
break
logging
.
debug
(
"worker got call"
)
call_json
=
call_encoded
.
decode
(
"utf-8"
)
call
=
json
.
loads
(
call_json
)
response
=
job_handler
(
call
)
response_json
=
json
.
dumps
(
response
)
+
"\n"
response_encoded
=
response_json
.
encode
(
"utf-8"
)
writer
.
write
(
response_encoded
)
logging
.
debug
(
"worker returned response"
)
except
KeyboardInterrupt
:
pass

Connects to the remote master and continuously receives calls executes them then returns a response until interrupted .


def worker_main(job_handler, host, port):
    """
    Starts an asyncio event loop to connect to the master and run jobs.
    """

    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(None)
    loop.run_until_complete(handle_jobs(job_handler, host, port, loop=loop))
    loop.close()

def worker_main ( job_handler , host , port ) : loop = asyncio . new_event_loop ( ) asyncio . set_event_loop ( None ) loop . run_until_complete ( handle_jobs ( job_handler , host , port , loop = loop ) ) loop . close ( )

Starts an asyncio event loop to connect to the master and run jobs .


def run_worker_pool(job_handler, host="localhost", port=48484,
                      *, max_workers=None):
    """
    Runs a pool of workers which connect to a remote HighFive master and begin
    executing calls.
    """

    if max_workers is None:
        max_workers = multiprocessing.cpu_count()

    processes = []
    for _ in range(max_workers):
        p = multiprocessing.Process(target=worker_main,
                args=(job_handler, host, port))
        p.start()
        processes.append(p)

    logger.debug("workers started")

    for p in processes:
        p.join()

    logger.debug("all workers completed")

def
run_worker_pool
(
job_handler
,
host
=
"localhost"
,
port
=
48484
,
*
,
max_workers
=
)
:
if
max_workers
is
:
max_workers
=
multiprocessing
.
cpu_count
(
)
processes
=
[
]
for
_
in
range
(
max_workers
)
:
p
=
multiprocessing
.
Process
(
target
=
worker_main
,
args
=
(
job_handler
,
host
,
port
)
)
p
.
start
(
)
processes
.
append
(
p
)
logger
.
debug
(
"workers started"
)
for
p
in
processes
:
p
.
join
(
)
logger
.
debug
(
"all workers completed"
)

Runs a pool of workers which connect to a remote HighFive master and begin executing calls .


def classification(self, classification):
        """
        Sets the classification of this CompanyDetailCompany.
        Classification of Company

        :param classification: The classification of this CompanyDetailCompany.
        :type: str
        """
        allowed_values = ["Public Limited Indian Non-Government Company", "Private Limited Indian Non-Government Company", "One Person Company", "Private Limited Foreign Company Incorporated in India", "Public Limited Foreign Company Incorporated in India", "Union Government Company", "State Government Company", "Guarantee & Association Public", "Guarantee & Association Private", "Not For Profit Company", "Unlimited Liabilities Public", "Unlimited Liabilities Private", "Undefined"]
        if classification not in allowed_values:
            raise ValueError(
                "Invalid value for `classification`, must be one of {0}"
                .format(allowed_values)
            )
        self._classification = classification

Sets the classification of this CompanyDetailCompany . Classification of Company


def _send_message(self, msg):
        """Add message to queue and start processing the queue."""
        LWLink.the_queue.put_nowait(msg)
        if LWLink.thread is None or not LWLink.thread.isAlive():
            LWLink.thread = Thread(target=self._send_queue)
            LWLink.thread.start()

def _send_message ( self , msg ) : LWLink . the_queue . put_nowait ( msg ) if LWLink . thread is None or not LWLink . thread . isAlive ( ) : LWLink . thread = Thread ( target = self . _send_queue ) LWLink . thread . start ( )

Add message to queue and start processing the queue .


def turn_on_light(self, device_id, name):
        """Create the message to turn light on."""
        msg = "!%sFdP32|Turn On|%s" % (device_id, name)
        self._send_message(msg)

def
turn_on_light
(
self
,
device_id
,
name
)
:
msg
=
"!%sFdP32|Turn On|%s"
%
(
device_id
,
name
)
self
.
_send_message
(
msg
)


def turn_on_switch(self, device_id, name):
        """Create the message to turn switch on."""
        msg = "!%sF1|Turn On|%s" % (device_id, name)
        self._send_message(msg)

def
turn_on_switch
(
self
,
device_id
,
name
)
:
msg
=
"!%sF1|Turn On|%s"
%
(
device_id
,
name
)
self
.
_send_message
(
msg
)


def turn_on_with_brightness(self, device_id, name, brightness):
        """Scale brightness from 0..255 to 1..32."""
        brightness_value = round((brightness * 31) / 255) + 1
        # F1 = Light on and F0 = light off. FdP[0..32] is brightness. 32 is
        # full. We want that when turning the light on.
        msg = "!%sFdP%d|Lights %d|%s" % (
            device_id, brightness_value, brightness_value, name)
        self._send_message(msg)

def
turn_on_with_brightness
(
self
,
device_id
,
name
,
brightness
)
:
brightness_value
=
round
(
(
brightness
*
31
)
/
255
)
+
1
msg
=
"!%sFdP%d|Lights %d|%s"
%
(
device_id
,
brightness_value
,
brightness_value
,
name
)
self
.
_send_message
(
msg
)


def turn_off(self, device_id, name):
        """Create the message to turn light or switch off."""
        msg = "!%sF0|Turn Off|%s" % (device_id, name)
        self._send_message(msg)

def
turn_off
(
self
,
device_id
,
name
)
:
msg
=
"!%sF0|Turn Off|%s"
%
(
device_id
,
name
)
self
.
_send_message
(
msg
)

Create the message to turn light or switch off .


def _send_queue(self):
        """If the queue is not empty, process the queue."""
        while not LWLink.the_queue.empty():
            self._send_reliable_message(LWLink.the_queue.get_nowait())

def _send_queue ( self ) : while not LWLink . the_queue . empty ( ) : self . _send_reliable_message ( LWLink . the_queue . get_nowait ( ) )

If the queue is not empty process the queue .


def _send_reliable_message(self, msg):
        """Send msg to LightwaveRF hub."""
        result = False
        max_retries = 15
        trans_id = next(LWLink.transaction_id)
        msg = "%d,%s" % (trans_id, msg)
        try:
            with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) \
                    as write_sock, \
                    socket.socket(socket.AF_INET, socket.SOCK_DGRAM) \
                    as read_sock:
                write_sock.setsockopt(
                    socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                read_sock.setsockopt(socket.SOL_SOCKET,
                                     socket.SO_BROADCAST, 1)
                read_sock.settimeout(self.SOCKET_TIMEOUT)
                read_sock.bind(('0.0.0.0', self.RX_PORT))
                while max_retries:
                    max_retries -= 1
                    write_sock.sendto(msg.encode(
                        'UTF-8'), (LWLink.link_ip, self.TX_PORT))
                    result = False
                    while True:
                        response, dummy = read_sock.recvfrom(1024)
                        response = response.decode('UTF-8')
                        if "Not yet registered." in response:
                            _LOGGER.error("Not yet registered")
                            self.register()
                            result = True
                            break

                        if response.startswith("%d,OK" % trans_id):
                            result = True
                            break
                        if response.startswith("%d,ERR" % trans_id):
                            _LOGGER.error(response)
                            break

                        _LOGGER.info(response)

                    if result:
                        break

                    time.sleep(0.25)

        except socket.timeout:
            _LOGGER.error("LW broker timeout!")
            return result

        except Exception as ex:
            _LOGGER.error(ex)
            raise

        if result:
            _LOGGER.info("LW broker OK!")
        else:
            _LOGGER.error("LW broker fail!")
        return result

def
_send_reliable_message
(
self
,
msg
)
:
result
=
False
max_retries
=
15
trans_id
=
next
(
LWLink
.
transaction_id
)
msg
=
"%d,%s"
%
(
trans_id
,
msg
)
try
:
with
socket
.
socket
(
socket
.
AF_INET
,
socket
.
SOCK_DGRAM
)
as
write_sock
,
socket
.
socket
(
socket
.
AF_INET
,
socket
.
SOCK_DGRAM
)
as
read_sock
:
write_sock
.
setsockopt
(
socket
.
SOL_SOCKET
,
socket
.
SO_REUSEADDR
,
1
)
read_sock
.
setsockopt
(
socket
.
SOL_SOCKET
,
socket
.
SO_BROADCAST
,
1
)
read_sock
.
settimeout
(
self
.
SOCKET_TIMEOUT
)
read_sock
.
bind
(
(
'0.0.0.0'
,
self
.
RX_PORT
)
)
while
max_retries
:
max_retries
-=
1
write_sock
.
sendto
(
msg
.
encode
(
'UTF-8'
)
,
(
LWLink
.
link_ip
,
self
.
TX_PORT
)
)
result
=
False
while
True
:
response
,
dummy
=
read_sock
.
recvfrom
(
1024
)
response
=
response
.
decode
(
'UTF-8'
)
if
"Not yet registered."
in
response
:
_LOGGER
.
error
(
"Not yet registered"
)
self
.
register
(
)
result
=
True
break
if
response
.
startswith
(
"%d,OK"
%
trans_id
)
:
result
=
True
break
if
response
.
startswith
(
"%d,ERR"
%
trans_id
)
:
_LOGGER
.
error
(
response
)
break
_LOGGER
.
info
(
response
)
if
result
:
break
time
.
sleep
(
0.25
)
except
socket
.
timeout
:
_LOGGER
.
error
(
"LW broker timeout!"
)
return
result
except
Exception
as
ex
:
_LOGGER
.
error
(
ex
)
raise
if
result
:
_LOGGER
.
info
(
"LW broker OK!"
)
else
:
_LOGGER
.
error
(
"LW broker fail!"
)
return
result


def create_adapter(cmph, ffi, obj):
    """ Generates a wrapped adapter for the given object

    Parameters
    ----------
    obj : list, buffer, array, or file

    Raises
    ------
    ValueError
        If presented with an object that cannot be adapted

    Returns
    -------
    CMPH capable adapter
    """

    # if arraylike and fixed unit size
    # if file
    # if buffer

    if is_file_location(obj):
        # The FP is captured for GC reasons inside the dtor closure
        # pylint: disable=invalid-name
        fd = open(obj)
        adapter = cmph.cmph_io_nlfile_adapter(fd)

        def dtor():
            cmph.cmph_io_nlfile_adapter_destroy(adapter)
            fd.close()

        # pylint: enable=invalid-name
        return _AdapterCxt(adapter, dtor)
    elif is_file(obj):
        adapter = cmph.cmph_io_nlfile_adapter(obj)
        dtor = lambda: cmph.cmph_io_nlfile_adapter_destroy(adapter)
        return _AdapterCxt(adapter, dtor)
    elif isinstance(obj, Sequence):
        if len(obj) == 0:
            raise ValueError("An empty sequence is already a perfect hash!")
        return _create_pyobj_adapter(cmph, ffi, obj)
    else:
        raise ValueError("data cannot have a cmph wrapper generated")


def nature(self, nature):
        """
        Sets the nature of this YearlyFinancials.
        Nature of the balancesheet

        :param nature: The nature of this YearlyFinancials.
        :type: str
        """
        allowed_values = ["STANDALONE"]
        if nature not in allowed_values:
            raise ValueError(
                "Invalid value for `nature`, must be one of {0}"
                .format(allowed_values)
            )
        self._nature = nature

Sets the nature of this YearlyFinancials . Nature of the balancesheet


def update_(self, sct_dict, conf_arg=True):
        """Update values of configuration section with dict.

        Args:
            sct_dict (dict): dict indexed with option names. Undefined
                options are discarded.
            conf_arg (bool): if True, only options that can be set in a config
                file are updated.
        """
        for opt, val in sct_dict.items():
            if opt not in self.def_:
                continue
            if not conf_arg or self.def_[opt].conf_arg:
                self[opt] = val

def update_ ( self , sct_dict , conf_arg = True ) : for opt , val in sct_dict . items ( ) : if opt not in self . def_ : continue if not conf_arg or self . def_ [ opt ] . conf_arg : self [ opt ] = val

Update values of configuration section with dict .


def reset_(self):
        """Restore default values of options in this section."""
        for opt, meta in self.defaults_():
            self[opt] = meta.default

def reset_ ( self ) : for opt , meta in self . defaults_ ( ) : self [ opt ] = meta . default

Restore default values of options in this section .


def set_config_files_(self, *config_files):
        """Set the list of config files.

        Args:
            config_files (pathlike): path of config files, given in the order
                of reading.
        """
        self._config_files = tuple(pathlib.Path(path) for path in config_files)

def set_config_files_ ( self , * config_files ) : self . _config_files = tuple ( pathlib . Path ( path ) for path in config_files )


def opt_vals_(self):
        """Iterator over sections, option names, and option values.

        This iterator is also implemented at the section level. The two loops
        produce the same output::

            for sct, opt, val in conf.opt_vals_():
                print(sct, opt, val)

            for sct in conf.sections_():
                for opt, val in conf[sct].opt_vals_():
                    print(sct, opt, val)

        Yields:
            tuples with sections, option names, and option values.
        """
        for sct, opt in self.options_():
            yield sct, opt, self[sct][opt]

def opt_vals_ ( self ) : for sct , opt in self . options_ ( ) : yield sct , opt , self [ sct ] [ opt ]

Iterator over sections option names and option values .


def defaults_(self):
        """Iterator over sections, option names, and option metadata.

        This iterator is also implemented at the section level. The two loops
        produce the same output::

            for sct, opt, meta in conf.defaults_():
                print(sct, opt, meta.default)

            for sct in conf.sections_():
                for opt, meta in conf[sct].defaults_():
                    print(sct, opt, meta.default)

        Yields:
            tuples with sections, option names, and :class:`Conf` instances
            holding option metadata.
        """
        for sct, opt in self.options_():
            yield sct, opt, self[sct].def_[opt]

def defaults_ ( self ) : for sct , opt in self . options_ ( ) : yield sct , opt , self [ sct ] . def_ [ opt ]

Iterator over sections option names and option metadata .


def create_config_(self, index=0, update=False):
        """Create config file.

        Create config file in :attr:`config_files_[index]`.

        Parameters:
            index(int): index of config file.
            update (bool): if set to True and :attr:`config_files_` already
                exists, its content is read and all the options it sets are
                kept in the produced config file.
        """
        if not self.config_files_[index:]:
            return
        path = self.config_files_[index]
        if not path.parent.exists():
            path.parent.mkdir(parents=True)
        conf_dict = {}
        for section in self.sections_():
            conf_opts = [o for o, m in self[section].defaults_() if m.conf_arg]
            if not conf_opts:
                continue
            conf_dict[section] = {}
            for opt in conf_opts:
                conf_dict[section][opt] = (self[section][opt] if update else
                                           self[section].def_[opt].default)
        with path.open('w') as cfile:
            toml.dump(conf_dict, cfile)

def create_config_ ( self , index = 0 , update = False ) : if not self . config_files_ [ index : ] : return path = self . config_files_ [ index ] if not path . parent . exists ( ) : path . parent . mkdir ( parents = True ) conf_dict = { } for section in self . sections_ ( ) : conf_opts = [ o for o , m in self [ section ] . defaults_ ( ) if m . conf_arg ] if not conf_opts : continue conf_dict [ section ] = { } for opt in conf_opts : conf_dict [ section ] [ opt ] = ( self [ section ] [ opt ] if update else self [ section ] . def_ [ opt ] . default ) with path . open ( 'w' ) as cfile : toml . dump ( conf_dict , cfile )


def update_(self, conf_dict, conf_arg=True):
        """Update values of configuration options with dict.

        Args:
            conf_dict (dict): dict of dict indexed with section and option
                names.
            conf_arg (bool): if True, only options that can be set in a config
                file are updated.
        """
        for section, secdict in conf_dict.items():
            self[section].update_(secdict, conf_arg)

def update_ ( self , conf_dict , conf_arg = True ) : for section , secdict in conf_dict . items ( ) : self [ section ] . update_ ( secdict , conf_arg )

Update values of configuration options with dict .


def read_config_(self, cfile):
        """Read a config file and set config values accordingly.

        Returns:
            dict: content of config file.
        """
        if not cfile.exists():
            return {}
        try:
            conf_dict = toml.load(str(cfile))
        except toml.TomlDecodeError:
            return None
        self.update_(conf_dict)
        return conf_dict

def read_config_ ( self , cfile ) : if not cfile . exists ( ) : return { } try : conf_dict = toml . load ( str ( cfile ) ) except toml . TomlDecodeError : return None self . update_ ( conf_dict ) return conf_dict

Read a config file and set config values accordingly .


def read_configs_(self):
        """Read config files and set config values accordingly.

        Returns:
            (dict, list, list): respectively content of files, list of
            missing/empty files and list of files for which a parsing error
            arised.
        """
        if not self.config_files_:
            return {}, [], []
        content = {section: {} for section in self}
        empty_files = []
        faulty_files = []
        for cfile in self.config_files_:
            conf_dict = self.read_config_(cfile)
            if conf_dict is None:
                faulty_files.append(cfile)
                continue
            elif not conf_dict:
                empty_files.append(cfile)
                continue
            for section, secdict in conf_dict.items():
                content[section].update(secdict)
        return content, empty_files, faulty_files

def read_configs_ ( self ) : if not self . config_files_ : return { } , [ ] , [ ] content = { section : { } for section in self } empty_files = [ ] faulty_files = [ ] for cfile in self . config_files_ : conf_dict = self . read_config_ ( cfile ) if conf_dict is None : faulty_files . append ( cfile ) continue elif not conf_dict : empty_files . append ( cfile ) continue for section , secdict in conf_dict . items ( ) : content [ section ] . update ( secdict ) return content , empty_files , faulty_files

Read config files and set config values accordingly .


def _names(section, option):
    """List of cli strings for a given option."""
    meta = section.def_[option]
    action = meta.cmd_kwargs.get('action')
    if action is internal.Switch:
        names = ['-{}'.format(option), '+{}'.format(option)]
        if meta.shortname is not None:
            names.append('-{}'.format(meta.shortname))
            names.append('+{}'.format(meta.shortname))
    else:
        names = ['--{}'.format(option)]
        if meta.shortname is not None:
            names.append('-{}'.format(meta.shortname))
    return names

def _names ( section , option ) : meta = section . def_ [ option ] action = meta . cmd_kwargs . get ( 'action' ) if action is internal . Switch : names = [ '-{}' . format ( option ) , '+{}' . format ( option ) ] if meta . shortname is not None : names . append ( '-{}' . format ( meta . shortname ) ) names . append ( '+{}' . format ( meta . shortname ) ) else : names = [ '--{}' . format ( option ) ] if meta . shortname is not None : names . append ( '-{}' . format ( meta . shortname ) ) return names

List of cli strings for a given option .


def sections_list(self, cmd=None):
        """List of config sections used by a command.

        Args:
            cmd (str): command name, set to ``None`` or ``''`` for the bare
                command.

        Returns:
            list of str: list of configuration sections used by that command.
        """
        sections = list(self.common.sections)
        if not cmd:
            if self.bare is not None:
                sections.extend(self.bare.sections)
                return sections
            return []
        sections.extend(self.subcmds[cmd].sections)
        if cmd in self._conf:
            sections.append(cmd)
        return sections

def sections_list ( self , cmd = None ) : sections = list ( self . common . sections ) if not cmd : if self . bare is not None : sections . extend ( self . bare . sections ) return sections return [ ] sections . extend ( self . subcmds [ cmd ] . sections ) if cmd in self . _conf : sections . append ( cmd ) return sections

List of config sections used by a command .


def _cmd_opts_solver(self, cmd_name):
        """Scan options related to one command and enrich _opt_cmds."""
        sections = self.sections_list(cmd_name)
        cmd_dict = self._opt_cmds[cmd_name] if cmd_name else self._opt_bare
        for sct in reversed(sections):
            for opt, opt_meta in self._conf[sct].def_.items():
                if not opt_meta.cmd_arg:
                    continue
                if opt not in cmd_dict:
                    cmd_dict[opt] = sct
                else:
                    warnings.warn(
                        'Command <{0}>: {1}.{2} shadowed by {3}.{2}'.format(
                            cmd_name, sct, opt, cmd_dict[opt]),
                        error.LoamWarning, stacklevel=4)

Scan options related to one command and enrich _opt_cmds .


def _add_options_to_parser(self, opts_dict, parser):
        """Add options to a parser."""
        store_bool = ('store_true', 'store_false')
        for opt, sct in opts_dict.items():
            meta = self._conf[sct].def_[opt]
            kwargs = copy.deepcopy(meta.cmd_kwargs)
            action = kwargs.get('action')
            if action is internal.Switch:
                kwargs.update(nargs=0)
            elif meta.default is not None and action not in store_bool:
                kwargs.setdefault('type', type(meta.default))
            kwargs.update(help=meta.help)
            kwargs.setdefault('default', self._conf[sct][opt])
            parser.add_argument(*_names(self._conf[sct], opt), **kwargs)

def _add_options_to_parser ( self , opts_dict , parser ) : store_bool = ( 'store_true' , 'store_false' ) for opt , sct in opts_dict . items ( ) : meta = self . _conf [ sct ] . def_ [ opt ] kwargs = copy . deepcopy ( meta . cmd_kwargs ) action = kwargs . get ( 'action' ) if action is internal . Switch : kwargs . update ( nargs = 0 ) elif meta . default is not None and action not in store_bool : kwargs . setdefault ( 'type' , type ( meta . default ) ) kwargs . update ( help = meta . help ) kwargs . setdefault ( 'default' , self . _conf [ sct ] [ opt ] ) parser . add_argument ( * _names ( self . _conf [ sct ] , opt ) , ** kwargs )


def _build_parser(self):
        """Build command line argument parser.

        Returns:
            :class:`argparse.ArgumentParser`: the command line argument parser.
            You probably won't need to use it directly. To parse command line
            arguments and update the :class:`ConfigurationManager` instance
            accordingly, use the :meth:`parse_args` method.
        """
        main_parser = argparse.ArgumentParser(description=self.common.help,
                                              prefix_chars='-+')

        self._add_options_to_parser(self._opt_bare, main_parser)
        main_parser.set_defaults(**self.common.defaults)
        if self.bare is not None:
            main_parser.set_defaults(**self.bare.defaults)

        subparsers = main_parser.add_subparsers(dest='loam_sub_name')
        for cmd_name, meta in self.subcmds.items():
            kwargs = {'prefix_chars': '+-', 'help': meta.help}
            dummy_parser = subparsers.add_parser(cmd_name, **kwargs)
            self._add_options_to_parser(self._opt_cmds[cmd_name], dummy_parser)
            dummy_parser.set_defaults(**meta.defaults)

        return main_parser

def _build_parser ( self ) : main_parser = argparse . ArgumentParser ( description = self . common . help , prefix_chars = '-+' ) self . _add_options_to_parser ( self . _opt_bare , main_parser ) main_parser . set_defaults ( ** self . common . defaults ) if self . bare is not None : main_parser . set_defaults ( ** self . bare . defaults ) subparsers = main_parser . add_subparsers ( dest = 'loam_sub_name' ) for cmd_name , meta in self . subcmds . items ( ) : kwargs = { 'prefix_chars' : '+-' , 'help' : meta . help } dummy_parser = subparsers . add_parser ( cmd_name , ** kwargs ) self . _add_options_to_parser ( self . _opt_cmds [ cmd_name ] , dummy_parser ) dummy_parser . set_defaults ( ** meta . defaults ) return main_parser


def parse_args(self, arglist=None):
        """Parse arguments and update options accordingly.

        Args:
            arglist (list of str): list of arguments to parse. If set to None,
                ``sys.argv[1:]`` is used.

        Returns:
            :class:`Namespace`: the argument namespace returned by the
            :class:`argparse.ArgumentParser`.
        """
        args = self._parser.parse_args(args=arglist)
        sub_cmd = args.loam_sub_name
        if sub_cmd is None:
            for opt, sct in self._opt_bare.items():
                self._conf[sct][opt] = getattr(args, opt, None)
        else:
            for opt, sct in self._opt_cmds[sub_cmd].items():
                self._conf[sct][opt] = getattr(args, opt, None)
        return args

def parse_args ( self , arglist = None ) : args = self . _parser . parse_args ( args = arglist ) sub_cmd = args . loam_sub_name if sub_cmd is None : for opt , sct in self . _opt_bare . items ( ) : self . _conf [ sct ] [ opt ] = getattr ( args , opt , None ) else : for opt , sct in self . _opt_cmds [ sub_cmd ] . items ( ) : self . _conf [ sct ] [ opt ] = getattr ( args , opt , None ) return args

Parse arguments and update options accordingly .


def _zsh_comp_command(self, zcf, cmd, grouping, add_help=True):
        """Write zsh _arguments compdef for a given command.

        Args:
            zcf (file): zsh compdef file.
            cmd (str): command name, set to None or '' for bare command.
            grouping (bool): group options (zsh>=5.4).
            add_help (bool): add an help option.
        """
        if add_help:
            if grouping:
                print("+ '(help)'", end=BLK, file=zcf)
            print("'--help[show help message]'", end=BLK, file=zcf)
            print("'-h[show help message]'", end=BLK, file=zcf)
        # could deal with duplicate by iterating in reverse and keep set of
        # already defined opts.
        no_comp = ('store_true', 'store_false')
        cmd_dict = self._opt_cmds[cmd] if cmd else self._opt_bare
        for opt, sct in cmd_dict.items():
            meta = self._conf[sct].def_[opt]
            if meta.cmd_kwargs.get('action') == 'append':
                grpfmt, optfmt = "+ '{}'", "'*{}[{}]{}'"
                if meta.comprule is None:
                    meta.comprule = ''
            else:
                grpfmt, optfmt = "+ '({})'", "'{}[{}]{}'"
            if meta.cmd_kwargs.get('action') in no_comp \
               or meta.cmd_kwargs.get('nargs') == 0:
                meta.comprule = None
            if meta.comprule is None:
                compstr = ''
            elif meta.comprule == '':
                optfmt = optfmt.split('[')
                optfmt = optfmt[0] + '=[' + optfmt[1]
                compstr = ': :( )'
            else:
                optfmt = optfmt.split('[')
                optfmt = optfmt[0] + '=[' + optfmt[1]
                compstr = ': :{}'.format(meta.comprule)
            if grouping:
                print(grpfmt.format(opt), end=BLK, file=zcf)
            for name in _names(self._conf[sct], opt):
                print(optfmt.format(name, meta.help.replace("'", "'\"'\"'"),
                                    compstr), end=BLK, file=zcf)

def
_zsh_comp_command
(
self
,
zcf
,
cmd
,
grouping
,
add_help
=
True
)
:
if
add_help
:
if
grouping
:
print
(
"+ '(help)'"
,
end
=
BLK
,
file
=
zcf
)
print
(
"'--help[show help message]'"
,
end
=
BLK
,
file
=
zcf
)
print
(
"'-h[show help message]'"
,
end
=
BLK
,
file
=
zcf
)
no_comp
=
(
'store_true'
,
'store_false'
)
cmd_dict
=
self
.
_opt_cmds
[
cmd
]
if
cmd
else
self
.
_opt_bare
for
opt
,
sct
in
cmd_dict
.
items
(
)
:
meta
=
self
.
_conf
[
sct
]
.
def_
[
opt
]
if
meta
.
cmd_kwargs
.
get
(
'action'
)
==
'append'
:
grpfmt
,
optfmt
=
"+ '{}'"
,
"'*{}[{}]{}'"
if
meta
.
comprule
is
:
meta
.
comprule
=
''
else
:
grpfmt
,
optfmt
=
"+ '({})'"
,
"'{}[{}]{}'"
if
meta
.
cmd_kwargs
.
get
(
'action'
)
in
no_comp
or
meta
.
cmd_kwargs
.
get
(
'nargs'
)
==
0
:
meta
.
comprule
=
if
meta
.
comprule
is
:
compstr
=
''
elif
meta
.
comprule
==
''
:
optfmt
=
optfmt
.
split
(
'['
)
optfmt
=
optfmt
[
0
]
+
'=['
+
optfmt
[
1
]
compstr
=
': :( )'
else
:
optfmt
=
optfmt
.
split
(
'['
)
optfmt
=
optfmt
[
0
]
+
'=['
+
optfmt
[
1
]
compstr
=
': :{}'
.
format
(
meta
.
comprule
)
if
grouping
:
print
(
grpfmt
.
format
(
opt
)
,
end
=
BLK
,
file
=
zcf
)
for
name
in
_names
(
self
.
_conf
[
sct
]
,
opt
)
:
print
(
optfmt
.
format
(
name
,
meta
.
help
.
replace
(
"'"
,
"'\"'\"'"
)
,
compstr
)
,
end
=
BLK
,
file
=
zcf
)

Write zsh _arguments compdef for a given command .


def zsh_complete(self, path, cmd, *cmds, sourceable=False):
        """Write zsh compdef script.

        Args:
            path (path-like): desired path of the compdef script.
            cmd (str): command name that should be completed.
            cmds (str): extra command names that should be completed.
            sourceable (bool): if True, the generated file will contain an
                explicit call to ``compdef``, which means it can be sourced
                to activate CLI completion.
        """
        grouping = internal.zsh_version() >= (5, 4)
        path = pathlib.Path(path)
        firstline = ['#compdef', cmd]
        firstline.extend(cmds)
        subcmds = list(self.subcmds.keys())
        with path.open('w') as zcf:
            print(*firstline, end='\n\n', file=zcf)
            # main function
            print('function _{} {{'.format(cmd), file=zcf)
            print('local line', file=zcf)
            print('_arguments -C', end=BLK, file=zcf)
            if subcmds:
                # list of subcommands and their description
                substrs = ["{}\\:'{}'".format(sub, self.subcmds[sub].help)
                           for sub in subcmds]
                print('"1:Commands:(({}))"'.format(' '.join(substrs)),
                      end=BLK, file=zcf)
            self._zsh_comp_command(zcf, None, grouping)
            if subcmds:
                print("'*::arg:->args'", file=zcf)
                print('case $line[1] in', file=zcf)
                for sub in subcmds:
                    print('{sub}) _{cmd}_{sub} ;;'.format(sub=sub, cmd=cmd),
                          file=zcf)
                print('esac', file=zcf)
            print('}', file=zcf)
            # all subcommand completion handlers
            for sub in subcmds:
                print('\nfunction _{}_{} {{'.format(cmd, sub), file=zcf)
                print('_arguments', end=BLK, file=zcf)
                self._zsh_comp_command(zcf, sub, grouping)
                print('}', file=zcf)
            if sourceable:
                print('\ncompdef _{0} {0}'.format(cmd), *cmds, file=zcf)

def
zsh_complete
(
self
,
path
,
cmd
,
*
cmds
,
sourceable
=
False
)
:
grouping
=
internal
.
zsh_version
(
)
>=
(
5
,
4
)
path
=
pathlib
.
Path
(
path
)
firstline
=
[
'#compdef'
,
cmd
]
firstline
.
extend
(
cmds
)
subcmds
=
list
(
self
.
subcmds
.
keys
(
)
)
with
path
.
open
(
'w'
)
as
zcf
:
print
(
*
firstline
,
end
=
'\n\n'
,
file
=
zcf
)
print
(
'function _{} {{'
.
format
(
cmd
)
,
file
=
zcf
)
print
(
'local line'
,
file
=
zcf
)
print
(
'_arguments -C'
,
end
=
BLK
,
file
=
zcf
)
if
subcmds
:
substrs
=
[
"{}\\:'{}'"
.
format
(
sub
,
self
.
subcmds
[
sub
]
.
help
)
for
sub
in
subcmds
]
print
(
'"1:Commands:(({}))"'
.
format
(
' '
.
join
(
substrs
)
)
,
end
=
BLK
,
file
=
zcf
)
self
.
_zsh_comp_command
(
zcf
,
,
grouping
)
if
subcmds
:
print
(
"'*::arg:->args'"
,
file
=
zcf
)
print
(
'case $line[1] in'
,
file
=
zcf
)
for
sub
in
subcmds
:
print
(
'{sub}) _{cmd}_{sub} ;;'
.
format
(
sub
=
sub
,
cmd
=
cmd
)
,
file
=
zcf
)
print
(
'esac'
,
file
=
zcf
)
print
(
'}'
,
file
=
zcf
)
for
sub
in
subcmds
:
print
(
'\nfunction _{}_{} {{'
.
format
(
cmd
,
sub
)
,
file
=
zcf
)
print
(
'_arguments'
,
end
=
BLK
,
file
=
zcf
)
self
.
_zsh_comp_command
(
zcf
,
sub
,
grouping
)
print
(
'}'
,
file
=
zcf
)
if
sourceable
:
print
(
'\ncompdef _{0} {0}'
.
format
(
cmd
)
,
*
cmds
,
file
=
zcf
)


def _bash_comp_command(self, cmd, add_help=True):
        """Build a list of all options for a given command.

        Args:
            cmd (str): command name, set to None or '' for bare command.
            add_help (bool): add an help option.

        Returns:
            list of str: list of CLI options strings.
        """
        out = ['-h', '--help'] if add_help else []
        cmd_dict = self._opt_cmds[cmd] if cmd else self._opt_bare
        for opt, sct in cmd_dict:
            out.extend(_names(self._conf[sct], opt))
        return out

def _bash_comp_command ( self , cmd , add_help = True ) : out = [ '-h' , '--help' ] if add_help else [ ] cmd_dict = self . _opt_cmds [ cmd ] if cmd else self . _opt_bare for opt , sct in cmd_dict : out . extend ( _names ( self . _conf [ sct ] , opt ) ) return out

Build a list of all options for a given command .


def bash_complete(self, path, cmd, *cmds):
        """Write bash complete script.

        Args:
            path (path-like): desired path of the complete script.
            cmd (str): command name that should be completed.
            cmds (str): extra command names that should be completed.
        """
        path = pathlib.Path(path)
        subcmds = list(self.subcmds.keys())
        with path.open('w') as bcf:
            # main function
            print('_{}() {{'.format(cmd), file=bcf)
            print('COMPREPLY=()', file=bcf)
            print(r'local cur=${COMP_WORDS[COMP_CWORD]}', end='\n\n', file=bcf)
            optstr = ' '.join(self._bash_comp_command(None))
            print(r'local options="{}"'.format(optstr), end='\n\n', file=bcf)
            if subcmds:
                print('local commands="{}"'.format(' '.join(subcmds)),
                      file=bcf)
                print('declare -A suboptions', file=bcf)
            for sub in subcmds:
                optstr = ' '.join(self._bash_comp_command(sub))
                print('suboptions[{}]="{}"'.format(sub, optstr), file=bcf)
            condstr = 'if'
            for sub in subcmds:
                print(condstr, r'[[ "${COMP_LINE}" == *"', sub, '"* ]] ; then',
                      file=bcf)
                print(r'COMPREPLY=( `compgen -W "${suboptions[', sub,
                      r']}" -- ${cur}` )', sep='', file=bcf)
                condstr = 'elif'
            print(condstr, r'[[ ${cur} == -* ]] ; then', file=bcf)
            print(r'COMPREPLY=( `compgen -W "${options}" -- ${cur}`)',
                  file=bcf)
            if subcmds:
                print(r'else', file=bcf)
                print(r'COMPREPLY=( `compgen -W "${commands}" -- ${cur}`)',
                      file=bcf)
            print('fi', file=bcf)
            print('}', end='\n\n', file=bcf)
            print('complete -F _{0} {0}'.format(cmd), *cmds, file=bcf)


async def start_master(host="", port=48484, *, loop=None):
    """
    Starts a new HighFive master at the given host and port, and returns it.
    """

    loop = loop if loop is not None else asyncio.get_event_loop()

    manager = jobs.JobManager(loop=loop)
    workers = set()
    server = await loop.create_server(
            lambda: WorkerProtocol(manager, workers), host, port)
    return Master(server, manager, workers, loop=loop)

async def start_master ( host = "" , port = 48484 , * , loop = None ) : loop = loop if loop is not None else asyncio . get_event_loop ( ) manager = jobs . JobManager ( loop = loop ) workers = set ( ) server = await loop . create_server ( lambda : WorkerProtocol ( manager , workers ) , host , port ) return Master ( server , manager , workers , loop = loop )

Starts a new HighFive master at the given host and port and returns it .


def connection_made(self, transport):
        """
        Called when a remote worker connection has been found. Finishes setting
        up the protocol object.
        """

        if self._manager.is_closed():
            logger.debug("worker tried to connect while manager was closed")
            return

        logger.debug("new worker connected")

        self._transport = transport
        self._buffer = bytearray()
        self._worker = Worker(self._transport, self._manager)
        self._workers.add(self._worker)

Called when a remote worker connection has been found . Finishes setting up the protocol object .


def line_received(self, line):
        """
        Called when a complete line is found from the remote worker. Decodes
        a response object from the line, then passes it to the worker object.
        """

        response = json.loads(line.decode("utf-8"))
        self._worker.response_received(response)

def line_received ( self , line ) : response = json . loads ( line . decode ( "utf-8" ) ) self . _worker . response_received ( response )

Called when a complete line is found from the remote worker . Decodes a response object from the line then passes it to the worker object .


def connection_lost(self, exc):
        """
        Called when the connection to the remote worker is broken. Closes the
        worker.
        """

        logger.debug("worker connection lost")

        self._worker.close()
        self._workers.remove(self._worker)

def
connection_lost
(
self
,
exc
)
:
logger
.
debug
(
"worker connection lost"
)
self
.
_worker
.
close
(
)
self
.
_workers
.
remove
(
self
.
_worker
)

Called when the connection to the remote worker is broken . Closes the worker .


def _job_loaded(self, job):
        """
        Called when a job has been found for the worker to run. Sends the job's
        RPC to the remote worker.
        """

        logger.debug("worker {} found a job".format(id(self)))

        if self._closed:
            self._manager.return_job(job)
            return

        self._job = job
        call_obj = self._job.get_call()
        call = (json.dumps(call_obj) + "\n").encode("utf-8")
        self._transport.write(call)

def
_job_loaded
(
self
,
job
)
:
logger
.
debug
(
"worker {} found a job"
.
format
(
id
(
self
)
)
)
if
self
.
_closed
:
self
.
_manager
.
return_job
(
job
)
return
self
.
_job
=
job
call_obj
=
self
.
_job
.
get_call
(
)
call
=
(
json
.
dumps
(
call_obj
)
+
"\n"
)
.
encode
(
"utf-8"
)
self
.
_transport
.
write
(
call
)

Called when a job has been found for the worker to run . Sends the job s RPC to the remote worker .


def response_received(self, response):
        """
        Called when a response to a job RPC has been received. Decodes the
        response and finalizes the result, then reports the result to the
        job manager.
        """

        if self._closed:
            return

        assert self._job is not None

        logger.debug("worker {} got response".format(id(self)))
        result = self._job.get_result(response)
        self._manager.add_result(self._job, result)

        self._load_job()

def
response_received
(
self
,
response
)
:
if
self
.
_closed
:
return
assert
self
.
_job
is
not
logger
.
debug
(
"worker {} got response"
.
format
(
id
(
self
)
)
)
result
=
self
.
_job
.
get_result
(
response
)
self
.
_manager
.
add_result
(
self
.
_job
,
result
)
self
.
_load_job
(
)

Called when a response to a job RPC has been received . Decodes the response and finalizes the result then reports the result to the job manager .


def close(self):
        """
        Closes the worker. No more jobs will be handled by the worker, and any
        running job is immediately returned to the job manager.
        """

        if self._closed:
            return

        self._closed = True

        if self._job is not None:
            self._manager.return_job(self._job)
            self._job = None

def close ( self ) : if self . _closed : return self . _closed = True if self . _job is not None : self . _manager . return_job ( self . _job ) self . _job = None

Closes the worker . No more jobs will be handled by the worker and any running job is immediately returned to the job manager .


def run(self, job_list):
        """
        Runs a job set which consists of the jobs in an iterable job list.
        """

        if self._closed:
            raise RuntimeError("master is closed")

        return self._manager.add_job_set(job_list)

def
run
(
self
,
job_list
)
:
if
self
.
_closed
:
raise
RuntimeError
(
"master is closed"
)
return
self
.
_manager
.
add_job_set
(
job_list
)

Runs a job set which consists of the jobs in an iterable job list .


def close(self):
        """
        Starts closing the HighFive master. The server will be closed and
        all queued job sets will be cancelled.
        """

        if self._closed:
            return

        self._closed = True

        self._server.close()
        self._manager.close()
        for worker in self._workers:
            worker.close()

def close ( self ) : if self . _closed : return self . _closed = True self . _server . close ( ) self . _manager . close ( ) for worker in self . _workers : worker . close ( )

Starts closing the HighFive master . The server will be closed and all queued job sets will be cancelled .


def _change(self):
        """
        Called when a state change has occurred. Waiters are notified that a
        change has occurred.
        """

        for waiter in self._waiters:
            if not waiter.done():
                waiter.set_result(None)
        self._waiters = []

def _change ( self ) : for waiter in self . _waiters : if not waiter . done ( ) : waiter . set_result ( None ) self . _waiters = [ ]

Called when a state change has occurred . Waiters are notified that a change has occurred .


def add(self, result):
        """
        Adds a new result.
        """

        assert not self._complete

        self._results.append(result)
        self._change()

def add ( self , result ) : assert not self . _complete self . _results . append ( result ) self . _change ( )


async def wait_changed(self):
        """
        Waits until the result set changes. Possible changes can be a result
        being added or the result set becoming complete. If the result set is
        already completed, this method returns immediately.
        """

        if not self.is_complete():
            waiter = self._loop.create_future()
            self._waiters.append(waiter)
            await waiter

async def wait_changed ( self ) : if not self . is_complete ( ) : waiter = self . _loop . create_future ( ) self . _waiters . append ( waiter ) await waiter

Waits until the result set changes . Possible changes can be a result being added or the result set becoming complete . If the result set is already completed this method returns immediately .