def _make_job_dict(job):
    """Creates the view for a job while calculating progress.

    Since a root job does not have a transaction id (TID) it will return its
    id as the TID.
    """
    body = {"id": job.get('id'),
            "action": job.get('action'),
            "completed": job.get('completed'),
            "tenant_id": job.get('tenant_id'),
            "created_at": job.get('created_at'),
            "transaction_id": job.get('transaction_id'),
            "parent_id": job.get('parent_id', None)}
    if not body['transaction_id']:
        body['transaction_id'] = job.get('id')
    completed = 0
    for sub in job.subtransactions:
        if sub.get('completed'):
            completed += 1
    pct = 100 if job.get('completed') else 0
    if len(job.subtransactions) > 0:
        pct = float(completed) / len(job.subtransactions) * 100.0
    body['transaction_percent'] = int(pct)
    body['completed_subtransactions'] = completed
    body['subtransactions'] = len(job.subtransactions)
    return body

def _make_job_dict ( job ) : body = { "id" : job . get ( 'id' ) , "action" : job . get ( 'action' ) , "completed" : job . get ( 'completed' ) , "tenant_id" : job . get ( 'tenant_id' ) , "created_at" : job . get ( 'created_at' ) , "transaction_id" : job . get ( 'transaction_id' ) , "parent_id" : job . get ( 'parent_id' , None ) } if not body [ 'transaction_id' ] : body [ 'transaction_id' ] = job . get ( 'id' ) completed = 0 for sub in job . subtransactions : if sub . get ( 'completed' ) : completed += 1 pct = 100 if job . get ( 'completed' ) else 0 if len ( job . subtransactions ) > 0 : pct = float ( completed ) / len ( job . subtransactions ) * 100.0 body [ 'transaction_percent' ] = int ( pct ) body [ 'completed_subtransactions' ] = completed body [ 'subtransactions' ] = len ( job . subtransactions ) return body

Creates the view for a job while calculating progress .


def get_mac_address_range(context, id, fields=None):
    """Retrieve a mac_address_range.

    : param context: neutron api request context
    : param id: UUID representing the network to fetch.
    : param fields: a list of strings that are valid keys in a
        network dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.
    """
    LOG.info("get_mac_address_range %s for tenant %s fields %s" %
             (id, context.tenant_id, fields))

    if not context.is_admin:
        raise n_exc.NotAuthorized()

    mac_address_range = db_api.mac_address_range_find(
        context, id=id, scope=db_api.ONE)

    if not mac_address_range:
        raise q_exc.MacAddressRangeNotFound(
            mac_address_range_id=id)
    return v._make_mac_range_dict(mac_address_range)


def delete_mac_address_range(context, id):
    """Delete a mac_address_range.

    : param context: neutron api request context
    : param id: UUID representing the mac_address_range to delete.
    """
    LOG.info("delete_mac_address_range %s for tenant %s" %
             (id, context.tenant_id))
    if not context.is_admin:
        raise n_exc.NotAuthorized()

    with context.session.begin():
        mar = db_api.mac_address_range_find(context, id=id, scope=db_api.ONE)
        if not mar:
            raise q_exc.MacAddressRangeNotFound(
                mac_address_range_id=id)
        _delete_mac_address_range(context, mar)


def delete_segment_allocation_range(context, sa_id):
    """Delete a segment_allocation_range.

    : param context: neutron api request context
    : param id: UUID representing the segment_allocation_range to delete.
    """
    LOG.info("delete_segment_allocation_range %s for tenant %s" %
             (sa_id, context.tenant_id))
    if not context.is_admin:
        raise n_exc.NotAuthorized()

    with context.session.begin():
        sa_range = db_api.segment_allocation_range_find(
            context, id=sa_id, scope=db_api.ONE)
        if not sa_range:
            raise q_exc.SegmentAllocationRangeNotFound(
                segment_allocation_range_id=sa_id)
        _delete_segment_allocation_range(context, sa_range)


def filter_factory(global_conf, **local_conf):
    """Returns a WSGI filter app for use with paste.deploy."""
    conf = global_conf.copy()
    conf.update(local_conf)

    def wrapper(app):
        return ResponseAsyncIdAdder(app, conf)

    return wrapper

def filter_factory ( global_conf , ** local_conf ) : conf = global_conf . copy ( ) conf . update ( local_conf ) def wrapper ( app ) : return ResponseAsyncIdAdder ( app , conf ) return wrapper

Returns a WSGI filter app for use with paste . deploy .


def get_used_ips(session, **kwargs):
    """Returns dictionary with keys segment_id and value used IPs count.

    Used IP address count is determined by:
    - allocated IPs
    - deallocated IPs whose `deallocated_at` is within the `reuse_after`
    window compared to the present time, excluding IPs that are accounted for
    in the current IP policy (because IP policy is mutable and deallocated IPs
    are not checked nor deleted on IP policy creation, thus deallocated IPs
    that don't fit the current IP policy can exist in the neutron database).
    """
    LOG.debug("Getting used IPs...")
    with session.begin():
        query = session.query(
            models.Subnet.segment_id,
            func.count(models.IPAddress.address))
        query = query.group_by(models.Subnet.segment_id)
        query = _filter(query, **kwargs)

        reuse_window = timeutils.utcnow() - datetime.timedelta(
            seconds=cfg.CONF.QUARK.ipam_reuse_after)
        # NOTE(asadoughi): This is an outer join instead of a regular join
        # to include subnets with zero IP addresses in the database.
        query = query.outerjoin(
            models.IPAddress,
            and_(models.Subnet.id == models.IPAddress.subnet_id,
                 or_(not_(models.IPAddress.lock_id.is_(None)),
                     models.IPAddress._deallocated.is_(None),
                     models.IPAddress._deallocated == 0,
                     models.IPAddress.deallocated_at > reuse_window)))

        query = query.outerjoin(
            models.IPPolicyCIDR,
            and_(
                models.Subnet.ip_policy_id == models.IPPolicyCIDR.ip_policy_id,
                models.IPAddress.address >= models.IPPolicyCIDR.first_ip,
                models.IPAddress.address <= models.IPPolicyCIDR.last_ip))
        # NOTE(asadoughi): (address is allocated) OR
        # (address is deallocated and not inside subnet's IP policy)
        query = query.filter(or_(
            models.IPAddress._deallocated.is_(None),
            models.IPAddress._deallocated == 0,
            models.IPPolicyCIDR.id.is_(None)))

        ret = ((segment_id, address_count)
               for segment_id, address_count in query.all())
        return dict(ret)

def
get_used_ips
(
session
,
**
kwargs
)
:
LOG
.
debug
(
"Getting used IPs..."
)
with
session
.
begin
(
)
:
query
=
session
.
query
(
models
.
Subnet
.
segment_id
,
func
.
count
(
models
.
IPAddress
.
address
)
)
query
=
query
.
group_by
(
models
.
Subnet
.
segment_id
)
query
=
_filter
(
query
,
**
kwargs
)
reuse_window
=
timeutils
.
utcnow
(
)
datetime
.
timedelta
(
seconds
=
cfg
.
CONF
.
QUARK
.
ipam_reuse_after
)
query
=
query
.
outerjoin
(
models
.
IPAddress
,
and_
(
models
.
Subnet
.
id
==
models
.
IPAddress
.
subnet_id
,
or_
(
not_
(
models
.
IPAddress
.
lock_id
.
is_
(
)
)
,
models
.
IPAddress
.
_deallocated
.
is_
(
)
,
models
.
IPAddress
.
_deallocated
==
0
,
models
.
IPAddress
.
deallocated_at
>
reuse_window
)
)
)
query
=
query
.
outerjoin
(
models
.
IPPolicyCIDR
,
and_
(
models
.
Subnet
.
ip_policy_id
==
models
.
IPPolicyCIDR
.
ip_policy_id
,
models
.
IPAddress
.
address
>=
models
.
IPPolicyCIDR
.
first_ip
,
models
.
IPAddress
.
address
<=
models
.
IPPolicyCIDR
.
last_ip
)
)
query
=
query
.
filter
(
or_
(
models
.
IPAddress
.
_deallocated
.
is_
(
)
,
models
.
IPAddress
.
_deallocated
==
0
,
models
.
IPPolicyCIDR
.
id
.
is_
(
)
)
)
ret
=
(
(
segment_id
,
address_count
)
for
segment_id
,
address_count
in
query
.
all
(
)
)
return
dict
(
ret
)

Returns dictionary with keys segment_id and value used IPs count .


def get_unused_ips(session, used_ips_counts, **kwargs):
    """Returns dictionary with key segment_id, and value unused IPs count.

    Unused IP address count is determined by:
    - adding subnet's cidr's size
    - subtracting IP policy exclusions on subnet
    - subtracting used ips per segment
    """
    LOG.debug("Getting unused IPs...")
    with session.begin():
        query = session.query(
            models.Subnet.segment_id,
            models.Subnet)
        query = _filter(query, **kwargs)
        query = query.group_by(models.Subnet.segment_id, models.Subnet.id)

        ret = defaultdict(int)
        for segment_id, subnet in query.all():
            net_size = netaddr.IPNetwork(subnet._cidr).size
            ip_policy = subnet["ip_policy"] or {"size": 0}
            ret[segment_id] += net_size - ip_policy["size"]

        for segment_id in used_ips_counts:
            ret[segment_id] -= used_ips_counts[segment_id]

        return ret

def
get_unused_ips
(
session
,
used_ips_counts
,
**
kwargs
)
:
LOG
.
debug
(
"Getting unused IPs..."
)
with
session
.
begin
(
)
:
query
=
session
.
query
(
models
.
Subnet
.
segment_id
,
models
.
Subnet
)
query
=
_filter
(
query
,
**
kwargs
)
query
=
query
.
group_by
(
models
.
Subnet
.
segment_id
,
models
.
Subnet
.
id
)
ret
=
defaultdict
(
int
)
for
segment_id
,
subnet
in
query
.
all
(
)
:
net_size
=
netaddr
.
IPNetwork
(
subnet
.
_cidr
)
.
size
ip_policy
=
subnet
[
"ip_policy"
]
or
{
"size"
:
0
}
ret
[
segment_id
]
+=
net_size
ip_policy
[
"size"
]
for
segment_id
in
used_ips_counts
:
ret
[
segment_id
]
-=
used_ips_counts
[
segment_id
]
return
ret

Returns dictionary with key segment_id and value unused IPs count .


def get_interfaces(self):
        """Returns a set of VIFs from `get_instances` return value."""
        LOG.debug("Getting interfaces from Xapi")

        with self.sessioned() as session:
            instances = self.get_instances(session)
            recs = session.xenapi.VIF.get_all_records()

        interfaces = set()
        for vif_ref, rec in recs.iteritems():
            vm = instances.get(rec["VM"])
            if not vm:
                continue
            device_id = vm.uuid
            interfaces.add(VIF(device_id, rec, vif_ref))
        return interfaces

def
get_interfaces
(
self
)
:
LOG
.
debug
(
"Getting interfaces from Xapi"
)
with
self
.
sessioned
(
)
as
session
:
instances
=
self
.
get_instances
(
session
)
recs
=
session
.
xenapi
.
VIF
.
get_all_records
(
)
interfaces
=
set
(
)
for
vif_ref
,
rec
in
recs
.
iteritems
(
)
:
vm
=
instances
.
get
(
rec
[
"VM"
]
)
if
not
vm
:
continue
device_id
=
vm
.
uuid
interfaces
.
add
(
VIF
(
device_id
,
rec
,
vif_ref
)
)
return
interfaces

Returns a set of VIFs from get_instances return value .


def update_interfaces(self, added_sg, updated_sg, removed_sg):
        """Handles changes to interfaces' security groups

        Calls refresh_interfaces on argument VIFs. Set security groups on
        added_sg's VIFs. Unsets security groups on removed_sg's VIFs.
        """
        if not (added_sg or updated_sg or removed_sg):
            return

        with self.sessioned() as session:
            self._set_security_groups(session, added_sg)
            self._unset_security_groups(session, removed_sg)
            combined = added_sg + updated_sg + removed_sg
            self._refresh_interfaces(session, combined)

def update_interfaces ( self , added_sg , updated_sg , removed_sg ) : if not ( added_sg or updated_sg or removed_sg ) : return with self . sessioned ( ) as session : self . _set_security_groups ( session , added_sg ) self . _unset_security_groups ( session , removed_sg ) combined = added_sg + updated_sg + removed_sg self . _refresh_interfaces ( session , combined )

Handles changes to interfaces security groups


def update_network(context, id, network):
    """Update values of a network.

    : param context: neutron api request context
    : param id: UUID representing the network to update.
    : param network: dictionary with keys indicating fields to update.
        valid keys are those that have a value of True for 'allow_put'
        as listed in the RESOURCE_ATTRIBUTE_MAP object in
        neutron/api/v2/attributes.py.
    """
    LOG.info("update_network %s for tenant %s" %
             (id, context.tenant_id))
    with context.session.begin():
        net = db_api.network_find(context, id=id, scope=db_api.ONE)
        if not net:
            raise n_exc.NetworkNotFound(net_id=id)
        net_dict = network["network"]
        utils.pop_param(net_dict, "network_plugin")
        if not context.is_admin and "ipam_strategy" in net_dict:
            utils.pop_param(net_dict, "ipam_strategy")
        net = db_api.network_update(context, net, **net_dict)

    return v._make_network_dict(net)

def
update_network
(
context
,
id
,
network
)
:
LOG
.
info
(
"update_network %s for tenant %s"
%
(
id
,
context
.
tenant_id
)
)
with
context
.
session
.
begin
(
)
:
net
=
db_api
.
network_find
(
context
,
id
=
id
,
scope
=
db_api
.
ONE
)
if
not
net
:
raise
n_exc
.
NetworkNotFound
(
net_id
=
id
)
net_dict
=
network
[
"network"
]
utils
.
pop_param
(
net_dict
,
"network_plugin"
)
if
not
context
.
is_admin
and
"ipam_strategy"
in
net_dict
:
utils
.
pop_param
(
net_dict
,
"ipam_strategy"
)
net
=
db_api
.
network_update
(
context
,
net
,
**
net_dict
)
return
v
.
_make_network_dict
(
net
)


def get_network(context, id, fields=None):
    """Retrieve a network.

    : param context: neutron api request context
    : param id: UUID representing the network to fetch.
    : param fields: a list of strings that are valid keys in a
        network dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.
    """
    LOG.info("get_network %s for tenant %s fields %s" %
             (id, context.tenant_id, fields))

    network = db_api.network_find(context=context, limit=None, sorts=['id'],
                                  marker=None, page_reverse=False,
                                  id=id, join_subnets=True, scope=db_api.ONE)
    if not network:
        raise n_exc.NetworkNotFound(net_id=id)
    return v._make_network_dict(network, fields=fields)


def get_networks(context, limit=None, sorts=['id'], marker=None,
                 page_reverse=False, filters=None, fields=None):
    """Retrieve a list of networks.

    The contents of the list depends on the identity of the user
    making the request (as indicated by the context) as well as any
    filters.
    : param context: neutron api request context
    : param filters: a dictionary with keys that are valid keys for
        a network as listed in the RESOURCE_ATTRIBUTE_MAP object
        in neutron/api/v2/attributes.py.  Values in this dictiontary
        are an iterable containing values that will be used for an exact
        match comparison for that value.  Each result returned by this
        function will have matched one of the values for each key in
        filters.
    : param fields: a list of strings that are valid keys in a
        network dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.
    """
    LOG.info("get_networks for tenant %s with filters %s, fields %s" %
             (context.tenant_id, filters, fields))
    filters = filters or {}
    nets = db_api.network_find(context, limit, sorts, marker, page_reverse,
                               join_subnets=True, **filters) or []
    nets = [v._make_network_dict(net, fields=fields) for net in nets]
    return nets


def get_networks_count(context, filters=None):
    """Return the number of networks.

    The result depends on the identity of the user making the request
    (as indicated by the context) as well as any filters.
    : param context: neutron api request context
    : param filters: a dictionary with keys that are valid keys for
        a network as listed in the RESOURCE_ATTRIBUTE_MAP object
        in neutron/api/v2/attributes.py.  Values in this dictiontary
        are an iterable containing values that will be used for an exact
        match comparison for that value.  Each result returned by this
        function will have matched one of the values for each key in
        filters.

    NOTE: this method is optional, as it was not part of the originally
          defined plugin API.
    """
    LOG.info("get_networks_count for tenant %s filters %s" %
             (context.tenant_id, filters))
    return db_api.network_count_all(context)


def delete_network(context, id):
    """Delete a network.

    : param context: neutron api request context
    : param id: UUID representing the network to delete.
    """
    LOG.info("delete_network %s for tenant %s" % (id, context.tenant_id))
    with context.session.begin():
        net = db_api.network_find(context=context, limit=None, sorts=['id'],
                                  marker=None, page_reverse=False, id=id,
                                  scope=db_api.ONE)
        if not net:
            raise n_exc.NetworkNotFound(net_id=id)
        if not context.is_admin:
            if STRATEGY.is_provider_network(net.id):
                raise n_exc.NotAuthorized(net_id=id)
        if net.ports:
            raise n_exc.NetworkInUse(net_id=id)
        net_driver = registry.DRIVER_REGISTRY.get_driver(net["network_plugin"])
        net_driver.delete_network(context, id)
        for subnet in net["subnets"]:
            subnets._delete_subnet(context, subnet)
        db_api.network_delete(context, net)

def
delete_network
(
context
,
id
)
:
LOG
.
info
(
"delete_network %s for tenant %s"
%
(
id
,
context
.
tenant_id
)
)
with
context
.
session
.
begin
(
)
:
net
=
db_api
.
network_find
(
context
=
context
,
limit
=
,
sorts
=
[
'id'
]
,
marker
=
,
page_reverse
=
False
,
id
=
id
,
scope
=
db_api
.
ONE
)
if
not
net
:
raise
n_exc
.
NetworkNotFound
(
net_id
=
id
)
if
not
context
.
is_admin
:
if
STRATEGY
.
is_provider_network
(
net
.
id
)
:
raise
n_exc
.
NotAuthorized
(
net_id
=
id
)
if
net
.
ports
:
raise
n_exc
.
NetworkInUse
(
net_id
=
id
)
net_driver
=
registry
.
DRIVER_REGISTRY
.
get_driver
(
net
[
"network_plugin"
]
)
net_driver
.
delete_network
(
context
,
id
)
for
subnet
in
net
[
"subnets"
]
:
subnets
.
_delete_subnet
(
context
,
subnet
)
db_api
.
network_delete
(
context
,
net
)


def make_case2(context):
    """This is a helper method for testing.

    When run with the current context, it will create a case 2 entries
    in the database. See top of file for what case 2 is.
    """
    query = context.session.query(models.IPAddress)
    period_start, period_end = billing.calc_periods()
    ip_list = billing.build_full_day_ips(query, period_start, period_end)
    import random
    ind = random.randint(0, len(ip_list) - 1)
    address = ip_list[ind]
    address.allocated_at = datetime.datetime.utcnow() -\
        datetime.timedelta(days=1)
    context.session.add(address)
    context.session.flush()

def make_case2 ( context ) : query = context . session . query ( models . IPAddress ) period_start , period_end = billing . calc_periods ( ) ip_list = billing . build_full_day_ips ( query , period_start , period_end ) import random ind = random . randint ( 0 , len ( ip_list ) - 1 ) address = ip_list [ ind ] address . allocated_at = datetime . datetime . utcnow ( ) - datetime . timedelta ( days = 1 ) context . session . add ( address ) context . session . flush ( )


def main(notify, hour, minute):
    """Runs billing report. Optionally sends notifications to billing"""

    # Read the config file and get the admin context
    config_opts = ['--config-file', '/etc/neutron/neutron.conf']
    config.init(config_opts)
    # Have to load the billing module _after_ config is parsed so
    # that we get the right network strategy
    network_strategy.STRATEGY.load()
    billing.PUBLIC_NETWORK_ID = network_strategy.STRATEGY.get_public_net_id()
    config.setup_logging()
    context = neutron_context.get_admin_context()

    # A query to get all IPAddress objects from the db
    query = context.session.query(models.IPAddress)

    (period_start, period_end) = billing.calc_periods(hour, minute)

    full_day_ips = billing.build_full_day_ips(query,
                                              period_start,
                                              period_end)
    partial_day_ips = billing.build_partial_day_ips(query,
                                                    period_start,
                                                    period_end)

    if notify:
        # '==================== Full Day ============================='
        for ipaddress in full_day_ips:
            click.echo('start: {}, end: {}'.format(period_start, period_end))
            payload = billing.build_payload(ipaddress,
                                            billing.IP_EXISTS,
                                            start_time=period_start,
                                            end_time=period_end)
            billing.do_notify(context,
                              billing.IP_EXISTS,
                              payload)
        # '==================== Part Day ============================='
        for ipaddress in partial_day_ips:
            click.echo('start: {}, end: {}'.format(period_start, period_end))
            payload = billing.build_payload(ipaddress,
                                            billing.IP_EXISTS,
                                            start_time=ipaddress.allocated_at,
                                            end_time=period_end)
            billing.do_notify(context,
                              billing.IP_EXISTS,
                              payload)
    else:
        click.echo('Case 1 ({}):\n'.format(len(full_day_ips)))
        for ipaddress in full_day_ips:
            pp(billing.build_payload(ipaddress,
                                     billing.IP_EXISTS,
                                     start_time=period_start,
                                     end_time=period_end))

        click.echo('\n===============================================\n')

        click.echo('Case 2 ({}):\n'.format(len(partial_day_ips)))
        for ipaddress in partial_day_ips:
            pp(billing.build_payload(ipaddress,
                                     billing.IP_EXISTS,
                                     start_time=ipaddress.allocated_at,
                                     end_time=period_end))

Runs billing report . Optionally sends notifications to billing


def start_rpc_listeners(self):
        """Configure all listeners here"""
        self._setup_rpc()
        if not self.endpoints:
            return []
        self.conn = n_rpc.create_connection()
        self.conn.create_consumer(self.topic, self.endpoints,
                                  fanout=False)
        return self.conn.consume_in_threads()

def start_rpc_listeners ( self ) : self . _setup_rpc ( ) if not self . endpoints : return [ ] self . conn = n_rpc . create_connection ( ) self . conn . create_consumer ( self . topic , self . endpoints , fanout = False ) return self . conn . consume_in_threads ( )


def context(self):
        """Provides an admin context for workers."""
        if not self._context:
            self._context = context.get_admin_context()
        return self._context

def context ( self ) : if not self . _context : self . _context = context . get_admin_context ( ) return self . _context


def update_sg(self, context, sg, rule_id, action):
        """Begins the async update process."""
        db_sg = db_api.security_group_find(context, id=sg, scope=db_api.ONE)
        if not db_sg:
            return None
        with context.session.begin():
            job_body = dict(action="%s sg rule %s" % (action, rule_id),
                            resource_id=rule_id,
                            tenant_id=db_sg['tenant_id'])
            job_body = dict(job=job_body)
            job = job_api.create_job(context.elevated(), job_body)
            rpc_client = QuarkSGAsyncProducerClient()
            try:
                rpc_client.populate_subtasks(context, sg, job['id'])
            except om_exc.MessagingTimeout:
                LOG.error("Failed to create subtasks. Rabbit running?")
                return None
        return {"job_id": job['id']}


def populate_subtasks(self, context, sg, parent_job_id):
        """Produces a list of ports to be updated async."""
        db_sg = db_api.security_group_find(context, id=sg, scope=db_api.ONE)
        if not db_sg:
            return None
        ports = db_api.sg_gather_associated_ports(context, db_sg)
        if len(ports) == 0:
            return {"ports": 0}
        for port in ports:
            job_body = dict(action="update port %s" % port['id'],
                            tenant_id=db_sg['tenant_id'],
                            resource_id=port['id'],
                            parent_id=parent_job_id)
            job_body = dict(job=job_body)
            job = job_api.create_job(context.elevated(), job_body)
            rpc_consumer = QuarkSGAsyncConsumerClient()
            try:
                rpc_consumer.update_port(context, port['id'], job['id'])
            except om_exc.MessagingTimeout:
                # TODO(roaet): Not too sure what can be done here other than
                # updating the job as a failure?
                LOG.error("Failed to update port. Rabbit running?")
        return None

Produces a list of ports to be updated async .


def update_ports_for_sg(self, context, portid, jobid):
        """Updates the ports through redis."""
        port = db_api.port_find(context, id=portid, scope=db_api.ONE)
        if not port:
            LOG.warning("Port not found")
            return
        net_driver = port_api._get_net_driver(port.network, port=port)
        base_net_driver = port_api._get_net_driver(port.network)
        sg_list = [sg for sg in port.security_groups]

        success = False
        error = None
        retries = 3
        retry_delay = 2
        for retry in xrange(retries):
            try:
                net_driver.update_port(context, port_id=port["backend_key"],
                                       mac_address=port["mac_address"],
                                       device_id=port["device_id"],
                                       base_net_driver=base_net_driver,
                                       security_groups=sg_list)
                success = True
                error = None
                break
            except Exception as error:
                LOG.warning("Could not connect to redis, but retrying soon")
                time.sleep(retry_delay)
        status_str = ""
        if not success:
            status_str = "Port %s update failed after %d tries. Error: %s" % (
                portid, retries, error)
        update_body = dict(completed=True, status=status_str)
        update_body = dict(job=update_body)
        job_api.update_job(context.elevated(), jobid, update_body)


def sg_gather_associated_ports(context, group):
    """Gather all ports associated to security group.

    Returns:
    * list, or None
    """
    if not group:
        return None
    if not hasattr(group, "ports") or len(group.ports) <= 0:
        return []
    return group.ports

def sg_gather_associated_ports ( context , group ) : if not group : return None if not hasattr ( group , "ports" ) or len ( group . ports ) <= 0 : return [ ] return group . ports

Gather all ports associated to security group .


def security_group_rule_update(context, rule, **kwargs):
    '''Updates a security group rule.

    NOTE(alexm) this is non-standard functionality.
    '''
    rule.update(kwargs)
    context.session.add(rule)
    return rule

def security_group_rule_update ( context , rule , ** kwargs ) : rule . update ( kwargs ) context . session . add ( rule ) return rule


def segment_allocation_find(context, lock_mode=False, **filters):
    """Query for segment allocations."""
    range_ids = filters.pop("segment_allocation_range_ids", None)

    query = context.session.query(models.SegmentAllocation)
    if lock_mode:
        query = query.with_lockmode("update")

    query = query.filter_by(**filters)

    # Optionally filter by given list of range ids
    if range_ids:
        query.filter(
            models.SegmentAllocation.segment_allocation_range_id.in_(
                range_ids))
    return query

def segment_allocation_find ( context , lock_mode = False , ** filters ) : range_ids = filters . pop ( "segment_allocation_range_ids" , None ) query = context . session . query ( models . SegmentAllocation ) if lock_mode : query = query . with_lockmode ( "update" ) query = query . filter_by ( ** filters ) if range_ids : query . filter ( models . SegmentAllocation . segment_allocation_range_id . in_ ( range_ids ) ) return query


def send(self, s):
        """
        Sends the given command to Niko Home Control and returns the output of
        the system.

        Aliases: write, put, sendall, send_all
        """
        self._socket.send(s.encode())
        return self.read()

def send ( self , s ) : self . _socket . send ( s . encode ( ) ) return self . read ( )

Sends the given command to Niko Home Control and returns the output of the system .


def if_(*args):
    """Implements the 'if' operator with support for multiple elseif-s."""
    for i in range(0, len(args) - 1, 2):
        if args[i]:
            return args[i + 1]
    if len(args) % 2:
        return args[-1]
    else:
        return None

def if_ ( * args ) : for i in range ( 0 , len ( args ) - 1 , 2 ) : if args [ i ] : return args [ i + 1 ] if len ( args ) % 2 : return args [ - 1 ] else : return None

Implements the if operator with support for multiple elseif - s .


def soft_equals(a, b):
    """Implements the '==' operator, which does type JS-style coertion."""
    if isinstance(a, str) or isinstance(b, str):
        return str(a) == str(b)
    if isinstance(a, bool) or isinstance(b, bool):
        return bool(a) is bool(b)
    return a == b

def soft_equals ( a , b ) : if isinstance ( a , str ) or isinstance ( b , str ) : return str ( a ) == str ( b ) if isinstance ( a , bool ) or isinstance ( b , bool ) : return bool ( a ) is bool ( b ) return a == b

Implements the == operator which does type JS - style coertion .


def less(a, b, *args):
    """Implements the '<' operator with JS-style type coertion."""
    types = set([type(a), type(b)])
    if float in types or int in types:
        try:
            a, b = float(a), float(b)
        except TypeError:
            # NaN
            return False
    return a < b and (not args or less(b, *args))

def less ( a , b , * args ) : types = set ( [ type ( a ) , type ( b ) ] ) if float in types or int in types : try : a , b = float ( a ) , float ( b ) except TypeError : return False return a < b and ( not args or less ( b , * args ) )

Implements the < operator with JS - style type coertion .


def less_or_equal(a, b, *args):
    """Implements the '<=' operator with JS-style type coertion."""
    return (
        less(a, b) or soft_equals(a, b)
    ) and (not args or less_or_equal(b, *args))

def less_or_equal ( a , b , * args ) : return ( less ( a , b ) or soft_equals ( a , b ) ) and ( not args or less_or_equal ( b , * args ) )

Implements the < = operator with JS - style type coertion .


def minus(*args):
    """Also, converts either to ints or to floats."""
    if len(args) == 1:
        return -to_numeric(args[0])
    return to_numeric(args[0]) - to_numeric(args[1])

def minus ( * args ) : if len ( args ) == 1 : return - to_numeric ( args [ 0 ] ) return to_numeric ( args [ 0 ] ) - to_numeric ( args [ 1 ] )

Also converts either to ints or to floats .


def merge(*args):
    """Implements the 'merge' operator for merging lists."""
    ret = []
    for arg in args:
        if isinstance(arg, list) or isinstance(arg, tuple):
            ret += list(arg)
        else:
            ret.append(arg)
    return ret

def merge ( * args ) : ret = [ ] for arg in args : if isinstance ( arg , list ) or isinstance ( arg , tuple ) : ret += list ( arg ) else : ret . append ( arg ) return ret

Implements the merge operator for merging lists .


def get_var(data, var_name, not_found=None):
    """Gets variable value from data dictionary."""
    try:
        for key in str(var_name).split('.'):
            try:
                data = data[key]
            except TypeError:
                data = data[int(key)]
    except (KeyError, TypeError, ValueError):
        return not_found
    else:
        return data

def get_var ( data , var_name , not_found = None ) : try : for key in str ( var_name ) . split ( '.' ) : try : data = data [ key ] except TypeError : data = data [ int ( key ) ] except ( KeyError , TypeError , ValueError ) : return not_found else : return data

Gets variable value from data dictionary .


def missing(data, *args):
    """Implements the missing operator for finding missing variables."""
    not_found = object()
    if args and isinstance(args[0], list):
        args = args[0]
    ret = []
    for arg in args:
        if get_var(data, arg, not_found) is not_found:
            ret.append(arg)
    return ret

def missing ( data , * args ) : not_found = object ( ) if args and isinstance ( args [ 0 ] , list ) : args = args [ 0 ] ret = [ ] for arg in args : if get_var ( data , arg , not_found ) is not_found : ret . append ( arg ) return ret

Implements the missing operator for finding missing variables .


def missing_some(data, min_required, args):
    """Implements the missing_some operator for finding missing variables."""
    if min_required < 1:
        return []
    found = 0
    not_found = object()
    ret = []
    for arg in args:
        if get_var(data, arg, not_found) is not_found:
            ret.append(arg)
        else:
            found += 1
            if found >= min_required:
                return []
    return ret

def missing_some ( data , min_required , args ) : if min_required < 1 : return [ ] found = 0 not_found = object ( ) ret = [ ] for arg in args : if get_var ( data , arg , not_found ) is not_found : ret . append ( arg ) else : found += 1 if found >= min_required : return [ ] return ret

Implements the missing_some operator for finding missing variables .


def jsonLogic(tests, data=None):
    """Executes the json-logic with given data."""
    # You've recursed to a primitive, stop!
    if tests is None or not isinstance(tests, dict):
        return tests

    data = data or {}

    operator = list(tests.keys())[0]
    values = tests[operator]

    # Easy syntax for unary operators, like {"var": "x"} instead of strict
    # {"var": ["x"]}
    if not isinstance(values, list) and not isinstance(values, tuple):
        values = [values]

    # Recursion!
    values = [jsonLogic(val, data) for val in values]

    if operator == 'var':
        return get_var(data, *values)
    if operator == 'missing':
        return missing(data, *values)
    if operator == 'missing_some':
        return missing_some(data, *values)

    if operator not in operations:
        raise ValueError("Unrecognized operation %s" % operator)

    return operations[operator](*values)

def
jsonLogic
(
tests
,
data
=
)
:
if
tests
is
or
not
isinstance
(
tests
,
dict
)
:
return
tests
data
=
data
or
{
}
operator
=
list
(
tests
.
keys
(
)
)
[
0
]
values
=
tests
[
operator
]
if
not
isinstance
(
values
,
list
)
and
not
isinstance
(
values
,
tuple
)
:
values
=
[
values
]
values
=
[
jsonLogic
(
val
,
data
)
for
val
in
values
]
if
operator
==
'var'
:
return
get_var
(
data
,
*
values
)
if
operator
==
'missing'
:
return
missing
(
data
,
*
values
)
if
operator
==
'missing_some'
:
return
missing_some
(
data
,
*
values
)
if
operator
not
in
operations
:
raise
ValueError
(
"Unrecognized operation %s"
%
operator
)
return
operations
[
operator
]
(
*
values
)

Executes the json - logic with given data .


def indent(self):
        """
        Performs an indentation
        """
        if not self.tab_always_indent:
            super(PyIndenterMode, self).indent()
        else:
            cursor = self.editor.textCursor()
            assert isinstance(cursor, QtGui.QTextCursor)
            if cursor.hasSelection():
                self.indent_selection(cursor)
            else:
                # simply insert indentation at the cursor position
                tab_len = self.editor.tab_length
                cursor.beginEditBlock()
                if self.editor.use_spaces_instead_of_tabs:
                    cursor.insertText(tab_len * " ")
                else:
                    cursor.insertText('\t')
                cursor.endEditBlock()
                self.editor.setTextCursor(cursor)

def
indent
(
self
)
:
if
not
self
.
tab_always_indent
:
super
(
PyIndenterMode
,
self
)
.
indent
(
)
else
:
cursor
=
self
.
editor
.
textCursor
(
)
assert
isinstance
(
cursor
,
QtGui
.
QTextCursor
)
if
cursor
.
hasSelection
(
)
:
self
.
indent_selection
(
cursor
)
else
:
tab_len
=
self
.
editor
.
tab_length
cursor
.
beginEditBlock
(
)
if
self
.
editor
.
use_spaces_instead_of_tabs
:
cursor
.
insertText
(
tab_len
*
" "
)
else
:
cursor
.
insertText
(
'\t'
)
cursor
.
endEditBlock
(
)
self
.
editor
.
setTextCursor
(
cursor
)


def unindent(self):
        """
        Performs an un-indentation
        """
        if self.tab_always_indent:
            cursor = self.editor.textCursor()
            if not cursor.hasSelection():
                cursor.select(cursor.LineUnderCursor)
            self.unindent_selection(cursor)
        else:
            super(PyIndenterMode, self).unindent()

def unindent ( self ) : if self . tab_always_indent : cursor = self . editor . textCursor ( ) if not cursor . hasSelection ( ) : cursor . select ( cursor . LineUnderCursor ) self . unindent_selection ( cursor ) else : super ( PyIndenterMode , self ) . unindent ( )


def _handle_indent_between_paren(self, column, line, parent_impl, tc):
        """
        Handle indent between symbols such as parenthesis, braces,...
        """
        pre, post = parent_impl
        next_char = self._get_next_char(tc)
        prev_char = self._get_prev_char(tc)
        prev_open = prev_char in ['[', '(', '{']
        next_close = next_char in [']', ')', '}']
        (open_line, open_symbol_col), (close_line, close_col) = \
            self._get_paren_pos(tc, column)
        open_line_txt = self._helper.line_text(open_line)
        open_line_indent = len(open_line_txt) - len(open_line_txt.lstrip())
        if prev_open:
            post = (open_line_indent + self.editor.tab_length) * ' '
        elif next_close and prev_char != ',':
            post = open_line_indent * ' '
        elif tc.block().blockNumber() == open_line:
            post = open_symbol_col * ' '

        # adapt indent if cursor on closing line and next line have same
        # indent -> PEP8 compliance
        if close_line and close_col:
            txt = self._helper.line_text(close_line)
            bn = tc.block().blockNumber()
            flg = bn == close_line
            next_indent = self._helper.line_indent(bn + 1) * ' '
            if flg and txt.strip().endswith(':') and next_indent == post:
                # | look at how the previous line ( ``':'):`` ) was
                # over-indented, this is actually what we are trying to
                # achieve here
                post += self.editor.tab_length * ' '

        # breaking string
        if next_char in ['"', "'"]:
            tc.movePosition(tc.Left)
        is_string = self._helper.is_comment_or_string(tc, formats=['string'])
        if next_char in ['"', "'"]:
            tc.movePosition(tc.Right)
        if is_string:
            trav = QTextCursor(tc)
            while self._helper.is_comment_or_string(
                    trav, formats=['string']):
                trav.movePosition(trav.Left)
            trav.movePosition(trav.Right)
            symbol = '%s' % self._get_next_char(trav)
            pre += symbol
            post += symbol

        return pre, post

def
_handle_indent_between_paren
(
self
,
column
,
line
,
parent_impl
,
tc
)
:
pre
,
post
=
parent_impl
next_char
=
self
.
_get_next_char
(
tc
)
prev_char
=
self
.
_get_prev_char
(
tc
)
prev_open
=
prev_char
in
[
'['
,
'('
,
'{'
]
next_close
=
next_char
in
[
']'
,
')'
,
'}'
]
(
open_line
,
open_symbol_col
)
,
(
close_line
,
close_col
)
=
self
.
_get_paren_pos
(
tc
,
column
)
open_line_txt
=
self
.
_helper
.
line_text
(
open_line
)
open_line_indent
=
len
(
open_line_txt
)
len
(
open_line_txt
.
lstrip
(
)
)
if
prev_open
:
post
=
(
open_line_indent
+
self
.
editor
.
tab_length
)
*
' '
elif
next_close
and
prev_char
!=
','
:
post
=
open_line_indent
*
' '
elif
tc
.
block
(
)
.
blockNumber
(
)
==
open_line
:
post
=
open_symbol_col
*
' '
if
close_line
and
close_col
:
txt
=
self
.
_helper
.
line_text
(
close_line
)
bn
=
tc
.
block
(
)
.
blockNumber
(
)
flg
=
bn
==
close_line
next_indent
=
self
.
_helper
.
line_indent
(
bn
+
1
)
*
' '
if
flg
and
txt
.
strip
(
)
.
endswith
(
':'
)
and
next_indent
==
post
:
post
+=
self
.
editor
.
tab_length
*
' '
if
next_char
in
[
'"'
,
"'"
]
:
tc
.
movePosition
(
tc
.
Left
)
is_string
=
self
.
_helper
.
is_comment_or_string
(
tc
,
formats
=
[
'string'
]
)
if
next_char
in
[
'"'
,
"'"
]
:
tc
.
movePosition
(
tc
.
Right
)
if
is_string
:
trav
=
QTextCursor
(
tc
)
while
self
.
_helper
.
is_comment_or_string
(
trav
,
formats
=
[
'string'
]
)
:
trav
.
movePosition
(
trav
.
Left
)
trav
.
movePosition
(
trav
.
Right
)
symbol
=
'%s'
%
self
.
_get_next_char
(
trav
)
pre
+=
symbol
post
+=
symbol
return
pre
,
post

Handle indent between symbols such as parenthesis braces ...


def _at_block_start(tc, line):
        """
        Improve QTextCursor.atBlockStart to ignore spaces
        """
        if tc.atBlockStart():
            return True
        column = tc.columnNumber()
        indentation = len(line) - len(line.lstrip())
        return column <= indentation

def _at_block_start ( tc , line ) : if tc . atBlockStart ( ) : return True column = tc . columnNumber ( ) indentation = len ( line ) - len ( line . lstrip ( ) ) return column <= indentation

Improve QTextCursor . atBlockStart to ignore spaces


def update_terminal_colors(self):
        """
        Update terminal color scheme based on the pygments color scheme colors
        """
        self.color_scheme = self.create_color_scheme(
            background=self.syntax_highlighter.color_scheme.background,
            foreground=self.syntax_highlighter.color_scheme.formats['normal'].foreground().color())

def update_terminal_colors ( self ) : self . color_scheme = self . create_color_scheme ( background = self . syntax_highlighter . color_scheme . background , foreground = self . syntax_highlighter . color_scheme . formats [ 'normal' ] . foreground ( ) . color ( ) )


def mouseMoveEvent(self, e):
        """
        Extends mouseMoveEvent to display a pointing hand cursor when the
        mouse cursor is over a file location
        """
        super(PyInteractiveConsole, self).mouseMoveEvent(e)
        cursor = self.cursorForPosition(e.pos())
        assert isinstance(cursor, QtGui.QTextCursor)
        p = cursor.positionInBlock()
        usd = cursor.block().userData()
        if usd and usd.start_pos_in_block <= p <= usd.end_pos_in_block:
            if QtWidgets.QApplication.overrideCursor() is None:
                QtWidgets.QApplication.setOverrideCursor(
                    QtGui.QCursor(QtCore.Qt.PointingHandCursor))
        else:
            if QtWidgets.QApplication.overrideCursor() is not None:
                QtWidgets.QApplication.restoreOverrideCursor()

def mouseMoveEvent ( self , e ) : super ( PyInteractiveConsole , self ) . mouseMoveEvent ( e ) cursor = self . cursorForPosition ( e . pos ( ) ) assert isinstance ( cursor , QtGui . QTextCursor ) p = cursor . positionInBlock ( ) usd = cursor . block ( ) . userData ( ) if usd and usd . start_pos_in_block <= p <= usd . end_pos_in_block : if QtWidgets . QApplication . overrideCursor ( ) is None : QtWidgets . QApplication . setOverrideCursor ( QtGui . QCursor ( QtCore . Qt . PointingHandCursor ) ) else : if QtWidgets . QApplication . overrideCursor ( ) is not None : QtWidgets . QApplication . restoreOverrideCursor ( )


def mousePressEvent(self, e):
        """
        Emits open_file_requested if the press event occured  over
        a file location string.
        """
        super(PyInteractiveConsole, self).mousePressEvent(e)
        cursor = self.cursorForPosition(e.pos())
        p = cursor.positionInBlock()
        usd = cursor.block().userData()
        if usd and usd.start_pos_in_block <= p <= usd.end_pos_in_block:
            if e.button() == QtCore.Qt.LeftButton:
                self.open_file_requested.emit(usd.filename, usd.line)

def mousePressEvent ( self , e ) : super ( PyInteractiveConsole , self ) . mousePressEvent ( e ) cursor = self . cursorForPosition ( e . pos ( ) ) p = cursor . positionInBlock ( ) usd = cursor . block ( ) . userData ( ) if usd and usd . start_pos_in_block <= p <= usd . end_pos_in_block : if e . button ( ) == QtCore . Qt . LeftButton : self . open_file_requested . emit ( usd . filename , usd . line )

Emits open_file_requested if the press event occured over a file location string .


def setup_actions(self):
        """ Connects slots to signals """
        self.actionOpen.triggered.connect(self.on_open)
        self.actionNew.triggered.connect(self.on_new)
        self.actionSave.triggered.connect(self.on_save)
        self.actionSave_as.triggered.connect(self.on_save_as)
        self.actionQuit.triggered.connect(
            QtWidgets.QApplication.instance().quit)
        self.tabWidget.current_changed.connect(self.on_current_tab_changed)
        self.tabWidget.last_tab_closed.connect(self.on_last_tab_closed)
        self.actionAbout.triggered.connect(self.on_about)
        self.actionRun.triggered.connect(self.on_run)
        self.interactiveConsole.process_finished.connect(
            self.on_process_finished)
        self.actionConfigure_run.triggered.connect(self.on_configure_run)

def setup_actions ( self ) : self . actionOpen . triggered . connect ( self . on_open ) self . actionNew . triggered . connect ( self . on_new ) self . actionSave . triggered . connect ( self . on_save ) self . actionSave_as . triggered . connect ( self . on_save_as ) self . actionQuit . triggered . connect ( QtWidgets . QApplication . instance ( ) . quit ) self . tabWidget . current_changed . connect ( self . on_current_tab_changed ) self . tabWidget . last_tab_closed . connect ( self . on_last_tab_closed ) self . actionAbout . triggered . connect ( self . on_about ) self . actionRun . triggered . connect ( self . on_run ) self . interactiveConsole . process_finished . connect ( self . on_process_finished ) self . actionConfigure_run . triggered . connect ( self . on_configure_run )


def setup_editor(self, editor):
        """
        Setup the python editor, run the server and connect a few signals.

        :param editor: editor to setup.
        """
        editor.cursorPositionChanged.connect(self.on_cursor_pos_changed)
        try:
            m = editor.modes.get(modes.GoToAssignmentsMode)
        except KeyError:
            pass
        else:
            assert isinstance(m, modes.GoToAssignmentsMode)
            m.out_of_doc.connect(self.on_goto_out_of_doc)

def setup_editor ( self , editor ) : editor . cursorPositionChanged . connect ( self . on_cursor_pos_changed ) try : m = editor . modes . get ( modes . GoToAssignmentsMode ) except KeyError : pass else : assert isinstance ( m , modes . GoToAssignmentsMode ) m . out_of_doc . connect ( self . on_goto_out_of_doc )

Setup the python editor run the server and connect a few signals .


def open_file(self, path, line=None):
        """
        Creates a new GenericCodeEdit, opens the requested file and adds it
        to the tab widget.

        :param path: Path of the file to open

        :return The opened editor if open succeeded.
        """
        editor = None
        if path:
            interpreter, pyserver, args = self._get_backend_parameters()
            editor = self.tabWidget.open_document(
                path, None, interpreter=interpreter, server_script=pyserver,
                args=args)
            if editor:
                self.setup_editor(editor)
            self.recent_files_manager.open_file(path)
            self.menu_recents.update_actions()
        if line is not None:
            TextHelper(self.tabWidget.current_widget()).goto_line(line)
        return editor

def open_file ( self , path , line = None ) : editor = None if path : interpreter , pyserver , args = self . _get_backend_parameters ( ) editor = self . tabWidget . open_document ( path , None , interpreter = interpreter , server_script = pyserver , args = args ) if editor : self . setup_editor ( editor ) self . recent_files_manager . open_file ( path ) self . menu_recents . update_actions ( ) if line is not None : TextHelper ( self . tabWidget . current_widget ( ) ) . goto_line ( line ) return editor

Creates a new GenericCodeEdit opens the requested file and adds it to the tab widget .


def on_new(self):
        """
        Add a new empty code editor to the tab widget
        """
        interpreter, pyserver, args = self._get_backend_parameters()
        self.setup_editor(self.tabWidget.create_new_document(
            extension='.py', interpreter=interpreter, server_script=pyserver,
            args=args))
        self.actionRun.setDisabled(True)
        self.actionConfigure_run.setDisabled(True)

def on_new ( self ) : interpreter , pyserver , args = self . _get_backend_parameters ( ) self . setup_editor ( self . tabWidget . create_new_document ( extension = '.py' , interpreter = interpreter , server_script = pyserver , args = args ) ) self . actionRun . setDisabled ( True ) self . actionConfigure_run . setDisabled ( True )


def on_open(self):
        """
        Shows an open file dialog and open the file if the dialog was
        accepted.

        """
        filename, filter = QtWidgets.QFileDialog.getOpenFileName(self, 'Open')
        if filename:
            self.open_file(filename)
        self.actionRun.setEnabled(True)
        self.actionConfigure_run.setEnabled(True)

def on_open ( self ) : filename , filter = QtWidgets . QFileDialog . getOpenFileName ( self , 'Open' ) if filename : self . open_file ( filename ) self . actionRun . setEnabled ( True ) self . actionConfigure_run . setEnabled ( True )

Shows an open file dialog and open the file if the dialog was accepted .


def on_save_as(self):
        """
        Save the current editor document as.
        """
        path = self.tabWidget.current_widget().file.path
        path = os.path.dirname(path) if path else ''
        filename, filter = QtWidgets.QFileDialog.getSaveFileName(
            self, 'Save', path)
        if filename:
            self.tabWidget.save_current(filename)
            self.recent_files_manager.open_file(filename)
            self.menu_recents.update_actions()
            self.actionRun.setEnabled(True)
            self.actionConfigure_run.setEnabled(True)
            self._update_status_bar(self.tabWidget.current_widget())

def on_save_as ( self ) : path = self . tabWidget . current_widget ( ) . file . path path = os . path . dirname ( path ) if path else '' filename , filter = QtWidgets . QFileDialog . getSaveFileName ( self , 'Save' , path ) if filename : self . tabWidget . save_current ( filename ) self . recent_files_manager . open_file ( filename ) self . menu_recents . update_actions ( ) self . actionRun . setEnabled ( True ) self . actionConfigure_run . setEnabled ( True ) self . _update_status_bar ( self . tabWidget . current_widget ( ) )


def setup_mnu_style(self, editor):
        """ setup the style menu for an editor tab """
        menu = QtWidgets.QMenu('Styles', self.menuEdit)
        group = QtWidgets.QActionGroup(self)
        self.styles_group = group
        current_style = editor.syntax_highlighter.color_scheme.name
        group.triggered.connect(self.on_style_changed)
        for s in sorted(PYGMENTS_STYLES):
            a = QtWidgets.QAction(menu)
            a.setText(s)
            a.setCheckable(True)
            if s == current_style:
                a.setChecked(True)
            group.addAction(a)
            menu.addAction(a)
        self.menuEdit.addMenu(menu)

def setup_mnu_style ( self , editor ) : menu = QtWidgets . QMenu ( 'Styles' , self . menuEdit ) group = QtWidgets . QActionGroup ( self ) self . styles_group = group current_style = editor . syntax_highlighter . color_scheme . name group . triggered . connect ( self . on_style_changed ) for s in sorted ( PYGMENTS_STYLES ) : a = QtWidgets . QAction ( menu ) a . setText ( s ) a . setCheckable ( True ) if s == current_style : a . setChecked ( True ) group . addAction ( a ) menu . addAction ( a ) self . menuEdit . addMenu ( menu )


def on_current_tab_changed(self):
        """
        Update action states when the current tab changed.
        """
        self.menuEdit.clear()
        self.menuModes.clear()
        self.menuPanels.clear()
        editor = self.tabWidget.current_widget()
        self.menuEdit.setEnabled(editor is not None)
        self.menuModes.setEnabled(editor is not None)
        self.menuPanels.setEnabled(editor is not None)
        self.actionSave.setEnabled(editor is not None)
        self.actionSave_as.setEnabled(editor is not None)
        self.actionConfigure_run.setEnabled(editor is not None)
        self.actionRun.setEnabled(editor is not None)
        if editor is not None:
            self.setup_mnu_edit(editor)
            self.setup_mnu_modes(editor)
            self.setup_mnu_panels(editor)
        self.widgetOutline.set_editor(editor)
        self._update_status_bar(editor)

def on_current_tab_changed ( self ) : self . menuEdit . clear ( ) self . menuModes . clear ( ) self . menuPanels . clear ( ) editor = self . tabWidget . current_widget ( ) self . menuEdit . setEnabled ( editor is not None ) self . menuModes . setEnabled ( editor is not None ) self . menuPanels . setEnabled ( editor is not None ) self . actionSave . setEnabled ( editor is not None ) self . actionSave_as . setEnabled ( editor is not None ) self . actionConfigure_run . setEnabled ( editor is not None ) self . actionRun . setEnabled ( editor is not None ) if editor is not None : self . setup_mnu_edit ( editor ) self . setup_mnu_modes ( editor ) self . setup_mnu_panels ( editor ) self . widgetOutline . set_editor ( editor ) self . _update_status_bar ( editor )

Update action states when the current tab changed .


def on_run(self):
        """
        Run the current current script
        """
        filename = self.tabWidget.current_widget().file.path
        wd = os.path.dirname(filename)
        args = Settings().get_run_config_for_file(filename)
        self.interactiveConsole.start_process(
            Settings().interpreter, args=[filename] + args, cwd=wd)
        self.dockWidget.show()
        self.actionRun.setEnabled(False)
        self.actionConfigure_run.setEnabled(False)

def on_run ( self ) : filename = self . tabWidget . current_widget ( ) . file . path wd = os . path . dirname ( filename ) args = Settings ( ) . get_run_config_for_file ( filename ) self . interactiveConsole . start_process ( Settings ( ) . interpreter , args = [ filename ] + args , cwd = wd ) self . dockWidget . show ( ) self . actionRun . setEnabled ( False ) self . actionConfigure_run . setEnabled ( False )


def on_goto_out_of_doc(self, assignment):
        """
        Open the a new tab when goto goes out of the current document.

        :param assignment: Destination
        """
        editor = self.open_file(assignment.module_path)
        if editor:
            TextHelper(editor).goto_line(assignment.line, assignment.column)

def on_goto_out_of_doc ( self , assignment ) : editor = self . open_file ( assignment . module_path ) if editor : TextHelper ( editor ) . goto_line ( assignment . line , assignment . column )

Open the a new tab when goto goes out of the current document .


def calltips(request_data):
    """
    Worker that returns a list of calltips.

    A calltips is a tuple made of the following parts:
      - module_name: name of the module of the function invoked
      - call_name: name of the function that is being called
      - params: the list of parameter names.
      - index: index of the current parameter
      - bracket_start

    :returns tuple(module_name, call_name, params)
    """
    code = request_data['code']
    line = request_data['line'] + 1
    column = request_data['column']
    path = request_data['path']
    # encoding = request_data['encoding']
    encoding = 'utf-8'
    # use jedi to get call signatures
    script = jedi.Script(code, line, column, path, encoding)
    signatures = script.call_signatures()
    for sig in signatures:
        results = (str(sig.module_name), str(sig.name),
                   [p.description for p in sig.params], sig.index,
                   sig.bracket_start, column)
        # todo: add support for multiple signatures, for that we need a custom
        # widget for showing calltips.
        return results
    return []

def calltips ( request_data ) : code = request_data [ 'code' ] line = request_data [ 'line' ] + 1 column = request_data [ 'column' ] path = request_data [ 'path' ] encoding = 'utf-8' script = jedi . Script ( code , line , column , path , encoding ) signatures = script . call_signatures ( ) for sig in signatures : results = ( str ( sig . module_name ) , str ( sig . name ) , [ p . description for p in sig . params ] , sig . index , sig . bracket_start , column ) return results return [ ]

Worker that returns a list of calltips .


def goto_assignments(request_data):
    """
    Go to assignements worker.
    """
    code = request_data['code']
    line = request_data['line'] + 1
    column = request_data['column']
    path = request_data['path']
    # encoding = request_data['encoding']
    encoding = 'utf-8'
    script = jedi.Script(code, line, column, path, encoding)
    try:
        definitions = script.goto_assignments()
    except jedi.NotFoundError:
        pass
    else:
        ret_val = [(d.module_path, d.line - 1 if d.line else None,
                    d.column, d.full_name)
                   for d in definitions]
        return ret_val

def goto_assignments ( request_data ) : code = request_data [ 'code' ] line = request_data [ 'line' ] + 1 column = request_data [ 'column' ] path = request_data [ 'path' ] encoding = 'utf-8' script = jedi . Script ( code , line , column , path , encoding ) try : definitions = script . goto_assignments ( ) except jedi . NotFoundError : pass else : ret_val = [ ( d . module_path , d . line - 1 if d . line else None , d . column , d . full_name ) for d in definitions ] return ret_val


def defined_names(request_data):
    """
    Returns the list of defined names for the document.
    """
    global _old_definitions
    ret_val = []
    path = request_data['path']
    toplvl_definitions = jedi.names(
        request_data['code'], path, 'utf-8')
    for d in toplvl_definitions:
        definition = _extract_def(d, path)
        if d.type != 'import':
            ret_val.append(definition)
    ret_val = [d.to_dict() for d in ret_val]
    return ret_val

def defined_names ( request_data ) : global _old_definitions ret_val = [ ] path = request_data [ 'path' ] toplvl_definitions = jedi . names ( request_data [ 'code' ] , path , 'utf-8' ) for d in toplvl_definitions : definition = _extract_def ( d , path ) if d . type != 'import' : ret_val . append ( definition ) ret_val = [ d . to_dict ( ) for d in ret_val ] return ret_val

Returns the list of defined names for the document .


def quick_doc(request_data):
    """
    Worker that returns the documentation of the symbol under cursor.
    """
    code = request_data['code']
    line = request_data['line'] + 1
    column = request_data['column']
    path = request_data['path']
    # encoding = 'utf-8'
    encoding = 'utf-8'
    script = jedi.Script(code, line, column, path, encoding)
    try:
        definitions = script.goto_definitions()
    except jedi.NotFoundError:
        return []
    else:
        ret_val = [d.docstring() for d in definitions]
        return ret_val

def quick_doc ( request_data ) : code = request_data [ 'code' ] line = request_data [ 'line' ] + 1 column = request_data [ 'column' ] path = request_data [ 'path' ] encoding = 'utf-8' script = jedi . Script ( code , line , column , path , encoding ) try : definitions = script . goto_definitions ( ) except jedi . NotFoundError : return [ ] else : ret_val = [ d . docstring ( ) for d in definitions ] return ret_val

Worker that returns the documentation of the symbol under cursor .


def run_pep8(request_data):
    """
    Worker that run the pep8 tool on the current editor text.

    :returns a list of tuples (msg, msg_type, line_number)
    """
    import pycodestyle
    from pyqode.python.backend.pep8utils import CustomChecker
    WARNING = 1
    code = request_data['code']
    path = request_data['path']
    max_line_length = request_data['max_line_length']
    ignore_rules = request_data['ignore_rules']
    ignore_rules += ['W291', 'W292', 'W293', 'W391']
    pycodestyle.MAX_LINE_LENGTH = max_line_length
    # setup our custom style guide with our custom checker which returns a list
    # of strings instread of spitting the results at stdout
    pep8style = pycodestyle.StyleGuide(parse_argv=False, config_file='',
                                       checker_class=CustomChecker)
    try:
        results = pep8style.input_file(path, lines=code.splitlines(True))
    except Exception:
        _logger().exception('Failed to run PEP8 analysis with data=%r'
                            % request_data)
        return []
    else:
        messages = []
        for line_number, offset, code, text, doc in results:
            if code in ignore_rules:
                continue
            messages.append(('[PEP8] %s: %s' % (code, text), WARNING,
                             line_number - 1))
        return messages

Worker that run the pep8 tool on the current editor text .


def icon_from_typename(name, icon_type):
    """
    Returns the icon resource filename that corresponds to the given typename.

    :param name: name of the completion. Use to make the distinction between
        public and private completions (using the count of starting '_')
    :pram typename: the typename reported by jedi

    :returns: The associate icon resource filename or None.
    """
    ICONS = {
        'CLASS': ICON_CLASS,
        'IMPORT': ICON_NAMESPACE,
        'STATEMENT': ICON_VAR,
        'FORFLOW': ICON_VAR,
        'FORSTMT': ICON_VAR,
        'WITHSTMT': ICON_VAR,
        'GLOBALSTMT': ICON_VAR,
        'MODULE': ICON_NAMESPACE,
        'KEYWORD': ICON_KEYWORD,
        'PARAM': ICON_VAR,
        'ARRAY': ICON_VAR,
        'INSTANCEELEMENT': ICON_VAR,
        'INSTANCE': ICON_VAR,
        'PARAM-PRIV': ICON_VAR,
        'PARAM-PROT': ICON_VAR,
        'FUNCTION': ICON_FUNC,
        'DEF': ICON_FUNC,
        'FUNCTION-PRIV': ICON_FUNC_PRIVATE,
        'FUNCTION-PROT': ICON_FUNC_PROTECTED
    }
    ret_val = None
    icon_type = icon_type.upper()
    # jedi 0.8 introduced NamedPart class, which have a string instead of being
    # one
    if hasattr(name, "string"):
        name = name.string
    if icon_type == "FORFLOW" or icon_type == "STATEMENT":
        icon_type = "PARAM"
    if icon_type == "PARAM" or icon_type == "FUNCTION":
        if name.startswith("__"):
            icon_type += "-PRIV"
        elif name.startswith("_"):
            icon_type += "-PROT"
    if icon_type in ICONS:
        ret_val = ICONS[icon_type]
    elif icon_type:
        _logger().warning("Unimplemented completion icon_type: %s", icon_type)
    return ret_val

Returns the icon resource filename that corresponds to the given typename .


def complete(code, line, column, path, encoding, prefix):
        """
        Completes python code using `jedi`_.

        :returns: a list of completion.
        """
        ret_val = []
        try:
            script = jedi.Script(code, line + 1, column, path, encoding)
            completions = script.completions()
            print('completions: %r' % completions)
        except jedi.NotFoundError:
            completions = []
        for completion in completions:
            ret_val.append({
                'name': completion.name,
                'icon': icon_from_typename(
                    completion.name, completion.type),
                'tooltip': completion.description})
        return ret_val

def
complete
(
code
,
line
,
column
,
path
,
encoding
,
prefix
)
:
ret_val
=
[
]
try
:
script
=
jedi
.
Script
(
code
,
line
+
1
,
column
,
path
,
encoding
)
completions
=
script
.
completions
(
)
print
(
'completions: %r'
%
completions
)
except
jedi
.
NotFoundError
:
completions
=
[
]
for
completion
in
completions
:
ret_val
.
append
(
{
'name'
:
completion
.
name
,
'icon'
:
icon_from_typename
(
completion
.
name
,
completion
.
type
)
,
'tooltip'
:
completion
.
description
}
)
return
ret_val


def make_python_patterns(additional_keywords=[], additional_builtins=[]):
    """Strongly inspired from idlelib.ColorDelegator.make_pat"""
    kw = r"\b" + any("keyword", kwlist + additional_keywords) + r"\b"
    kw_namespace = r"\b" + any("namespace", kw_namespace_list) + r"\b"
    word_operators = r"\b" + any("operator_word", wordop_list) + r"\b"
    builtinlist = [str(name) for name in dir(builtins)
                   if not name.startswith('_')] + additional_builtins
    for v in ['None', 'True', 'False']:
        builtinlist.remove(v)
    builtin = r"([^.'\"\\#]\b|^)" + any("builtin", builtinlist) + r"\b"
    builtin_fct = any("builtin_fct", [r'_{2}[a-zA-Z_]*_{2}'])
    comment = any("comment", [r"#[^\n]*"])
    instance = any("instance", [r"\bself\b", r"\bcls\b"])
    decorator = any('decorator', [r'@\w*', r'.setter'])
    number = any("number",
                 [r"\b[+-]?[0-9]+[lLjJ]?\b",
                  r"\b[+-]?0[xX][0-9A-Fa-f]+[lL]?\b",
                  r"\b[+-]?0[oO][0-7]+[lL]?\b",
                  r"\b[+-]?0[bB][01]+[lL]?\b",
                  r"\b[+-]?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?[jJ]?\b"])
    sqstring = r"(\b[rRuU])?'[^'\\\n]*(\\.[^'\\\n]*)*'?"
    dqstring = r'(\b[rRuU])?"[^"\\\n]*(\\.[^"\\\n]*)*"?'
    uf_sqstring = r"(\b[rRuU])?'[^'\\\n]*(\\.[^'\\\n]*)*(\\)$(?!')$"
    uf_dqstring = r'(\b[rRuU])?"[^"\\\n]*(\\.[^"\\\n]*)*(\\)$(?!")$'
    sq3string = r"(\b[rRuU])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
    dq3string = r'(\b[rRuU])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
    uf_sq3string = r"(\b[rRuU])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(\\)?(?!''')$"
    uf_dq3string = r'(\b[rRuU])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(\\)?(?!""")$'
    string = any("string", [sq3string, dq3string, sqstring, dqstring])
    ufstring1 = any("uf_sqstring", [uf_sqstring])
    ufstring2 = any("uf_dqstring", [uf_dqstring])
    ufstring3 = any("uf_sq3string", [uf_sq3string])
    ufstring4 = any("uf_dq3string", [uf_dq3string])
    return "|".join([instance, decorator, kw, kw_namespace, builtin,
                     word_operators, builtin_fct, comment,
                     ufstring1, ufstring2, ufstring3, ufstring4, string,
                     number, any("SYNC", [r"\n"])])

Strongly inspired from idlelib . ColorDelegator . make_pat


def _check_word_cursor(self, tc=None):
        """
        Request a go to assignment.

        :param tc: Text cursor which contains the text that we must look for
                   its assignment. Can be None to go to the text that is under
                   the text cursor.
        :type tc: QtGui.QTextCursor
        """
        if not tc:
            tc = TextHelper(self.editor).word_under_cursor()

        request_data = {
            'code': self.editor.toPlainText(),
            'line': tc.blockNumber(),
            'column': tc.columnNumber(),
            'path': self.editor.file.path,
            'encoding': self.editor.file.encoding
        }
        try:
            self.editor.backend.send_request(
                workers.goto_assignments, request_data,
                on_receive=self._on_results_available)
        except NotRunning:
            pass

def _check_word_cursor ( self , tc = None ) : if not tc : tc = TextHelper ( self . editor ) . word_under_cursor ( ) request_data = { 'code' : self . editor . toPlainText ( ) , 'line' : tc . blockNumber ( ) , 'column' : tc . columnNumber ( ) , 'path' : self . editor . file . path , 'encoding' : self . editor . file . encoding } try : self . editor . backend . send_request ( workers . goto_assignments , request_data , on_receive = self . _on_results_available ) except NotRunning : pass


def _unique(self, seq):
        """
        Not performant but works.
        """
        # order preserving
        checked = []
        for e in seq:
            present = False
            for c in checked:
                if str(c) == str(e):
                    present = True
                    break
            if not present:
                checked.append(e)
        return checked

def _unique ( self , seq ) : checked = [ ] for e in seq : present = False for c in checked : if str ( c ) == str ( e ) : present = True break if not present : checked . append ( e ) return checked


def read_bgen(filepath, metafile_filepath=None, samples_filepath=None, verbose=True):
    r""" Read a given BGEN file.

    Parameters
    ----------
    filepath : str
        A bgen file path.
    metafile_filepath : str, optional
        If ``None``, it will try to read the ``filepath + ".metadata"`` file. If this is
        not possible, it will create one. It tries to create one at
        ``filepath + ".metadata"``. If that is also no possible, it tries to create one
        at a temporary folder.
    samples_filepath : str, optional
        A sample file in `gen format <https://goo.gl/bCzo7m>`_.
        If ``samples_filepath`` is provided, sample ids are read from this file.
        Otherwise, it reads from the bgen file itself if possible. Defaults to ``None``.
    verbose : bool, optional
        ``True`` to show progress; ``False`` otherwise. Defaults to ``True``.

    Returns
    -------
    variants : :class:`dask.dataFrame.DataFrame`
        Variant position, chromosomes, rsids, etc.
    samples : :class:`pandas.Series`
        Sample identifications.
    genotype : list
        List of genotypes.

    Examples
    --------
    .. doctest::

        >>> from bgen_reader import example_files, read_bgen
        >>>
        >>> with example_files("haplotypes.bgen") as filepath:
        ...     bgen = read_bgen(filepath, verbose=False)
        ...     variants = bgen["variants"]
        ...     samples = bgen["samples"]
        ...
        ...     v = variants.loc[0].compute()
        ...     g = bgen["genotype"][0].compute()
        ...     print(v)
        ...     print(samples)
        ...     print(g["probs"][0])
             id rsid chrom  pos  nalleles allele_ids  vaddr
        0  SNP1  RS1     1    1         2        A,G    102
        0    sample_0
        1    sample_1
        2    sample_2
        3    sample_3
        Name: id, dtype: object
        [1. 0. 1. 0.]
    """

    assert_file_exist(filepath)
    assert_file_readable(filepath)

    metafile_filepath = _get_valid_metafile_filepath(filepath, metafile_filepath)
    if not os.path.exists(metafile_filepath):
        if verbose:
            print(
                f"We will create the metafile `{metafile_filepath}`. This file will "
                "speed up further\nreads and only need to be created once. So, please, "
                "bear with me."
            )
        create_metafile(filepath, metafile_filepath, verbose)

    samples = get_samples(filepath, samples_filepath, verbose)
    variants = map_metadata(filepath, metafile_filepath)
    genotype = map_genotype(filepath, metafile_filepath, verbose)

    return dict(variants=variants, samples=samples, genotype=genotype)


def create_metafile(bgen_filepath, metafile_filepath, verbose=True):
    r"""Create variants metadata file.

    Variants metadata file helps speed up subsequent reads of the associated
    bgen file.

    Parameters
    ----------
    bgen_filepath : str
        Bgen file path.
    metafile_file : str
        Metafile file path.
    verbose : bool
        ``True`` to show progress; ``False`` otherwise.

    Examples
    --------
    .. doctest::

        >>> import os
        >>> from bgen_reader import create_metafile, example_files
        >>>
        >>> with example_files("example.32bits.bgen") as filepath:
        ...     folder = os.path.dirname(filepath)
        ...     metafile_filepath = os.path.join(folder, filepath + ".metadata")
        ...
        ...     try:
        ...         create_metafile(filepath, metafile_filepath, verbose=False)
        ...     finally:
        ...         if os.path.exists(metafile_filepath):
        ...             os.remove(metafile_filepath)
    """
    if verbose:
        verbose = 1
    else:
        verbose = 0

    bgen_filepath = make_sure_bytes(bgen_filepath)
    metafile_filepath = make_sure_bytes(metafile_filepath)

    assert_file_exist(bgen_filepath)
    assert_file_readable(bgen_filepath)

    if exists(metafile_filepath):
        raise ValueError(f"The file {metafile_filepath} already exists.")

    with bgen_file(bgen_filepath) as bgen:
        nparts = _estimate_best_npartitions(lib.bgen_nvariants(bgen))
        metafile = lib.bgen_create_metafile(bgen, metafile_filepath, nparts, verbose)
        if metafile == ffi.NULL:
            raise RuntimeError(f"Error while creating metafile: {metafile_filepath}.")

        if lib.bgen_close_metafile(metafile) != 0:
            raise RuntimeError(f"Error while closing metafile: {metafile_filepath}.")


def match(self, subsetLines, offsetOfSubset, fileName):
        """
            Search through lines for match.
            Raise an Exception if fail to match
            If match is succesful return the position the match was found
        """

        for (offset,l) in enumerate(subsetLines):
            column = l.find(self.literal)
            if column != -1:
                truePosition = offset + offsetOfSubset
                _logger.debug('Found match on line {}, col {}'.format(str(truePosition+ 1), column))
                _logger.debug('Line is {}'.format(l))
                self.matchLocation = CheckFileParser.FileLocation(fileName, truePosition +1)
                return truePosition

        # No Match found
        self.failed = True
        raise DirectiveException(self)

def
match
(
self
,
subsetLines
,
offsetOfSubset
,
fileName
)
:
for
(
offset
,
l
)
in
enumerate
(
subsetLines
)
:
column
=
l
.
find
(
self
.
literal
)
if
column
!=
1
:
truePosition
=
offset
+
offsetOfSubset
_logger
.
debug
(
'Found match on line {}, col {}'
.
format
(
str
(
truePosition
+
1
)
,
column
)
)
_logger
.
debug
(
'Line is {}'
.
format
(
l
)
)
self
.
matchLocation
=
CheckFileParser
.
FileLocation
(
fileName
,
truePosition
+
1
)
return
truePosition
self
.
failed
=
True
raise
DirectiveException
(
self
)

Search through lines for match . Raise an Exception if fail to match If match is succesful return the position the match was found


def match(self, subsetLines, offsetOfSubset, fileName):
        """
            Search through lines for match.
            Raise an Exception if a match
        """
        for (offset,l) in enumerate(subsetLines):
            for t in self.regex:
                m = t.Regex.search(l)
                if m != None:
                    truePosition = offset + offsetOfSubset
                    _logger.debug('Found match on line {}'.format(str(truePosition+ 1)))
                    _logger.debug('Line is {}'.format(l))
                    self.failed = True
                    self.matchLocation = CheckFileParser.FileLocation(fileName, truePosition +1)
                    raise DirectiveException(self)

def
match
(
self
,
subsetLines
,
offsetOfSubset
,
fileName
)
:
for
(
offset
,
l
)
in
enumerate
(
subsetLines
)
:
for
t
in
self
.
regex
:
m
=
t
.
Regex
.
search
(
l
)
if
m
!=
:
truePosition
=
offset
+
offsetOfSubset
_logger
.
debug
(
'Found match on line {}'
.
format
(
str
(
truePosition
+
1
)
)
)
_logger
.
debug
(
'Line is {}'
.
format
(
l
)
)
self
.
failed
=
True
self
.
matchLocation
=
CheckFileParser
.
FileLocation
(
fileName
,
truePosition
+
1
)
raise
DirectiveException
(
self
)

Search through lines for match . Raise an Exception if a match


def isA(instance, typeList):
    """
        Return true if ``instance`` is an instance of any the Directive
        types in ``typeList``
    """
    return any(map(lambda iType: isinstance(instance,iType), typeList))

def isA ( instance , typeList ) : return any ( map ( lambda iType : isinstance ( instance , iType ) , typeList ) )

Return true if instance is an instance of any the Directive types in typeList


def _touch(fname, mode=0o666, dir_fd=None, **kwargs):
    """ Touch a file.

    Credits to <https://stackoverflow.com/a/1160227>.
    """
    flags = os.O_CREAT | os.O_APPEND
    with os.fdopen(os.open(fname, flags=flags, mode=mode, dir_fd=dir_fd)) as f:
        os.utime(
            f.fileno() if os.utime in os.supports_fd else fname,
            dir_fd=None if os.supports_fd else dir_fd,
            **kwargs,
        )

def _touch ( fname , mode = 0o666 , dir_fd = None , ** kwargs ) : flags = os . O_CREAT | os . O_APPEND with os . fdopen ( os . open ( fname , flags = flags , mode = mode , dir_fd = dir_fd ) ) as f : os . utime ( f . fileno ( ) if os . utime in os . supports_fd else fname , dir_fd = None if os . supports_fd else dir_fd , ** kwargs , )


def allele_frequency(expec):
    r""" Compute allele frequency from its expectation.

    Parameters
    ----------
    expec : array_like
        Allele expectations encoded as a samples-by-alleles matrix.

    Returns
    -------
    :class:`numpy.ndarray`
        Allele frequencies encoded as a variants-by-alleles matrix.

    Examples
    --------
    .. doctest::

        >>> from bgen_reader import read_bgen, example_files
        >>> from bgen_reader import allele_expectation, allele_frequency
        >>>
        >>> # Download an example
        >>> example = example_files("example.32bits.bgen")
        >>> filepath = example.filepath
        >>>
        >>> bgen = read_bgen(filepath, verbose=False)
        >>>
        >>> variants = bgen["variants"]
        >>> samples = bgen["samples"]
        >>> genotype = bgen["genotype"]
        >>>
        >>> variant = variants[variants["rsid"] == "RSID_6"].compute()
        >>> variant_idx = variant.index.item()
        >>>
        >>> p = genotype[variant_idx].compute()["probs"]
        >>> # For unphased genotypes only.
        >>> e = allele_expectation(bgen, variant_idx)
        >>> f = allele_frequency(e)
        >>>
        >>> alleles = variant["allele_ids"].item().split(",")
        >>> print(alleles[0] + ": {}".format(f[0]))
        A: 229.23103218810434
        >>> print(alleles[1] + ": {}".format(f[1]))
        G: 270.7689678118956
        >>> print(variant)
                id    rsid chrom   pos  nalleles allele_ids  vaddr
        4  SNPID_6  RSID_6    01  6000         2        A,G  19377
        >>>
        >>> # Clean-up the example
        >>> example.close()
    """
    expec = asarray(expec, float)
    if expec.ndim != 2:
        raise ValueError("Expectation matrix must be bi-dimensional.")
    ploidy = expec.shape[-1]
    return expec.sum(-2) / ploidy

r Compute allele frequency from its expectation .


def compute_dosage(expec, alt=None):
    r""" Compute dosage from allele expectation.

    Parameters
    ----------
    expec : array_like
        Allele expectations encoded as a samples-by-alleles matrix.
    alt : array_like, optional
        Alternative allele index. If ``None``, the allele having the minor
        allele frequency for the provided ``expec`` is used as the alternative.
        Defaults to ``None``.

    Returns
    -------
    :class:`numpy.ndarray`
        Dosage encoded as an array of size equal to the number of samples.

    Examples
    --------
    .. code-block:: python
        :caption: First a quick-start example.

        >>> from bgen_reader import allele_expectation, compute_dosage
        >>> from bgen_reader import example_files, read_bgen
        >>>
        >>> # Download an example.
        >>> example = example_files("example.32bits.bgen")
        >>> filepath = example.filepath
        >>>
        >>> # Read the example.
        >>> bgen = read_bgen(filepath, verbose=False)
        >>>
        >>> # Extract the allele expectations of the fourth variant.
        >>> variant_idx = 3
        >>> e = allele_expectation(bgen, variant_idx)
        >>>
        >>> # Compute the dosage when considering the first allele
        >>> # as the reference/alternative one.
        >>> alt_allele_idx = 1
        >>> d = compute_dosage(e, alt=alt_allele_idx)
        >>>
        >>> # Print the dosage of the first five samples only.
        >>> print(d[:5])
        [1.96185308 0.00982666 0.01745552 1.00347899 1.01153563]
        >>>
        >>> # Clean-up the example
        >>> example.close()

    .. code-block:: python
        :caption: Genotype probabilities, allele expectations and frequencies.

        >>> from bgen_reader import (
        ...     allele_expectation,
        ...     allele_frequency,
        ...     compute_dosage,
        ...     example_files,
        ...     read_bgen,
        ... )
        >>> from pandas import DataFrame
        >>> from xarray import DataArray
        >>>
        >>> # Download an example
        >>> example = example_files("example.32bits.bgen")
        >>> filepath = example.filepath
        >>>
        >>> # Open the bgen file.
        >>> bgen = read_bgen(filepath, verbose=False)
        >>> variants = bgen["variants"]
        >>> genotype = bgen["genotype"]
        >>> samples = bgen["samples"]
        >>>
        >>> variant_idx = 3
        >>> variant = variants.loc[variant_idx].compute()
        >>> # Print the metadata of the fourth variant.
        >>> print(variant)
                id    rsid chrom   pos  nalleles allele_ids  vaddr
        3  SNPID_5  RSID_5    01  5000         2        A,G  16034

        >>> geno = bgen["genotype"][variant_idx].compute()
        >>> metageno = DataFrame({k: geno[k] for k in ["ploidy", "missing"]},
        ...                      index=samples)
        >>> metageno.index.name = "sample"
        >>> print(metageno) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
                    ploidy  missing
        sample
        sample_001       2    False
        sample_002       2    False
        sample_003       2    False
        sample_004       2    False
        ...            ...      ...
        sample_497       2    False
        sample_498       2    False
        sample_499       2    False
        sample_500       2    False
        <BLANKLINE>
        [500 rows x 2 columns]
        >>> p = DataArray(
        ...     geno["probs"],
        ...     name="probability",
        ...     coords={"sample": samples},
        ...     dims=["sample", "genotype"],
        ... )
        >>> # Print the genotype probabilities.
        >>> print(p.to_series().unstack(level=-1)) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
        genotype          0        1        2
        sample
        sample_001  0.00488  0.02838  0.96674
        sample_002  0.99045  0.00928  0.00027
        sample_003  0.98932  0.00391  0.00677
        sample_004  0.00662  0.98328  0.01010
        ...             ...      ...      ...
        sample_497  0.00137  0.01312  0.98550
        sample_498  0.00552  0.99423  0.00024
        sample_499  0.01266  0.01154  0.97580
        sample_500  0.00021  0.98431  0.01547
        <BLANKLINE>
        [500 rows x 3 columns]
        >>> alleles = variant["allele_ids"].item().split(",")
        >>> e = DataArray(
        ...     allele_expectation(bgen, variant_idx),
        ...     name="expectation",
        ...     coords={"sample": samples, "allele": alleles},
        ...     dims=["sample", "allele"],
        ... )
        >>> # Print the allele expectations.
        >>> print(e.to_series().unstack(level=-1)) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
        allele            A        G
        sample
        sample_001  0.03815  1.96185
        sample_002  1.99017  0.00983
        sample_003  1.98254  0.01746
        sample_004  0.99652  1.00348
        ...             ...      ...
        sample_497  0.01587  1.98413
        sample_498  1.00528  0.99472
        sample_499  0.03687  1.96313
        sample_500  0.98474  1.01526
        <BLANKLINE>
        [500 rows x 2 columns]
        >>> rsid = variant["rsid"].item()
        >>> chrom = variant["chrom"].item()
        >>> variant_name = f"{chrom}:{rsid}"
        >>> f = DataFrame(allele_frequency(e), columns=[variant_name], index=alleles)
        >>> f.index.name = "allele"
        >>> # Allele frequencies.
        >>> print(f) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
                01:RSID_5
        allele
        A       305.97218
        G       194.02782
        >>> alt = f.idxmin().item()
        >>> alt_idx = alleles.index(alt)
        >>> d = compute_dosage(e, alt=alt_idx).to_series()
        >>> d = DataFrame(d.values, columns=[f"alt={alt}"], index=d.index)
        >>> # Dosages when considering G as the alternative allele.
        >>> print(d) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
                      alt=G
        sample
        sample_001  1.96185
        sample_002  0.00983
        sample_003  0.01746
        sample_004  1.00348
        ...             ...
        sample_497  1.98413
        sample_498  0.99472
        sample_499  1.96313
        sample_500  1.01526
        <BLANKLINE>
        [500 rows x 1 columns]
        >>>
        >>> # Clean-up the example
        >>> example.close()
    """
    if alt is None:
        return expec[..., -1]
    try:
        return expec[:, alt]
    except NotImplementedError:
        alt = asarray(alt, int)
        return asarray(expec, float)[:, alt]

def compute_dosage ( expec , alt = None ) : r if alt is None : return expec [ ... , - 1 ] try : return expec [ : , alt ] except NotImplementedError : alt = asarray ( alt , int ) return asarray ( expec , float ) [ : , alt ]

r Compute dosage from allele expectation .


def allele_expectation(bgen, variant_idx):
    r""" Allele expectation.

    Compute the expectation of each allele from the genotype probabilities.

    Parameters
    ----------
    bgen : bgen_file
        Bgen file handler.
    variant_idx : int
        Variant index.

    Returns
    -------
    :class:`numpy.ndarray`
        Samples-by-alleles matrix of allele expectations.

    Note
    ----
    This function supports unphased genotypes only.

    Examples
    --------
    .. doctest::

        >>> from bgen_reader import allele_expectation, example_files, read_bgen
        >>>
        >>> from texttable import Texttable
        >>>
        >>> # Download an example.
        >>> example = example_files("example.32bits.bgen")
        >>> filepath = example.filepath
        >>>
        >>> # Read the example.
        >>> bgen = read_bgen(filepath, verbose=False)
        >>>
        >>> variants = bgen["variants"]
        >>> samples = bgen["samples"]
        >>> genotype = bgen["genotype"]
        >>>
        >>> genotype = bgen["genotype"]
        >>> # This `compute` call will return a pandas data frame,
        >>> variant = variants[variants["rsid"] == "RSID_6"].compute()
        >>> # from which we retrieve the variant index.
        >>> variant_idx = variant.index.item()
        >>> print(variant)
                id    rsid chrom   pos  nalleles allele_ids  vaddr
        4  SNPID_6  RSID_6    01  6000         2        A,G  19377
        >>> genotype = bgen["genotype"]
        >>> # Samples is a pandas series, and we retrieve the
        >>> # sample index from the sample name.
        >>> sample_idx = samples[samples == "sample_005"].index.item()
        >>>
        >>> genotype = bgen["genotype"]
        >>> # This `compute` call will return a dictionary from which
        >>> # we can get the probability matrix the corresponding
        >>> # variant.
        >>> p = genotype[variant_idx].compute()["probs"][sample_idx]
        >>>
        >>> genotype = bgen["genotype"]
        >>> # Allele expectation makes sense for unphased genotypes only,
        >>> # which is the case here.
        >>> e = allele_expectation(bgen, variant_idx)[sample_idx]
        >>>
        >>> genotype = bgen["genotype"]
        >>> alleles = variant["allele_ids"].item().split(",")
        >>>
        >>> genotype = bgen["genotype"]
        >>>
        >>> # Print what we have got in a nice format.
        >>> table = Texttable()
        >>> table = table.add_rows(
        ...     [
        ...         ["", "AA", "AG", "GG", "E[.]"],
        ...         ["p"] + list(p) + ["na"],
        ...         ["#" + alleles[0], 2, 1, 0, e[0]],
        ...         ["#" + alleles[1], 0, 1, 2, e[1]],
        ...     ]
        ... )
        >>> print(table.draw())
        +----+-------+-------+-------+-------+
        |    |  AA   |  AG   |  GG   | E[.]  |
        +====+=======+=======+=======+=======+
        | p  | 0.012 | 0.987 | 0.001 | na    |
        +----+-------+-------+-------+-------+
        | #A | 2     | 1     | 0     | 1.011 |
        +----+-------+-------+-------+-------+
        | #G | 0     | 1     | 2     | 0.989 |
        +----+-------+-------+-------+-------+
        >>>
        >>> # Clean-up.
        >>> example.close()
    """
    geno = bgen["genotype"][variant_idx].compute()
    if geno["phased"]:
        raise ValueError("Allele expectation is define for unphased genotypes only.")

    nalleles = bgen["variants"].loc[variant_idx, "nalleles"].compute().item()
    genotypes = get_genotypes(geno["ploidy"], nalleles)
    expec = []
    for i in range(len(genotypes)):
        count = asarray(genotypes_to_allele_counts(genotypes[i]), float)
        n = count.shape[0]
        expec.append((count.T * geno["probs"][i, :n]).sum(1))

    return stack(expec, axis=0)


def find_libname(self, name):
        """Try to infer the correct library name."""
        names = ["{}.lib", "lib{}.lib", "{}lib.lib"]
        names = [n.format(name) for n in names]
        dirs = self.get_library_dirs()
        for d in dirs:
            for n in names:
                if exists(join(d, n)):
                    return n[:-4]
        msg = "Could not find the {} library.".format(name)
        raise ValueError(msg)

def
find_libname
(
self
,
name
)
:
names
=
[
"{}.lib"
,
"lib{}.lib"
,
"{}lib.lib"
]
names
=
[
n
.
format
(
name
)
for
n
in
names
]
dirs
=
self
.
get_library_dirs
(
)
for
d
in
dirs
:
for
n
in
names
:
if
exists
(
join
(
d
,
n
)
)
:
return
n
[
:
4
]
msg
=
"Could not find the {} library."
.
format
(
name
)
raise
ValueError
(
msg
)


def fit(self, X, y=None):
        """Fit distance-based AD.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            The input samples. Use ``dtype=np.float32`` for maximum
            efficiency.

        Returns
        -------
        self : object
            Returns self.
        """
        # Check data
        X = check_array(X)
        self.tree = BallTree(X, leaf_size=self.leaf_size, metric=self.metric)
        dist_train = self.tree.query(X, k=2)[0]
        if self.threshold == 'auto':
            self.threshold_value = 0.5 * sqrt(var(dist_train[:, 1])) + mean(dist_train[:, 1])
        elif self.threshold == 'cv':
            if y is None:
                raise ValueError("Y must be specified to find the optimal threshold.")
            y = check_array(y, accept_sparse='csc', ensure_2d=False, dtype=None)
            self.threshold_value = 0
            score = 0
            Y_pred, Y_true, AD = [], [], []
            cv = KFold(n_splits=5, random_state=1, shuffle=True)
            for train_index, test_index in cv.split(X):
                x_train = safe_indexing(X, train_index)
                x_test = safe_indexing(X, test_index)
                y_train = safe_indexing(y, train_index)
                y_test = safe_indexing(y, test_index)
                data_test = safe_indexing(dist_train[:, 1], test_index)
                if self.reg_model is None:
                    reg_model = RandomForestRegressor(n_estimators=500, random_state=1).fit(x_train, y_train)
                else:
                    reg_model = clone(self.reg_model).fit(x_train, y_train)
                Y_pred.append(reg_model.predict(x_test))
                Y_true.append(y_test)
                AD.append(data_test)
            AD_ = unique(hstack(AD))
            for z in AD_:
                AD_new = hstack(AD) <= z
                if self.score == 'ba_ad':
                    val = balanced_accuracy_score_with_ad(Y_true=hstack(Y_true), Y_pred=hstack(Y_pred), AD=AD_new)
                elif self.score == 'rmse_ad':
                    val = rmse_score_with_ad(Y_true=hstack(Y_true), Y_pred=hstack(Y_pred), AD=AD_new)
                if val >= score:
                    score = val
                    self.threshold_value = z
        else:
            self.threshold_value = self.threshold
        return self


def predict_proba(self, X):
        """Returns the value of the nearest neighbor from the training set.

        Parameters
        ----------
         X : array-like or sparse matrix, shape (n_samples, n_features)
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        y : array, shape (n_samples,)
        """
        # Check is fit had been called
        check_is_fitted(self, ['tree'])
        # Check data
        X = check_array(X)
        return self.tree.query(X)[0].flatten()

def predict_proba ( self , X ) : check_is_fitted ( self , [ 'tree' ] ) X = check_array ( X ) return self . tree . query ( X ) [ 0 ] . flatten ( )

Returns the value of the nearest neighbor from the training set .


def fit(self, X, y=None):
        """Learning is to find the inverse matrix for X and calculate the threshold.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            The input samples. Use ``dtype=np.float32`` for maximum
            efficiency.
        y : array-like, shape = [n_samples] or [n_samples, n_outputs]
            The target values (real numbers in regression).

        Returns
        -------
        self : object
        """
        # Check that X have correct shape
        X = check_array(X)
        self.inverse_influence_matrix = self.__make_inverse_matrix(X)
        if self.threshold == 'auto':
            self.threshold_value = 3 * (1 + X.shape[1]) / X.shape[0]
        elif self.threshold == 'cv':
            if y is None:
                raise ValueError("Y must be specified to find the optimal threshold.")
            y = check_array(y, accept_sparse='csc', ensure_2d=False, dtype=None)
            self.threshold_value = 0
            score = 0
            Y_pred, Y_true, AD = [], [], []
            cv = KFold(n_splits=5, random_state=1, shuffle=True)
            for train_index, test_index in cv.split(X):
                x_train = safe_indexing(X, train_index)
                x_test = safe_indexing(X, test_index)
                y_train = safe_indexing(y, train_index)
                y_test = safe_indexing(y, test_index)
                if self.reg_model is None:
                    reg_model = RandomForestRegressor(n_estimators=500, random_state=1).fit(x_train, y_train)
                else:
                    reg_model = clone(self.reg_model).fit(x_train, y_train)
                Y_pred.append(reg_model.predict(x_test))
                Y_true.append(y_test)
                ad_model = self.__make_inverse_matrix(x_train)
                AD.append(self.__find_leverages(x_test, ad_model))
            AD_ = unique(hstack(AD))
            for z in AD_:
                AD_new = hstack(AD) <= z
                if self.score == 'ba_ad':
                    val = balanced_accuracy_score_with_ad(Y_true=hstack(Y_true), Y_pred=hstack(Y_pred), AD=AD_new)
                elif self.score == 'rmse_ad':
                    val = rmse_score_with_ad(Y_true=hstack(Y_true), Y_pred=hstack(Y_pred), AD=AD_new)
                if val >= score:
                    score = val
                    self.threshold_value = z
        else:
            self.threshold_value = self.threshold
        return self

Learning is to find the inverse matrix for X and calculate the threshold .


def predict_proba(self, X):
        """Predict the distances for X to center of the training set.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        leverages: array of shape = [n_samples]
                   The objects distances to center of the training set.
        """
        # Check is fit had been called
        check_is_fitted(self, ['inverse_influence_matrix'])
        # Check that X have correct shape
        X = check_array(X)
        return self.__find_leverages(X, self.inverse_influence_matrix)

def predict_proba ( self , X ) : check_is_fitted ( self , [ 'inverse_influence_matrix' ] ) X = check_array ( X ) return self . __find_leverages ( X , self . inverse_influence_matrix )

Predict the distances for X to center of the training set .


def predict(self, X):
        """Predict inside or outside AD for X.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        ad : array of shape = [n_samples]
            Array contains True (reaction in AD) and False (reaction residing outside AD).
        """
        # Check is fit had been called
        check_is_fitted(self, ['inverse_influence_matrix'])
        # Check that X have correct shape
        X = check_array(X)
        return self.__find_leverages(X, self.inverse_influence_matrix) <= self.threshold_value

def predict ( self , X ) : check_is_fitted ( self , [ 'inverse_influence_matrix' ] ) X = check_array ( X ) return self . __find_leverages ( X , self . inverse_influence_matrix ) <= self . threshold_value