def r_annotations(self):
        """ Route to retrieve annotations by target

        :param target_urn: The CTS URN for which to retrieve annotations
        :type target_urn: str
        :return: a JSON string containing count and list of resources
        :rtype: {str: Any}
        """

        target = request.args.get("target", None)
        wildcard = request.args.get("wildcard", ".", type=str)
        include = request.args.get("include")
        exclude = request.args.get("exclude")
        limit = request.args.get("limit", None, type=int)
        start = request.args.get("start", 1, type=int)
        expand = request.args.get("expand", False, type=bool)

        if target:

            try:
                urn = MyCapytain.common.reference.URN(target)
            except ValueError:
                return "invalid urn", 400

            count, annotations = self.__queryinterface__.getAnnotations(urn, wildcard=wildcard, include=include,
                                                                        exclude=exclude, limit=limit, start=start,
                                                                        expand=expand)
        else:
            #  Note that this implementation is not done for too much annotations
            #  because we do not implement pagination here
            count, annotations = self.__queryinterface__.getAnnotations(None, limit=limit, start=start, expand=expand)
        mapped = []
        response = {
            "@context": type(self).JSONLD_CONTEXT,
            "id": url_for(".r_annotations", start=start, limit=limit),
            "type": "AnnotationCollection",
            "startIndex": start,
            "items": [
            ],
            "total": count
        }
        for a in annotations:
            mapped.append({
                "id": url_for(".r_annotation", sha=a.sha),
                "body": url_for(".r_annotation_body", sha=a.sha),
                "type": "Annotation",
                "target": a.target.to_json(),
                "dc:type": a.type_uri,
                "owl:sameAs": [a.uri],
                "nemo:slug": a.slug
            })
        response["items"] = mapped
        response = jsonify(response)
        return response

def
r_annotations
(
self
)
:
target
=
request
.
args
.
get
(
"target"
,
)
wildcard
=
request
.
args
.
get
(
"wildcard"
,
"."
,
type
=
str
)
include
=
request
.
args
.
get
(
"include"
)
exclude
=
request
.
args
.
get
(
"exclude"
)
limit
=
request
.
args
.
get
(
"limit"
,
,
type
=
int
)
start
=
request
.
args
.
get
(
"start"
,
1
,
type
=
int
)
expand
=
request
.
args
.
get
(
"expand"
,
False
,
type
=
bool
)
if
target
:
try
:
urn
=
MyCapytain
.
common
.
reference
.
URN
(
target
)
except
ValueError
:
return
"invalid urn"
,
400
count
,
annotations
=
self
.
__queryinterface__
.
getAnnotations
(
urn
,
wildcard
=
wildcard
,
include
=
include
,
exclude
=
exclude
,
limit
=
limit
,
start
=
start
,
expand
=
expand
)
else
:
count
,
annotations
=
self
.
__queryinterface__
.
getAnnotations
(
,
limit
=
limit
,
start
=
start
,
expand
=
expand
)
mapped
=
[
]
response
=
{
"@context"
:
type
(
self
)
.
JSONLD_CONTEXT
,
"id"
:
url_for
(
".r_annotations"
,
start
=
start
,
limit
=
limit
)
,
"type"
:
"AnnotationCollection"
,
"startIndex"
:
start
,
"items"
:
[
]
,
"total"
:
count
}
for
a
in
annotations
:
mapped
.
append
(
{
"id"
:
url_for
(
".r_annotation"
,
sha
=
a
.
sha
)
,
"body"
:
url_for
(
".r_annotation_body"
,
sha
=
a
.
sha
)
,
"type"
:
"Annotation"
,
"target"
:
a
.
target
.
to_json
(
)
,
"dc:type"
:
a
.
type_uri
,
"owl:sameAs"
:
[
a
.
uri
]
,
"nemo:slug"
:
a
.
slug
}
)
response
[
"items"
]
=
mapped
response
=
jsonify
(
response
)
return
response


def lookup(cls, key, get=False):
        """Returns the label for a given Enum key"""
        if get:
            item = cls._item_dict.get(key)
            return item.name if item else key
        return cls._item_dict[key].name

def lookup ( cls , key , get = False ) : if get : item = cls . _item_dict . get ( key ) return item . name if item else key return cls . _item_dict [ key ] . name


def verbose(cls, key=False, default=''):
        """Returns the verbose name for a given enum value"""
        if key is False:
            items = cls._item_dict.values()
            return [(x.key, x.value) for x in sorted(items, key=lambda x:x.sort or x.key)]

        item = cls._item_dict.get(key)
        return item.value if item else default

def verbose ( cls , key = False , default = '' ) : if key is False : items = cls . _item_dict . values ( ) return [ ( x . key , x . value ) for x in sorted ( items , key = lambda x : x . sort or x . key ) ] item = cls . _item_dict . get ( key ) return item . value if item else default


def get_configured_dns():
    """
        Returns the configured DNS servers with the use f nmcli.
    """
    ips = []
    try:
        output = subprocess.check_output(['nmcli', 'device', 'show'])
        output = output.decode('utf-8')

        for line in output.split('\n'):
            if 'DNS' in line:
                pattern = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"
                for hit in re.findall(pattern, line):
                    ips.append(hit)
    except FileNotFoundError:
        pass
    return ips

def get_configured_dns ( ) : ips = [ ] try : output = subprocess . check_output ( [ 'nmcli' , 'device' , 'show' ] ) output = output . decode ( 'utf-8' ) for line in output . split ( '\n' ) : if 'DNS' in line : pattern = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}" for hit in re . findall ( pattern , line ) : ips . append ( hit ) except FileNotFoundError : pass return ips

Returns the configured DNS servers with the use f nmcli .


def zone_transfer(address, dns_name):
    """
        Tries to perform a zone transfer.
    """
    ips = []
    try:
        print_notification("Attempting dns zone transfer for {} on {}".format(dns_name, address))
        z = dns.zone.from_xfr(dns.query.xfr(address, dns_name))
    except dns.exception.FormError:
        print_notification("Zone transfer not allowed")
        return ips
    names = z.nodes.keys()
    print_success("Zone transfer successfull for {}, found {} entries".format(address, len(names)))
    for n in names:
        node = z[n]
        data = node.get_rdataset(dns.rdataclass.IN, dns.rdatatype.A)
        if data:
            # TODO add hostnames to entries.
            # hostname = n.to_text()
            for item in data.items:
                address = item.address
                ips.append(address)
    return ips


def resolve_domains(domains, disable_zone=False):
    """
        Resolves the list of domains and returns the ips.
    """
    dnsresolver = dns.resolver.Resolver()

    ips = []

    for domain in domains:
        print_notification("Resolving {}".format(domain))
        try:
            result = dnsresolver.query(domain, 'A')
            for a in result.response.answer[0]:
                ips.append(str(a))
                if not disable_zone:
                    ips.extend(zone_transfer(str(a), domain))
        except dns.resolver.NXDOMAIN as e:
            print_error(e)
    return ips

def
resolve_domains
(
domains
,
disable_zone
=
False
)
:
dnsresolver
=
dns
.
resolver
.
Resolver
(
)
ips
=
[
]
for
domain
in
domains
:
print_notification
(
"Resolving {}"
.
format
(
domain
)
)
try
:
result
=
dnsresolver
.
query
(
domain
,
'A'
)
for
a
in
result
.
response
.
answer
[
0
]
:
ips
.
append
(
str
(
a
)
)
if
not
disable_zone
:
ips
.
extend
(
zone_transfer
(
str
(
a
)
,
domain
)
)
except
dns
.
resolver
.
NXDOMAIN
as
e
:
print_error
(
e
)
return
ips

Resolves the list of domains and returns the ips .


def parse_ips(ips, netmask, include_public):
    """
        Parses the list of ips, turns these into ranges based on the netmask given.
        Set include_public to True to include public IP adresses.
    """
    hs = HostSearch()
    rs = RangeSearch()
    ranges = []
    ips = list(set(ips))
    included_ips = []
    print_success("Found {} ips".format(len(ips)))
    for ip in ips:
        ip_address = ipaddress.ip_address(ip)
        if include_public or ip_address.is_private:
            # To stop the screen filling with ranges.
            if len(ips) < 15:
                print_success("Found ip: {}".format(ip))
            host = hs.id_to_object(ip)
            host.add_tag('dns_discover')
            host.save()
            r = str(ipaddress.IPv4Network("{}/{}".format(ip, netmask), strict=False))
            ranges.append(r)
            included_ips.append(ip)
        else:
            print_notification("Excluding ip {}".format(ip))

    ranges = list(set(ranges))
    print_success("Found {} ranges".format(len(ranges)))
    for rng in ranges:
        # To stop the screen filling with ranges.
        if len(ranges) < 15:
            print_success("Found range: {}".format(rng))
        r = rs.id_to_object(rng)
        r.add_tag('dns_discover')
        r.save()

    stats = {}
    stats['ips'] = included_ips
    stats['ranges'] = ranges
    return stats

def
parse_ips
(
ips
,
netmask
,
include_public
)
:
hs
=
HostSearch
(
)
rs
=
RangeSearch
(
)
ranges
=
[
]
ips
=
list
(
set
(
ips
)
)
included_ips
=
[
]
print_success
(
"Found {} ips"
.
format
(
len
(
ips
)
)
)
for
ip
in
ips
:
ip_address
=
ipaddress
.
ip_address
(
ip
)
if
include_public
or
ip_address
.
is_private
:
if
len
(
ips
)
<
15
:
print_success
(
"Found ip: {}"
.
format
(
ip
)
)
host
=
hs
.
id_to_object
(
ip
)
host
.
add_tag
(
'dns_discover'
)
host
.
save
(
)
r
=
str
(
ipaddress
.
IPv4Network
(
"{}/{}"
.
format
(
ip
,
netmask
)
,
strict
=
False
)
)
ranges
.
append
(
r
)
included_ips
.
append
(
ip
)
else
:
print_notification
(
"Excluding ip {}"
.
format
(
ip
)
)
ranges
=
list
(
set
(
ranges
)
)
print_success
(
"Found {} ranges"
.
format
(
len
(
ranges
)
)
)
for
rng
in
ranges
:
if
len
(
ranges
)
<
15
:
print_success
(
"Found range: {}"
.
format
(
rng
)
)
r
=
rs
.
id_to_object
(
rng
)
r
.
add_tag
(
'dns_discover'
)
r
.
save
(
)
stats
=
{
}
stats
[
'ips'
]
=
included_ips
stats
[
'ranges'
]
=
ranges
return
stats

Parses the list of ips turns these into ranges based on the netmask given . Set include_public to True to include public IP adresses .


def create_connection(conf):
    """
        Creates a connection based upon the given configuration object.
    """
    host_config = {}
    host_config['hosts'] = [conf.get('jackal', 'host')]
    if int(conf.get('jackal', 'use_ssl')):
        host_config['use_ssl'] = True
    if conf.get('jackal', 'ca_certs'):
        host_config['ca_certs'] = conf.get('jackal', 'ca_certs')
    if int(conf.get('jackal', 'client_certs')):
        host_config['client_cert'] = conf.get('jackal', 'client_cert')
        host_config['client_key'] = conf.get('jackal', 'client_key')

    # Disable hostname checking for now.
    host_config['ssl_assert_hostname'] = False

    connections.create_connection(**host_config)

def create_connection ( conf ) : host_config = { } host_config [ 'hosts' ] = [ conf . get ( 'jackal' , 'host' ) ] if int ( conf . get ( 'jackal' , 'use_ssl' ) ) : host_config [ 'use_ssl' ] = True if conf . get ( 'jackal' , 'ca_certs' ) : host_config [ 'ca_certs' ] = conf . get ( 'jackal' , 'ca_certs' ) if int ( conf . get ( 'jackal' , 'client_certs' ) ) : host_config [ 'client_cert' ] = conf . get ( 'jackal' , 'client_cert' ) host_config [ 'client_key' ] = conf . get ( 'jackal' , 'client_key' ) host_config [ 'ssl_assert_hostname' ] = False connections . create_connection ( ** host_config )

Creates a connection based upon the given configuration object .


def search(self, number=None, *args, **kwargs):
        """
            Searches the elasticsearch instance to retrieve the requested documents.
        """
        search = self.create_search(*args, **kwargs)
        try:
            if number:
                response = search[0:number]
            else:
                args, _ = self.core_parser.parse_known_args()
                if args.number:
                    response = search[0:args.number]
                else:
                    response = search.scan()

            return [hit for hit in response]
        except NotFoundError:
            print_error("The index was not found, have you initialized the index?")
            return []
        except (ConnectionError, TransportError):
            print_error("Cannot connect to elasticsearch")
            return []

Searches the elasticsearch instance to retrieve the requested documents .


def argument_search(self):
        """
            Uses the command line arguments to fill the search function and call it.
        """
        arguments, _ = self.argparser.parse_known_args()
        return self.search(**vars(arguments))

def argument_search ( self ) : arguments , _ = self . argparser . parse_known_args ( ) return self . search ( ** vars ( arguments ) )

Uses the command line arguments to fill the search function and call it .


def count(self, *args, **kwargs):
        """
            Returns the number of results after filtering with the given arguments.
        """
        search = self.create_search(*args, **kwargs)
        try:
            return search.count()
        except NotFoundError:
            print_error("The index was not found, have you initialized the index?")
        except (ConnectionError, TransportError):
            print_error("Cannot connect to elasticsearch")

Returns the number of results after filtering with the given arguments .


def argument_count(self):
        """
            Uses the command line arguments to fill the count function and call it.
        """
        arguments, _ = self.argparser.parse_known_args()
        return self.count(**vars(arguments))

def argument_count ( self ) : arguments , _ = self . argparser . parse_known_args ( ) return self . count ( ** vars ( arguments ) )

Uses the command line arguments to fill the count function and call it .


def get_pipe(self, object_type):
        """
            Returns a generator that maps the input of the pipe to an elasticsearch object.
            Will call id_to_object if it cannot serialize the data from json.
        """
        for line in sys.stdin:
            try:
                data = json.loads(line.strip())
                obj = object_type(**data)
                yield obj
            except ValueError:
                yield self.id_to_object(line.strip())

def get_pipe ( self , object_type ) : for line in sys . stdin : try : data = json . loads ( line . strip ( ) ) obj = object_type ( ** data ) yield obj except ValueError : yield self . id_to_object ( line . strip ( ) )

Returns a generator that maps the input of the pipe to an elasticsearch object . Will call id_to_object if it cannot serialize the data from json .


def id_to_object(self, line):
        """
            Resolves an ip adres to a range object, creating it if it doesn't exists.
        """
        result = Range.get(line, ignore=404)
        if not result:
            result = Range(range=line)
            result.save()
        return result

def id_to_object ( self , line ) : result = Range . get ( line , ignore = 404 ) if not result : result = Range ( range = line ) result . save ( ) return result

Resolves an ip adres to a range object creating it if it doesn t exists .


def argparser(self):
        """
            Argparser option with search functionality specific for ranges.
        """
        core_parser = self.core_parser
        core_parser.add_argument('-r', '--range', type=str, help="The range to search for use")
        return core_parser

def
argparser
(
self
)
:
core_parser
=
self
.
core_parser
core_parser
.
add_argument
(
'-r'
,
'--range'
,
type
=
str
,
help
=
"The range to search for use"
)
return
core_parser

Argparser option with search functionality specific for ranges .


def object_to_id(self, obj):
        """
            Searches elasticsearch for objects with the same address, protocol, port and state.
        """
        search = Service.search()
        search = search.filter("term", address=obj.address)
        search = search.filter("term", protocol=obj.protocol)
        search = search.filter("term", port=obj.port)
        search = search.filter("term", state=obj.state)
        if search.count():
            result = search[0].execute()[0]
            return result.meta.id
        else:
            return None

def object_to_id ( self , obj ) : search = Service . search ( ) search = search . filter ( "term" , address = obj . address ) search = search . filter ( "term" , protocol = obj . protocol ) search = search . filter ( "term" , port = obj . port ) search = search . filter ( "term" , state = obj . state ) if search . count ( ) : result = search [ 0 ] . execute ( ) [ 0 ] return result . meta . id else : return None

Searches elasticsearch for objects with the same address protocol port and state .


def id_to_object(self, line):
        """
            Resolves the given id to a user object, if it doesn't exists it will be created.
        """
        user = User.get(line, ignore=404)
        if not user:
            user = User(username=line)
            user.save()
        return user

def id_to_object ( self , line ) : user = User . get ( line , ignore = 404 ) if not user : user = User ( username = line ) user . save ( ) return user

Resolves the given id to a user object if it doesn t exists it will be created .


def get_domains(self):
        """
            Retrieves the domains of the users from elastic.
        """
        search = User.search()
        search.aggs.bucket('domains', 'terms', field='domain', order={'_count': 'desc'}, size=100)
        response = search.execute()
        return [entry.key for entry in response.aggregations.domains.buckets]

def get_domains ( self ) : search = User . search ( ) search . aggs . bucket ( 'domains' , 'terms' , field = 'domain' , order = { '_count' : 'desc' } , size = 100 ) response = search . execute ( ) return [ entry . key for entry in response . aggregations . domains . buckets ]

Retrieves the domains of the users from elastic .


def get_pipe(self):
        """
            Returns a list that maps the input of the pipe to an elasticsearch object.
            Will call id_to_object if it cannot serialize the data from json.
        """
        lines = []
        for line in sys.stdin:
            try:
                lines.append(self.line_to_object(line.strip()))
            except ValueError:
                pass
            except KeyError:
                pass
        return lines

def get_pipe ( self ) : lines = [ ] for line in sys . stdin : try : lines . append ( self . line_to_object ( line . strip ( ) ) ) except ValueError : pass except KeyError : pass return lines

Returns a list that maps the input of the pipe to an elasticsearch object . Will call id_to_object if it cannot serialize the data from json .


def tree2commands(self, adapter, session, lastcmds, xsync):
    '''Consumes an ET protocol tree and converts it to state.Command commands'''

    # do some preliminary sanity checks...
    # todo: do i really want to be using assert statements?...

    assert xsync.tag == constants.NODE_SYNCML
    assert len(xsync) == 2
    assert xsync[0].tag == constants.CMD_SYNCHDR
    assert xsync[1].tag == constants.NODE_SYNCBODY

    version = xsync[0].findtext('VerProto')
    if version != constants.SYNCML_VERSION_1_2:
      raise common.FeatureNotSupported('unsupported SyncML version "%s" (expected "%s")' \
                                       % (version, constants.SYNCML_VERSION_1_2))
    verdtd = xsync[0].findtext('VerDTD')
    if verdtd != constants.SYNCML_DTD_VERSION_1_2:
      raise common.FeatureNotSupported('unsupported SyncML DTD version "%s" (expected "%s")' \
                                       % (verdtd, constants.SYNCML_DTD_VERSION_1_2))

    ret = self.initialize(adapter, session, xsync)
    hdrcmd = ret[0]

    if session.isServer:
      log.debug('received request SyncML message from "%s" (s%s.m%s)',
                hdrcmd.target, hdrcmd.sessionID, hdrcmd.msgID)
    else:
      log.debug('received response SyncML message from "%s" (s%s.m%s)',
                lastcmds[0].target, lastcmds[0].sessionID, lastcmds[0].msgID)

    try:
      return self._tree2commands(adapter, session, lastcmds, xsync, ret)
    except Exception, e:
      if not session.isServer:
        raise
      # TODO: make this configurable as to whether or not any error
      #       is sent back to the peer as a SyncML "standardized" error
      #       status...
      code = '%s.%s' % (e.__class__.__module__, e.__class__.__name__)
      msg  = ''.join(traceback.format_exception_only(type(e), e)).strip()
      log.exception('failed while interpreting command tree: %s', msg)
      # TODO: for some reason, the active exception is not being logged...
      return [
        hdrcmd,
        state.Command(
          name       = constants.CMD_STATUS,
          cmdID      = '1',
          msgRef     = session.pendingMsgID,
          cmdRef     = 0,
          sourceRef  = xsync[0].findtext('Source/LocURI'),
          targetRef  = xsync[0].findtext('Target/LocURI'),
          statusOf   = constants.CMD_SYNCHDR,
          statusCode = constants.STATUS_COMMAND_FAILED,
          errorCode  = code,
          errorMsg   = msg,
          errorTrace = ''.join(traceback.format_exception(type(e), e, sys.exc_info()[2])),
          ),
        state.Command(name=constants.CMD_FINAL)]

Consumes an ET protocol tree and converts it to state . Command commands


def initialize_indices():
    """
        Initializes the indices
    """
    Host.init()
    Range.init()
    Service.init()
    User.init()
    Credential.init()
    Log.init()

def initialize_indices ( ) : Host . init ( ) Range . init ( ) Service . init ( ) User . init ( ) Credential . init ( ) Log . init ( )


def parse_single_computer(entry):
    """
        Parse the entry into a computer object.
    """
    computer = Computer(dns_hostname=get_field(entry, 'dNSHostName'), description=get_field(
        entry, 'description'), os=get_field(entry, 'operatingSystem'), group_id=get_field(entry, 'primaryGroupID'))
    try:
        ip = str(ipaddress.ip_address(get_field(entry, 'IPv4')))
    except ValueError:
        ip = ''

    if ip:
        computer.ip = ip
    elif computer.dns_hostname:
        computer.ip = resolve_ip(computer.dns_hostname)
    return computer

def parse_single_computer ( entry ) : computer = Computer ( dns_hostname = get_field ( entry , 'dNSHostName' ) , description = get_field ( entry , 'description' ) , os = get_field ( entry , 'operatingSystem' ) , group_id = get_field ( entry , 'primaryGroupID' ) ) try : ip = str ( ipaddress . ip_address ( get_field ( entry , 'IPv4' ) ) ) except ValueError : ip = '' if ip : computer . ip = ip elif computer . dns_hostname : computer . ip = resolve_ip ( computer . dns_hostname ) return computer

Parse the entry into a computer object .


def parse_domain_computers(filename):
    """
        Parse the file and extract the computers, import the computers that resolve into jackal.
    """
    with open(filename) as f:
        data = json.loads(f.read())
    hs = HostSearch()
    count = 0
    entry_count = 0
    print_notification("Parsing {} entries".format(len(data)))
    for system in data:
        entry_count += 1
        parsed = parse_single_computer(system)
        if parsed.ip:
            try:
                host = hs.id_to_object(parsed.ip)
                host.description.append(parsed.description)
                host.hostname.append(parsed.dns_hostname)
                if parsed.os:
                    host.os = parsed.os
                host.domain_controller = parsed.dc
                host.add_tag('domaindump')
                host.save()
                count += 1
            except ValueError:
                pass
        sys.stdout.write('\r')
        sys.stdout.write(
            "[{}/{}] {} resolved".format(entry_count, len(data), count))
        sys.stdout.flush()
    sys.stdout.write('\r')
    return count

def
parse_domain_computers
(
filename
)
:
with
open
(
filename
)
as
f
:
data
=
json
.
loads
(
f
.
read
(
)
)
hs
=
HostSearch
(
)
count
=
0
entry_count
=
0
print_notification
(
"Parsing {} entries"
.
format
(
len
(
data
)
)
)
for
system
in
data
:
entry_count
+=
1
parsed
=
parse_single_computer
(
system
)
if
parsed
.
ip
:
try
:
host
=
hs
.
id_to_object
(
parsed
.
ip
)
host
.
description
.
append
(
parsed
.
description
)
host
.
hostname
.
append
(
parsed
.
dns_hostname
)
if
parsed
.
os
:
host
.
os
=
parsed
.
os
host
.
domain_controller
=
parsed
.
dc
host
.
add_tag
(
'domaindump'
)
host
.
save
(
)
count
+=
1
except
ValueError
:
pass
sys
.
stdout
.
write
(
'\r'
)
sys
.
stdout
.
write
(
"[{}/{}] {} resolved"
.
format
(
entry_count
,
len
(
data
)
,
count
)
)
sys
.
stdout
.
flush
(
)
sys
.
stdout
.
write
(
'\r'
)
return
count

Parse the file and extract the computers import the computers that resolve into jackal .


def parse_user(entry, domain_groups):
    """
        Parses a single entry from the domaindump
    """
    result = {}
    distinguished_name = get_field(entry, 'distinguishedName')
    result['domain'] = ".".join(distinguished_name.split(',DC=')[1:])
    result['name'] = get_field(entry, 'name')
    result['username'] = get_field(entry, 'sAMAccountName')
    result['description'] = get_field(entry, 'description')
    result['sid'] = get_field(entry, 'objectSid').split('-')[-1]

    primary_group = get_field(entry, 'primaryGroupID')
    member_of = entry['attributes'].get('memberOf', [])
    groups = []
    for member in member_of:
        for e in member.split(','):
            if e.startswith('CN='):
                groups.append(e[3:])
    groups.append(domain_groups.get(primary_group, ''))
    result['groups'] = groups

    flags = []
    try:
        uac = int(get_field(entry, 'userAccountControl'))

        for flag, value in uac_flags.items():
            if uac & value:
                flags.append(flag)
    except ValueError:
        pass
    result['flags'] = flags
    return result

def parse_user ( entry , domain_groups ) : result = { } distinguished_name = get_field ( entry , 'distinguishedName' ) result [ 'domain' ] = "." . join ( distinguished_name . split ( ',DC=' ) [ 1 : ] ) result [ 'name' ] = get_field ( entry , 'name' ) result [ 'username' ] = get_field ( entry , 'sAMAccountName' ) result [ 'description' ] = get_field ( entry , 'description' ) result [ 'sid' ] = get_field ( entry , 'objectSid' ) . split ( '-' ) [ - 1 ] primary_group = get_field ( entry , 'primaryGroupID' ) member_of = entry [ 'attributes' ] . get ( 'memberOf' , [ ] ) groups = [ ] for member in member_of : for e in member . split ( ',' ) : if e . startswith ( 'CN=' ) : groups . append ( e [ 3 : ] ) groups . append ( domain_groups . get ( primary_group , '' ) ) result [ 'groups' ] = groups flags = [ ] try : uac = int ( get_field ( entry , 'userAccountControl' ) ) for flag , value in uac_flags . items ( ) : if uac & value : flags . append ( flag ) except ValueError : pass result [ 'flags' ] = flags return result


def parse_domain_users(domain_users_file, domain_groups_file):
    """
        Parses the domain users and groups files.
    """
    with open(domain_users_file) as f:
        users = json.loads(f.read())

    domain_groups = {}
    if domain_groups_file:
        with open(domain_groups_file) as f:
            groups = json.loads(f.read())
            for group in groups:
                sid = get_field(group, 'objectSid')
                domain_groups[int(sid.split('-')[-1])] = get_field(group, 'cn')

    user_search = UserSearch()
    count = 0
    total = len(users)
    print_notification("Importing {} users".format(total))
    for entry in users:
        result = parse_user(entry, domain_groups)
        user = user_search.id_to_object(result['username'])
        user.name = result['name']
        user.domain.append(result['domain'])
        user.description = result['description']
        user.groups.extend(result['groups'])
        user.flags.extend(result['flags'])
        user.sid = result['sid']
        user.add_tag("domaindump")
        user.save()
        count += 1
        sys.stdout.write('\r')
        sys.stdout.write("[{}/{}]".format(count, total))
        sys.stdout.flush()
    sys.stdout.write('\r')
    return count

def
parse_domain_users
(
domain_users_file
,
domain_groups_file
)
:
with
open
(
domain_users_file
)
as
f
:
users
=
json
.
loads
(
f
.
read
(
)
)
domain_groups
=
{
}
if
domain_groups_file
:
with
open
(
domain_groups_file
)
as
f
:
groups
=
json
.
loads
(
f
.
read
(
)
)
for
group
in
groups
:
sid
=
get_field
(
group
,
'objectSid'
)
domain_groups
[
int
(
sid
.
split
(
'-'
)
[
1
]
)
]
=
get_field
(
group
,
'cn'
)
user_search
=
UserSearch
(
)
count
=
0
total
=
len
(
users
)
print_notification
(
"Importing {} users"
.
format
(
total
)
)
for
entry
in
users
:
result
=
parse_user
(
entry
,
domain_groups
)
user
=
user_search
.
id_to_object
(
result
[
'username'
]
)
user
.
name
=
result
[
'name'
]
user
.
domain
.
append
(
result
[
'domain'
]
)
user
.
description
=
result
[
'description'
]
user
.
groups
.
extend
(
result
[
'groups'
]
)
user
.
flags
.
extend
(
result
[
'flags'
]
)
user
.
sid
=
result
[
'sid'
]
user
.
add_tag
(
"domaindump"
)
user
.
save
(
)
count
+=
1
sys
.
stdout
.
write
(
'\r'
)
sys
.
stdout
.
write
(
"[{}/{}]"
.
format
(
count
,
total
)
)
sys
.
stdout
.
flush
(
)
sys
.
stdout
.
write
(
'\r'
)
return
count

Parses the domain users and groups files .


def import_domaindump():
    """
        Parses ldapdomaindump files and stores hosts and users in elasticsearch.
    """
    parser = argparse.ArgumentParser(
        description="Imports users, groups and computers result files from the ldapdomaindump tool, will resolve the names from domain_computers output for IPs")
    parser.add_argument("files", nargs='+',
                        help="The domaindump files to import")
    arguments = parser.parse_args()
    domain_users_file = ''
    domain_groups_file = ''
    computer_count = 0
    user_count = 0
    stats = {}
    for filename in arguments.files:
        if filename.endswith('domain_computers.json'):
            print_notification('Parsing domain computers')
            computer_count = parse_domain_computers(filename)
            if computer_count:
                stats['hosts'] = computer_count
                print_success("{} hosts imported".format(computer_count))
        elif filename.endswith('domain_users.json'):
            domain_users_file = filename
        elif filename.endswith('domain_groups.json'):
            domain_groups_file = filename
    if domain_users_file:
        print_notification("Parsing domain users")
        user_count = parse_domain_users(domain_users_file, domain_groups_file)
        if user_count:
            print_success("{} users imported".format(user_count))
            stats['users'] = user_count
    Logger().log("import_domaindump", 'Imported domaindump, found {} user, {} systems'.format(user_count, computer_count), stats)

Parses ldapdomaindump files and stores hosts and users in elasticsearch .


def autocomplete(query, country=None, hurricanes=False, cities=True, timeout=5):
    """Make an autocomplete API request

    This can be used to find cities and/or hurricanes by name

    :param string query: city
    :param string country: restrict search to a specific country. Must be a two letter country code
    :param boolean hurricanes: whether to search for hurricanes or not
    :param boolean cities: whether to search for cities or not
    :param integer timeout: timeout of the api request
    :returns: result of the autocomplete API request
    :rtype: dict

    """
    data = {}
    data['query'] = quote(query)
    data['country'] = country or ''
    data['hurricanes'] = 1 if hurricanes else 0
    data['cities'] = 1 if cities else 0
    data['format'] = 'JSON'
    r = requests.get(AUTOCOMPLETE_URL.format(**data), timeout=timeout)
    results = json.loads(r.content)['RESULTS']
    return results

def autocomplete ( query , country = None , hurricanes = False , cities = True , timeout = 5 ) : data = { } data [ 'query' ] = quote ( query ) data [ 'country' ] = country or '' data [ 'hurricanes' ] = 1 if hurricanes else 0 data [ 'cities' ] = 1 if cities else 0 data [ 'format' ] = 'JSON' r = requests . get ( AUTOCOMPLETE_URL . format ( ** data ) , timeout = timeout ) results = json . loads ( r . content ) [ 'RESULTS' ] return results


def request(key, features, query, timeout=5):
    """Make an API request

    :param string key: API key to use
    :param list features: features to request. It must be a subset of :data:`FEATURES`
    :param string query: query to send
    :param integer timeout: timeout of the request
    :returns: result of the API request
    :rtype: dict

    """
    data = {}
    data['key'] = key
    data['features'] = '/'.join([f for f in features if f in FEATURES])
    data['query'] = quote(query)
    data['format'] = 'json'
    r = requests.get(API_URL.format(**data), timeout=timeout)
    results = json.loads(_unicode(r.content))
    return results

def request ( key , features , query , timeout = 5 ) : data = { } data [ 'key' ] = key data [ 'features' ] = '/' . join ( [ f for f in features if f in FEATURES ] ) data [ 'query' ] = quote ( query ) data [ 'format' ] = 'json' r = requests . get ( API_URL . format ( ** data ) , timeout = timeout ) results = json . loads ( _unicode ( r . content ) ) return results


def _unicode(string):
    """Try to convert a string to unicode using different encodings"""
    for encoding in ['utf-8', 'latin1']:
        try:
            result = unicode(string, encoding)
            return result
        except UnicodeDecodeError:
            pass
    result = unicode(string, 'utf-8', 'replace')
    return result

def _unicode ( string ) : for encoding in [ 'utf-8' , 'latin1' ] : try : result = unicode ( string , encoding ) return result except UnicodeDecodeError : pass result = unicode ( string , 'utf-8' , 'replace' ) return result


def http_get_provider(provider,
                      request_url, params, token_secret, token_cookie = None):
    '''Handle HTTP GET requests on an authentication endpoint.

    Authentication flow begins when ``params`` has a ``login`` key with a value
    of ``start``. For instance, ``/auth/twitter?login=start``.

    :param str provider: An provider to obtain a user ID from.
    :param str request_url: The authentication endpoint/callback.
    :param dict params: GET parameters from the query string.
    :param str token_secret: An app secret to encode/decode JSON web tokens.
    :param str token_cookie: The current JSON web token, if available.
    :return: A dict containing any of the following possible keys:

        ``status``: an HTTP status code the server should sent

        ``redirect``: where the client should be directed to continue the flow

        ``set_token_cookie``: contains a JSON web token and should be stored by
        the client and passed in the next call.

        ``provider_user_id``: the user ID from the login provider

        ``provider_user_name``: the user name from the login provider
    '''

    if not validate_provider(provider):
        raise InvalidUsage('Provider not supported')

    klass    = getattr(socialauth.providers, provider.capitalize())
    provider = klass(request_url, params, token_secret, token_cookie)
    if provider.status == 302:
        ret = dict(status = 302, redirect = provider.redirect)
        tc  = getattr(provider, 'set_token_cookie', None)
        if tc is not None:
            ret['set_token_cookie'] = tc

        return ret

    if provider.status == 200 and provider.user_id is not None:
        ret = dict(status = 200, provider_user_id = provider.user_id)
        if provider.user_name is not None:
            ret['provider_user_name'] = provider.user_name

        return ret

    raise InvalidUsage('Invalid request')

def
http_get_provider
(
provider
,
request_url
,
params
,
token_secret
,
token_cookie
=
)
:
if
not
validate_provider
(
provider
)
:
raise
InvalidUsage
(
'Provider not supported'
)
klass
=
getattr
(
socialauth
.
providers
,
provider
.
capitalize
(
)
)
provider
=
klass
(
request_url
,
params
,
token_secret
,
token_cookie
)
if
provider
.
status
==
302
:
ret
=
dict
(
status
=
302
,
redirect
=
provider
.
redirect
)
tc
=
getattr
(
provider
,
'set_token_cookie'
,
)
if
tc
is
not
:
ret
[
'set_token_cookie'
]
=
tc
return
ret
if
provider
.
status
==
200
and
provider
.
user_id
is
not
:
ret
=
dict
(
status
=
200
,
provider_user_id
=
provider
.
user_id
)
if
provider
.
user_name
is
not
:
ret
[
'provider_user_name'
]
=
provider
.
user_name
return
ret
raise
InvalidUsage
(
'Invalid request'
)

Handle HTTP GET requests on an authentication endpoint .


def to_json(self):
        """ Method to call to get a serializable object for json.dump or jsonify based on the target

        :return: dict
        """
        if self.subreference is not None:
            return {
                "source": self.objectId,
                "selector": {
                    "type": "FragmentSelector",
                    "conformsTo": "http://ontology-dts.org/terms/subreference",
                    "value": self.subreference
                }
            }
        else:
            return {"source": self.objectId}

Method to call to get a serializable object for json . dump or jsonify based on the target


def read(self):
        """ Read the contents of the Annotation Resource

        :return: the contents of the resource
        :rtype: str or bytes or flask.response
        """
        if not self.__content__:
            self.__retriever__ = self.__resolver__.resolve(self.uri)
            self.__content__, self.__mimetype__ = self.__retriever__.read(self.uri)
        return self.__content__

def read ( self ) : if not self . __content__ : self . __retriever__ = self . __resolver__ . resolve ( self . uri ) self . __content__ , self . __mimetype__ = self . __retriever__ . read ( self . uri ) return self . __content__


def build_index_and_mapping(triples):
    """index all triples into indexes and return their mappings"""
    ents = bidict()
    rels = bidict()
    ent_id = 0
    rel_id = 0

    collected = []
    for t in triples:
        for e in (t.head, t.tail):
            if e not in ents:
                ents[e] = ent_id
                ent_id += 1
        if t.relation not in rels:
            rels[t.relation] = rel_id
            rel_id += 1
        collected.append(kgedata.TripleIndex(ents[t.head], rels[t.relation], ents[t.tail]))

    return collected, ents, rels

def build_index_and_mapping ( triples ) : ents = bidict ( ) rels = bidict ( ) ent_id = 0 rel_id = 0 collected = [ ] for t in triples : for e in ( t . head , t . tail ) : if e not in ents : ents [ e ] = ent_id ent_id += 1 if t . relation not in rels : rels [ t . relation ] = rel_id rel_id += 1 collected . append ( kgedata . TripleIndex ( ents [ t . head ] , rels [ t . relation ] , ents [ t . tail ] ) ) return collected , ents , rels


def recover_triples_from_mapping(indexes, ents: bidict, rels: bidict):
    """recover triples from mapping."""
    triples = []
    for t in indexes:
        triples.append(kgedata.Triple(ents.inverse[t.head], rels.inverse[t.relation], ents.inverse[t.tail]))
    return triples

def recover_triples_from_mapping ( indexes , ents : bidict , rels : bidict ) : triples = [ ] for t in indexes : triples . append ( kgedata . Triple ( ents . inverse [ t . head ] , rels . inverse [ t . relation ] , ents . inverse [ t . tail ] ) ) return triples


def _transform_triple_numpy(x):
    """Transform triple index into a 1-D numpy array."""
    return np.array([x.head, x.relation, x.tail], dtype=np.int64)

def _transform_triple_numpy ( x ) : return np . array ( [ x . head , x . relation , x . tail ] , dtype = np . int64 )

Transform triple index into a 1 - D numpy array .


def pack_triples_numpy(triples):
    """Packs a list of triple indexes into a 2D numpy array."""
    if len(triples) == 0:
        return np.array([], dtype=np.int64)
    return np.stack(list(map(_transform_triple_numpy, triples)), axis=0)

def pack_triples_numpy ( triples ) : if len ( triples ) == 0 : return np . array ( [ ] , dtype = np . int64 ) return np . stack ( list ( map ( _transform_triple_numpy , triples ) ) , axis = 0 )

Packs a list of triple indexes into a 2D numpy array .


def remove_near_duplicate_relation(triples, threshold=0.97):
    """If entity pairs in a relation is as close as another relations, only keep one relation of such set."""
    logging.debug("remove duplicate")

    _assert_threshold(threshold)

    duplicate_rel_counter = defaultdict(list)
    relations = set()
    for t in triples:
        duplicate_rel_counter[t.relation].append(f"{t.head} {t.tail}")
        relations.add(t.relation)
    relations = list(relations)

    num_triples = len(triples)
    removal_relation_set = set()

    for rel, values in duplicate_rel_counter.items():
        duplicate_rel_counter[rel] = Superminhash(values)
    for i in relations:
        for j in relations:
            if i == j or i in removal_relation_set or j in removal_relation_set: continue
            close_relations = [i]
            if _set_close_to(duplicate_rel_counter[i], duplicate_rel_counter[j], threshold):
                close_relations.append(j)
        if len(close_relations) > 1:
            close_relations.pop(np.random.randint(len(close_relations)))
            removal_relation_set |= set(close_relations)
    logging.info("Removing {} relations: {}".format(len(removal_relation_set), str(removal_relation_set)))

    return list(filterfalse(lambda x: x.relation in removal_relation_set, triples))

def
remove_near_duplicate_relation
(
triples
,
threshold
=
0.97
)
:
logging
.
debug
(
"remove duplicate"
)
_assert_threshold
(
threshold
)
duplicate_rel_counter
=
defaultdict
(
list
)
relations
=
set
(
)
for
t
in
triples
:
duplicate_rel_counter
[
t
.
relation
]
.
append
(
f"{t.head} {t.tail}"
)
relations
.
add
(
t
.
relation
)
relations
=
list
(
relations
)
num_triples
=
len
(
triples
)
removal_relation_set
=
set
(
)
for
rel
,
values
in
duplicate_rel_counter
.
items
(
)
:
duplicate_rel_counter
[
rel
]
=
Superminhash
(
values
)
for
i
in
relations
:
for
j
in
relations
:
if
i
==
j
or
i
in
removal_relation_set
or
j
in
removal_relation_set
:
continue
close_relations
=
[
i
]
if
_set_close_to
(
duplicate_rel_counter
[
i
]
,
duplicate_rel_counter
[
j
]
,
threshold
)
:
close_relations
.
append
(
j
)
if
len
(
close_relations
)
>
1
:
close_relations
.
pop
(
np
.
random
.
randint
(
len
(
close_relations
)
)
)
removal_relation_set
|=
set
(
close_relations
)
logging
.
info
(
"Removing {} relations: {}"
.
format
(
len
(
removal_relation_set
)
,
str
(
removal_relation_set
)
)
)
return
list
(
filterfalse
(
lambda
x
:
x
.
relation
in
removal_relation_set
,
triples
)
)

If entity pairs in a relation is as close as another relations only keep one relation of such set .


def remove_direct_link_triples(train, valid, test):
    """Remove direct links in the training sets."""
    pairs = set()
    merged = valid + test
    for t in merged:
        pairs.add((t.head, t.tail))

    filtered = filterfalse(lambda t: (t.head, t.tail) in pairs or (t.tail, t.head) in pairs, train)
    return list(filtered)

def remove_direct_link_triples ( train , valid , test ) : pairs = set ( ) merged = valid + test for t in merged : pairs . add ( ( t . head , t . tail ) ) filtered = filterfalse ( lambda t : ( t . head , t . tail ) in pairs or ( t . tail , t . head ) in pairs , train ) return list ( filtered )

Remove direct links in the training sets .


def shrink_indexes_in_place(self, triples):
        """Uses a union find to find segment."""

        _ent_roots = self.UnionFind(self._ent_id)
        _rel_roots = self.UnionFind(self._rel_id)

        for t in triples:
            _ent_roots.add(t.head)
            _ent_roots.add(t.tail)
            _rel_roots.add(t.relation)

        for i, t in enumerate(triples):
            h = _ent_roots.find(t.head)
            r = _rel_roots.find(t.relation)
            t = _ent_roots.find(t.tail)
            triples[i] = kgedata.TripleIndex(h, r, t)

        ents = bidict()
        available_ent_idx = 0
        for previous_idx, ent_exist in enumerate(_ent_roots.roots()):
            if not ent_exist:
                self._ents.inverse.pop(previous_idx)
            else:
                ents[self._ents.inverse[previous_idx]] = available_ent_idx
            available_ent_idx += 1
        rels = bidict()
        available_rel_idx = 0
        for previous_idx, rel_exist in enumerate(_rel_roots.roots()):
            if not rel_exist:
                self._rels.inverse.pop(previous_idx)
            else:
                rels[self._rels.inverse[previous_idx]] = available_rel_idx
            available_rel_idx += 1
        self._ents = ents
        self._rels = rels
        self._ent_id = available_ent_idx
        self._rel_id = available_rel_idx

def shrink_indexes_in_place ( self , triples ) : _ent_roots = self . UnionFind ( self . _ent_id ) _rel_roots = self . UnionFind ( self . _rel_id ) for t in triples : _ent_roots . add ( t . head ) _ent_roots . add ( t . tail ) _rel_roots . add ( t . relation ) for i , t in enumerate ( triples ) : h = _ent_roots . find ( t . head ) r = _rel_roots . find ( t . relation ) t = _ent_roots . find ( t . tail ) triples [ i ] = kgedata . TripleIndex ( h , r , t ) ents = bidict ( ) available_ent_idx = 0 for previous_idx , ent_exist in enumerate ( _ent_roots . roots ( ) ) : if not ent_exist : self . _ents . inverse . pop ( previous_idx ) else : ents [ self . _ents . inverse [ previous_idx ] ] = available_ent_idx available_ent_idx += 1 rels = bidict ( ) available_rel_idx = 0 for previous_idx , rel_exist in enumerate ( _rel_roots . roots ( ) ) : if not rel_exist : self . _rels . inverse . pop ( previous_idx ) else : rels [ self . _rels . inverse [ previous_idx ] ] = available_rel_idx available_rel_idx += 1 self . _ents = ents self . _rels = rels self . _ent_id = available_ent_idx self . _rel_id = available_rel_idx


def freeze(self):
        """Create a usable data structure for serializing."""
        data = super(IndexBuilder, self).freeze()
        try:
            # Sphinx >= 1.5 format
            # Due to changes from github.com/sphinx-doc/sphinx/pull/2454
            base_file_names = data['docnames']
        except KeyError:
            # Sphinx < 1.5 format
            base_file_names = data['filenames']

        store = {}
        c = itertools.count()
        for prefix, items in iteritems(data['objects']):
            for name, (index, typeindex, _, shortanchor) in iteritems(items):
                objtype = data['objtypes'][typeindex]
                if objtype.startswith('cpp:'):
                    split =  name.rsplit('::', 1)
                    if len(split) != 2:
                        warnings.warn("What's up with %s?" % str((prefix, name, objtype)))
                        continue
                    prefix, name = split
                    last_prefix = prefix.split('::')[-1]
                else:
                    last_prefix = prefix.split('.')[-1]

                store[next(c)] = {
                    'filename': base_file_names[index],
                    'objtype': objtype,
                    'prefix': prefix,
                    'last_prefix': last_prefix,
                    'name': name,
                    'shortanchor': shortanchor,
                }

        data.update({'store': store})
        return data

def
freeze
(
self
)
:
data
=
super
(
IndexBuilder
,
self
)
.
freeze
(
)
try
:
base_file_names
=
data
[
'docnames'
]
except
KeyError
:
base_file_names
=
data
[
'filenames'
]
store
=
{
}
c
=
itertools
.
count
(
)
for
prefix
,
items
in
iteritems
(
data
[
'objects'
]
)
:
for
name
,
(
index
,
typeindex
,
_
,
shortanchor
)
in
iteritems
(
items
)
:
objtype
=
data
[
'objtypes'
]
[
typeindex
]
if
objtype
.
startswith
(
'cpp:'
)
:
split
=
name
.
rsplit
(
'::'
,
1
)
if
len
(
split
)
!=
2
:
warnings
.
warn
(
"What's up with %s?"
%
str
(
(
prefix
,
name
,
objtype
)
)
)
continue
prefix
,
name
=
split
last_prefix
=
prefix
.
split
(
'::'
)
[
1
]
else
:
last_prefix
=
prefix
.
split
(
'.'
)
[
1
]
store
[
next
(
c
)
]
=
{
'filename'
:
base_file_names
[
index
]
,
'objtype'
:
objtype
,
'prefix'
:
prefix
,
'last_prefix'
:
last_prefix
,
'name'
:
name
,
'shortanchor'
:
shortanchor
,
}
data
.
update
(
{
'store'
:
store
}
)
return
data

Create a usable data structure for serializing .


def log_operation(entities, operation_name, params=None):
    """Logs an operation done on an entity, possibly with other arguments
    """
    if isinstance(entities, (list, tuple)):
        entities = list(entities)
    else:
        entities = [entities]

    p = {'name': operation_name, 'on': entities}
    if params:
        p['params'] = params
    _log(TYPE_CODES.OPERATION, p)

def log_operation ( entities , operation_name , params = None ) : if isinstance ( entities , ( list , tuple ) ) : entities = list ( entities ) else : entities = [ entities ] p = { 'name' : operation_name , 'on' : entities } if params : p [ 'params' ] = params _log ( TYPE_CODES . OPERATION , p )

Logs an operation done on an entity possibly with other arguments


def log_state(entity, state):
    """Logs a new state of an entity
    """
    p = {'on': entity, 'state': state}
    _log(TYPE_CODES.STATE, p)

def log_state ( entity , state ) : p = { 'on' : entity , 'state' : state } _log ( TYPE_CODES . STATE , p )


def log_update(entity, update):
    """Logs an update done on an entity
    """
    p = {'on': entity, 'update': update}
    _log(TYPE_CODES.UPDATE, p)

def log_update ( entity , update ) : p = { 'on' : entity , 'update' : update } _log ( TYPE_CODES . UPDATE , p )


def log_error(error, result):
    """Logs an error
    """
    p = {'error': error, 'result':result}
    _log(TYPE_CODES.ERROR, p)

def log_error ( error , result ) : p = { 'error' : error , 'result' : result } _log ( TYPE_CODES . ERROR , p )


def dict_cursor(func):
    """
    Decorator that provides a dictionary cursor to the calling function

    Adds the cursor as the second argument to the calling functions

    Requires that the function being decorated is an instance of a class or object
    that yields a cursor from a get_cursor(cursor_type=CursorType.DICT) coroutine or provides such an object
    as the first argument in its signature

    Yields:
        A client-side dictionary cursor
    """

    @wraps(func)
    def wrapper(cls, *args, **kwargs):
        with (yield from cls.get_cursor(_CursorType.DICT)) as c:
            return (yield from func(cls, c, *args, **kwargs))

    return wrapper

def dict_cursor ( func ) : @ wraps ( func ) def wrapper ( cls , * args , ** kwargs ) : with ( yield from cls . get_cursor ( _CursorType . DICT ) ) as c : return ( yield from func ( cls , c , * args , ** kwargs ) ) return wrapper


def cursor(func):
    """
    Decorator that provides a cursor to the calling function

    Adds the cursor as the second argument to the calling functions

    Requires that the function being decorated is an instance of a class or object
    that yields a cursor from a get_cursor() coroutine or provides such an object
    as the first argument in its signature

    Yields:
        A client-side cursor
    """

    @wraps(func)
    def wrapper(cls, *args, **kwargs):
        with (yield from cls.get_cursor()) as c:
            return (yield from func(cls, c, *args, **kwargs))

    return wrapper

def cursor ( func ) : @ wraps ( func ) def wrapper ( cls , * args , ** kwargs ) : with ( yield from cls . get_cursor ( ) ) as c : return ( yield from func ( cls , c , * args , ** kwargs ) ) return wrapper


def nt_cursor(func):
    """
    Decorator that provides a namedtuple cursor to the calling function

    Adds the cursor as the second argument to the calling functions

    Requires that the function being decorated is an instance of a class or object
    that yields a cursor from a get_cursor(cursor_type=CursorType.NAMEDTUPLE) coroutine or provides such an object
    as the first argument in its signature

    Yields:
        A client-side namedtuple cursor
    """

    @wraps(func)
    def wrapper(cls, *args, **kwargs):
        with (yield from cls.get_cursor(_CursorType.NAMEDTUPLE)) as c:
            return (yield from func(cls, c, *args, **kwargs))

    return wrapper

def nt_cursor ( func ) : @ wraps ( func ) def wrapper ( cls , * args , ** kwargs ) : with ( yield from cls . get_cursor ( _CursorType . NAMEDTUPLE ) ) as c : return ( yield from func ( cls , c , * args , ** kwargs ) ) return wrapper


def transaction(func):
    """
    Provides a transacted cursor which will run in autocommit=false mode

    For any exception the transaction will be rolled back.
    Requires that the function being decorated is an instance of a class or object
    that yields a cursor from a get_cursor(cursor_type=CursorType.NAMEDTUPLE) coroutine or provides such an object
    as the first argument in its signature

    Yields:
        A client-side transacted named cursor
    """

    @wraps(func)
    def wrapper(cls, *args, **kwargs):
        with (yield from cls.get_cursor(_CursorType.NAMEDTUPLE)) as c:
            try:
                yield from c.execute('BEGIN')
                result = (yield from func(cls, c, *args, **kwargs))
            except Exception:
                yield from c.execute('ROLLBACK')
            else:
                yield from c.execute('COMMIT')
                return result

    return wrapper

def transaction ( func ) : @ wraps ( func ) def wrapper ( cls , * args , ** kwargs ) : with ( yield from cls . get_cursor ( _CursorType . NAMEDTUPLE ) ) as c : try : yield from c . execute ( 'BEGIN' ) result = ( yield from func ( cls , c , * args , ** kwargs ) ) except Exception : yield from c . execute ( 'ROLLBACK' ) else : yield from c . execute ( 'COMMIT' ) return result return wrapper

Provides a transacted cursor which will run in autocommit = false mode


def count(cls, cur, table:str, where_keys: list=None):
        """
        gives the number of records in the table

        Args:
            table: a string indicating the name of the table

        Returns:
            an integer indicating the number of records in the table

        """

        if where_keys:
            where_clause, values = cls._get_where_clause_with_values(where_keys)
            query = cls._count_query_where.format(table, where_clause)
            q, t = query, values
        else:
            query = cls._count_query.format(table)
            q, t = query, ()
        yield from cur.execute(q, t)
        result = yield from cur.fetchone()
        return int(result[0])

def count ( cls , cur , table : str , where_keys : list = None ) : if where_keys : where_clause , values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _count_query_where . format ( table , where_clause ) q , t = query , values else : query = cls . _count_query . format ( table ) q , t = query , ( ) yield from cur . execute ( q , t ) result = yield from cur . fetchone ( ) return int ( result [ 0 ] )


def insert(cls, cur, table: str, values: dict):
        """
        Creates an insert statement with only chosen fields

        Args:
            table: a string indicating the name of the table
            values: a dict of fields and values to be inserted

        Returns:
            A 'Record' object with table columns as properties

        """
        keys = cls._COMMA.join(values.keys())
        value_place_holder = cls._PLACEHOLDER * len(values)
        query = cls._insert_string.format(table, keys, value_place_holder[:-1])
        yield from cur.execute(query, tuple(values.values()))
        return (yield from cur.fetchone())

def insert ( cls , cur , table : str , values : dict ) : keys = cls . _COMMA . join ( values . keys ( ) ) value_place_holder = cls . _PLACEHOLDER * len ( values ) query = cls . _insert_string . format ( table , keys , value_place_holder [ : - 1 ] ) yield from cur . execute ( query , tuple ( values . values ( ) ) ) return ( yield from cur . fetchone ( ) )


def update(cls, cur, table: str, values: dict, where_keys: list) -> tuple:
        """
        Creates an update query with only chosen fields
        Supports only a single field where clause

        Args:
            table: a string indicating the name of the table
            values: a dict of fields and values to be inserted
            where_keys: list of dictionary
            example of where keys: [{'name':('>', 'cip'),'url':('=', 'cip.com'},{'type':{'<=', 'manufacturer'}}]
            where_clause will look like ((name>%s and url=%s) or (type <= %s))
            items within each dictionary get 'AND'-ed and dictionaries themselves get 'OR'-ed

        Returns:
            an integer indicating count of rows deleted

        """
        keys = cls._COMMA.join(values.keys())
        value_place_holder = cls._PLACEHOLDER * len(values)
        where_clause, where_values = cls._get_where_clause_with_values(where_keys)
        query = cls._update_string.format(table, keys, value_place_holder[:-1], where_clause)
        yield from cur.execute(query, (tuple(values.values()) + where_values))
        return (yield from cur.fetchall())

def update ( cls , cur , table : str , values : dict , where_keys : list ) -> tuple : keys = cls . _COMMA . join ( values . keys ( ) ) value_place_holder = cls . _PLACEHOLDER * len ( values ) where_clause , where_values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _update_string . format ( table , keys , value_place_holder [ : - 1 ] , where_clause ) yield from cur . execute ( query , ( tuple ( values . values ( ) ) + where_values ) ) return ( yield from cur . fetchall ( ) )


def delete(cls, cur, table: str, where_keys: list):
        """
        Creates a delete query with where keys
        Supports multiple where clause with and or or both

        Args:
            table: a string indicating the name of the table
            where_keys: list of dictionary
            example of where keys: [{'name':('>', 'cip'),'url':('=', 'cip.com'},{'type':{'<=', 'manufacturer'}}]
            where_clause will look like ((name>%s and url=%s) or (type <= %s))
            items within each dictionary get 'AND'-ed and dictionaries themselves get 'OR'-ed

        Returns:
            an integer indicating count of rows deleted

        """
        where_clause, values = cls._get_where_clause_with_values(where_keys)
        query = cls._delete_query.format(table, where_clause)
        yield from cur.execute(query, values)
        return cur.rowcount

def delete ( cls , cur , table : str , where_keys : list ) : where_clause , values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _delete_query . format ( table , where_clause ) yield from cur . execute ( query , values ) return cur . rowcount


def select(cls, cur, table: str, order_by: str, columns: list=None, where_keys: list=None, limit=100,
               offset=0):
        """
        Creates a select query for selective columns with where keys
        Supports multiple where claus with and or or both

        Args:
            table: a string indicating the name of the table
            order_by: a string indicating column name to order the results on
            columns: list of columns to select from
            where_keys: list of dictionary
            limit: the limit on the number of results
            offset: offset on the results

            example of where keys: [{'name':('>', 'cip'),'url':('=', 'cip.com'},{'type':{'<=', 'manufacturer'}}]
            where_clause will look like ((name>%s and url=%s) or (type <= %s))
            items within each dictionary get 'AND'-ed and across dictionaries get 'OR'-ed

        Returns:
            A list of 'Record' object with table columns as properties

        """
        if columns:
            columns_string = cls._COMMA.join(columns)
            if where_keys:
                where_clause, values = cls._get_where_clause_with_values(where_keys)
                query = cls._select_selective_column_with_condition.format(columns_string, table, where_clause,
                                                                           order_by, limit, offset)
                q, t = query, values
            else:
                query = cls._select_selective_column.format(columns_string, table, order_by, limit, offset)
                q, t = query, ()
        else:
            if where_keys:
                where_clause, values = cls._get_where_clause_with_values(where_keys)
                query = cls._select_all_string_with_condition.format(table, where_clause, order_by, limit, offset)
                q, t = query, values
            else:
                query = cls._select_all_string.format(table, order_by, limit, offset)
                q, t = query, ()

        yield from cur.execute(q, t)
        return (yield from cur.fetchall())

def select ( cls , cur , table : str , order_by : str , columns : list = None , where_keys : list = None , limit = 100 , offset = 0 ) : if columns : columns_string = cls . _COMMA . join ( columns ) if where_keys : where_clause , values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _select_selective_column_with_condition . format ( columns_string , table , where_clause , order_by , limit , offset ) q , t = query , values else : query = cls . _select_selective_column . format ( columns_string , table , order_by , limit , offset ) q , t = query , ( ) else : if where_keys : where_clause , values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _select_all_string_with_condition . format ( table , where_clause , order_by , limit , offset ) q , t = query , values else : query = cls . _select_all_string . format ( table , order_by , limit , offset ) q , t = query , ( ) yield from cur . execute ( q , t ) return ( yield from cur . fetchall ( ) )


def raw_sql(cls, cur, query: str, values: tuple):
        """
        Run a raw sql query

        Args:
            query : query string to execute
            values : tuple of values to be used with the query

        Returns:
            result of query as list of named tuple

        """
        yield from cur.execute(query, values)
        return (yield from cur.fetchall())

def raw_sql ( cls , cur , query : str , values : tuple ) : yield from cur . execute ( query , values ) return ( yield from cur . fetchall ( ) )


def serialize_text(out, text):
    """This method is used to append content of the `text`
    argument to the `out` argument.

    Depending on how many lines in the text, a
    padding can be added to all lines except the first
    one.

    Concatenation result is appended to the `out` argument.
    """
    padding = len(out)
    # we need to add padding to all lines
    # except the first one
    add_padding = padding_adder(padding)
    text = add_padding(text, ignore_first_line=True)

    return out + text

def serialize_text ( out , text ) : padding = len ( out ) add_padding = padding_adder ( padding ) text = add_padding ( text , ignore_first_line = True ) return out + text

This method is used to append content of the text argument to the out argument .


def format_value(value):
    """This function should return unicode representation of the value
    """
    value_id = id(value)

    if value_id in recursion_breaker.processed:
        return u'<recursion>'

    recursion_breaker.processed.add(value_id)

    try:
        if isinstance(value, six.binary_type):
            # suppose, all byte strings are in unicode
            # don't know if everybody in the world uses anything else?
            return u"'{0}'".format(value.decode('utf-8'))

        elif isinstance(value, six.text_type):
            return u"u'{0}'".format(value)

        elif isinstance(value, (list, tuple)):
            # long lists or lists with multiline items
            # will be shown vertically
            values = list(map(format_value, value))
            result = serialize_list(u'[', values, delimiter=u',') + u']'
            return force_unicode(result)

        elif isinstance(value, dict):
            items = six.iteritems(value)

            # format each key/value pair as a text,
            # calling format_value recursively
            items = (tuple(map(format_value, item))
                     for item in items)

            items = list(items)
            # sort by keys for readability
            items.sort()

            # for each item value
            items = [
                serialize_text(
                    u'{0}: '.format(key),
                    item_value)
                for key, item_value in items]

            # and serialize these pieces as a list, enclosing
            # them into a curve brackets
            result = serialize_list(u'{', items, delimiter=u',') + u'}'
            return force_unicode(result)
        return force_unicode(repr(value))

    finally:
        recursion_breaker.processed.remove(value_id)

def
format_value
(
value
)
:
value_id
=
id
(
value
)
if
value_id
in
recursion_breaker
.
processed
:
return
u'<recursion>'
recursion_breaker
.
processed
.
add
(
value_id
)
try
:
if
isinstance
(
value
,
six
.
binary_type
)
:
return
u"'{0}'"
.
format
(
value
.
decode
(
'utf-8'
)
)
elif
isinstance
(
value
,
six
.
text_type
)
:
return
u"u'{0}'"
.
format
(
value
)
elif
isinstance
(
value
,
(
list
,
tuple
)
)
:
values
=
list
(
map
(
format_value
,
value
)
)
result
=
serialize_list
(
u'['
,
values
,
delimiter
=
u','
)
+
u']'
return
force_unicode
(
result
)
elif
isinstance
(
value
,
dict
)
:
items
=
six
.
iteritems
(
value
)
items
=
(
tuple
(
map
(
format_value
,
item
)
)
for
item
in
items
)
items
=
list
(
items
)
items
.
sort
(
)
items
=
[
serialize_text
(
u'{0}: '
.
format
(
key
)
,
item_value
)
for
key
,
item_value
in
items
]
result
=
serialize_list
(
u'{'
,
items
,
delimiter
=
u','
)
+
u'}'
return
force_unicode
(
result
)
return
force_unicode
(
repr
(
value
)
)
finally
:
recursion_breaker
.
processed
.
remove
(
value_id
)


def traverse(element, query, deep=False):
    """
    Helper function to traverse an element tree rooted at element, yielding nodes matching the query.
    """
    # Grab the next part of the query (it will be chopped from the front each iteration).
    part = query[0]
    if not part:
        # If the part is blank, we encountered a //, meaning search all sub-nodes.
        query = query[1:]
        part = query[0]
        deep = True
    # Parse out any predicate (tag[pred]) from this part of the query.
    part, predicate = xpath_re.match(query[0]).groups()
    for c in element._children:
        if part in ('*', c.tagname) and c._match(predicate):
            # A potential matching branch: this child matches the next query part (and predicate).
            if len(query) == 1:
                # If this is the last part of the query, we found a matching element, yield it.
                yield c
            else:
                # Otherwise, check the children of this child against the next query part.
                for e in traverse(c, query[1:]):
                    yield e
        if deep:
            # If we're searching all sub-nodes, traverse with the same query, regardless of matching.
            # This basically creates a recursion branch to search EVERYWHERE for anything after //.
            for e in traverse(c, query, deep=True):
                yield e

def traverse ( element , query , deep = False ) : part = query [ 0 ] if not part : query = query [ 1 : ] part = query [ 0 ] deep = True part , predicate = xpath_re . match ( query [ 0 ] ) . groups ( ) for c in element . _children : if part in ( '*' , c . tagname ) and c . _match ( predicate ) : if len ( query ) == 1 : yield c else : for e in traverse ( c , query [ 1 : ] ) : yield e if deep : for e in traverse ( c , query , deep = True ) : yield e

Helper function to traverse an element tree rooted at element yielding nodes matching the query .


def parse_query(query):
    """
    Given a simplified XPath query string, returns an array of normalized query parts.
    """
    parts = query.split('/')
    norm = []
    for p in parts:
        p = p.strip()
        if p:
            norm.append(p)
        elif '' not in norm:
            norm.append('')
    return norm

def parse_query ( query ) : parts = query . split ( '/' ) norm = [ ] for p in parts : p = p . strip ( ) if p : norm . append ( p ) elif '' not in norm : norm . append ( '' ) return norm

Given a simplified XPath query string returns an array of normalized query parts .


def insert(self, before, name, attrs=None, data=None):
        """
        Inserts a new element as a child of this element, before the specified index or sibling.

        :param before: An :class:`XmlElement` or a numeric index to insert the new node before
        :param name: The tag name to add
        :param attrs: Attributes for the new tag
        :param data: CDATA for the new tag
        :returns: The newly-created element
        :rtype: :class:`XmlElement`
        """
        if isinstance(before, self.__class__):
            if before.parent != self:
                raise ValueError('Cannot insert before an element with a different parent.')
            before = before.index
        # Make sure 0 <= before <= len(_children).
        before = min(max(0, before), len(self._children))
        elem = self.__class__(name, attrs, data, parent=self, index=before)
        self._children.insert(before, elem)
        # Re-index all the children.
        for idx, c in enumerate(self._children):
            c.index = idx
        return elem

Inserts a new element as a child of this element before the specified index or sibling .


def children(self, name=None, reverse=False):
        """
        A generator yielding children of this node.

        :param name: If specified, only consider elements with this tag name
        :param reverse: If ``True``, children will be yielded in reverse declaration order
        """
        elems = self._children
        if reverse:
            elems = reversed(elems)
        for elem in elems:
            if name is None or elem.tagname == name:
                yield elem

def children ( self , name = None , reverse = False ) : elems = self . _children if reverse : elems = reversed ( elems ) for elem in elems : if name is None or elem . tagname == name : yield elem

A generator yielding children of this node .


def _match(self, pred):
        """
        Helper function to determine if this node matches the given predicate.
        """
        if not pred:
            return True
        # Strip off the [ and ]
        pred = pred[1:-1]
        if pred.startswith('@'):
            # An attribute predicate checks the existence (and optionally value) of an attribute on this tag.
            pred = pred[1:]
            if '=' in pred:
                attr, value = pred.split('=', 1)
                if value[0] in ('"', "'"):
                    value = value[1:]
                if value[-1] in ('"', "'"):
                    value = value[:-1]
                return self.attrs.get(attr) == value
            else:
                return pred in self.attrs
        elif num_re.match(pred):
            # An index predicate checks whether we are the n-th child of our parent (0-based).
            index = int(pred)
            if index < 0:
                if self.parent:
                    # For negative indexes, count from the end of the list.
                    return self.index == (len(self.parent._children) + index)
                else:
                    # If we're the root node, the only index we could be is 0.
                    return index == 0
            else:
                return index == self.index
        else:
            if '=' in pred:
                tag, value = pred.split('=', 1)
                if value[0] in ('"', "'"):
                    value = value[1:]
                if value[-1] in ('"', "'"):
                    value = value[:-1]
                for c in self._children:
                    if c.tagname == tag and c.data == value:
                        return True
            else:
                # A plain [tag] predicate means we match if we have a child with tagname "tag".
                for c in self._children:
                    if c.tagname == pred:
                        return True
        return False

def _match ( self , pred ) : if not pred : return True pred = pred [ 1 : - 1 ] if pred . startswith ( '@' ) : pred = pred [ 1 : ] if '=' in pred : attr , value = pred . split ( '=' , 1 ) if value [ 0 ] in ( '"' , "'" ) : value = value [ 1 : ] if value [ - 1 ] in ( '"' , "'" ) : value = value [ : - 1 ] return self . attrs . get ( attr ) == value else : return pred in self . attrs elif num_re . match ( pred ) : index = int ( pred ) if index < 0 : if self . parent : return self . index == ( len ( self . parent . _children ) + index ) else : return index == 0 else : return index == self . index else : if '=' in pred : tag , value = pred . split ( '=' , 1 ) if value [ 0 ] in ( '"' , "'" ) : value = value [ 1 : ] if value [ - 1 ] in ( '"' , "'" ) : value = value [ : - 1 ] for c in self . _children : if c . tagname == tag and c . data == value : return True else : for c in self . _children : if c . tagname == pred : return True return False

Helper function to determine if this node matches the given predicate .


def path(self, include_root=False):
        """
        Returns a canonical path to this element, relative to the root node.

        :param include_root: If ``True``, include the root node in the path. Defaults to ``False``.
        """
        path = '%s[%d]' % (self.tagname, self.index or 0)
        p = self.parent
        while p is not None:
            if p.parent or include_root:
                path = '%s[%d]/%s' % (p.tagname, p.index or 0, path)
            p = p.parent
        return path

def path ( self , include_root = False ) : path = '%s[%d]' % ( self . tagname , self . index or 0 ) p = self . parent while p is not None : if p . parent or include_root : path = '%s[%d]/%s' % ( p . tagname , p . index or 0 , path ) p = p . parent return path

Returns a canonical path to this element relative to the root node .


def iter(self, name=None):
        """
        Recursively find any descendants of this node with the given tag name. If a tag name is omitted, this will
        yield every descendant node.

        :param name: If specified, only consider elements with this tag name
        :returns: A generator yielding descendants of this node
        """
        for c in self._children:
            if name is None or c.tagname == name:
                yield c
            for gc in c.find(name):
                yield gc

def iter ( self , name = None ) : for c in self . _children : if name is None or c . tagname == name : yield c for gc in c . find ( name ) : yield gc

Recursively find any descendants of this node with the given tag name . If a tag name is omitted this will yield every descendant node .


def last(self, name=None):
        """
        Returns the last child of this node.

        :param name: If specified, only consider elements with this tag name
        :rtype: :class:`XmlElement`
        """
        for c in self.children(name, reverse=True):
            return c

def last ( self , name = None ) : for c in self . children ( name , reverse = True ) : return c


def parents(self, name=None):
        """
        Yields all parents of this element, back to the root element.

        :param name: If specified, only consider elements with this tag name
        """
        p = self.parent
        while p is not None:
            if name is None or p.tagname == name:
                yield p
            p = p.parent

def parents ( self , name = None ) : p = self . parent while p is not None : if name is None or p . tagname == name : yield p p = p . parent

Yields all parents of this element back to the root element .


def next(self, name=None):
        """
        Returns the next sibling of this node.

        :param name: If specified, only consider elements with this tag name
        :rtype: :class:`XmlElement`
        """
        if self.parent is None or self.index is None:
            return None
        for idx in xrange(self.index + 1, len(self.parent)):
            if name is None or self.parent[idx].tagname == name:
                return self.parent[idx]

def next ( self , name = None ) : if self . parent is None or self . index is None : return None for idx in xrange ( self . index + 1 , len ( self . parent ) ) : if name is None or self . parent [ idx ] . tagname == name : return self . parent [ idx ]


def prev(self, name=None):
        """
        Returns the previous sibling of this node.

        :param name: If specified, only consider elements with this tag name
        :rtype: :class:`XmlElement`
        """
        if self.parent is None or self.index is None:
            return None
        for idx in xrange(self.index - 1, -1, -1):
            if name is None or self.parent[idx].tagname == name:
                return self.parent[idx]

def prev ( self , name = None ) : if self . parent is None or self . index is None : return None for idx in xrange ( self . index - 1 , - 1 , - 1 ) : if name is None or self . parent [ idx ] . tagname == name : return self . parent [ idx ]

Returns the previous sibling of this node .


def get_observations(self):
        """
        Parses the HTML table into a list of dictionaries, each of which
        represents a single observation.
        """
        if self.empty:
            return []
        rows = list(self.tbody)
        observations = []
        for row_observation, row_details in zip(rows[::2], rows[1::2]):
            data = {}
            cells = OBSERVATION_XPATH(row_observation)
            data['name'] = _clean_cell(cells[0])
            data['date'] = _clean_cell(cells[1])
            data['magnitude'] = _clean_cell(cells[3])
            data['obscode'] = _clean_cell(cells[6])
            cells = DETAILS_XPATH(row_details)
            data['comp1'] = _clean_cell(cells[0])
            data['chart'] = _clean_cell(cells[3]).replace('None', '')
            data['comment_code'] = _clean_cell(cells[4])
            data['notes'] = _clean_cell(cells[5])
            observations.append(data)
        return observations

def get_observations ( self ) : if self . empty : return [ ] rows = list ( self . tbody ) observations = [ ] for row_observation , row_details in zip ( rows [ : : 2 ] , rows [ 1 : : 2 ] ) : data = { } cells = OBSERVATION_XPATH ( row_observation ) data [ 'name' ] = _clean_cell ( cells [ 0 ] ) data [ 'date' ] = _clean_cell ( cells [ 1 ] ) data [ 'magnitude' ] = _clean_cell ( cells [ 3 ] ) data [ 'obscode' ] = _clean_cell ( cells [ 6 ] ) cells = DETAILS_XPATH ( row_details ) data [ 'comp1' ] = _clean_cell ( cells [ 0 ] ) data [ 'chart' ] = _clean_cell ( cells [ 3 ] ) . replace ( 'None' , '' ) data [ 'comment_code' ] = _clean_cell ( cells [ 4 ] ) data [ 'notes' ] = _clean_cell ( cells [ 5 ] ) observations . append ( data ) return observations

Parses the HTML table into a list of dictionaries each of which represents a single observation .


def get_cache_key(prefix, *args, **kwargs):
    """
    Calculates cache key based on `args` and `kwargs`.
    `args` and `kwargs` must be instances of hashable types.
    """
    hash_args_kwargs = hash(tuple(kwargs.iteritems()) + args)
    return '{}_{}'.format(prefix, hash_args_kwargs)

def get_cache_key ( prefix , * args , ** kwargs ) : hash_args_kwargs = hash ( tuple ( kwargs . iteritems ( ) ) + args ) return '{}_{}' . format ( prefix , hash_args_kwargs )

Calculates cache key based on args and kwargs . args and kwargs must be instances of hashable types .


def cache_func(prefix, method=False):
    """
    Cache result of function execution into the django cache backend.
    Calculate cache key based on `prefix`, `args` and `kwargs` of the function.
    For using like object method set `method=True`.
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            cache_args = args
            if method:
                cache_args = args[1:]
            cache_key = get_cache_key(prefix, *cache_args, **kwargs)
            cached_value = cache.get(cache_key)
            if cached_value is None:
                cached_value = func(*args, **kwargs)
                cache.set(cache_key, cached_value)
            return cached_value
        return wrapper
    return decorator

def cache_func ( prefix , method = False ) : def decorator ( func ) : @ wraps ( func ) def wrapper ( * args , ** kwargs ) : cache_args = args if method : cache_args = args [ 1 : ] cache_key = get_cache_key ( prefix , * cache_args , ** kwargs ) cached_value = cache . get ( cache_key ) if cached_value is None : cached_value = func ( * args , ** kwargs ) cache . set ( cache_key , cached_value ) return cached_value return wrapper return decorator

Cache result of function execution into the django cache backend . Calculate cache key based on prefix args and kwargs of the function . For using like object method set method = True .


def get_or_default(func=None, default=None):
    """
    Wrapper around Django's ORM `get` functionality.
    Wrap anything that raises ObjectDoesNotExist exception
    and provide the default value if necessary.
    `default` by default is None. `default` can be any callable,
    if it is callable it will be called when ObjectDoesNotExist
    exception will be raised.
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except ObjectDoesNotExist:
                if callable(default):
                    return default()
                else:
                    return default
        return wrapper
    if func is None:
        return decorator
    else:
        return decorator(func)

def get_or_default ( func = None , default = None ) : def decorator ( func ) : @ wraps ( func ) def wrapper ( * args , ** kwargs ) : try : return func ( * args , ** kwargs ) except ObjectDoesNotExist : if callable ( default ) : return default ( ) else : return default return wrapper if func is None : return decorator else : return decorator ( func )

Wrapper around Django s ORM get functionality . Wrap anything that raises ObjectDoesNotExist exception and provide the default value if necessary . default by default is None . default can be any callable if it is callable it will be called when ObjectDoesNotExist exception will be raised .


def _get_column_nums_from_args(columns):
    """Turn column inputs from user into list of simple numbers.

    Inputs can be:

      - individual number: 1
      - range: 1-3
      - comma separated list: 1,2,3,4-6
    """
    nums = []
    for c in columns:
        for p in c.split(','):
            p = p.strip()
            try:
                c = int(p)
                nums.append(c)
            except (TypeError, ValueError):
                start, ignore, end = p.partition('-')
                try:
                    start = int(start)
                    end = int(end)
                except (TypeError, ValueError):
                    raise ValueError(
                        'Did not understand %r, expected digit-digit' % c
                    )
                inc = 1 if start < end else -1
                nums.extend(range(start, end + inc, inc))
    # The user will pass us 1-based indexes, but we need to use
    # 0-based indexing with the row.
    return [n - 1 for n in nums]

Turn column inputs from user into list of simple numbers .


def _get_printable_columns(columns, row):
    """Return only the part of the row which should be printed.
    """
    if not columns:
        return row

    # Extract the column values, in the order specified.
    return tuple(row[c] for c in columns)

def _get_printable_columns ( columns , row ) : if not columns : return row return tuple ( row [ c ] for c in columns )

Return only the part of the row which should be printed .


def writerow(self, observation_data):
        """
        Writes a single observation to the output file.

        If the ``observation_data`` parameter is a dictionary, it is
        converted to a list to keep a consisted field order (as described
        in format specification). Otherwise it is assumed that the data
        is a raw record ready to be written to file.

        :param observation_data: a single observation as a dictionary or list
        """
        if isinstance(observation_data, (list, tuple)):
            row = observation_data
        else:
            row = self.dict_to_row(observation_data)
        self.writer.writerow(row)

def writerow ( self , observation_data ) : if isinstance ( observation_data , ( list , tuple ) ) : row = observation_data else : row = self . dict_to_row ( observation_data ) self . writer . writerow ( row )

Writes a single observation to the output file .


def dict_to_row(cls, observation_data):
        """
        Takes a dictionary of observation data and converts it to a list
        of fields according to AAVSO visual format specification.

        :param cls: current class
        :param observation_data: a single observation as a dictionary
        """
        row = []
        row.append(observation_data['name'])
        row.append(observation_data['date'])
        row.append(observation_data['magnitude'])
        comment_code = observation_data.get('comment_code', 'na')
        if not comment_code:
            comment_code = 'na'
        row.append(comment_code)
        comp1 = observation_data.get('comp1', 'na')
        if not comp1:
            comp1 = 'na'
        row.append(comp1)
        comp2 = observation_data.get('comp2', 'na')
        if not comp2:
            comp2 = 'na'
        row.append(comp2)
        chart = observation_data.get('chart', 'na')
        if not chart:
            chart = 'na'
        row.append(chart)
        notes = observation_data.get('notes', 'na')
        if not notes:
            notes = 'na'
        row.append(notes)
        return row

def dict_to_row ( cls , observation_data ) : row = [ ] row . append ( observation_data [ 'name' ] ) row . append ( observation_data [ 'date' ] ) row . append ( observation_data [ 'magnitude' ] ) comment_code = observation_data . get ( 'comment_code' , 'na' ) if not comment_code : comment_code = 'na' row . append ( comment_code ) comp1 = observation_data . get ( 'comp1' , 'na' ) if not comp1 : comp1 = 'na' row . append ( comp1 ) comp2 = observation_data . get ( 'comp2' , 'na' ) if not comp2 : comp2 = 'na' row . append ( comp2 ) chart = observation_data . get ( 'chart' , 'na' ) if not chart : chart = 'na' row . append ( chart ) notes = observation_data . get ( 'notes' , 'na' ) if not notes : notes = 'na' row . append ( notes ) return row

Takes a dictionary of observation data and converts it to a list of fields according to AAVSO visual format specification .


def row_to_dict(cls, row):
        """
        Converts a raw input record to a dictionary of observation data.

        :param cls: current class
        :param row: a single observation as a list or tuple
        """
        comment_code = row[3]
        if comment_code.lower() == 'na':
            comment_code = ''
        comp1 = row[4]
        if comp1.lower() == 'na':
            comp1 = ''
        comp2 = row[5]
        if comp2.lower() == 'na':
            comp2 = ''
        chart = row[6]
        if chart.lower() == 'na':
            chart = ''
        notes = row[7]
        if notes.lower() == 'na':
            notes = ''
        return {
            'name': row[0],
            'date': row[1],
            'magnitude': row[2],
            'comment_code': comment_code,
            'comp1': comp1,
            'comp2': comp2,
            'chart': chart,
            'notes': notes,
        }

def row_to_dict ( cls , row ) : comment_code = row [ 3 ] if comment_code . lower ( ) == 'na' : comment_code = '' comp1 = row [ 4 ] if comp1 . lower ( ) == 'na' : comp1 = '' comp2 = row [ 5 ] if comp2 . lower ( ) == 'na' : comp2 = '' chart = row [ 6 ] if chart . lower ( ) == 'na' : chart = '' notes = row [ 7 ] if notes . lower ( ) == 'na' : notes = '' return { 'name' : row [ 0 ] , 'date' : row [ 1 ] , 'magnitude' : row [ 2 ] , 'comment_code' : comment_code , 'comp1' : comp1 , 'comp2' : comp2 , 'chart' : chart , 'notes' : notes , }

Converts a raw input record to a dictionary of observation data .


def get_default_tag(app):
    '''Get the name of the view function used to prevent having to set the tag
    manually for every endpoint'''
    view_func = get_view_function(app, request.path, request.method)
    if view_func:
        return view_func.__name__

def get_default_tag ( app ) : view_func = get_view_function ( app , request . path , request . method ) if view_func : return view_func . __name__


def download_observations(observer_code):
    """
    Downloads all variable star observations by a given observer.

    Performs a series of HTTP requests to AAVSO's WebObs search and
    downloads the results page by page. Each page is then passed to
    :py:class:`~pyaavso.parsers.webobs.WebObsResultsParser` and parse results
    are added to the final observation list.
    """
    page_number = 1
    observations = []
    while True:
        logger.info('Downloading page %d...', page_number)
        response = requests.get(WEBOBS_RESULTS_URL, params={
            'obscode': observer_code,
            'num_results': 200,
            'obs_types': 'all',
            'page': page_number,
        })
        logger.debug(response.request.url)
        parser = WebObsResultsParser(response.text)
        observations.extend(parser.get_observations())
        # kinda silly, but there's no need for lxml machinery here
        if '>Next</a>' not in response.text:
            break
        page_number += 1
    return observations

def
download_observations
(
observer_code
)
:
page_number
=
1
observations
=
[
]
while
True
:
logger
.
info
(
'Downloading page %d...'
,
page_number
)
response
=
requests
.
get
(
WEBOBS_RESULTS_URL
,
params
=
{
'obscode'
:
observer_code
,
'num_results'
:
200
,
'obs_types'
:
'all'
,
'page'
:
page_number
,
}
)
logger
.
debug
(
response
.
request
.
url
)
parser
=
WebObsResultsParser
(
response
.
text
)
observations
.
extend
(
parser
.
get_observations
(
)
)
if
'>Next</a>'
not
in
response
.
text
:
break
page_number
+=
1
return
observations

Downloads all variable star observations by a given observer .


def image_path(instance, filename):
    """Generates likely unique image path using md5 hashes"""
    filename, ext = os.path.splitext(filename.lower())
    instance_id_hash = hashlib.md5(str(instance.id)).hexdigest()
    filename_hash = ''.join(random.sample(hashlib.md5(filename.encode('utf-8')).hexdigest(), 8))
    return '{}/{}{}'.format(instance_id_hash, filename_hash, ext)

def image_path ( instance , filename ) : filename , ext = os . path . splitext ( filename . lower ( ) ) instance_id_hash = hashlib . md5 ( str ( instance . id ) ) . hexdigest ( ) filename_hash = '' . join ( random . sample ( hashlib . md5 ( filename . encode ( 'utf-8' ) ) . hexdigest ( ) , 8 ) ) return '{}/{}{}' . format ( instance_id_hash , filename_hash , ext )


async def process_lander_page(session, github_api_token, ltd_product_data,
                              mongo_collection=None):
    """Extract, transform, and load metadata from Lander-based projects.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    github_api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    ltd_product_data : `dict`
        Contents of ``metadata.yaml``, obtained via `download_metadata_yaml`.
        Data for this technote from the LTD Keeper API
        (``GET /products/<slug>``). Usually obtained via
        `lsstprojectmeta.ltd.get_ltd_product`.
    mongo_collection : `motor.motor_asyncio.AsyncIOMotorCollection`, optional
        MongoDB collection. This should be the common MongoDB collection for
        LSST projectmeta JSON-LD records. If provided, ths JSON-LD is upserted
        into the MongoDB collection.

    Returns
    -------
    metadata : `dict`
        JSON-LD-formatted dictionary.

    Raises
    ------
    NotLanderPageError
        Raised when the LTD product cannot be interpreted as a Lander page
        because the ``/metadata.jsonld`` file is absent. This implies that
        the LTD product *could* be of a different format.

    .. `GitHub personal access token guide`: https://ls.st/41d
    """
    logger = logging.getLogger(__name__)

    # Try to download metadata.jsonld from the Landing page site.
    published_url = ltd_product_data['published_url']
    jsonld_url = urljoin(published_url, '/metadata.jsonld')
    try:
        async with session.get(jsonld_url) as response:
            logger.debug('%s response status %r', jsonld_url, response.status)
            response.raise_for_status()
            json_data = await response.text()
    except aiohttp.ClientResponseError as err:
        logger.debug('Tried to download %s, got status %d',
                     jsonld_url, err.code)
        raise NotLanderPageError()
    # Use our own json parser to get datetimes
    metadata = decode_jsonld(json_data)

    if mongo_collection is not None:
        await _upload_to_mongodb(mongo_collection, metadata)

    return metadata

async
def
process_lander_page
(
session
,
github_api_token
,
ltd_product_data
,
mongo_collection
=
)
:
logger
=
logging
.
getLogger
(
__name__
)
published_url
=
ltd_product_data
[
'published_url'
]
jsonld_url
=
urljoin
(
published_url
,
'/metadata.jsonld'
)
try
:
async
with
session
.
get
(
jsonld_url
)
as
response
:
logger
.
debug
(
'%s response status %r'
,
jsonld_url
,
response
.
status
)
response
.
raise_for_status
(
)
json_data
=
await
response
.
text
(
)
except
aiohttp
.
ClientResponseError
as
err
:
logger
.
debug
(
'Tried to download %s, got status %d'
,
jsonld_url
,
err
.
code
)
raise
NotLanderPageError
(
)
metadata
=
decode_jsonld
(
json_data
)
if
mongo_collection
is
not
:
await
_upload_to_mongodb
(
mongo_collection
,
metadata
)
return
metadata

Extract transform and load metadata from Lander - based projects .


async def _upload_to_mongodb(collection, jsonld):
    """Upsert the technote resource into the projectmeta MongoDB collection.

    Parameters
    ----------
    collection : `motor.motor_asyncio.AsyncIOMotorCollection`
        The MongoDB collection.
    jsonld : `dict`
        The JSON-LD document that represents the document resource.
    """
    document = {
        'data': jsonld
    }
    query = {
        'data.reportNumber': jsonld['reportNumber']
    }
    await collection.update(query, document, upsert=True, multi=False)

async def _upload_to_mongodb ( collection , jsonld ) : document = { 'data' : jsonld } query = { 'data.reportNumber' : jsonld [ 'reportNumber' ] } await collection . update ( query , document , upsert = True , multi = False )

Upsert the technote resource into the projectmeta MongoDB collection .


def json_doc_to_xml(json_obj, lang='en', custom_namespace=None):
    """Converts a Open511 JSON document to XML.

    lang: the appropriate language code

    Takes a dict deserialized from JSON, returns an lxml Element.

    Accepts only the full root-level JSON object from an Open511 response."""
    if 'meta' not in json_obj:
        raise Exception("This function requires a conforming Open511 JSON document with a 'meta' section.")
    json_obj = dict(json_obj)
    meta = json_obj.pop('meta')
    elem = get_base_open511_element(lang=lang, version=meta.pop('version'))

    pagination = json_obj.pop('pagination', None)

    json_struct_to_xml(json_obj, elem, custom_namespace=custom_namespace)

    if pagination:
        elem.append(json_struct_to_xml(pagination, 'pagination', custom_namespace=custom_namespace))

    json_struct_to_xml(meta, elem)

    return elem

Converts a Open511 JSON document to XML .


def json_struct_to_xml(json_obj, root, custom_namespace=None):
    """Converts a Open511 JSON fragment to XML.

    Takes a dict deserialized from JSON, returns an lxml Element.

    This won't provide a conforming document if you pass in a full JSON document;
    it's for translating little fragments, and is mostly used internally."""
    if isinstance(root, (str, unicode)):
        if root.startswith('!'):
            root = etree.Element('{%s}%s' % (NS_PROTECTED, root[1:]))
        elif root.startswith('+'):
            if not custom_namespace:
                raise Exception("JSON fields starts with +, but no custom namespace provided")
            root = etree.Element('{%s}%s' % (custom_namespace, root[1:]))
        else:
            root = etree.Element(root)
    if root.tag in ('attachments', 'grouped_events', 'media_files'):
        for link in json_obj:
            root.append(json_link_to_xml(link))
    elif isinstance(json_obj, (str, unicode)):
        root.text = json_obj
    elif isinstance(json_obj, (int, float)):
        root.text = unicode(json_obj)
    elif isinstance(json_obj, dict):
        if frozenset(json_obj.keys()) == frozenset(('type', 'coordinates')):
            root.append(geojson_to_gml(json_obj))
        else:
            for key, val in json_obj.items():
                if key == 'url' or key.endswith('_url'):
                    el = json_link_to_xml(val, json_link_key_to_xml_rel(key))
                else:
                    el = json_struct_to_xml(val, key, custom_namespace=custom_namespace)
                if el is not None:
                    root.append(el)
    elif isinstance(json_obj, list):
        tag_name = root.tag
        if tag_name.endswith('ies'):
            tag_name = tag_name[:-3] + 'y'
        elif tag_name.endswith('s'):
            tag_name = tag_name[:-1]
        for val in json_obj:
            el = json_struct_to_xml(val, tag_name, custom_namespace=custom_namespace)
            if el is not None:
                root.append(el)
    elif json_obj is None:
        return None
    else:
        raise NotImplementedError
    return root

Converts a Open511 JSON fragment to XML .