def serve_forever(self, poll_interval=0.5):
        """
        Start serving HTTP requests

        This method blocks the current thread.

        :param poll_interval: polling timeout
        :return:
        """
        logger.info('Starting server on {}:{}...'.format(
            self.server_name, self.server_port)
        )
        while True:
            try:
                self.poll_once(poll_interval)
            except (KeyboardInterrupt, SystemExit):
                break
        self.handle_close()
        logger.info('Server stopped.')

def
serve_forever
(
self
,
poll_interval
=
0.5
)
:
logger
.
info
(
'Starting server on {}:{}...'
.
format
(
self
.
server_name
,
self
.
server_port
)
)
while
True
:
try
:
self
.
poll_once
(
poll_interval
)
except
(
KeyboardInterrupt
,
SystemExit
)
:
break
self
.
handle_close
(
)
logger
.
info
(
'Server stopped.'
)


def write_index_translation(translation_filename, entity_ids, relation_ids):
    """write triples into a translation file."""
    translation = triple_pb.Translation()
    entities = []
    for name, index in entity_ids.items():
        translation.entities.add(element=name, index=index)
    relations = []
    for name, index in relation_ids.items():
        translation.relations.add(element=name, index=index)
    with open(translation_filename, "wb") as f:
        f.write(translation.SerializeToString())

def write_index_translation ( translation_filename , entity_ids , relation_ids ) : translation = triple_pb . Translation ( ) entities = [ ] for name , index in entity_ids . items ( ) : translation . entities . add ( element = name , index = index ) relations = [ ] for name , index in relation_ids . items ( ) : translation . relations . add ( element = name , index = index ) with open ( translation_filename , "wb" ) as f : f . write ( translation . SerializeToString ( ) )


def write_triples(filename, triples, delimiter=DEFAULT_DELIMITER, triple_order="hrt"):
    """write triples to file."""
    with open(filename, 'w') as f:
        for t in triples:
            line = t.serialize(delimiter, triple_order)
            f.write(line + "\n")

def write_triples ( filename , triples , delimiter = DEFAULT_DELIMITER , triple_order = "hrt" ) : with open ( filename , 'w' ) as f : for t in triples : line = t . serialize ( delimiter , triple_order ) f . write ( line + "\n" )


def read_translation(filename):
    """Returns protobuf mapcontainer. Read from translation file."""
    translation = triple_pb.Translation()
    with open(filename, "rb") as f:
        translation.ParseFromString(f.read())

    def unwrap_translation_units(units):
        for u in units: yield u.element, u.index

    return (list(unwrap_translation_units(translation.entities)),
        list(unwrap_translation_units(translation.relations)))

def read_translation ( filename ) : translation = triple_pb . Translation ( ) with open ( filename , "rb" ) as f : translation . ParseFromString ( f . read ( ) ) def unwrap_translation_units ( units ) : for u in units : yield u . element , u . index return ( list ( unwrap_translation_units ( translation . entities ) ) , list ( unwrap_translation_units ( translation . relations ) ) )

Returns protobuf mapcontainer . Read from translation file .


def read_openke_translation(filename, delimiter='\t', entity_first=True):
    """Returns map with entity or relations from plain text."""
    result = {}
    with open(filename, "r") as f:
        _ = next(f) # pass the total entry number
        for line in f:
            line_slice = line.rstrip().split(delimiter)
            if not entity_first:
                line_slice = list(reversed(line_slice))
            result[line_slice[0]] = line_slice[1]

    return result

def read_openke_translation ( filename , delimiter = '\t' , entity_first = True ) : result = { } with open ( filename , "r" ) as f : _ = next ( f ) for line in f : line_slice = line . rstrip ( ) . split ( delimiter ) if not entity_first : line_slice = list ( reversed ( line_slice ) ) result [ line_slice [ 0 ] ] = line_slice [ 1 ] return result

Returns map with entity or relations from plain text .


def overview():
    """
        Prints an overview of the tags of the hosts.
    """
    doc = Host()
    search = doc.search()
    search.aggs.bucket('tag_count', 'terms', field='tags', order={'_count': 'desc'}, size=100)
    response = search.execute()
    print_line("{0:<25} {1}".format('Tag', 'Count'))
    print_line("-" * 30)
    for entry in response.aggregations.tag_count.buckets:
        print_line("{0:<25} {1}".format(entry.key, entry.doc_count))

def
overview
(
)
:
doc
=
Host
(
)
search
=
doc
.
search
(
)
search
.
aggs
.
bucket
(
'tag_count'
,
'terms'
,
field
=
'tags'
,
order
=
{
'_count'
:
'desc'
}
,
size
=
100
)
response
=
search
.
execute
(
)
print_line
(
"{0:<25} {1}"
.
format
(
'Tag'
,
'Count'
)
)
print_line
(
"-"
*
30
)
for
entry
in
response
.
aggregations
.
tag_count
.
buckets
:
print_line
(
"{0:<25} {1}"
.
format
(
entry
.
key
,
entry
.
doc_count
)
)

Prints an overview of the tags of the hosts .


def main():
    """
        Main credentials tool
    """
    cred_search = CredentialSearch()
    arg = argparse.ArgumentParser(parents=[cred_search.argparser], conflict_handler='resolve')
    arg.add_argument('-c', '--count', help="Only show the number of results", action="store_true")
    arguments = arg.parse_args()

    if arguments.count:
        print_line("Number of credentials: {}".format(cred_search.argument_count()))
    else:
        response = cred_search.get_credentials()
        for hit in response:
            print_json(hit.to_dict(include_meta=True))

def
main
(
)
:
cred_search
=
CredentialSearch
(
)
arg
=
argparse
.
ArgumentParser
(
parents
=
[
cred_search
.
argparser
]
,
conflict_handler
=
'resolve'
)
arg
.
add_argument
(
'-c'
,
'--count'
,
help
=
"Only show the number of results"
,
action
=
"store_true"
)
arguments
=
arg
.
parse_args
(
)
if
arguments
.
count
:
print_line
(
"Number of credentials: {}"
.
format
(
cred_search
.
argument_count
(
)
)
)
else
:
response
=
cred_search
.
get_credentials
(
)
for
hit
in
response
:
print_json
(
hit
.
to_dict
(
include_meta
=
True
)
)


def overview():
    """
        Provides an overview of the duplicate credentials.
    """
    search = Credential.search()
    search.aggs.bucket('password_count', 'terms', field='secret', order={'_count': 'desc'}, size=20)\
        .metric('username_count', 'cardinality', field='username') \
        .metric('host_count', 'cardinality', field='host_ip') \
        .metric('top_hits', 'top_hits', docvalue_fields=['username'], size=100)
    response = search.execute()
    print_line("{0:65} {1:5} {2:5} {3:5} {4}".format("Secret", "Count", "Hosts", "Users", "Usernames"))
    print_line("-"*100)
    for entry in response.aggregations.password_count.buckets:
        usernames = []
        for creds in entry.top_hits:
            usernames.append(creds.username[0])
        usernames = list(set(usernames))
        print_line("{0:65} {1:5} {2:5} {3:5} {4}".format(entry.key, entry.doc_count, entry.host_count.value, entry.username_count.value, usernames))

def
overview
(
)
:
search
=
Credential
.
search
(
)
search
.
aggs
.
bucket
(
'password_count'
,
'terms'
,
field
=
'secret'
,
order
=
{
'_count'
:
'desc'
}
,
size
=
20
)
.
metric
(
'username_count'
,
'cardinality'
,
field
=
'username'
)
.
metric
(
'host_count'
,
'cardinality'
,
field
=
'host_ip'
)
.
metric
(
'top_hits'
,
'top_hits'
,
docvalue_fields
=
[
'username'
]
,
size
=
100
)
response
=
search
.
execute
(
)
print_line
(
"{0:65} {1:5} {2:5} {3:5} {4}"
.
format
(
"Secret"
,
"Count"
,
"Hosts"
,
"Users"
,
"Usernames"
)
)
print_line
(
"-"
*
100
)
for
entry
in
response
.
aggregations
.
password_count
.
buckets
:
usernames
=
[
]
for
creds
in
entry
.
top_hits
:
usernames
.
append
(
creds
.
username
[
0
]
)
usernames
=
list
(
set
(
usernames
)
)
print_line
(
"{0:65} {1:5} {2:5} {3:5} {4}"
.
format
(
entry
.
key
,
entry
.
doc_count
,
entry
.
host_count
.
value
,
entry
.
username_count
.
value
,
usernames
)
)

Provides an overview of the duplicate credentials .


def process(self, nemo):
        """ Register nemo and parses annotations

        .. note:: Process parses the annotation and extends informations about the target URNs by retrieving resource in range

        :param nemo: Nemo
        """
        self.__nemo__ = nemo
        for annotation in self.__annotations__:
            annotation.target.expanded = frozenset(
                self.__getinnerreffs__(
                    objectId=annotation.target.objectId,
                    subreference=annotation.target.subreference
                )
            )

def process ( self , nemo ) : self . __nemo__ = nemo for annotation in self . __annotations__ : annotation . target . expanded = frozenset ( self . __getinnerreffs__ ( objectId = annotation . target . objectId , subreference = annotation . target . subreference ) )


def pipe_worker(pipename, filename, object_type, query, format_string, unique=False):
    """
        Starts the loop to provide the data from jackal.
    """
    print_notification("[{}] Starting pipe".format(pipename))
    object_type = object_type()
    try:
        while True:
            uniq = set()
            # Remove the previous file if it exists
            if os.path.exists(filename):
                os.remove(filename)

            # Create the named pipe
            os.mkfifo(filename)
            # This function will block until a process opens it
            with open(filename, 'w') as pipe:
                print_success("[{}] Providing data".format(pipename))
                # Search the database
                objects = object_type.search(**query)
                for obj in objects:
                    data = fmt.format(format_string, **obj.to_dict())
                    if unique:
                        if not data in uniq:
                            uniq.add(data)
                            pipe.write(data + '\n')
                    else:
                        pipe.write(data + '\n')
            os.unlink(filename)
    except KeyboardInterrupt:
        print_notification("[{}] Shutting down named pipe".format(pipename))
    except Exception as e:
        print_error("[{}] Error: {}, stopping named pipe".format(e, pipename))
    finally:
        os.remove(filename)

def
pipe_worker
(
pipename
,
filename
,
object_type
,
query
,
format_string
,
unique
=
False
)
:
print_notification
(
"[{}] Starting pipe"
.
format
(
pipename
)
)
object_type
=
object_type
(
)
try
:
while
True
:
uniq
=
set
(
)
if
os
.
path
.
exists
(
filename
)
:
os
.
remove
(
filename
)
os
.
mkfifo
(
filename
)
with
open
(
filename
,
'w'
)
as
pipe
:
print_success
(
"[{}] Providing data"
.
format
(
pipename
)
)
objects
=
object_type
.
search
(
**
query
)
for
obj
in
objects
:
data
=
fmt
.
format
(
format_string
,
**
obj
.
to_dict
(
)
)
if
unique
:
if
not
data
in
uniq
:
uniq
.
add
(
data
)
pipe
.
write
(
data
+
'\n'
)
else
:
pipe
.
write
(
data
+
'\n'
)
os
.
unlink
(
filename
)
except
KeyboardInterrupt
:
print_notification
(
"[{}] Shutting down named pipe"
.
format
(
pipename
)
)
except
Exception
as
e
:
print_error
(
"[{}] Error: {}, stopping named pipe"
.
format
(
e
,
pipename
)
)
finally
:
os
.
remove
(
filename
)

Starts the loop to provide the data from jackal .


def create_query(section):
    """
        Creates a search query based on the section of the config file.
    """
    query = {}

    if 'ports' in section:
        query['ports'] = [section['ports']]
    if 'up' in section:
        query['up'] = bool(section['up'])
    if 'search' in section:
        query['search'] = [section['search']]
    if 'tags' in section:
        query['tags'] = [section['tags']]
    if 'groups' in section:
        query['groups'] = [section['groups']]

    return query

def create_query ( section ) : query = { } if 'ports' in section : query [ 'ports' ] = [ section [ 'ports' ] ] if 'up' in section : query [ 'up' ] = bool ( section [ 'up' ] ) if 'search' in section : query [ 'search' ] = [ section [ 'search' ] ] if 'tags' in section : query [ 'tags' ] = [ section [ 'tags' ] ] if 'groups' in section : query [ 'groups' ] = [ section [ 'groups' ] ] return query

Creates a search query based on the section of the config file .


def create_pipe_workers(configfile, directory):
    """
        Creates the workers based on the given configfile to provide named pipes in the directory.
    """
    type_map = {'service': ServiceSearch,
                'host': HostSearch, 'range': RangeSearch,
                'user': UserSearch}
    config = configparser.ConfigParser()
    config.read(configfile)

    if not len(config.sections()):
        print_error("No named pipes configured")
        return

    print_notification("Starting {} pipes in directory {}".format(
        len(config.sections()), directory))

    workers = []
    for name in config.sections():
        section = config[name]
        query = create_query(section)
        object_type = type_map[section['type']]
        args = (name, os.path.join(directory, name), object_type, query,
                section['format'], bool(section.get('unique', 0)))
        workers.append(multiprocessing.Process(target=pipe_worker, args=args))

    return workers

def
create_pipe_workers
(
configfile
,
directory
)
:
type_map
=
{
'service'
:
ServiceSearch
,
'host'
:
HostSearch
,
'range'
:
RangeSearch
,
'user'
:
UserSearch
}
config
=
configparser
.
ConfigParser
(
)
config
.
read
(
configfile
)
if
not
len
(
config
.
sections
(
)
)
:
print_error
(
"No named pipes configured"
)
return
print_notification
(
"Starting {} pipes in directory {}"
.
format
(
len
(
config
.
sections
(
)
)
,
directory
)
)
workers
=
[
]
for
name
in
config
.
sections
(
)
:
section
=
config
[
name
]
query
=
create_query
(
section
)
object_type
=
type_map
[
section
[
'type'
]
]
args
=
(
name
,
os
.
path
.
join
(
directory
,
name
)
,
object_type
,
query
,
section
[
'format'
]
,
bool
(
section
.
get
(
'unique'
,
0
)
)
)
workers
.
append
(
multiprocessing
.
Process
(
target
=
pipe_worker
,
args
=
args
)
)
return
workers

Creates the workers based on the given configfile to provide named pipes in the directory .


def main():
    """
        Loads the config and handles the workers.
    """
    config = Config()
    pipes_dir = config.get('pipes', 'directory')
    pipes_config = config.get('pipes', 'config_file')
    pipes_config_path = os.path.join(config.config_dir, pipes_config)
    if not os.path.exists(pipes_config_path):
        print_error("Please configure the named pipes first")
        return

    workers = create_pipe_workers(pipes_config_path, pipes_dir)
    if workers:
        for worker in workers:
            worker.start()

        try:
            for worker in workers:
                worker.join()
        except KeyboardInterrupt:
            print_notification("Shutting down")
            for worker in workers:
                worker.terminate()
                worker.join()

Loads the config and handles the workers .


def f_i18n_iso(isocode, lang="eng"):
    """ Replace isocode by its language equivalent

    :param isocode: Three character long language code
    :param lang: Lang in which to return the language name
    :return: Full Text Language Name
    """
    if lang not in flask_nemo._data.AVAILABLE_TRANSLATIONS:
        lang = "eng"

    try:
        return flask_nemo._data.ISOCODES[isocode][lang]
    except KeyError:
        return "Unknown"

def f_i18n_iso ( isocode , lang = "eng" ) : if lang not in flask_nemo . _data . AVAILABLE_TRANSLATIONS : lang = "eng" try : return flask_nemo . _data . ISOCODES [ isocode ] [ lang ] except KeyError : return "Unknown"


def f_hierarchical_passages(reffs, citation):
    """ A function to construct a hierarchical dictionary representing the different citation layers of a text

    :param reffs: passage references with human-readable equivalent
    :type reffs: [(str, str)]
    :param citation: Main Citation
    :type citation: Citation
    :return: nested dictionary representing where keys represent the names of the levels and the final values represent the passage reference
    :rtype: OrderedDict
    """
    d = OrderedDict()
    levels = [x for x in citation]
    for cit, name in reffs:
        ref = cit.split('-')[0]
        levs = ['%{}|{}%'.format(levels[i].name, v) for i, v in enumerate(ref.split('.'))]
        getFromDict(d, levs[:-1])[name] = cit
    return d

def f_hierarchical_passages ( reffs , citation ) : d = OrderedDict ( ) levels = [ x for x in citation ] for cit , name in reffs : ref = cit . split ( '-' ) [ 0 ] levs = [ '%{}|{}%' . format ( levels [ i ] . name , v ) for i , v in enumerate ( ref . split ( '.' ) ) ] getFromDict ( d , levs [ : - 1 ] ) [ name ] = cit return d


def f_i18n_citation_type(string, lang="eng"):
    """ Take a string of form %citation_type|passage% and format it for human

    :param string: String of formation %citation_type|passage%
    :param lang: Language to translate to
    :return: Human Readable string

    .. note :: To Do : Use i18n tools and provide real i18n
    """
    s = " ".join(string.strip("%").split("|"))
    return s.capitalize()

def
f_i18n_citation_type
(
string
,
lang
=
"eng"
)
:
s
=
" "
.
join
(
string
.
strip
(
"%"
)
.
split
(
"|"
)
)
return
s
.
capitalize
(
)


def f_annotation_filter(annotations, type_uri, number):
    """ Annotation filtering filter

    :param annotations: List of annotations
    :type annotations: [AnnotationResource]
    :param type_uri: URI Type on which to filter
    :type type_uri: str
    :param number: Number of the annotation to return
    :type number: int
    :return: Annotation(s) matching the request
    :rtype: [AnnotationResource] or AnnotationResource
    """
    filtered = [
        annotation
        for annotation in annotations
        if annotation.type_uri == type_uri
    ]
    number = min([len(filtered), number])
    if number == 0:
        return None
    else:
        return filtered[number-1]

def f_annotation_filter ( annotations , type_uri , number ) : filtered = [ annotation for annotation in annotations if annotation . type_uri == type_uri ] number = min ( [ len ( filtered ) , number ] ) if number == 0 : return None else : return filtered [ number - 1 ]


def check_service(service):
    """
        Connect to a service to see if it is a http or https server.
    """
    # Try HTTP
    service.add_tag('header_scan')
    http = False
    try:
        result = requests.head('http://{}:{}'.format(service.address, service.port), timeout=1)
        print_success("Found http service on {}:{}".format(service.address, service.port))
        service.add_tag('http')
        http = True
        try:
            service.banner = result.headers['Server']
        except KeyError:
            pass
    except (ConnectionError, ConnectTimeout, ReadTimeout, Error):
        pass

    if not http:
        # Try HTTPS
        try:
            result = requests.head('https://{}:{}'.format(service.address, service.port), verify=False, timeout=3)
            service.add_tag('https')
            print_success("Found https service on {}:{}".format(service.address, service.port))
            try:
                service.banner = result.headers['Server']
            except KeyError:
                pass
        except (ConnectionError, ConnectTimeout, ReadTimeout, Error):
            pass
    service.save()

def
check_service
(
service
)
:
service
.
add_tag
(
'header_scan'
)
http
=
False
try
:
result
=
requests
.
head
(
'http://{}:{}'
.
format
(
service
.
address
,
service
.
port
)
,
timeout
=
1
)
print_success
(
"Found http service on {}:{}"
.
format
(
service
.
address
,
service
.
port
)
)
service
.
add_tag
(
'http'
)
http
=
True
try
:
service
.
banner
=
result
.
headers
[
'Server'
]
except
KeyError
:
pass
except
(
ConnectionError
,
ConnectTimeout
,
ReadTimeout
,
Error
)
:
pass
if
not
http
:
try
:
result
=
requests
.
head
(
'https://{}:{}'
.
format
(
service
.
address
,
service
.
port
)
,
verify
=
False
,
timeout
=
3
)
service
.
add_tag
(
'https'
)
print_success
(
"Found https service on {}:{}"
.
format
(
service
.
address
,
service
.
port
)
)
try
:
service
.
banner
=
result
.
headers
[
'Server'
]
except
KeyError
:
pass
except
(
ConnectionError
,
ConnectTimeout
,
ReadTimeout
,
Error
)
:
pass
service
.
save
(
)

Connect to a service to see if it is a http or https server .


def main():
    """
        Retrieves services starts check_service in a gevent pool of 100.
    """
    search = ServiceSearch()
    services = search.get_services(up=True, tags=['!header_scan'])
    print_notification("Scanning {} services".format(len(services)))

    # Disable the insecure request warning
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

    pool = Pool(100)
    count = 0
    for service in services:
        count += 1
        if count % 50 == 0:
            print_notification("Checking {}/{} services".format(count, len(services)))
        pool.spawn(check_service, service)

    pool.join()
    print_notification("Completed, 'http' tag added to services that respond to http, 'https' tag added to services that respond to https.")

Retrieves services starts check_service in a gevent pool of 100 .


def import_nmap(result, tag, check_function=all_hosts, import_services=False):
    """
        Imports the given nmap result.
    """
    host_search = HostSearch(arguments=False)
    service_search = ServiceSearch()
    parser = NmapParser()
    report = parser.parse_fromstring(result)
    imported_hosts = 0
    imported_services = 0
    for nmap_host in report.hosts:
        if check_function(nmap_host):
            imported_hosts += 1
            host = host_search.id_to_object(nmap_host.address)
            host.status = nmap_host.status
            host.add_tag(tag)
            if nmap_host.os_fingerprinted:
                host.os = nmap_host.os_fingerprint
            if nmap_host.hostnames:
                host.hostname.extend(nmap_host.hostnames)
            if import_services:
                for service in nmap_host.services:
                    imported_services += 1
                    serv = Service(**service.get_dict())
                    serv.address = nmap_host.address
                    service_id = service_search.object_to_id(serv)
                    if service_id:
                        # Existing object, save the banner and script results.
                        serv_old = Service.get(service_id)
                        if service.banner:
                            serv_old.banner = service.banner
                        # TODO implement
                        # if service.script_results:
                            # serv_old.script_results.extend(service.script_results)
                        serv_old.save()
                    else:
                        # New object
                        serv.address = nmap_host.address
                        serv.save()
                    if service.state == 'open':
                        host.open_ports.append(service.port)
                    if service.state == 'closed':
                        host.closed_ports.append(service.port)
                    if service.state == 'filtered':
                        host.filtered_ports.append(service.port)
            host.save()
    if imported_hosts:
        print_success("Imported {} hosts, with tag {}".format(imported_hosts, tag))
    else:
        print_error("No hosts found")
    return {'hosts': imported_hosts, 'services': imported_services}

def
import_nmap
(
result
,
tag
,
check_function
=
all_hosts
,
import_services
=
False
)
:
host_search
=
HostSearch
(
arguments
=
False
)
service_search
=
ServiceSearch
(
)
parser
=
NmapParser
(
)
report
=
parser
.
parse_fromstring
(
result
)
imported_hosts
=
0
imported_services
=
0
for
nmap_host
in
report
.
hosts
:
if
check_function
(
nmap_host
)
:
imported_hosts
+=
1
host
=
host_search
.
id_to_object
(
nmap_host
.
address
)
host
.
status
=
nmap_host
.
status
host
.
add_tag
(
tag
)
if
nmap_host
.
os_fingerprinted
:
host
.
os
=
nmap_host
.
os_fingerprint
if
nmap_host
.
hostnames
:
host
.
hostname
.
extend
(
nmap_host
.
hostnames
)
if
import_services
:
for
service
in
nmap_host
.
services
:
imported_services
+=
1
serv
=
Service
(
**
service
.
get_dict
(
)
)
serv
.
address
=
nmap_host
.
address
service_id
=
service_search
.
object_to_id
(
serv
)
if
service_id
:
serv_old
=
Service
.
get
(
service_id
)
if
service
.
banner
:
serv_old
.
banner
=
service
.
banner
serv_old
.
save
(
)
else
:
serv
.
address
=
nmap_host
.
address
serv
.
save
(
)
if
service
.
state
==
'open'
:
host
.
open_ports
.
append
(
service
.
port
)
if
service
.
state
==
'closed'
:
host
.
closed_ports
.
append
(
service
.
port
)
if
service
.
state
==
'filtered'
:
host
.
filtered_ports
.
append
(
service
.
port
)
host
.
save
(
)
if
imported_hosts
:
print_success
(
"Imported {} hosts, with tag {}"
.
format
(
imported_hosts
,
tag
)
)
else
:
print_error
(
"No hosts found"
)
return
{
'hosts'
:
imported_hosts
,
'services'
:
imported_services
}


def nmap(nmap_args, ips):
    """
        Start an nmap process with the given args on the given ips.
    """
    config = Config()
    arguments = ['nmap', '-Pn']
    arguments.extend(ips)
    arguments.extend(nmap_args)
    output_file = ''
    now = datetime.datetime.now()
    if not '-oA' in nmap_args:
        output_name = 'nmap_jackal_{}'.format(now.strftime("%Y-%m-%d %H:%M"))
        path_name = os.path.join(config.get('nmap', 'directory'), output_name)
        print_notification("Writing output of nmap to {}".format(path_name))
        if not os.path.exists(config.get('nmap', 'directory')):
            os.makedirs(config.get('nmap', 'directory'))
        output_file = path_name + '.xml'
        arguments.extend(['-oA', path_name])
    else:
        output_file = nmap_args[nmap_args.index('-oA') + 1] + '.xml'

    print_notification("Starting nmap")
    subprocess.call(arguments)

    with open(output_file, 'r') as f:
        return f.read()

def
nmap
(
nmap_args
,
ips
)
:
config
=
Config
(
)
arguments
=
[
'nmap'
,
'-Pn'
]
arguments
.
extend
(
ips
)
arguments
.
extend
(
nmap_args
)
output_file
=
''
now
=
datetime
.
datetime
.
now
(
)
if
not
'-oA'
in
nmap_args
:
output_name
=
'nmap_jackal_{}'
.
format
(
now
.
strftime
(
"%Y-%m-%d %H:%M"
)
)
path_name
=
os
.
path
.
join
(
config
.
get
(
'nmap'
,
'directory'
)
,
output_name
)
print_notification
(
"Writing output of nmap to {}"
.
format
(
path_name
)
)
if
not
os
.
path
.
exists
(
config
.
get
(
'nmap'
,
'directory'
)
)
:
os
.
makedirs
(
config
.
get
(
'nmap'
,
'directory'
)
)
output_file
=
path_name
+
'.xml'
arguments
.
extend
(
[
'-oA'
,
path_name
]
)
else
:
output_file
=
nmap_args
[
nmap_args
.
index
(
'-oA'
)
+
1
]
+
'.xml'
print_notification
(
"Starting nmap"
)
subprocess
.
call
(
arguments
)
with
open
(
output_file
,
'r'
)
as
f
:
return
f
.
read
(
)

Start an nmap process with the given args on the given ips .


def nmap_scan():
    """
        Scans the given hosts with nmap.
    """
    # Create the search and config objects
    hs = HostSearch()
    config = Config()

    # Static options to be able to figure out what options to use depending on the input the user gives.
    nmap_types = ['top10', 'top100', 'custom', 'top1000', 'all']
    options = {'top10':'--top-ports 10', 'top100':'--top-ports 100', 'custom': config.get('nmap', 'options'), 'top1000': '--top-ports 1000', 'all': '-p-'}

    # Create an argument parser
    hs_parser = hs.argparser
    argparser = argparse.ArgumentParser(parents=[hs_parser], conflict_handler='resolve', \
    description="Scans hosts from the database using nmap, any arguments that are not in the help are passed to nmap")
    argparser.add_argument('type', metavar='type', \
        help='The number of ports to scan: top10, top100, custom, top1000 (default) or all', \
        type=str, choices=nmap_types, default='top1000', const='top1000', nargs='?')
    arguments, extra_nmap_args = argparser.parse_known_args()

    # Fix the tags for the search
    tags = nmap_types[nmap_types.index(arguments.type):]
    tags = ["!nmap_" + tag  for tag in tags]

    hosts = hs.get_hosts(tags=tags)
    hosts = [host for host in hosts]

    # Create the nmap arguments
    nmap_args = []
    nmap_args.extend(extra_nmap_args)
    nmap_args.extend(options[arguments.type].split(' '))

    # Run nmap
    print_notification("Running nmap with args: {} on {} hosts(s)".format(nmap_args, len(hosts)))
    if len(hosts):
        result = nmap(nmap_args, [str(h.address) for h in hosts])
        # Import the nmap result
        for host in hosts:
            host.add_tag("nmap_{}".format(arguments.type))
            host.save()
        print_notification("Nmap done, importing results")
        stats = import_nmap(result, "nmap_{}".format(arguments.type), check_function=all_hosts, import_services=True)
        stats['scanned_hosts'] = len(hosts)
        stats['type'] = arguments.type

        Logger().log('nmap_scan', "Performed nmap {} scan on {} hosts".format(arguments.type, len(hosts)), stats)
    else:
        print_notification("No hosts found")


def nmap_smb_vulnscan():
    """
        Scans available smb services in the database for smb signing and ms17-010.
    """
    service_search = ServiceSearch()
    services = service_search.get_services(ports=['445'], tags=['!smb_vulnscan'], up=True)
    services = [service for service in services]
    service_dict = {}
    for service in services:
        service.add_tag('smb_vulnscan')
        service_dict[str(service.address)] = service

    nmap_args = "-Pn -n --disable-arp-ping --script smb-security-mode.nse,smb-vuln-ms17-010.nse -p 445".split(" ")

    if services:
        result = nmap(nmap_args, [str(s.address) for s in services])
        parser = NmapParser()
        report = parser.parse_fromstring(result)
        smb_signing = 0
        ms17 = 0
        for nmap_host in report.hosts:
            for script_result in nmap_host.scripts_results:
                script_result = script_result.get('elements', {})
                service = service_dict[str(nmap_host.address)]
                if script_result.get('message_signing', '') == 'disabled':
                    print_success("({}) SMB Signing disabled".format(nmap_host.address))
                    service.add_tag('smb_signing_disabled')
                    smb_signing += 1
                if script_result.get('CVE-2017-0143', {}).get('state', '') == 'VULNERABLE':
                    print_success("({}) Vulnerable for MS17-010".format(nmap_host.address))
                    service.add_tag('MS17-010')
                    ms17 += 1
                service.update(tags=service.tags)

        print_notification("Completed, 'smb_signing_disabled' tag added to systems with smb signing disabled, 'MS17-010' tag added to systems that did not apply MS17-010.")
        stats = {'smb_signing': smb_signing, 'MS17_010': ms17, 'scanned_services': len(services)}

        Logger().log('smb_vulnscan', 'Scanned {} smb services for vulnerabilities'.format(len(services)), stats)
    else:
        print_notification("No services found to scan.")

Scans available smb services in the database for smb signing and ms17 - 010 .


def overview():
    """
        Function to create an overview of the services.
        Will print a list of ports found an the number of times the port was seen.
    """
    search = Service.search()
    search = search.filter("term", state='open')
    search.aggs.bucket('port_count', 'terms', field='port', order={'_count': 'desc'}, size=100) \
        .metric('unique_count', 'cardinality', field='address')
    response = search.execute()
    print_line("Port     Count")
    print_line("---------------")
    for entry in response.aggregations.port_count.buckets:
        print_line("{0:<7}  {1}".format(entry.key, entry.unique_count.value))

def
overview
(
)
:
search
=
Service
.
search
(
)
search
=
search
.
filter
(
"term"
,
state
=
'open'
)
search
.
aggs
.
bucket
(
'port_count'
,
'terms'
,
field
=
'port'
,
order
=
{
'_count'
:
'desc'
}
,
size
=
100
)
.
metric
(
'unique_count'
,
'cardinality'
,
field
=
'address'
)
response
=
search
.
execute
(
)
print_line
(
"Port     Count"
)
print_line
(
"---------------"
)
for
entry
in
response
.
aggregations
.
port_count
.
buckets
:
print_line
(
"{0:<7}  {1}"
.
format
(
entry
.
key
,
entry
.
unique_count
.
value
)
)

Function to create an overview of the services . Will print a list of ports found an the number of times the port was seen .


def _plugin_endpoint_rename(fn_name, instance):
    """ Rename endpoint function name to avoid conflict when namespacing is set to true

    :param fn_name: Name of the route function
    :param instance: Instance bound to the function
    :return: Name of the new namespaced function name
    """

    if instance and instance.namespaced:
        fn_name = "r_{0}_{1}".format(instance.name, fn_name[2:])
    return fn_name

def _plugin_endpoint_rename ( fn_name , instance ) : if instance and instance . namespaced : fn_name = "r_{0}_{1}" . format ( instance . name , fn_name [ 2 : ] ) return fn_name


def get_locale(self):
        """ Retrieve the best matching locale using request headers

        .. note:: Probably one of the thing to enhance quickly.

        :rtype: str
        """
        best_match = request.accept_languages.best_match(['de', 'fr', 'en', 'la'])
        if best_match is None:
            if len(request.accept_languages) > 0:
                best_match = request.accept_languages[0][0][:2]
            else:
                return self.__default_lang__
        lang = self.__default_lang__
        if best_match == "de":
            lang = "ger"
        elif best_match == "fr":
            lang = "fre"
        elif best_match == "en":
            lang = "eng"
        elif best_match == "la":
            lang = "lat"
        return lang

def get_locale ( self ) : best_match = request . accept_languages . best_match ( [ 'de' , 'fr' , 'en' , 'la' ] ) if best_match is None : if len ( request . accept_languages ) > 0 : best_match = request . accept_languages [ 0 ] [ 0 ] [ : 2 ] else : return self . __default_lang__ lang = self . __default_lang__ if best_match == "de" : lang = "ger" elif best_match == "fr" : lang = "fre" elif best_match == "en" : lang = "eng" elif best_match == "la" : lang = "lat" return lang


def transform(self, work, xml, objectId, subreference=None):
        """ Transform input according to potentially registered XSLT

        .. note:: Since 1.0.0, transform takes an objectId parameter which represent the passage which is called

        .. note:: Due to XSLT not being able to be used twice, we rexsltise the xml at every call of xslt

        .. warning:: Until a C libxslt error is fixed ( https://bugzilla.gnome.org/show_bug.cgi?id=620102 ), \
        it is not possible to use strip tags in the xslt given to this application

        :param work: Work object containing metadata about the xml
        :type work: MyCapytains.resources.inventory.Text
        :param xml: XML to transform
        :type xml: etree._Element
        :param objectId: Object Identifier
        :type objectId: str
        :param subreference: Subreference
        :type subreference: str
        :return: String representation of transformed resource
        :rtype: str
        """
        # We check first that we don't have
        if str(objectId) in self._transform:
            func = self._transform[str(objectId)]
        else:
            func = self._transform["default"]

        # If we have a string, it means we get a XSL filepath
        if isinstance(func, str):
            with open(func) as f:
                xslt = etree.XSLT(etree.parse(f))
            return etree.tostring(
                xslt(xml),
                encoding=str, method="html",
                xml_declaration=None, pretty_print=False, with_tail=True, standalone=None
            )

        # If we have a function, it means we return the result of the function
        elif isinstance(func, Callable):
            return func(work, xml, objectId, subreference)
        # If we have None, it means we just give back the xml
        elif func is None:
            return etree.tostring(xml, encoding=str)

def transform ( self , work , xml , objectId , subreference = None ) : if str ( objectId ) in self . _transform : func = self . _transform [ str ( objectId ) ] else : func = self . _transform [ "default" ] if isinstance ( func , str ) : with open ( func ) as f : xslt = etree . XSLT ( etree . parse ( f ) ) return etree . tostring ( xslt ( xml ) , encoding = str , method = "html" , xml_declaration = None , pretty_print = False , with_tail = True , standalone = None ) elif isinstance ( func , Callable ) : return func ( work , xml , objectId , subreference ) elif func is None : return etree . tostring ( xml , encoding = str )


def get_inventory(self):
        """ Request the api endpoint to retrieve information about the inventory

        :return: Main Collection
        :rtype: Collection
        """
        if self._inventory is not None:
            return self._inventory

        self._inventory = self.resolver.getMetadata()
        return self._inventory

def get_inventory ( self ) : if self . _inventory is not None : return self . _inventory self . _inventory = self . resolver . getMetadata ( ) return self . _inventory


def get_reffs(self, objectId, subreference=None, collection=None, export_collection=False):
        """ Retrieve and transform a list of references.

        Returns the inventory collection object with its metadata and a callback function taking a level parameter \
        and returning a list of strings.

        :param objectId: Collection Identifier
        :type objectId: str
        :param subreference: Subreference from which to retrieve children
        :type subreference: str
        :param collection: Collection object bearing metadata
        :type collection: Collection
        :param export_collection: Return collection metadata
        :type export_collection: bool
        :return: Returns either the list of references, or the text collection object with its references as tuple
        :rtype: (Collection, [str]) or [str]
        """
        if collection is not None:
            text = collection
        else:
            text = self.get_collection(objectId)
        reffs = self.chunk(
            text,
            lambda level: self.resolver.getReffs(objectId, level=level, subreference=subreference)
        )
        if export_collection is True:
            return text, reffs
        return reffs

def get_reffs ( self , objectId , subreference = None , collection = None , export_collection = False ) : if collection is not None : text = collection else : text = self . get_collection ( objectId ) reffs = self . chunk ( text , lambda level : self . resolver . getReffs ( objectId , level = level , subreference = subreference ) ) if export_collection is True : return text , reffs return reffs

Retrieve and transform a list of references .


def get_passage(self, objectId, subreference):
        """ Retrieve the passage identified by the parameters

        :param objectId: Collection Identifier
        :type objectId: str
        :param subreference: Subreference of the passage
        :type subreference: str
        :return: An object bearing metadata and its text
        :rtype: InteractiveTextualNode
        """
        passage = self.resolver.getTextualNode(
            textId=objectId,
            subreference=subreference,
            metadata=True
        )
        return passage

def get_passage ( self , objectId , subreference ) : passage = self . resolver . getTextualNode ( textId = objectId , subreference = subreference , metadata = True ) return passage


def get_siblings(self, objectId, subreference, passage):
        """ Get siblings of a browsed subreference

        .. note:: Since 1.0.0c, there is no more prevnext dict. Nemo uses the list of original\
        chunked references to retrieve next and previous, or simply relies on the resolver to get siblings\
        when the subreference is not found in given original chunks.

        :param objectId: Id of the object
        :param subreference: Subreference of the object
        :param passage: Current Passage
        :return: Previous and next references
        :rtype: (str, str)
        """
        reffs = [reff for reff, _ in self.get_reffs(objectId)]
        if subreference in reffs:
            index = reffs.index(subreference)
            # Not the first item and not the last one
            if 0 < index < len(reffs) - 1:
                return reffs[index-1], reffs[index+1]
            elif index == 0 and index < len(reffs) - 1:
                return None, reffs[1]
            elif index > 0 and index == len(reffs) - 1:
                return reffs[index-1], None
            else:
                return None, None
        else:
            return passage.siblingsId

def get_siblings ( self , objectId , subreference , passage ) : reffs = [ reff for reff , _ in self . get_reffs ( objectId ) ] if subreference in reffs : index = reffs . index ( subreference ) if 0 < index < len ( reffs ) - 1 : return reffs [ index - 1 ] , reffs [ index + 1 ] elif index == 0 and index < len ( reffs ) - 1 : return None , reffs [ 1 ] elif index > 0 and index == len ( reffs ) - 1 : return reffs [ index - 1 ] , None else : return None , None else : return passage . siblingsId


def semantic(self, collection, parent=None):
        """ Generates a SEO friendly string for given collection

        :param collection: Collection object to generate string for
        :param parent: Current collection parent
        :return: SEO/URL Friendly string
        """
        if parent is not None:
            collections = parent.parents[::-1] + [parent, collection]
        else:
            collections = collection.parents[::-1] + [collection]

        return filters.slugify("--".join([item.get_label() for item in collections if item.get_label()]))

def semantic ( self , collection , parent = None ) : if parent is not None : collections = parent . parents [ : : - 1 ] + [ parent , collection ] else : collections = collection . parents [ : : - 1 ] + [ collection ] return filters . slugify ( "--" . join ( [ item . get_label ( ) for item in collections if item . get_label ( ) ] ) )


def make_coins(self, collection, text, subreference="", lang=None):
        """ Creates a CoINS Title string from information

        :param collection: Collection to create coins from
        :param text: Text/Passage object
        :param subreference: Subreference
        :param lang: Locale information
        :return: Coins HTML title value
        """
        if lang is None:
            lang = self.__default_lang__
        return "url_ver=Z39.88-2004"\
                 "&ctx_ver=Z39.88-2004"\
                 "&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook"\
                 "&rft_id={cid}"\
                 "&rft.genre=bookitem"\
                 "&rft.btitle={title}"\
                 "&rft.edition={edition}"\
                 "&rft.au={author}"\
                 "&rft.atitle={pages}"\
                 "&rft.language={language}"\
                 "&rft.pages={pages}".format(
                    title=quote(str(text.get_title(lang))), author=quote(str(text.get_creator(lang))),
                    cid=url_for(".r_collection", objectId=collection.id, _external=True),
                    language=collection.lang, pages=quote(subreference), edition=quote(str(text.get_description(lang)))
                 )


def expose_ancestors_or_children(self, member, collection, lang=None):
        """ Build an ancestor or descendant dict view based on selected information

        :param member: Current Member to build for
        :param collection: Collection from which we retrieved it
        :param lang: Language to express data in
        :return:
        """
        x = {
            "id": member.id,
            "label": str(member.get_label(lang)),
            "model": str(member.model),
            "type": str(member.type),
            "size": member.size,
            "semantic": self.semantic(member, parent=collection)
        }
        if isinstance(member, ResourceCollection):
            x["lang"] = str(member.lang)
        return x

def expose_ancestors_or_children ( self , member , collection , lang = None ) : x = { "id" : member . id , "label" : str ( member . get_label ( lang ) ) , "model" : str ( member . model ) , "type" : str ( member . type ) , "size" : member . size , "semantic" : self . semantic ( member , parent = collection ) } if isinstance ( member , ResourceCollection ) : x [ "lang" ] = str ( member . lang ) return x


def make_members(self, collection, lang=None):
        """ Build member list for given collection

        :param collection: Collection to build dict view of for its members
        :param lang: Language to express data in
        :return: List of basic objects
        """
        objects = sorted([
                self.expose_ancestors_or_children(member, collection, lang=lang)
                for member in collection.members
                if member.get_label()
            ],
            key=itemgetter("label")
        )
        return objects

def make_members ( self , collection , lang = None ) : objects = sorted ( [ self . expose_ancestors_or_children ( member , collection , lang = lang ) for member in collection . members if member . get_label ( ) ] , key = itemgetter ( "label" ) ) return objects


def make_parents(self, collection, lang=None):
        """ Build parents list for given collection

        :param collection: Collection to build dict view of for its members
        :param lang: Language to express data in
        :return: List of basic objects
        """
        return [
            {
                "id": member.id,
                "label": str(member.get_label(lang)),
                "model": str(member.model),
                "type": str(member.type),
                "size": member.size
            }
            for member in collection.parents
            if member.get_label()
        ]

def make_parents ( self , collection , lang = None ) : return [ { "id" : member . id , "label" : str ( member . get_label ( lang ) ) , "model" : str ( member . model ) , "type" : str ( member . type ) , "size" : member . size } for member in collection . parents if member . get_label ( ) ]


def r_collections(self, lang=None):
        """ Retrieve the top collections of the inventory

        :param lang: Lang in which to express main data
        :type lang: str
        :return: Collections information and template
        :rtype: {str: Any}
        """
        collection = self.resolver.getMetadata()
        return {
            "template": "main::collection.html",
            "current_label": collection.get_label(lang),
            "collections": {
                "members": self.make_members(collection, lang=lang)
            }
        }

def r_collections ( self , lang = None ) : collection = self . resolver . getMetadata ( ) return { "template" : "main::collection.html" , "current_label" : collection . get_label ( lang ) , "collections" : { "members" : self . make_members ( collection , lang = lang ) } }


def r_collection(self, objectId, lang=None):
        """ Collection content browsing route function

        :param objectId: Collection identifier
        :type objectId: str
        :param lang: Lang in which to express main data
        :type lang: str
        :return: Template and collections contained in given collection
        :rtype: {str: Any}
        """
        collection = self.resolver.getMetadata(objectId)
        return {
            "template": "main::collection.html",
            "collections": {
                "current": {
                    "label": str(collection.get_label(lang)),
                    "id": collection.id,
                    "model": str(collection.model),
                    "type": str(collection.type),
                },
                "members": self.make_members(collection, lang=lang),
                "parents": self.make_parents(collection, lang=lang)
            },
        }

def r_collection ( self , objectId , lang = None ) : collection = self . resolver . getMetadata ( objectId ) return { "template" : "main::collection.html" , "collections" : { "current" : { "label" : str ( collection . get_label ( lang ) ) , "id" : collection . id , "model" : str ( collection . model ) , "type" : str ( collection . type ) , } , "members" : self . make_members ( collection , lang = lang ) , "parents" : self . make_parents ( collection , lang = lang ) } , }


def r_references(self, objectId, lang=None):
        """ Text exemplar references browsing route function

        :param objectId: Collection identifier
        :type objectId: str
        :param lang: Lang in which to express main data
        :type lang: str
        :return: Template and required information about text with its references
        """
        collection, reffs = self.get_reffs(objectId=objectId, export_collection=True)
        return {
            "template": "main::references.html",
            "objectId": objectId,
            "citation": collection.citation,
            "collections": {
                "current": {
                    "label": collection.get_label(lang),
                    "id": collection.id,
                    "model": str(collection.model),
                    "type": str(collection.type),
                },
                "parents": self.make_parents(collection, lang=lang)
            },
            "reffs": reffs
        }

def r_references ( self , objectId , lang = None ) : collection , reffs = self . get_reffs ( objectId = objectId , export_collection = True ) return { "template" : "main::references.html" , "objectId" : objectId , "citation" : collection . citation , "collections" : { "current" : { "label" : collection . get_label ( lang ) , "id" : collection . id , "model" : str ( collection . model ) , "type" : str ( collection . type ) , } , "parents" : self . make_parents ( collection , lang = lang ) } , "reffs" : reffs }


def r_first_passage(self, objectId):
        """ Provides a redirect to the first passage of given objectId

        :param objectId: Collection identifier
        :type objectId: str
        :return: Redirection to the first passage of given text
        """
        collection, reffs = self.get_reffs(objectId=objectId, export_collection=True)
        first, _ = reffs[0]
        return redirect(
            url_for(".r_passage_semantic", objectId=objectId, subreference=first, semantic=self.semantic(collection))
        )

def r_first_passage ( self , objectId ) : collection , reffs = self . get_reffs ( objectId = objectId , export_collection = True ) first , _ = reffs [ 0 ] return redirect ( url_for ( ".r_passage_semantic" , objectId = objectId , subreference = first , semantic = self . semantic ( collection ) ) )


def r_passage(self, objectId, subreference, lang=None):
        """ Retrieve the text of the passage

        :param objectId: Collection identifier
        :type objectId: str
        :param lang: Lang in which to express main data
        :type lang: str
        :param subreference: Reference identifier
        :type subreference: str
        :return: Template, collections metadata and Markup object representing the text
        :rtype: {str: Any}
        """
        collection = self.get_collection(objectId)
        if isinstance(collection, CtsWorkMetadata):
            editions = [t for t in collection.children.values() if isinstance(t, CtsEditionMetadata)]
            if len(editions) == 0:
                raise UnknownCollection("This work has no default edition")
            return redirect(url_for(".r_passage", objectId=str(editions[0].id), subreference=subreference))
        text = self.get_passage(objectId=objectId, subreference=subreference)
        passage = self.transform(text, text.export(Mimetypes.PYTHON.ETREE), objectId)
        prev, next = self.get_siblings(objectId, subreference, text)
        return {
            "template": "main::text.html",
            "objectId": objectId,
            "subreference": subreference,
            "collections": {
                "current": {
                    "label": collection.get_label(lang),
                    "id": collection.id,
                    "model": str(collection.model),
                    "type": str(collection.type),
                    "author": text.get_creator(lang),
                    "title": text.get_title(lang),
                    "description": text.get_description(lang),
                    "citation": collection.citation,
                    "coins": self.make_coins(collection, text, subreference, lang=lang)
                },
                "parents": self.make_parents(collection, lang=lang)
            },
            "text_passage": Markup(passage),
            "prev": prev,
            "next": next
        }

def
r_passage
(
self
,
objectId
,
subreference
,
lang
=
)
:
collection
=
self
.
get_collection
(
objectId
)
if
isinstance
(
collection
,
CtsWorkMetadata
)
:
editions
=
[
t
for
t
in
collection
.
children
.
values
(
)
if
isinstance
(
t
,
CtsEditionMetadata
)
]
if
len
(
editions
)
==
0
:
raise
UnknownCollection
(
"This work has no default edition"
)
return
redirect
(
url_for
(
".r_passage"
,
objectId
=
str
(
editions
[
0
]
.
id
)
,
subreference
=
subreference
)
)
text
=
self
.
get_passage
(
objectId
=
objectId
,
subreference
=
subreference
)
passage
=
self
.
transform
(
text
,
text
.
export
(
Mimetypes
.
PYTHON
.
ETREE
)
,
objectId
)
prev
,
next
=
self
.
get_siblings
(
objectId
,
subreference
,
text
)
return
{
"template"
:
"main::text.html"
,
"objectId"
:
objectId
,
"subreference"
:
subreference
,
"collections"
:
{
"current"
:
{
"label"
:
collection
.
get_label
(
lang
)
,
"id"
:
collection
.
id
,
"model"
:
str
(
collection
.
model
)
,
"type"
:
str
(
collection
.
type
)
,
"author"
:
text
.
get_creator
(
lang
)
,
"title"
:
text
.
get_title
(
lang
)
,
"description"
:
text
.
get_description
(
lang
)
,
"citation"
:
collection
.
citation
,
"coins"
:
self
.
make_coins
(
collection
,
text
,
subreference
,
lang
=
lang
)
}
,
"parents"
:
self
.
make_parents
(
collection
,
lang
=
lang
)
}
,
"text_passage"
:
Markup
(
passage
)
,
"prev"
:
prev
,
"next"
:
next
}


def r_assets(self, filetype, asset):
        """ Route for specific assets.

        :param filetype: Asset Type
        :param asset: Filename of an asset
        :return: Response
        """
        if filetype in self.assets and asset in self.assets[filetype] and self.assets[filetype][asset]:
            return send_from_directory(
                directory=self.assets[filetype][asset],
                filename=asset
            )
        abort(404)

def r_assets ( self , filetype , asset ) : if filetype in self . assets and asset in self . assets [ filetype ] and self . assets [ filetype ] [ asset ] : return send_from_directory ( directory = self . assets [ filetype ] [ asset ] , filename = asset ) abort ( 404 )


def register_assets(self):
        """ Merge and register assets, both as routes and dictionary

        :return: None
        """
        self.blueprint.add_url_rule(
            # Register another path to ensure assets compatibility
            "{0}.secondary/<filetype>/<asset>".format(self.static_url_path),
            view_func=self.r_assets,
            endpoint="secondary_assets",
            methods=["GET"]
        )

def register_assets ( self ) : self . blueprint . add_url_rule ( "{0}.secondary/<filetype>/<asset>" . format ( self . static_url_path ) , view_func = self . r_assets , endpoint = "secondary_assets" , methods = [ "GET" ] )

Merge and register assets both as routes and dictionary


def create_blueprint(self):
        """ Create blueprint and register rules

        :return: Blueprint of the current nemo app
        :rtype: flask.Blueprint
        """
        self.register_plugins()

        self.blueprint = Blueprint(
            self.name,
            "nemo",
            url_prefix=self.prefix,
            template_folder=self.template_folder,
            static_folder=self.static_folder,
            static_url_path=self.static_url_path
        )

        for url, name, methods, instance in self._urls:
            self.blueprint.add_url_rule(
                url,
                view_func=self.view_maker(name, instance),
                endpoint=_plugin_endpoint_rename(name, instance),
                methods=methods
            )

        for url, name, methods, instance in self._semantic_url:
            self.blueprint.add_url_rule(
                url,
                view_func=self.view_maker(name, instance),
                endpoint=_plugin_endpoint_rename(name, instance)+"_semantic",
                methods=methods
            )

        self.register_assets()
        self.register_filters()

        # We extend the loading list by the instance value
        self.__templates_namespaces__.extend(self.__instance_templates__)
        # We generate a template loader
        for namespace, directory in self.__templates_namespaces__[::-1]:
            if namespace not in self.__template_loader__:
                self.__template_loader__[namespace] = []
            self.__template_loader__[namespace].append(
                jinja2.FileSystemLoader(op.abspath(directory))
            )
        self.blueprint.jinja_loader = jinja2.PrefixLoader(
            {namespace: jinja2.ChoiceLoader(paths) for namespace, paths in self.__template_loader__.items()},
            "::"
        )

        if self.cache is not None:
            for func, instance in self.cached:
                setattr(instance, func.__name__, self.cache.memoize()(func))

        return self.blueprint

def create_blueprint ( self ) : self . register_plugins ( ) self . blueprint = Blueprint ( self . name , "nemo" , url_prefix = self . prefix , template_folder = self . template_folder , static_folder = self . static_folder , static_url_path = self . static_url_path ) for url , name , methods , instance in self . _urls : self . blueprint . add_url_rule ( url , view_func = self . view_maker ( name , instance ) , endpoint = _plugin_endpoint_rename ( name , instance ) , methods = methods ) for url , name , methods , instance in self . _semantic_url : self . blueprint . add_url_rule ( url , view_func = self . view_maker ( name , instance ) , endpoint = _plugin_endpoint_rename ( name , instance ) + "_semantic" , methods = methods ) self . register_assets ( ) self . register_filters ( ) self . __templates_namespaces__ . extend ( self . __instance_templates__ ) for namespace , directory in self . __templates_namespaces__ [ : : - 1 ] : if namespace not in self . __template_loader__ : self . __template_loader__ [ namespace ] = [ ] self . __template_loader__ [ namespace ] . append ( jinja2 . FileSystemLoader ( op . abspath ( directory ) ) ) self . blueprint . jinja_loader = jinja2 . PrefixLoader ( { namespace : jinja2 . ChoiceLoader ( paths ) for namespace , paths in self . __template_loader__ . items ( ) } , "::" ) if self . cache is not None : for func , instance in self . cached : setattr ( instance , func . __name__ , self . cache . memoize ( ) ( func ) ) return self . blueprint


def view_maker(self, name, instance=None):
        """ Create a view

        :param name: Name of the route function to use for the view.
        :type name: str
        :return: Route function which makes use of Nemo context (such as menu informations)
        :rtype: function
        """
        if instance is None:
            instance = self
        sig = "lang" in [
            parameter.name
            for parameter in inspect.signature(getattr(instance, name)).parameters.values()
        ]

        def route(**kwargs):
            if sig and "lang" not in kwargs:
                kwargs["lang"] = self.get_locale()
            if "semantic" in kwargs:
                del kwargs["semantic"]
            return self.route(getattr(instance, name), **kwargs)
        return route

def view_maker ( self , name , instance = None ) : if instance is None : instance = self sig = "lang" in [ parameter . name for parameter in inspect . signature ( getattr ( instance , name ) ) . parameters . values ( ) ] def route ( ** kwargs ) : if sig and "lang" not in kwargs : kwargs [ "lang" ] = self . get_locale ( ) if "semantic" in kwargs : del kwargs [ "semantic" ] return self . route ( getattr ( instance , name ) , ** kwargs ) return route


def main_collections(self, lang=None):
        """ Retrieve main parent collections of a repository

        :param lang: Language to retrieve information in
        :return: Sorted collections representations
        """
        return sorted([
            {
                "id": member.id,
                "label": str(member.get_label(lang=lang)),
                "model": str(member.model),
                "type": str(member.type),
                "size": member.size
            }
            for member in self.resolver.getMetadata().members
        ], key=itemgetter("label"))

def main_collections ( self , lang = None ) : return sorted ( [ { "id" : member . id , "label" : str ( member . get_label ( lang = lang ) ) , "model" : str ( member . model ) , "type" : str ( member . type ) , "size" : member . size } for member in self . resolver . getMetadata ( ) . members ] , key = itemgetter ( "label" ) )


def make_cache_keys(self, endpoint, kwargs):
        """ This function is built to provide cache keys for templates

        :param endpoint: Current endpoint
        :param kwargs: Keyword Arguments
        :return: tuple of i18n dependant cache key and i18n ignoring cache key
        :rtype: tuple(str)
        """
        keys = sorted(kwargs.keys())
        i18n_cache_key = endpoint+"|"+"|".join([kwargs[k] for k in keys])
        if "lang" in keys:
            cache_key = endpoint+"|" + "|".join([kwargs[k] for k in keys if k != "lang"])
        else:
            cache_key = i18n_cache_key
        return i18n_cache_key, cache_key

def make_cache_keys ( self , endpoint , kwargs ) : keys = sorted ( kwargs . keys ( ) ) i18n_cache_key = endpoint + "|" + "|" . join ( [ kwargs [ k ] for k in keys ] ) if "lang" in keys : cache_key = endpoint + "|" + "|" . join ( [ kwargs [ k ] for k in keys if k != "lang" ] ) else : cache_key = i18n_cache_key return i18n_cache_key , cache_key


def render(self, template, **kwargs):
        """ Render a route template and adds information to this route.

        :param template: Template name.
        :type template: str
        :param kwargs: dictionary of named arguments used to be passed to the template
        :type kwargs: dict
        :return: Http Response with rendered template
        :rtype: flask.Response
        """

        kwargs["cache_key"] = "%s" % kwargs["url"].values()
        kwargs["lang"] = self.get_locale()
        kwargs["assets"] = self.assets
        kwargs["main_collections"] = self.main_collections(kwargs["lang"])
        kwargs["cache_active"] = self.cache is not None
        kwargs["cache_time"] = 0
        kwargs["cache_key"], kwargs["cache_key_i18n"] = self.make_cache_keys(request.endpoint, kwargs["url"])
        kwargs["template"] = template

        for plugin in self.__plugins_render_views__:
            kwargs.update(plugin.render(**kwargs))

        return render_template(kwargs["template"], **kwargs)

def render ( self , template , ** kwargs ) : kwargs [ "cache_key" ] = "%s" % kwargs [ "url" ] . values ( ) kwargs [ "lang" ] = self . get_locale ( ) kwargs [ "assets" ] = self . assets kwargs [ "main_collections" ] = self . main_collections ( kwargs [ "lang" ] ) kwargs [ "cache_active" ] = self . cache is not None kwargs [ "cache_time" ] = 0 kwargs [ "cache_key" ] , kwargs [ "cache_key_i18n" ] = self . make_cache_keys ( request . endpoint , kwargs [ "url" ] ) kwargs [ "template" ] = template for plugin in self . __plugins_render_views__ : kwargs . update ( plugin . render ( ** kwargs ) ) return render_template ( kwargs [ "template" ] , ** kwargs )

Render a route template and adds information to this route .


def register(self):
        """ Register the app using Blueprint

        :return: Nemo blueprint
        :rtype: flask.Blueprint
        """
        if self.app is not None:
            if not self.blueprint:
                self.blueprint = self.create_blueprint()
            self.app.register_blueprint(self.blueprint)
            if self.cache is None:
                # We register a fake cache extension.
                setattr(self.app.jinja_env, "_fake_cache_extension", self)
                self.app.jinja_env.add_extension(FakeCacheExtension)
            return self.blueprint
        return None

def register ( self ) : if self . app is not None : if not self . blueprint : self . blueprint = self . create_blueprint ( ) self . app . register_blueprint ( self . blueprint ) if self . cache is None : setattr ( self . app . jinja_env , "_fake_cache_extension" , self ) self . app . jinja_env . add_extension ( FakeCacheExtension ) return self . blueprint return None


def register_filters(self):
        """ Register filters for Jinja to use

       .. note::  Extends the dictionary filters of jinja_env using self._filters list
        """
        for _filter, instance in self._filters:
            if not instance:
                self.app.jinja_env.filters[
                    _filter.replace("f_", "")
                ] = getattr(flask_nemo.filters, _filter)
            else:
                self.app.jinja_env.filters[
                    _filter.replace("f_", "")
                ] = getattr(instance, _filter.replace("_{}".format(instance.name), ""))

def register_filters ( self ) : for _filter , instance in self . _filters : if not instance : self . app . jinja_env . filters [ _filter . replace ( "f_" , "" ) ] = getattr ( flask_nemo . filters , _filter ) else : self . app . jinja_env . filters [ _filter . replace ( "f_" , "" ) ] = getattr ( instance , _filter . replace ( "_{}" . format ( instance . name ) , "" ) )


def register_plugins(self):
        """ Register plugins in Nemo instance

        - Clear routes first if asked by one plugin
        - Clear assets if asked by one plugin and replace by the last plugin registered static_folder
        - Register each plugin
            - Append plugin routes to registered routes
            - Append plugin filters to registered filters
            - Append templates directory to given namespaces
            - Append assets (CSS, JS, statics) to given resources
            - Append render view (if exists) to Nemo.render stack
        """
        if len([plugin for plugin in self.__plugins__.values() if plugin.clear_routes]) > 0:  # Clear current routes
            self._urls = list()
            self.cached = list()

        clear_assets = [plugin for plugin in self.__plugins__.values() if plugin.clear_assets]
        if len(clear_assets) > 0 and not self.prevent_plugin_clearing_assets:  # Clear current Assets
            self.__assets__ = copy(type(self).ASSETS)
            static_path = [plugin.static_folder for plugin in clear_assets if plugin.static_folder]
            if len(static_path) > 0:
                self.static_folder = static_path[-1]

        for plugin in self.__plugins__.values():
            self._urls.extend([(url, function, methods, plugin) for url, function, methods in plugin.routes])
            self._filters.extend([(filt, plugin) for filt in plugin.filters])
            self.__templates_namespaces__.extend(
                [(namespace, directory) for namespace, directory in plugin.templates.items()]
            )
            for asset_type in self.__assets__:
                for key, value in plugin.assets[asset_type].items():
                    self.__assets__[asset_type][key] = value
            if plugin.augment:
                self.__plugins_render_views__.append(plugin)

            if hasattr(plugin, "CACHED"):
                for func in plugin.CACHED:
                    self.cached.append((getattr(plugin, func), plugin))
            plugin.register_nemo(self)

def register_plugins ( self ) : if len ( [ plugin for plugin in self . __plugins__ . values ( ) if plugin . clear_routes ] ) > 0 : self . _urls = list ( ) self . cached = list ( ) clear_assets = [ plugin for plugin in self . __plugins__ . values ( ) if plugin . clear_assets ] if len ( clear_assets ) > 0 and not self . prevent_plugin_clearing_assets : self . __assets__ = copy ( type ( self ) . ASSETS ) static_path = [ plugin . static_folder for plugin in clear_assets if plugin . static_folder ] if len ( static_path ) > 0 : self . static_folder = static_path [ - 1 ] for plugin in self . __plugins__ . values ( ) : self . _urls . extend ( [ ( url , function , methods , plugin ) for url , function , methods in plugin . routes ] ) self . _filters . extend ( [ ( filt , plugin ) for filt in plugin . filters ] ) self . __templates_namespaces__ . extend ( [ ( namespace , directory ) for namespace , directory in plugin . templates . items ( ) ] ) for asset_type in self . __assets__ : for key , value in plugin . assets [ asset_type ] . items ( ) : self . __assets__ [ asset_type ] [ key ] = value if plugin . augment : self . __plugins_render_views__ . append ( plugin ) if hasattr ( plugin , "CACHED" ) : for func in plugin . CACHED : self . cached . append ( ( getattr ( plugin , func ) , plugin ) ) plugin . register_nemo ( self )


def chunk(self, text, reffs):
        """ Handle a list of references depending on the text identifier using the chunker dictionary.

        :param text: Text object from which comes the references
        :type text: MyCapytains.resources.texts.api.Text
        :param reffs: List of references to transform
        :type reffs: References
        :return: Transformed list of references
        :rtype: [str]
        """
        if str(text.id) in self.chunker:
            return self.chunker[str(text.id)](text, reffs)
        return self.chunker["default"](text, reffs)

def chunk ( self , text , reffs ) : if str ( text . id ) in self . chunker : return self . chunker [ str ( text . id ) ] ( text , reffs ) return self . chunker [ "default" ] ( text , reffs )

Handle a list of references depending on the text identifier using the chunker dictionary .


def add_tag():
    """
        Obtains the data from the pipe and appends the given tag.
    """
    if len(sys.argv) > 1:
        tag = sys.argv[1]
        doc_mapper = DocMapper()
        if doc_mapper.is_pipe:
            count = 0
            for obj in doc_mapper.get_pipe():
                obj.add_tag(tag)
                obj.update(tags=obj.tags)
                count += 1
            print_success("Added tag '{}' to {} object(s)".format(tag, count))
        else:
            print_error("Please use this script with pipes")
    else:
        print_error("Usage: jk-add-tag <tag>")
        sys.exit()

def
add_tag
(
)
:
if
len
(
sys
.
argv
)
>
1
:
tag
=
sys
.
argv
[
1
]
doc_mapper
=
DocMapper
(
)
if
doc_mapper
.
is_pipe
:
count
=
0
for
obj
in
doc_mapper
.
get_pipe
(
)
:
obj
.
add_tag
(
tag
)
obj
.
update
(
tags
=
obj
.
tags
)
count
+=
1
print_success
(
"Added tag '{}' to {} object(s)"
.
format
(
tag
,
count
)
)
else
:
print_error
(
"Please use this script with pipes"
)
else
:
print_error
(
"Usage: jk-add-tag <tag>"
)
sys
.
exit
(
)

Obtains the data from the pipe and appends the given tag .


def set(self, section, key, value):
        """
            Creates the section value if it does not exists and sets the value.
            Use write_config to actually set the value.
        """
        if not section in self.config:
            self.config.add_section(section)
        self.config.set(section, key, value)

def set ( self , section , key , value ) : if not section in self . config : self . config . add_section ( section ) self . config . set ( section , key , value )

Creates the section value if it does not exists and sets the value . Use write_config to actually set the value .


def get(self, section, key):
        """
            This function tries to retrieve the value from the configfile
            otherwise will return a default.
        """
        try:
            return self.config.get(section, key)
        except configparser.NoSectionError:
            pass
        except configparser.NoOptionError:
            pass
        return self.defaults[section][key]

def get ( self , section , key ) : try : return self . config . get ( section , key ) except configparser . NoSectionError : pass except configparser . NoOptionError : pass return self . defaults [ section ] [ key ]

This function tries to retrieve the value from the configfile otherwise will return a default .


def config_dir(self):
        """
            Returns the configuration directory
        """
        home = expanduser('~')
        config_dir = os.path.join(home, '.jackal')
        return config_dir

def config_dir ( self ) : home = expanduser ( '~' ) config_dir = os . path . join ( home , '.jackal' ) return config_dir


def write_config(self, initialize_indices=False):
        """
            Write the current config to disk to store them.
        """
        if not os.path.exists(self.config_dir):
            os.mkdir(self.config_dir)

        with open(self.config_file, 'w') as configfile:
            self.config.write(configfile)

        if initialize_indices:
            index = self.get('jackal', 'index')
            from jackal import Host, Range, Service, User, Credential, Log
            from jackal.core import create_connection
            create_connection(self)
            Host.init(index="{}-hosts".format(index))
            Range.init(index="{}-ranges".format(index))
            Service.init(index="{}-services".format(index))
            User.init(index="{}-users".format(index))
            Credential.init(index="{}-creds".format(index))
            Log.init(index="{}-log".format(index))

def write_config ( self , initialize_indices = False ) : if not os . path . exists ( self . config_dir ) : os . mkdir ( self . config_dir ) with open ( self . config_file , 'w' ) as configfile : self . config . write ( configfile ) if initialize_indices : index = self . get ( 'jackal' , 'index' ) from jackal import Host , Range , Service , User , Credential , Log from jackal . core import create_connection create_connection ( self ) Host . init ( index = "{}-hosts" . format ( index ) ) Range . init ( index = "{}-ranges" . format ( index ) ) Service . init ( index = "{}-services" . format ( index ) ) User . init ( index = "{}-users" . format ( index ) ) Credential . init ( index = "{}-creds" . format ( index ) ) Log . init ( index = "{}-log" . format ( index ) )

Write the current config to disk to store them .


def ensure_remote_branch_is_tracked(branch):
    """Track the specified remote branch if it is not already tracked."""
    if branch == MASTER_BRANCH:
        # We don't need to explicitly track the master branch, so we're done.
        return

    # Ensure the specified branch is in the local branch list.
    output = subprocess.check_output(['git', 'branch', '--list'])
    for line in output.split('\n'):
        if line.strip() == branch:
            # We are already tracking the remote branch
            break
    else:
        # We are not tracking the remote branch, so track it.
        try:
            sys.stdout.write(subprocess.check_output(
                ['git', 'checkout', '--track', 'origin/%s' % branch]))
        except subprocess.CalledProcessError:
            # Bail gracefully.
            raise SystemExit(1)

def ensure_remote_branch_is_tracked ( branch ) : if branch == MASTER_BRANCH : return output = subprocess . check_output ( [ 'git' , 'branch' , '--list' ] ) for line in output . split ( '\n' ) : if line . strip ( ) == branch : break else : try : sys . stdout . write ( subprocess . check_output ( [ 'git' , 'checkout' , '--track' , 'origin/%s' % branch ] ) ) except subprocess . CalledProcessError : raise SystemExit ( 1 )

Track the specified remote branch if it is not already tracked .


def main(branch):
    """Checkout, update and branch from the specified branch."""
    try:
        # Ensure that we're in a git repository. This command is silent unless
        # you're not actually in a git repository, in which case, you receive a
        # "Not a git repository" error message.
        output = subprocess.check_output(['git', 'rev-parse']).decode('utf-8')
        sys.stdout.write(output)
    except subprocess.CalledProcessError:
        # Bail if we're not in a git repository.
        return

    # This behavior ensures a better user experience for those that aren't
    # intimately familiar with git.
    ensure_remote_branch_is_tracked(branch)

    # Switch to the specified branch and update it.
    subprocess.check_call(['git', 'checkout', '--quiet', branch])

    # Pulling is always safe here, because we never commit to this branch.
    subprocess.check_call(['git', 'pull', '--quiet'])

    # Checkout the top commit in the branch, effectively going "untracked."
    subprocess.check_call(['git', 'checkout', '--quiet', '%s~0' % branch])

    # Clean up the repository of Python cruft. Because we've just switched
    # branches and compiled Python files should not be version controlled,
    # there are likely leftover compiled Python files sitting on disk which may
    # confuse some tools, such as sqlalchemy-migrate.
    subprocess.check_call(['find', '.', '-name', '"*.pyc"', '-delete'])

    # For the sake of user experience, give some familiar output.
    print('Your branch is up to date with branch \'origin/%s\'.' % branch)

Checkout update and branch from the specified branch .


def get_interface_name():
    """
        Returns the interface name of the first not link_local and not loopback interface.
    """
    interface_name = ''
    interfaces = psutil.net_if_addrs()
    for name, details in interfaces.items():
        for detail in details:
            if detail.family == socket.AF_INET:
                ip_address = ipaddress.ip_address(detail.address)
                if not (ip_address.is_link_local or ip_address.is_loopback):
                    interface_name = name
                    break
    return interface_name

def get_interface_name ( ) : interface_name = '' interfaces = psutil . net_if_addrs ( ) for name , details in interfaces . items ( ) : for detail in details : if detail . family == socket . AF_INET : ip_address = ipaddress . ip_address ( detail . address ) if not ( ip_address . is_link_local or ip_address . is_loopback ) : interface_name = name break return interface_name

Returns the interface name of the first not link_local and not loopback interface .


def load_targets(self):
        """
            load_targets will load the services with smb signing disabled and if ldap is enabled the services with the ldap port open.
        """
        ldap_services = []
        if self.ldap:
            ldap_services = self.search.get_services(ports=[389], up=True)

        self.ldap_strings = ["ldap://{}".format(service.address) for service in ldap_services]
        self.services = self.search.get_services(tags=['smb_signing_disabled'])
        self.ips = [str(service.address) for service in self.services]

def load_targets ( self ) : ldap_services = [ ] if self . ldap : ldap_services = self . search . get_services ( ports = [ 389 ] , up = True ) self . ldap_strings = [ "ldap://{}" . format ( service . address ) for service in ldap_services ] self . services = self . search . get_services ( tags = [ 'smb_signing_disabled' ] ) self . ips = [ str ( service . address ) for service in self . services ]

load_targets will load the services with smb signing disabled and if ldap is enabled the services with the ldap port open .


def write_targets(self):
        """
            write_targets will write the contents of ips and ldap_strings to the targets_file.
        """
        if len(self.ldap_strings) == 0 and len(self.ips) == 0:
            print_notification("No targets left")
            if self.auto_exit:
                if self.notifier:
                    self.notifier.stop()
                self.terminate_processes()

        with open(self.targets_file, 'w') as f:
            f.write('\n'.join(self.ldap_strings + self.ips))

def
write_targets
(
self
)
:
if
len
(
self
.
ldap_strings
)
==
0
and
len
(
self
.
ips
)
==
0
:
print_notification
(
"No targets left"
)
if
self
.
auto_exit
:
if
self
.
notifier
:
self
.
notifier
.
stop
(
)
self
.
terminate_processes
(
)
with
open
(
self
.
targets_file
,
'w'
)
as
f
:
f
.
write
(
'\n'
.
join
(
self
.
ldap_strings
+
self
.
ips
)
)

write_targets will write the contents of ips and ldap_strings to the targets_file .


def start_processes(self):
        """
            Starts the ntlmrelayx.py and responder processes.
            Assumes you have these programs in your path.
        """
        self.relay = subprocess.Popen(['ntlmrelayx.py', '-6', '-tf', self.targets_file, '-w', '-l', self.directory, '-of', self.output_file], cwd=self.directory)
        self.responder = subprocess.Popen(['responder', '-I', self.interface_name])

def start_processes ( self ) : self . relay = subprocess . Popen ( [ 'ntlmrelayx.py' , '-6' , '-tf' , self . targets_file , '-w' , '-l' , self . directory , '-of' , self . output_file ] , cwd = self . directory ) self . responder = subprocess . Popen ( [ 'responder' , '-I' , self . interface_name ] )

Starts the ntlmrelayx . py and responder processes . Assumes you have these programs in your path .


def callback(self, event):
        """
            Function that gets called on each event from pyinotify.
        """
        # IN_CLOSE_WRITE -> 0x00000008
        if event.mask == 0x00000008:
            if event.name.endswith('.json'):
                print_success("Ldapdomaindump file found")
                if event.name in ['domain_groups.json', 'domain_users.json']:
                    if event.name == 'domain_groups.json':
                        self.domain_groups_file = event.pathname
                    if event.name == 'domain_users.json':
                        self.domain_users_file = event.pathname
                    if self.domain_groups_file and self.domain_users_file:
                        print_success("Importing users")
                        subprocess.Popen(['jk-import-domaindump', self.domain_groups_file, self.domain_users_file])
                elif event.name == 'domain_computers.json':
                    print_success("Importing computers")
                    subprocess.Popen(['jk-import-domaindump', event.pathname])

                # Ldap has been dumped, so remove the ldap targets.
                self.ldap_strings = []
                self.write_targets()

            if event.name.endswith('_samhashes.sam'):
                host = event.name.replace('_samhashes.sam', '')
                # TODO import file.
                print_success("Secretsdump file, host ip: {}".format(host))
                subprocess.Popen(['jk-import-secretsdump', event.pathname])

                # Remove this system from this ip list.
                self.ips.remove(host)
                self.write_targets()

def
callback
(
self
,
event
)
:
if
event
.
mask
==
0x00000008
:
if
event
.
name
.
endswith
(
'.json'
)
:
print_success
(
"Ldapdomaindump file found"
)
if
event
.
name
in
[
'domain_groups.json'
,
'domain_users.json'
]
:
if
event
.
name
==
'domain_groups.json'
:
self
.
domain_groups_file
=
event
.
pathname
if
event
.
name
==
'domain_users.json'
:
self
.
domain_users_file
=
event
.
pathname
if
self
.
domain_groups_file
and
self
.
domain_users_file
:
print_success
(
"Importing users"
)
subprocess
.
Popen
(
[
'jk-import-domaindump'
,
self
.
domain_groups_file
,
self
.
domain_users_file
]
)
elif
event
.
name
==
'domain_computers.json'
:
print_success
(
"Importing computers"
)
subprocess
.
Popen
(
[
'jk-import-domaindump'
,
event
.
pathname
]
)
self
.
ldap_strings
=
[
]
self
.
write_targets
(
)
if
event
.
name
.
endswith
(
'_samhashes.sam'
)
:
host
=
event
.
name
.
replace
(
'_samhashes.sam'
,
''
)
print_success
(
"Secretsdump file, host ip: {}"
.
format
(
host
)
)
subprocess
.
Popen
(
[
'jk-import-secretsdump'
,
event
.
pathname
]
)
self
.
ips
.
remove
(
host
)
self
.
write_targets
(
)

Function that gets called on each event from pyinotify .


def watch(self):
        """
            Watches directory for changes
        """
        wm = pyinotify.WatchManager()
        self.notifier = pyinotify.Notifier(wm, default_proc_fun=self.callback)
        wm.add_watch(self.directory, pyinotify.ALL_EVENTS)
        try:
            self.notifier.loop()
        except (KeyboardInterrupt, AttributeError):
            print_notification("Stopping")
        finally:
            self.notifier.stop()
            self.terminate_processes()

def watch ( self ) : wm = pyinotify . WatchManager ( ) self . notifier = pyinotify . Notifier ( wm , default_proc_fun = self . callback ) wm . add_watch ( self . directory , pyinotify . ALL_EVENTS ) try : self . notifier . loop ( ) except ( KeyboardInterrupt , AttributeError ) : print_notification ( "Stopping" ) finally : self . notifier . stop ( ) self . terminate_processes ( )


def terminate_processes(self):
        """
            Terminate the processes.
        """
        if self.relay:
            self.relay.terminate()
        if self.responder:
            self.responder.terminate()

def terminate_processes ( self ) : if self . relay : self . relay . terminate ( ) if self . responder : self . responder . terminate ( )


def wait(self):
        """
            This function waits for the relay and responding processes to exit.
            Captures KeyboardInterrupt to shutdown these processes.
        """
        try:
            self.relay.wait()
            self.responder.wait()
        except KeyboardInterrupt:
            print_notification("Stopping")
        finally:
            self.terminate_processes()

def wait ( self ) : try : self . relay . wait ( ) self . responder . wait ( ) except KeyboardInterrupt : print_notification ( "Stopping" ) finally : self . terminate_processes ( )

This function waits for the relay and responding processes to exit . Captures KeyboardInterrupt to shutdown these processes .


def getAnnotations(self, targets, wildcard=".", include=None, exclude=None, limit=None, start=1, expand=False,
                       **kwargs):
        """ Retrieve annotations from the query provider

        :param targets: The CTS URN(s) to query as the target of annotations
        :type targets: [MyCapytain.common.reference.URN], URN or None
        :param wildcard: Wildcard specifier for how to match the URN
        :type wildcard: str
        :param include: URI(s) of Annotation types to include in the results
        :type include: list(str)
        :param exclude: URI(s) of Annotation types to include in the results
        :type exclude: list(str)
        :param limit: The max number of results to return (Default is None for no limit)
        :type limit: int
        :param start: the starting record to return (Default is 1)
        :type start: int
        :param expand: Flag to state whether Annotations are expanded (Default is False)
        :type expand: bool

        :return: Tuple representing the query results. The first element
                 The first element is the number of total Annotations found
                 The second element is the list of Annotations
        :rtype: (int, list(Annotation)

        .. note::

            Wildcard should be one of the following value

            - '.' to match exact,
            - '.%' to match exact plus lower in the hierarchy
            - '%.' to match exact + higher in the hierarchy
            - '-' to match in the range
            - '%.%' to match all

        """
        return 0, []

def getAnnotations ( self , targets , wildcard = "." , include = None , exclude = None , limit = None , start = 1 , expand = False , ** kwargs ) : return 0 , [ ]


def render(self, **kwargs):
        """ Make breadcrumbs for a route

        :param kwargs: dictionary of named arguments used to construct the view
        :type kwargs: dict
        :return: List of dict items the view can use to construct the link.
        :rtype: {str: list({ "link": str, "title", str, "args", dict})}
        """
        breadcrumbs = []
        # this is the list of items we want to accumulate in the breadcrumb trail.
        # item[0] is the key into the kwargs["url"] object and item[1] is the  name of the route
        # setting a route name to None means that it's needed to construct the route of the next item in the list
        # but shouldn't be included in the list itself (this is currently the case for work --
        # at some point we probably should include work in the navigation)
        breadcrumbs = []
        if "collections" in kwargs:
            breadcrumbs = [{
                "title": "Text Collections",
                "link": ".r_collections",
                "args": {}
            }]

            if "parents" in kwargs["collections"]:
                breadcrumbs += [
                        {
                            "title": parent["label"],
                            "link": ".r_collection_semantic",
                            "args": {
                                "objectId": parent["id"],
                                "semantic": f_slugify(parent["label"]),
                            },
                        }
                        for parent in kwargs["collections"]["parents"]
                  ][::-1]

            if "current" in kwargs["collections"]:
                breadcrumbs.append({
                    "title": kwargs["collections"]["current"]["label"],
                    "link": None,
                    "args": {}
                })

        # don't link the last item in the trail
        if len(breadcrumbs) > 0:
            breadcrumbs[-1]["link"] = None

        return {"breadcrumbs": breadcrumbs}

def
render
(
self
,
**
kwargs
)
:
breadcrumbs
=
[
]
breadcrumbs
=
[
]
if
"collections"
in
kwargs
:
breadcrumbs
=
[
{
"title"
:
"Text Collections"
,
"link"
:
".r_collections"
,
"args"
:
{
}
}
]
if
"parents"
in
kwargs
[
"collections"
]
:
breadcrumbs
+=
[
{
"title"
:
parent
[
"label"
]
,
"link"
:
".r_collection_semantic"
,
"args"
:
{
"objectId"
:
parent
[
"id"
]
,
"semantic"
:
f_slugify
(
parent
[
"label"
]
)
,
}
,
}
for
parent
in
kwargs
[
"collections"
]
[
"parents"
]
]
[
:
:
1
]
if
"current"
in
kwargs
[
"collections"
]
:
breadcrumbs
.
append
(
{
"title"
:
kwargs
[
"collections"
]
[
"current"
]
[
"label"
]
,
"link"
:
,
"args"
:
{
}
}
)
if
len
(
breadcrumbs
)
>
0
:
breadcrumbs
[
1
]
[
"link"
]
=
return
{
"breadcrumbs"
:
breadcrumbs
}


def main():
    """
        This function obtains hosts from core and starts a nessus scan on these hosts.
        The nessus tag is appended to the host tags.
    """
    config = Config()
    core = HostSearch()
    hosts = core.get_hosts(tags=['!nessus'], up=True)
    hosts = [host for host in hosts]
    host_ips = ",".join([str(host.address) for host in hosts])

    url = config.get('nessus', 'host')
    access = config.get('nessus', 'access_key')
    secret = config.get('nessus', 'secret_key')
    template_name = config.get('nessus', 'template_name')

    nessus = Nessus(access, secret, url, template_name)

    scan_id = nessus.create_scan(host_ips)
    nessus.start_scan(scan_id)

    for host in hosts:
        host.add_tag('nessus')
        host.save()

    Logger().log("nessus", "Nessus scan started on {} hosts".format(len(hosts)), {'scanned_hosts': len(hosts)})

def
main
(
)
:
config
=
Config
(
)
core
=
HostSearch
(
)
hosts
=
core
.
get_hosts
(
tags
=
[
'!nessus'
]
,
up
=
True
)
hosts
=
[
host
for
host
in
hosts
]
host_ips
=
","
.
join
(
[
str
(
host
.
address
)
for
host
in
hosts
]
)
url
=
config
.
get
(
'nessus'
,
'host'
)
access
=
config
.
get
(
'nessus'
,
'access_key'
)
secret
=
config
.
get
(
'nessus'
,
'secret_key'
)
template_name
=
config
.
get
(
'nessus'
,
'template_name'
)
nessus
=
Nessus
(
access
,
secret
,
url
,
template_name
)
scan_id
=
nessus
.
create_scan
(
host_ips
)
nessus
.
start_scan
(
scan_id
)
for
host
in
hosts
:
host
.
add_tag
(
'nessus'
)
host
.
save
(
)
Logger
(
)
.
log
(
"nessus"
,
"Nessus scan started on {} hosts"
.
format
(
len
(
hosts
)
)
,
{
'scanned_hosts'
:
len
(
hosts
)
}
)

This function obtains hosts from core and starts a nessus scan on these hosts . The nessus tag is appended to the host tags .


def get_template_uuid(self):
        """
            Retrieves the uuid of the given template name.
        """
        response = requests.get(self.url + 'editor/scan/templates', headers=self.headers, verify=False)
        templates = json.loads(response.text)
        for template in templates['templates']:
            if template['name'] == self.template_name:
                return template['uuid']

def get_template_uuid ( self ) : response = requests . get ( self . url + 'editor/scan/templates' , headers = self . headers , verify = False ) templates = json . loads ( response . text ) for template in templates [ 'templates' ] : if template [ 'name' ] == self . template_name : return template [ 'uuid' ]

Retrieves the uuid of the given template name .


def create_scan(self, host_ips):
        """
            Creates a scan with the given host ips
            Returns the scan id of the created object.
        """
        now = datetime.datetime.now()
        data = {
            "uuid": self.get_template_uuid(),
            "settings": {
                "name": "jackal-" + now.strftime("%Y-%m-%d %H:%M"),
                "text_targets": host_ips
            }
        }
        response = requests.post(self.url + 'scans', data=json.dumps(data), verify=False, headers=self.headers)
        if response:
            result = json.loads(response.text)
            return result['scan']['id']

def
create_scan
(
self
,
host_ips
)
:
now
=
datetime
.
datetime
.
now
(
)
data
=
{
"uuid"
:
self
.
get_template_uuid
(
)
,
"settings"
:
{
"name"
:
"jackal-"
+
now
.
strftime
(
"%Y-%m-%d %H:%M"
)
,
"text_targets"
:
host_ips
}
}
response
=
requests
.
post
(
self
.
url
+
'scans'
,
data
=
json
.
dumps
(
data
)
,
verify
=
False
,
headers
=
self
.
headers
)
if
response
:
result
=
json
.
loads
(
response
.
text
)
return
result
[
'scan'
]
[
'id'
]

Creates a scan with the given host ips Returns the scan id of the created object .


def start_scan(self, scan_id):
        """
            Starts the scan identified by the scan_id.s
        """
        requests.post(self.url + 'scans/{}/launch'.format(scan_id), verify=False, headers=self.headers)

def start_scan ( self , scan_id ) : requests . post ( self . url + 'scans/{}/launch' . format ( scan_id ) , verify = False , headers = self . headers )

Starts the scan identified by the scan_id . s


def cmpToDataStore_uri(base, ds1, ds2):
  '''Bases the comparison of the datastores on URI alone.'''
  ret = difflib.get_close_matches(base.uri, [ds1.uri, ds2.uri], 1, cutoff=0.5)
  if len(ret) <= 0:
    return 0
  if ret[0] == ds1.uri:
    return -1
  return 1

def cmpToDataStore_uri ( base , ds1 , ds2 ) : ret = difflib . get_close_matches ( base . uri , [ ds1 . uri , ds2 . uri ] , 1 , cutoff = 0.5 ) if len ( ret ) <= 0 : return 0 if ret [ 0 ] == ds1 . uri : return - 1 return 1

Bases the comparison of the datastores on URI alone .


def add_tag(self, tag):
        """
            Adds a tag to the list of tags and makes sure the result list contains only unique results.
        """
        self.tags = list(set(self.tags or []) | set([tag]))

def add_tag ( self , tag ) : self . tags = list ( set ( self . tags or [ ] ) | set ( [ tag ] ) )

Adds a tag to the list of tags and makes sure the result list contains only unique results .


def remove_tag(self, tag):
        """
            Removes a tag from this object
        """
        self.tags = list(set(self.tags or []) - set([tag]))

def remove_tag ( self , tag ) : self . tags = list ( set ( self . tags or [ ] ) - set ( [ tag ] ) )


def to_dict(self, include_meta=False):
        """
            Returns the result as a dictionary, provide the include_meta flag to als show information like index and doctype.
        """
        result = super(JackalDoc, self).to_dict(include_meta=include_meta)
        if include_meta:
            source = result.pop('_source')
            return {**result, **source}
        else:
            return result

def to_dict ( self , include_meta = False ) : result = super ( JackalDoc , self ) . to_dict ( include_meta = include_meta ) if include_meta : source = result . pop ( '_source' ) return { ** result , ** source } else : return result

Returns the result as a dictionary provide the include_meta flag to als show information like index and doctype .