def nearest(self, ver):
        """
        Retrieve the official version nearest the one given.
        """
        if not isinstance(ver, Version):
            ver = Version(ver)

        if ver in OFFICIAL_VERSIONS:
            return ver

        # We might not have an exact match for that.
        # See if we have one that's newer than the grid we're looking at.
        versions = list(OFFICIAL_VERSIONS)
        versions.sort(reverse=True)
        best = None
        for candidate in versions:
            # Due to ambiguities, we might have an exact match and not know it.
            # '2.0' will not hash to the same value as '2.0.0', but both are
            # equivalent.
            if candidate == ver:
                # We can't beat this, make a note of the match for later
                return candidate

            # If we have not seen a better candidate, and this is older
            # then we may have to settle for that.
            if (best is None) and (candidate < ver):
                warnings.warn('This version of hszinc does not yet '\
                            'support version %s, please seek a newer version '\
                            'or file a bug.  Closest (older) version supported is %s.'\
                            % (ver, candidate))
                return candidate

            # Probably the best so far, but see if we can go closer
            if candidate > ver:
                best = candidate

        # Unhappy path, no best option?  This should not happen.
        assert best is not None
        warnings.warn('This version of hszinc does not yet '\
                    'support version %s, please seek a newer version '\
                    'or file a bug.  Closest (newer) version supported is %s.'\
                    % (ver, best))
        return best

Retrieve the official version nearest the one given .


def encrypt_files(selected_host, only_link, file_name):
    """
    Encrypts file with gpg and random generated password
    """
    if ENCRYPTION_DISABLED:
        print('For encryption please install gpg')
        exit()
    passphrase = '%030x' % random.randrange(16**30)
    source_filename = file_name
    cmd = 'gpg --batch --symmetric --cipher-algo AES256 --passphrase-fd 0 ' \
          '--output - {}'.format(source_filename)
    encrypted_output = Popen(shlex.split(cmd), stdout=PIPE, stdin=PIPE, stderr=PIPE)
    encrypted_data = encrypted_output.communicate(passphrase.encode())[0]
    return upload_files(encrypted_data, selected_host, only_link, file_name)+'#'+passphrase


def check_max_filesize(chosen_file, max_size):
    """
    Checks file sizes for host
    """
    if os.path.getsize(chosen_file) > max_size:
        return False
    else:
        return True

def check_max_filesize ( chosen_file , max_size ) : if os . path . getsize ( chosen_file ) > max_size : return False else : return True


def parse_arguments(args, clone_list):
    """
    Makes parsing arguments a function.
    """
    returned_string=""
    host_number = args.host
    if args.show_list:
        print(generate_host_string(clone_list, "Available hosts: "))
        exit()
    if args.decrypt:
        for i in args.files:
            print(decrypt_files(i))
            exit()
    if args.files:
        for i in args.files:
            if args.limit_size:
                if args.host == host_number and host_number is not None:
                    if not check_max_filesize(i, clone_list[host_number][3]):
                        host_number = None
                for n, host in enumerate(clone_list):
                    if not check_max_filesize(i, host[3]):
                        clone_list[n] = None
                if not clone_list:
                    print('None of the clones is able to support so big file.')
            if args.no_cloudflare:
                if args.host == host_number and host_number is not None and not clone_list[host_number][4]:
                    print("This host uses Cloudflare, please choose different host.")
                    exit(1)
                else:
                    for n, host in enumerate(clone_list):
                        if not host[4]:
                            clone_list[n] = None
            clone_list = list(filter(None, clone_list))
            if host_number is None or args.host != host_number:
                host_number = random.randrange(0, len(clone_list))
            while True:
                try:
                    if args.encrypt:
                        returned_string = encrypt_files(clone_list[host_number], args.only_link, i)
                    else:
                        returned_string = upload_files(open(i, 'rb'), \
                              clone_list[host_number], args.only_link, i)
                    if args.only_link:
                        print(returned_string[0])
                    else:
                        print(returned_string)
                except IndexError:
                    #print('Selected server (' + clone_list[host_number][0] + ') is offline.')
                    #print('Trying other host.')
                    host_number = random.randrange(0, len(clone_list))
                    continue
                except IsADirectoryError:
                    print('limf does not support directory upload, if you want to upload ' \
                          'every file in directory use limf {}/*.'.format(i.replace('/', '')))

                if args.log:
                    with open(os.path.expanduser(args.logfile), "a+") as logfile:
                        if args.only_link:
                            logfile.write(returned_string[1])
                        else:
                            logfile.write(returned_string)
                        logfile.write("\n")
                break
    else:
        print("limf: try 'limf -h' for more information")


def upload_files(selected_file, selected_host, only_link, file_name):
    """
    Uploads selected file to the host, thanks to the fact that
    every pomf.se based site has pretty much the same architecture.
    """
    try:
        answer = requests.post(
            url=selected_host[0]+"upload.php",
            files={'files[]':selected_file})
        file_name_1 = re.findall(r'"url": *"((h.+\/){0,1}(.+?))"[,\}]', \
            answer.text.replace("\\", ""))[0][2]
        if only_link:
            return [selected_host[1]+file_name_1, "{}: {}{}".format(file_name, selected_host[1], file_name_1)]
        else:
            return "{}: {}{}".format(file_name, selected_host[1], file_name_1)
    except requests.exceptions.ConnectionError:
        print(file_name + ' couldn\'t be uploaded to ' + selected_host[0])

def
upload_files
(
selected_file
,
selected_host
,
only_link
,
file_name
)
:
try
:
answer
=
requests
.
post
(
url
=
selected_host
[
0
]
+
"upload.php"
,
files
=
{
'files[]'
:
selected_file
}
)
file_name_1
=
re
.
findall
(
r'"url": *"((h.+\/){0,1}(.+?))"[,\}]'
,
answer
.
text
.
replace
(
"\\"
,
""
)
)
[
0
]
[
2
]
if
only_link
:
return
[
selected_host
[
1
]
+
file_name_1
,
"{}: {}{}"
.
format
(
file_name
,
selected_host
[
1
]
,
file_name_1
)
]
else
:
return
"{}: {}{}"
.
format
(
file_name
,
selected_host
[
1
]
,
file_name_1
)
except
requests
.
exceptions
.
ConnectionError
:
print
(
file_name
+
' couldn\'t be uploaded to '
+
selected_host
[
0
]
)

Uploads selected file to the host thanks to the fact that every pomf . se based site has pretty much the same architecture .


def decrypt_files(file_link):
    """
    Decrypts file from entered links
    """
    if ENCRYPTION_DISABLED:
        print('For decryption please install gpg')
        exit()
    try:
        parsed_link = re.findall(r'(.*/(.*))#(.{30})', file_link)[0]
        req = urllib.request.Request(
            parsed_link[0],
            data=None,
            headers={
                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) ' \
                ' AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
                }
        )
        #downloads the file using fake useragent
        file_response = urllib.request.urlopen(req)
        file_to_decrypt = file_response.read()
        #decrypts the data using piping to ggp
        decrypt_r, decrypt_w = os.pipe()
        cmd = 'gpg --batch --decrypt --passphrase-fd {}'.format(decrypt_r)
        decrypt_output = Popen(shlex.split(cmd), stdout=PIPE, stdin=PIPE, stderr=PIPE, \
                         pass_fds=(decrypt_r,))
        os.close(decrypt_r)
        open(decrypt_w, 'w').write(parsed_link[2])
        decrypted_data, stderr = decrypt_output.communicate(file_to_decrypt)
        with open(parsed_link[1], 'wb') as decrypted_file:
            decrypted_file.write(decrypted_data)
        return parsed_link[1] + ' is decrypted and saved.'
    except IndexError:
        return 'Please enter valid link.'


def from_schema(self, schema_node, base_name=None):
        """
        Creates a Swagger definition from a colander schema.

        :param schema_node:
            Colander schema to be transformed into a Swagger definition.
        :param base_name:
            Schema alternative title.

        :rtype: dict
        :returns: Swagger schema.
        """
        return self._ref_recursive(self.type_converter(schema_node), self.ref, base_name)

def from_schema ( self , schema_node , base_name = None ) : return self . _ref_recursive ( self . type_converter ( schema_node ) , self . ref , base_name )

Creates a Swagger definition from a colander schema .


def from_schema(self, schema_node):
        """
        Creates a list of Swagger params from a colander request schema.

        :param schema_node:
            Request schema to be transformed into Swagger.
        :param validators:
            Validators used in colander with the schema.

        :rtype: list
        :returns: List of Swagger parameters.
        """

        params = []

        for param_schema in schema_node.children:
            location = param_schema.name
            if location is 'body':
                name = param_schema.__class__.__name__
                if name == 'body':
                    name = schema_node.__class__.__name__ + 'Body'
                param = self.parameter_converter(location,
                                                 param_schema)
                param['name'] = name
                if self.ref:
                    param = self._ref(param)
                params.append(param)

            elif location in (('path', 'header', 'headers', 'querystring', 'GET')):
                for node_schema in param_schema.children:
                    param = self.parameter_converter(location, node_schema)
                    if self.ref:
                        param = self._ref(param)
                    params.append(param)

        return params

def from_schema ( self , schema_node ) : params = [ ] for param_schema in schema_node . children : location = param_schema . name if location is 'body' : name = param_schema . __class__ . __name__ if name == 'body' : name = schema_node . __class__ . __name__ + 'Body' param = self . parameter_converter ( location , param_schema ) param [ 'name' ] = name if self . ref : param = self . _ref ( param ) params . append ( param ) elif location in ( ( 'path' , 'header' , 'headers' , 'querystring' , 'GET' ) ) : for node_schema in param_schema . children : param = self . parameter_converter ( location , node_schema ) if self . ref : param = self . _ref ( param ) params . append ( param ) return params

Creates a list of Swagger params from a colander request schema .


def from_path(self, path):
        """
        Create a list of Swagger path params from a cornice service path.

        :type path: string
        :rtype: list
        """
        path_components = path.split('/')
        param_names = [comp[1:-1] for comp in path_components
                       if comp.startswith('{') and comp.endswith('}')]

        params = []
        for name in param_names:
            param_schema = colander.SchemaNode(colander.String(), name=name)
            param = self.parameter_converter('path', param_schema)
            if self.ref:
                param = self._ref(param)
            params.append(param)

        return params

def from_path ( self , path ) : path_components = path . split ( '/' ) param_names = [ comp [ 1 : - 1 ] for comp in path_components if comp . startswith ( '{' ) and comp . endswith ( '}' ) ] params = [ ] for name in param_names : param_schema = colander . SchemaNode ( colander . String ( ) , name = name ) param = self . parameter_converter ( 'path' , param_schema ) if self . ref : param = self . _ref ( param ) params . append ( param ) return params

Create a list of Swagger path params from a cornice service path .


def _ref(self, param, base_name=None):
        """
        Store a parameter schema and return a reference to it.

        :param schema:
            Swagger parameter definition.
        :param base_name:
            Name that should be used for the reference.

        :rtype: dict
        :returns: JSON pointer to the original parameter definition.
        """

        name = base_name or param.get('title', '') or param.get('name', '')

        pointer = self.json_pointer + name
        self.parameter_registry[name] = param

        return {'$ref': pointer}

def _ref ( self , param , base_name = None ) : name = base_name or param . get ( 'title' , '' ) or param . get ( 'name' , '' ) pointer = self . json_pointer + name self . parameter_registry [ name ] = param return { '$ref' : pointer }

Store a parameter schema and return a reference to it .


def from_schema_mapping(self, schema_mapping):
        """
        Creates a Swagger response object from a dict of response schemas.

        :param schema_mapping:
            Dict with entries matching ``{status_code: response_schema}``.
        :rtype: dict
        :returns: Response schema.
        """

        responses = {}

        for status, response_schema in schema_mapping.items():

            response = {}
            if response_schema.description:
                response['description'] = response_schema.description
            else:
                raise CorniceSwaggerException('Responses must have a description.')

            for field_schema in response_schema.children:
                location = field_schema.name

                if location == 'body':
                    title = field_schema.__class__.__name__
                    if title == 'body':
                        title = response_schema.__class__.__name__ + 'Body'
                    field_schema.title = title
                    response['schema'] = self.definitions.from_schema(field_schema)

                elif location in ('header', 'headers'):
                    header_schema = self.type_converter(field_schema)
                    headers = header_schema.get('properties')
                    if headers:
                        # Response headers doesn't accept titles
                        for header in headers.values():
                            header.pop('title')

                        response['headers'] = headers

            pointer = response_schema.__class__.__name__
            if self.ref:
                response = self._ref(response, pointer)
            responses[status] = response

        return responses

def
from_schema_mapping
(
self
,
schema_mapping
)
:
responses
=
{
}
for
status
,
response_schema
in
schema_mapping
.
items
(
)
:
response
=
{
}
if
response_schema
.
description
:
response
[
'description'
]
=
response_schema
.
description
else
:
raise
CorniceSwaggerException
(
'Responses must have a description.'
)
for
field_schema
in
response_schema
.
children
:
location
=
field_schema
.
name
if
location
==
'body'
:
title
=
field_schema
.
__class__
.
__name__
if
title
==
'body'
:
title
=
response_schema
.
__class__
.
__name__
+
'Body'
field_schema
.
title
=
title
response
[
'schema'
]
=
self
.
definitions
.
from_schema
(
field_schema
)
elif
location
in
(
'header'
,
'headers'
)
:
header_schema
=
self
.
type_converter
(
field_schema
)
headers
=
header_schema
.
get
(
'properties'
)
if
headers
:
for
header
in
headers
.
values
(
)
:
header
.
pop
(
'title'
)
response
[
'headers'
]
=
headers
pointer
=
response_schema
.
__class__
.
__name__
if
self
.
ref
:
response
=
self
.
_ref
(
response
,
pointer
)
responses
[
status
]
=
response
return
responses

Creates a Swagger response object from a dict of response schemas .


def _ref(self, resp, base_name=None):
        """
        Store a response schema and return a reference to it.

        :param schema:
            Swagger response definition.
        :param base_name:
            Name that should be used for the reference.

        :rtype: dict
        :returns: JSON pointer to the original response definition.
        """

        name = base_name or resp.get('title', '') or resp.get('name', '')

        pointer = self.json_pointer + name
        self.response_registry[name] = resp

        return {'$ref': pointer}

def _ref ( self , resp , base_name = None ) : name = base_name or resp . get ( 'title' , '' ) or resp . get ( 'name' , '' ) pointer = self . json_pointer + name self . response_registry [ name ] = resp return { '$ref' : pointer }

Store a response schema and return a reference to it .


def generate(self, title=None, version=None, base_path=None,
                 info=None, swagger=None, **kwargs):
        """Generate a Swagger 2.0 documentation. Keyword arguments may be used
        to provide additional information to build methods as such ignores.

        :param title:
            The name presented on the swagger document.
        :param version:
            The version of the API presented on the swagger document.
        :param base_path:
            The path that all requests to the API must refer to.
        :param info:
            Swagger info field.
        :param swagger:
            Extra fields that should be provided on the swagger documentation.

        :rtype: dict
        :returns: Full OpenAPI/Swagger compliant specification for the application.
        """
        title = title or self.api_title
        version = version or self.api_version
        info = info or self.swagger.get('info', {})
        swagger = swagger or self.swagger
        base_path = base_path or self.base_path

        swagger = swagger.copy()
        info.update(title=title, version=version)
        swagger.update(swagger='2.0', info=info, basePath=base_path)

        paths, tags = self._build_paths()

        # Update the provided tags with the extracted ones preserving order
        if tags:
            swagger.setdefault('tags', [])
            tag_names = {t['name'] for t in swagger['tags']}
            for tag in tags:
                if tag['name'] not in tag_names:
                    swagger['tags'].append(tag)

        # Create/Update swagger sections with extracted values where not provided
        if paths:
            swagger.setdefault('paths', {})
            merge_dicts(swagger['paths'], paths)

        definitions = self.definitions.definition_registry
        if definitions:
            swagger.setdefault('definitions', {})
            merge_dicts(swagger['definitions'], definitions)

        parameters = self.parameters.parameter_registry
        if parameters:
            swagger.setdefault('parameters', {})
            merge_dicts(swagger['parameters'], parameters)

        responses = self.responses.response_registry
        if responses:
            swagger.setdefault('responses', {})
            merge_dicts(swagger['responses'], responses)

        return swagger

def generate ( self , title = None , version = None , base_path = None , info = None , swagger = None , ** kwargs ) : title = title or self . api_title version = version or self . api_version info = info or self . swagger . get ( 'info' , { } ) swagger = swagger or self . swagger base_path = base_path or self . base_path swagger = swagger . copy ( ) info . update ( title = title , version = version ) swagger . update ( swagger = '2.0' , info = info , basePath = base_path ) paths , tags = self . _build_paths ( ) if tags : swagger . setdefault ( 'tags' , [ ] ) tag_names = { t [ 'name' ] for t in swagger [ 'tags' ] } for tag in tags : if tag [ 'name' ] not in tag_names : swagger [ 'tags' ] . append ( tag ) if paths : swagger . setdefault ( 'paths' , { } ) merge_dicts ( swagger [ 'paths' ] , paths ) definitions = self . definitions . definition_registry if definitions : swagger . setdefault ( 'definitions' , { } ) merge_dicts ( swagger [ 'definitions' ] , definitions ) parameters = self . parameters . parameter_registry if parameters : swagger . setdefault ( 'parameters' , { } ) merge_dicts ( swagger [ 'parameters' ] , parameters ) responses = self . responses . response_registry if responses : swagger . setdefault ( 'responses' , { } ) merge_dicts ( swagger [ 'responses' ] , responses ) return swagger

Generate a Swagger 2 . 0 documentation . Keyword arguments may be used to provide additional information to build methods as such ignores .


def _build_paths(self):
        """
        Build the Swagger "paths" and "tags" attributes from cornice service
        definitions.
        """
        paths = {}
        tags = []

        for service in self.services:
            path, path_obj = self._extract_path_from_service(service)

            service_tags = getattr(service, 'tags', [])
            self._check_tags(service_tags)
            tags = self._get_tags(tags, service_tags)

            for method, view, args in service.definitions:

                if method.lower() in map(str.lower, self.ignore_methods):
                    continue

                op = self._extract_operation_from_view(view, args)

                if any(ctype in op.get('consumes', []) for ctype in self.ignore_ctypes):
                    continue

                # XXX: Swagger doesn't support different schemas for for a same method
                # with different ctypes as cornice. If this happens, you may ignore one
                # content-type from the documentation otherwise we raise an Exception
                # Related to https://github.com/OAI/OpenAPI-Specification/issues/146
                previous_definition = path_obj.get(method.lower())
                if previous_definition:
                    raise CorniceSwaggerException(("Swagger doesn't support multiple "
                                                   "views for a same method. You may "
                                                   "ignore one."))

                # If tag not defined and a default tag is provided
                if 'tags' not in op and self.default_tags:
                    if callable(self.default_tags):
                        op['tags'] = self.default_tags(service, method)
                    else:
                        op['tags'] = self.default_tags

                op_tags = op.get('tags', [])
                self._check_tags(op_tags)

                # Add service tags
                if service_tags:
                    new_tags = service_tags + op_tags
                    op['tags'] = list(OrderedDict.fromkeys(new_tags))

                # Add method tags to root tags
                tags = self._get_tags(tags, op_tags)

                # If operation id is not defined and a default generator is provided
                if 'operationId' not in op and self.default_op_ids:
                    if not callable(self.default_op_ids):
                        raise CorniceSwaggerException('default_op_id should be a callable.')
                    op['operationId'] = self.default_op_ids(service, method)

                # If security options not defined and default is provided
                if 'security' not in op and self.default_security:
                    if callable(self.default_security):
                        op['security'] = self.default_security(service, method)
                    else:
                        op['security'] = self.default_security

                if not isinstance(op.get('security', []), list):
                    raise CorniceSwaggerException('security should be a list or callable')

                path_obj[method.lower()] = op
            paths[path] = path_obj

        return paths, tags

def
_build_paths
(
self
)
:
paths
=
{
}
tags
=
[
]
for
service
in
self
.
services
:
path
,
path_obj
=
self
.
_extract_path_from_service
(
service
)
service_tags
=
getattr
(
service
,
'tags'
,
[
]
)
self
.
_check_tags
(
service_tags
)
tags
=
self
.
_get_tags
(
tags
,
service_tags
)
for
method
,
view
,
args
in
service
.
definitions
:
if
method
.
lower
(
)
in
map
(
str
.
lower
,
self
.
ignore_methods
)
:
continue
op
=
self
.
_extract_operation_from_view
(
view
,
args
)
if
any
(
ctype
in
op
.
get
(
'consumes'
,
[
]
)
for
ctype
in
self
.
ignore_ctypes
)
:
continue
previous_definition
=
path_obj
.
get
(
method
.
lower
(
)
)
if
previous_definition
:
raise
CorniceSwaggerException
(
(
"Swagger doesn't support multiple "
"views for a same method. You may "
"ignore one."
)
)
if
'tags'
not
in
op
and
self
.
default_tags
:
if
callable
(
self
.
default_tags
)
:
op
[
'tags'
]
=
self
.
default_tags
(
service
,
method
)
else
:
op
[
'tags'
]
=
self
.
default_tags
op_tags
=
op
.
get
(
'tags'
,
[
]
)
self
.
_check_tags
(
op_tags
)
if
service_tags
:
new_tags
=
service_tags
+
op_tags
op
[
'tags'
]
=
list
(
OrderedDict
.
fromkeys
(
new_tags
)
)
tags
=
self
.
_get_tags
(
tags
,
op_tags
)
if
'operationId'
not
in
op
and
self
.
default_op_ids
:
if
not
callable
(
self
.
default_op_ids
)
:
raise
CorniceSwaggerException
(
'default_op_id should be a callable.'
)
op
[
'operationId'
]
=
self
.
default_op_ids
(
service
,
method
)
if
'security'
not
in
op
and
self
.
default_security
:
if
callable
(
self
.
default_security
)
:
op
[
'security'
]
=
self
.
default_security
(
service
,
method
)
else
:
op
[
'security'
]
=
self
.
default_security
if
not
isinstance
(
op
.
get
(
'security'
,
[
]
)
,
list
)
:
raise
CorniceSwaggerException
(
'security should be a list or callable'
)
path_obj
[
method
.
lower
(
)
]
=
op
paths
[
path
]
=
path_obj
return
paths
,
tags

Build the Swagger paths and tags attributes from cornice service definitions .


def _extract_path_from_service(self, service):
        """
        Extract path object and its parameters from service definitions.

        :param service:
            Cornice service to extract information from.

        :rtype: dict
        :returns: Path definition.
        """

        path_obj = {}
        path = service.path
        route_name = getattr(service, 'pyramid_route', None)
        # handle services that don't create fresh routes,
        # we still need the paths so we need to grab pyramid introspector to
        # extract that information
        if route_name:
            # avoid failure if someone forgets to pass registry
            registry = self.pyramid_registry or get_current_registry()
            route_intr = registry.introspector.get('routes', route_name)
            if route_intr:
                path = route_intr['pattern']
            else:
                msg = 'Route `{}` is not found by ' \
                      'pyramid introspector'.format(route_name)
                raise ValueError(msg)

        # handle traverse and subpath as regular parameters
        # docs.pylonsproject.org/projects/pyramid/en/latest/narr/hybrid.html
        for subpath_marker in ('*subpath', '*traverse'):
            path = path.replace(subpath_marker, '{subpath}')

        # Extract path parameters
        parameters = self.parameters.from_path(path)
        if parameters:
            path_obj['parameters'] = parameters

        return path, path_obj

def
_extract_path_from_service
(
self
,
service
)
:
path_obj
=
{
}
path
=
service
.
path
route_name
=
getattr
(
service
,
'pyramid_route'
,
)
if
route_name
:
registry
=
self
.
pyramid_registry
or
get_current_registry
(
)
route_intr
=
registry
.
introspector
.
get
(
'routes'
,
route_name
)
if
route_intr
:
path
=
route_intr
[
'pattern'
]
else
:
msg
=
'Route `{}` is not found by '
'pyramid introspector'
.
format
(
route_name
)
raise
ValueError
(
msg
)
for
subpath_marker
in
(
'*subpath'
,
'*traverse'
)
:
path
=
path
.
replace
(
subpath_marker
,
'{subpath}'
)
parameters
=
self
.
parameters
.
from_path
(
path
)
if
parameters
:
path_obj
[
'parameters'
]
=
parameters
return
path
,
path_obj

Extract path object and its parameters from service definitions .


def _extract_operation_from_view(self, view, args):
        """
        Extract swagger operation details from colander view definitions.

        :param view:
            View to extract information from.
        :param args:
            Arguments from the view decorator.

        :rtype: dict
        :returns: Operation definition.
        """

        op = {
            'responses': {
                'default': {
                    'description': 'UNDOCUMENTED RESPONSE'
                }
            },
        }

        # If 'produces' are not defined in the view, try get from renderers
        renderer = args.get('renderer', '')

        if "json" in renderer:  # allows for "json" or "simplejson"
            produces = ['application/json']
        elif renderer == 'xml':
            produces = ['text/xml']
        else:
            produces = None

        if produces:
            op.setdefault('produces', produces)

        # Get explicit accepted content-types
        consumes = args.get('content_type')

        if consumes is not None:
            # convert to a list, if it's not yet one
            consumes = to_list(consumes)

            # It is possible to add callables for content_type, so we have to
            # to filter those out, since we cannot evaluate those here.
            consumes = [x for x in consumes if not callable(x)]
            op['consumes'] = consumes

        # Get parameters from view schema
        is_colander = self._is_colander_schema(args)
        if is_colander:
            schema = self._extract_transform_colander_schema(args)
            parameters = self.parameters.from_schema(schema)
        else:
            # Bail out for now
            parameters = None
        if parameters:
            op['parameters'] = parameters

        # Get summary from docstring
        if isinstance(view, six.string_types):
            if 'klass' in args:
                ob = args['klass']
                view_ = getattr(ob, view.lower())
                docstring = trim(view_.__doc__)
        else:
            docstring = str(trim(view.__doc__))

        if docstring and self.summary_docstrings:
            op['summary'] = docstring

        # Get response definitions
        if 'response_schemas' in args:
            op['responses'] = self.responses.from_schema_mapping(args['response_schemas'])

        # Get response tags
        if 'tags' in args:
            op['tags'] = args['tags']

        # Get response operationId
        if 'operation_id' in args:
            op['operationId'] = args['operation_id']

        # Get security policies
        if 'api_security' in args:
            op['security'] = args['api_security']

        return op

def
_extract_operation_from_view
(
self
,
view
,
args
)
:
op
=
{
'responses'
:
{
'default'
:
{
'description'
:
'UNDOCUMENTED RESPONSE'
}
}
,
}
renderer
=
args
.
get
(
'renderer'
,
''
)
if
"json"
in
renderer
:
produces
=
[
'application/json'
]
elif
renderer
==
'xml'
:
produces
=
[
'text/xml'
]
else
:
produces
=
if
produces
:
op
.
setdefault
(
'produces'
,
produces
)
consumes
=
args
.
get
(
'content_type'
)
if
consumes
is
not
:
consumes
=
to_list
(
consumes
)
consumes
=
[
x
for
x
in
consumes
if
not
callable
(
x
)
]
op
[
'consumes'
]
=
consumes
is_colander
=
self
.
_is_colander_schema
(
args
)
if
is_colander
:
schema
=
self
.
_extract_transform_colander_schema
(
args
)
parameters
=
self
.
parameters
.
from_schema
(
schema
)
else
:
parameters
=
if
parameters
:
op
[
'parameters'
]
=
parameters
if
isinstance
(
view
,
six
.
string_types
)
:
if
'klass'
in
args
:
ob
=
args
[
'klass'
]
view_
=
getattr
(
ob
,
view
.
lower
(
)
)
docstring
=
trim
(
view_
.
__doc__
)
else
:
docstring
=
str
(
trim
(
view
.
__doc__
)
)
if
docstring
and
self
.
summary_docstrings
:
op
[
'summary'
]
=
docstring
if
'response_schemas'
in
args
:
op
[
'responses'
]
=
self
.
responses
.
from_schema_mapping
(
args
[
'response_schemas'
]
)
if
'tags'
in
args
:
op
[
'tags'
]
=
args
[
'tags'
]
if
'operation_id'
in
args
:
op
[
'operationId'
]
=
args
[
'operation_id'
]
if
'api_security'
in
args
:
op
[
'security'
]
=
args
[
'api_security'
]
return
op

Extract swagger operation details from colander view definitions .


def _extract_transform_colander_schema(self, args):
        """
        Extract schema from view args and transform it using
        the pipeline of schema transformers

        :param args:
            Arguments from the view decorator.

        :rtype: colander.MappingSchema()
        :returns: View schema cloned and transformed
        """

        schema = args.get('schema', colander.MappingSchema())
        if not isinstance(schema, colander.Schema):
            schema = schema()
        schema = schema.clone()
        for transformer in self.schema_transformers:
            schema = transformer(schema, args)
        return schema

def _extract_transform_colander_schema ( self , args ) : schema = args . get ( 'schema' , colander . MappingSchema ( ) ) if not isinstance ( schema , colander . Schema ) : schema = schema ( ) schema = schema . clone ( ) for transformer in self . schema_transformers : schema = transformer ( schema , args ) return schema


def convert(self, schema_node, definition_handler):
        """
        Convert node schema into a parameter object.
        """

        converted = {
            'name': schema_node.name,
            'in': self._in,
            'required': schema_node.required
        }
        if schema_node.description:
            converted['description'] = schema_node.description

        if schema_node.default:
            converted['default'] = schema_node.default

        schema = definition_handler(schema_node)
        # Parameters shouldn't have a title
        schema.pop('title', None)
        converted.update(schema)

        if schema.get('type') == 'array':
            converted['items'] = {'type': schema['items']['type']}

        return converted

def convert ( self , schema_node , definition_handler ) : converted = { 'name' : schema_node . name , 'in' : self . _in , 'required' : schema_node . required } if schema_node . description : converted [ 'description' ] = schema_node . description if schema_node . default : converted [ 'default' ] = schema_node . default schema = definition_handler ( schema_node ) schema . pop ( 'title' , None ) converted . update ( schema ) if schema . get ( 'type' ) == 'array' : converted [ 'items' ] = { 'type' : schema [ 'items' ] [ 'type' ] } return converted

Convert node schema into a parameter object .


def merge_dicts(base, changes):
    """Merge b into a recursively, without overwriting values.

    :param base: the dict that will be altered.
    :param changes: changes to update base.
    """
    for k, v in changes.items():
        if isinstance(v, dict):
            merge_dicts(base.setdefault(k, {}), v)
        else:
            base.setdefault(k, v)

def merge_dicts ( base , changes ) : for k , v in changes . items ( ) : if isinstance ( v , dict ) : merge_dicts ( base . setdefault ( k , { } ) , v ) else : base . setdefault ( k , v )

Merge b into a recursively without overwriting values .


def get_transition_viewset_method(transition_name, **kwargs):
    '''
    Create a viewset method for the provided `transition_name`
    '''
    @detail_route(methods=['post'], **kwargs)
    def inner_func(self, request, pk=None, **kwargs):
        object = self.get_object()
        transition_method = getattr(object, transition_name)

        transition_method(by=self.request.user)

        if self.save_after_transition:
            object.save()

        serializer = self.get_serializer(object)
        return Response(serializer.data)

    return inner_func

def get_transition_viewset_method ( transition_name , ** kwargs ) : @ detail_route ( methods = [ 'post' ] , ** kwargs ) def inner_func ( self , request , pk = None , ** kwargs ) : object = self . get_object ( ) transition_method = getattr ( object , transition_name ) transition_method ( by = self . request . user ) if self . save_after_transition : object . save ( ) serializer = self . get_serializer ( object ) return Response ( serializer . data ) return inner_func

Create a viewset method for the provided transition_name


def get_viewset_transition_action_mixin(model, **kwargs):
    '''
    Find all transitions defined on `model`, then create a corresponding
    viewset action method for each and apply it to `Mixin`. Finally, return
    `Mixin`
    '''
    instance = model()

    class Mixin(object):
        save_after_transition = True

    transitions = instance.get_all_status_transitions()
    transition_names = set(x.name for x in transitions)
    for transition_name in transition_names:
        setattr(
            Mixin,
            transition_name,
            get_transition_viewset_method(transition_name, **kwargs)
        )

    return Mixin

def get_viewset_transition_action_mixin ( model , ** kwargs ) : instance = model ( ) class Mixin ( object ) : save_after_transition = True transitions = instance . get_all_status_transitions ( ) transition_names = set ( x . name for x in transitions ) for transition_name in transition_names : setattr ( Mixin , transition_name , get_transition_viewset_method ( transition_name , ** kwargs ) ) return Mixin

Find all transitions defined on model then create a corresponding viewset action method for each and apply it to Mixin . Finally return Mixin


def fresh_cookies(ctx, mold=''):
    """Refresh the project from the original cookiecutter template."""
    mold = mold or "https://github.com/Springerle/py-generic-project.git"  # TODO: URL from config
    tmpdir = os.path.join(tempfile.gettempdir(), "cc-upgrade-pygments-markdown-lexer")

    if os.path.isdir('.git'):
        # TODO: Ensure there are no local unstashed changes
        pass

    # Make a copy of the new mold version
    if os.path.isdir(tmpdir):
        shutil.rmtree(tmpdir)
    if os.path.exists(mold):
        shutil.copytree(mold, tmpdir, ignore=shutil.ignore_patterns(
            ".git", ".svn", "*~",
        ))
    else:
        ctx.run("git clone {} {}".format(mold, tmpdir))

    # Copy recorded "cookiecutter.json" into mold
    shutil.copy2("project.d/cookiecutter.json", tmpdir)

    with pushd('..'):
        ctx.run("cookiecutter --no-input {}".format(tmpdir))
    if os.path.exists('.git'):
        ctx.run("git status")

Refresh the project from the original cookiecutter template .


def ci(ctx):
    """Perform continuous integration tasks."""
    opts = ['']

    # 'tox' makes no sense in Travis
    if os.environ.get('TRAVIS', '').lower() == 'true':
        opts += ['test.pytest']
    else:
        opts += ['test.tox']

    ctx.run("invoke --echo --pty clean --all build --docs check --reports{}".format(' '.join(opts)))


def py_hash(key, num_buckets):
    """Generate a number in the range [0, num_buckets).

    Args:
        key (int): The key to hash.
        num_buckets (int): Number of buckets to use.

    Returns:
        The bucket number `key` computes to.

    Raises:
        ValueError: If `num_buckets` is not a positive number.
    """
    b, j = -1, 0

    if num_buckets < 1:
        raise ValueError('num_buckets must be a positive number')

    while j < num_buckets:
        b = int(j)
        key = ((key * long(2862933555777941757)) + 1) & 0xffffffffffffffff
        j = float(b + 1) * (float(1 << 31) / float((key >> 33) + 1))

    return int(b)

def
py_hash
(
key
,
num_buckets
)
:
b
,
j
=
1
,
0
if
num_buckets
<
1
:
raise
ValueError
(
'num_buckets must be a positive number'
)
while
j
<
num_buckets
:
b
=
int
(
j
)
key
=
(
(
key
*
long
(
2862933555777941757
)
)
+
1
)
&
0xffffffffffffffff
j
=
float
(
b
+
1
)
*
(
float
(
1
<<
31
)
/
float
(
(
key
>>
33
)
+
1
)
)
return
int
(
b
)

Generate a number in the range [ 0 num_buckets ) .


def setup(app):
    """ Initializer for Sphinx extension API.

        See http://www.sphinx-doc.org/en/stable/extdev/index.html#dev-extensions.
    """
    lexer = MarkdownLexer()
    for alias in lexer.aliases:
        app.add_lexer(alias, lexer)

    return dict(version=__version__)

def setup ( app ) : lexer = MarkdownLexer ( ) for alias in lexer . aliases : app . add_lexer ( alias , lexer ) return dict ( version = __version__ )


def load(self):
        """Return a dict of stats."""
        ret = {}

        # Read the mdstat file
        with open(self.get_path(), 'r') as f:
            # lines is a list of line (with \n)
            lines = f.readlines()

        # First line: get the personalities
        # The "Personalities" line tells you what RAID level the kernel currently supports.
        # This can be changed by either changing the raid modules or recompiling the kernel.
        # Possible personalities include: [raid0] [raid1] [raid4] [raid5] [raid6] [linear] [multipath] [faulty]
        ret['personalities'] = self.get_personalities(lines[0])

        # Second to last before line: Array definition
        ret['arrays'] = self.get_arrays(lines[1:-1], ret['personalities'])

        # Save the file content as it for the __str__ method
        self.content = reduce(lambda x, y: x + y, lines)

        return ret

def load ( self ) : ret = { } with open ( self . get_path ( ) , 'r' ) as f : lines = f . readlines ( ) ret [ 'personalities' ] = self . get_personalities ( lines [ 0 ] ) ret [ 'arrays' ] = self . get_arrays ( lines [ 1 : - 1 ] , ret [ 'personalities' ] ) self . content = reduce ( lambda x , y : x + y , lines ) return ret


def get_personalities(self, line):
        """Return a list of personalities readed from the input line."""
        return [split('\W+', i)[1] for i in line.split(':')[1].split(' ') if i.startswith('[')]

def
get_personalities
(
self
,
line
)
:
return
[
split
(
'\W+'
,
i
)
[
1
]
for
i
in
line
.
split
(
':'
)
[
1
]
.
split
(
' '
)
if
i
.
startswith
(
'['
)
]

Return a list of personalities readed from the input line .


def get_arrays(self, lines, personalities=[]):
        """Return a dict of arrays."""
        ret = {}

        i = 0
        while i < len(lines):
            try:
                # First array line: get the md device
                md_device = self.get_md_device_name(lines[i])
            except IndexError:
                # No array detected
                pass
            else:
                # Array detected
                if md_device is not None:
                    # md device line
                    ret[md_device] = self.get_md_device(lines[i], personalities)
                    # md config/status line
                    i += 1
                    ret[md_device].update(self.get_md_status(lines[i]))
            i += 1

        return ret

def get_arrays ( self , lines , personalities = [ ] ) : ret = { } i = 0 while i < len ( lines ) : try : md_device = self . get_md_device_name ( lines [ i ] ) except IndexError : pass else : if md_device is not None : ret [ md_device ] = self . get_md_device ( lines [ i ] , personalities ) i += 1 ret [ md_device ] . update ( self . get_md_status ( lines [ i ] ) ) i += 1 return ret


def get_md_device(self, line, personalities=[]):
        """Return a dict of md device define in the line."""
        ret = {}

        splitted = split('\W+', line)
        # Raid status
        # Active or 'started'. An inactive array is usually faulty.
        # Stopped arrays aren't visible here.
        ret['status'] = splitted[1]
        if splitted[2] in personalities:
            # Raid type (ex: RAID5)
            ret['type'] = splitted[2]
            # Array's components
            ret['components'] = self.get_components(line, with_type=True)
        else:
            # Raid type (ex: RAID5)
            ret['type'] = None
            # Array's components
            ret['components'] = self.get_components(line, with_type=False)

        return ret

def get_md_device ( self , line , personalities = [ ] ) : ret = { } splitted = split ( '\W+' , line ) ret [ 'status' ] = splitted [ 1 ] if splitted [ 2 ] in personalities : ret [ 'type' ] = splitted [ 2 ] ret [ 'components' ] = self . get_components ( line , with_type = True ) else : ret [ 'type' ] = None ret [ 'components' ] = self . get_components ( line , with_type = False ) return ret

Return a dict of md device define in the line .


def get_md_status(self, line):
        """Return a dict of md status define in the line."""
        ret = {}

        splitted = split('\W+', line)
        if len(splitted) < 7:
            ret['available'] = None
            ret['used'] = None
            ret['config'] = None
        else:
            # The final 2 entries on this line: [n/m] [UUUU_]
            # [n/m] means that ideally the array would have n devices however, currently, m devices are in use.
            # Obviously when m >= n then things are good.
            ret['available'] = splitted[-4]
            ret['used'] = splitted[-3]
            # [UUUU_] represents the status of each device, either U for up or _ for down.
            ret['config'] = splitted[-2]

        return ret

def get_md_status ( self , line ) : ret = { } splitted = split ( '\W+' , line ) if len ( splitted ) < 7 : ret [ 'available' ] = None ret [ 'used' ] = None ret [ 'config' ] = None else : ret [ 'available' ] = splitted [ - 4 ] ret [ 'used' ] = splitted [ - 3 ] ret [ 'config' ] = splitted [ - 2 ] return ret

Return a dict of md status define in the line .


def get_components(self, line, with_type=True):
        """Return a dict of components in the line.

        key: device name (ex: 'sdc1')
        value: device role number
        """
        ret = {}

        # Ignore (F) (see test 08)
        line2 = reduce(lambda x, y: x + y, split('\(.+\)', line))
        if with_type:
            splitted = split('\W+', line2)[3:]
        else:
            splitted = split('\W+', line2)[2:]
        ret = dict(zip(splitted[0::2], splitted[1::2]))

        return ret

def get_components ( self , line , with_type = True ) : ret = { } line2 = reduce ( lambda x , y : x + y , split ( '\(.+\)' , line ) ) if with_type : splitted = split ( '\W+' , line2 ) [ 3 : ] else : splitted = split ( '\W+' , line2 ) [ 2 : ] ret = dict ( zip ( splitted [ 0 : : 2 ] , splitted [ 1 : : 2 ] ) ) return ret

Return a dict of components in the line .


def register_receivers(app, config):
    """Register signal receivers which send events."""
    for event_name, event_config in config.items():
        event_builders = [
            obj_or_import_string(func)
            for func in event_config.get('event_builders', [])
        ]

        signal = obj_or_import_string(event_config['signal'])
        signal.connect(
            EventEmmiter(event_name, event_builders), sender=app, weak=False
        )

def register_receivers ( app , config ) : for event_name , event_config in config . items ( ) : event_builders = [ obj_or_import_string ( func ) for func in event_config . get ( 'event_builders' , [ ] ) ] signal = obj_or_import_string ( event_config [ 'signal' ] ) signal . connect ( EventEmmiter ( event_name , event_builders ) , sender = app , weak = False )

Register signal receivers which send events .


def set_scheduled(self):
        """
        Returns True if state was successfully changed from idle to scheduled.
        """
        with self._idle_lock:
            if self._idle:
                self._idle = False
                return True
        return False

def set_scheduled ( self ) : with self . _idle_lock : if self . _idle : self . _idle = False return True return False

Returns True if state was successfully changed from idle to scheduled .


def post(self, **kwargs):
        """Get statistics."""
        data = request.get_json(force=False)
        if data is None:
            data = {}
        result = {}
        for query_name, config in data.items():
            if config is None or not isinstance(config, dict) \
                    or (set(config.keys()) != {'stat', 'params'} and
                        set(config.keys()) != {'stat'}):
                raise InvalidRequestInputError(
                    'Invalid Input. It should be of the form '
                    '{ STATISTIC_NAME: { "stat": STAT_TYPE, '
                    '"params": STAT_PARAMS \}}'
                )
            stat = config['stat']
            params = config.get('params', {})
            try:
                query_cfg = current_stats.queries[stat]
            except KeyError:
                raise UnknownQueryError(stat)

            permission = current_stats.permission_factory(stat, params)
            if permission is not None and not permission.can():
                message = ('You do not have a permission to query the '
                           'statistic "{}" with those '
                           'parameters'.format(stat))
                if current_user.is_authenticated:
                    abort(403, message)
                abort(401, message)
            try:
                query = query_cfg.query_class(**query_cfg.query_config)
                result[query_name] = query.run(**params)
            except ValueError as e:
                raise InvalidRequestInputError(e.args[0])
            except NotFoundError as e:
                return None
        return self.make_response(result)


def _get_oldest_event_timestamp(self):
        """Search for the oldest event timestamp."""
        # Retrieve the oldest event in order to start aggregation
        # from there
        query_events = Search(
            using=self.client,
            index=self.event_index
        )[0:1].sort(
            {'timestamp': {'order': 'asc'}}
        )
        result = query_events.execute()
        # There might not be any events yet if the first event have been
        # indexed but the indices have not been refreshed yet.
        if len(result) == 0:
            return None
        return parser.parse(result[0]['timestamp'])

def _get_oldest_event_timestamp ( self ) : query_events = Search ( using = self . client , index = self . event_index ) [ 0 : 1 ] . sort ( { 'timestamp' : { 'order' : 'asc' } } ) result = query_events . execute ( ) if len ( result ) == 0 : return None return parser . parse ( result [ 0 ] [ 'timestamp' ] )


def get_bookmark(self):
        """Get last aggregation date."""
        if not Index(self.aggregation_alias,
                     using=self.client).exists():
            if not Index(self.event_index,
                         using=self.client).exists():
                return datetime.date.today()
            return self._get_oldest_event_timestamp()

        # retrieve the oldest bookmark
        query_bookmark = Search(
            using=self.client,
            index=self.aggregation_alias,
            doc_type=self.bookmark_doc_type
        )[0:1].sort(
            {'date': {'order': 'desc'}}
        )
        bookmarks = query_bookmark.execute()
        # if no bookmark is found but the index exist, the bookmark was somehow
        # lost or never written, so restart from the beginning
        if len(bookmarks) == 0:
            return self._get_oldest_event_timestamp()

        # change it to doc_id_suffix
        bookmark = datetime.datetime.strptime(bookmarks[0].date,
                                              self.doc_id_suffix)
        return bookmark

def get_bookmark ( self ) : if not Index ( self . aggregation_alias , using = self . client ) . exists ( ) : if not Index ( self . event_index , using = self . client ) . exists ( ) : return datetime . date . today ( ) return self . _get_oldest_event_timestamp ( ) query_bookmark = Search ( using = self . client , index = self . aggregation_alias , doc_type = self . bookmark_doc_type ) [ 0 : 1 ] . sort ( { 'date' : { 'order' : 'desc' } } ) bookmarks = query_bookmark . execute ( ) if len ( bookmarks ) == 0 : return self . _get_oldest_event_timestamp ( ) bookmark = datetime . datetime . strptime ( bookmarks [ 0 ] . date , self . doc_id_suffix ) return bookmark


def set_bookmark(self):
        """Set bookmark for starting next aggregation."""
        def _success_date():
            bookmark = {
                'date': self.new_bookmark or datetime.datetime.utcnow().
                strftime(self.doc_id_suffix)
            }

            yield dict(_index=self.last_index_written,
                       _type=self.bookmark_doc_type,
                       _source=bookmark)
        if self.last_index_written:
            bulk(self.client,
                 _success_date(),
                 stats_only=True)

def set_bookmark ( self ) : def _success_date ( ) : bookmark = { 'date' : self . new_bookmark or datetime . datetime . utcnow ( ) . strftime ( self . doc_id_suffix ) } yield dict ( _index = self . last_index_written , _type = self . bookmark_doc_type , _source = bookmark ) if self . last_index_written : bulk ( self . client , _success_date ( ) , stats_only = True )

Set bookmark for starting next aggregation .


def _format_range_dt(self, d):
        """Format range filter datetime to the closest aggregation interval."""
        if not isinstance(d, six.string_types):
            d = d.isoformat()
        return '{0}||/{1}'.format(
            d, self.dt_rounding_map[self.aggregation_interval])

def _format_range_dt ( self , d ) : if not isinstance ( d , six . string_types ) : d = d . isoformat ( ) return '{0}||/{1}' . format ( d , self . dt_rounding_map [ self . aggregation_interval ] )

Format range filter datetime to the closest aggregation interval .


def agg_iter(self, lower_limit=None, upper_limit=None):
        """Aggregate and return dictionary to be indexed in ES."""
        lower_limit = lower_limit or self.get_bookmark().isoformat()
        upper_limit = upper_limit or (
            datetime.datetime.utcnow().replace(microsecond=0).isoformat())
        aggregation_data = {}

        self.agg_query = Search(using=self.client,
                                index=self.event_index).\
            filter('range', timestamp={
                'gte': self._format_range_dt(lower_limit),
                'lte': self._format_range_dt(upper_limit)})

        # apply query modifiers
        for modifier in self.query_modifiers:
            self.agg_query = modifier(self.agg_query)

        hist = self.agg_query.aggs.bucket(
            'histogram',
            'date_histogram',
            field='timestamp',
            interval=self.aggregation_interval
        )
        terms = hist.bucket(
            'terms', 'terms', field=self.aggregation_field, size=0
        )
        top = terms.metric(
            'top_hit', 'top_hits', size=1, sort={'timestamp': 'desc'}
        )
        for dst, (metric, src, opts) in self.metric_aggregation_fields.items():
            terms.metric(dst, metric, field=src, **opts)

        results = self.agg_query.execute()
        index_name = None
        for interval in results.aggregations['histogram'].buckets:
            interval_date = datetime.datetime.strptime(
                interval['key_as_string'], '%Y-%m-%dT%H:%M:%S')
            for aggregation in interval['terms'].buckets:
                aggregation_data['timestamp'] = interval_date.isoformat()
                aggregation_data[self.aggregation_field] = aggregation['key']
                aggregation_data['count'] = aggregation['doc_count']

                if self.metric_aggregation_fields:
                    for f in self.metric_aggregation_fields:
                        aggregation_data[f] = aggregation[f]['value']

                doc = aggregation.top_hit.hits.hits[0]['_source']
                for destination, source in self.copy_fields.items():
                    if isinstance(source, six.string_types):
                        aggregation_data[destination] = doc[source]
                    else:
                        aggregation_data[destination] = source(
                            doc,
                            aggregation_data
                        )

                index_name = 'stats-{0}-{1}'.\
                             format(self.event,
                                    interval_date.strftime(
                                        self.index_name_suffix))
                self.indices.add(index_name)
                yield dict(_id='{0}-{1}'.
                           format(aggregation['key'],
                                  interval_date.strftime(
                                      self.doc_id_suffix)),
                           _index=index_name,
                           _type=self.aggregation_doc_type,
                           _source=aggregation_data)
        self.last_index_written = index_name

def agg_iter ( self , lower_limit = None , upper_limit = None ) : lower_limit = lower_limit or self . get_bookmark ( ) . isoformat ( ) upper_limit = upper_limit or ( datetime . datetime . utcnow ( ) . replace ( microsecond = 0 ) . isoformat ( ) ) aggregation_data = { } self . agg_query = Search ( using = self . client , index = self . event_index ) . filter ( 'range' , timestamp = { 'gte' : self . _format_range_dt ( lower_limit ) , 'lte' : self . _format_range_dt ( upper_limit ) } ) for modifier in self . query_modifiers : self . agg_query = modifier ( self . agg_query ) hist = self . agg_query . aggs . bucket ( 'histogram' , 'date_histogram' , field = 'timestamp' , interval = self . aggregation_interval ) terms = hist . bucket ( 'terms' , 'terms' , field = self . aggregation_field , size = 0 ) top = terms . metric ( 'top_hit' , 'top_hits' , size = 1 , sort = { 'timestamp' : 'desc' } ) for dst , ( metric , src , opts ) in self . metric_aggregation_fields . items ( ) : terms . metric ( dst , metric , field = src , ** opts ) results = self . agg_query . execute ( ) index_name = None for interval in results . aggregations [ 'histogram' ] . buckets : interval_date = datetime . datetime . strptime ( interval [ 'key_as_string' ] , '%Y-%m-%dT%H:%M:%S' ) for aggregation in interval [ 'terms' ] . buckets : aggregation_data [ 'timestamp' ] = interval_date . isoformat ( ) aggregation_data [ self . aggregation_field ] = aggregation [ 'key' ] aggregation_data [ 'count' ] = aggregation [ 'doc_count' ] if self . metric_aggregation_fields : for f in self . metric_aggregation_fields : aggregation_data [ f ] = aggregation [ f ] [ 'value' ] doc = aggregation . top_hit . hits . hits [ 0 ] [ '_source' ] for destination , source in self . copy_fields . items ( ) : if isinstance ( source , six . string_types ) : aggregation_data [ destination ] = doc [ source ] else : aggregation_data [ destination ] = source ( doc , aggregation_data ) index_name = 'stats-{0}-{1}' . format ( self . event , interval_date . strftime ( self . index_name_suffix ) ) self . indices . add ( index_name ) yield dict ( _id = '{0}-{1}' . format ( aggregation [ 'key' ] , interval_date . strftime ( self . doc_id_suffix ) ) , _index = index_name , _type = self . aggregation_doc_type , _source = aggregation_data ) self . last_index_written = index_name

Aggregate and return dictionary to be indexed in ES .


def run(self, start_date=None, end_date=None, update_bookmark=True):
        """Calculate statistics aggregations."""
        # If no events have been indexed there is nothing to aggregate
        if not Index(self.event_index, using=self.client).exists():
            return
        lower_limit = start_date or self.get_bookmark()
        # Stop here if no bookmark could be estimated.
        if lower_limit is None:
            return
        upper_limit = min(
            end_date or datetime.datetime.max,  # ignore if `None`
            datetime.datetime.utcnow().replace(microsecond=0),
            datetime.datetime.combine(
                lower_limit + datetime.timedelta(self.batch_size),
                datetime.datetime.min.time())
        )
        while upper_limit <= datetime.datetime.utcnow():
            self.indices = set()
            self.new_bookmark = upper_limit.strftime(self.doc_id_suffix)
            bulk(self.client,
                 self.agg_iter(lower_limit, upper_limit),
                 stats_only=True,
                 chunk_size=50)
            # Flush all indices which have been modified
            current_search_client.indices.flush(
                index=','.join(self.indices),
                wait_if_ongoing=True
            )
            if update_bookmark:
                self.set_bookmark()
            self.indices = set()
            lower_limit = lower_limit + datetime.timedelta(self.batch_size)
            upper_limit = min(
                end_date or datetime.datetime.max,  # ignore if `None``
                datetime.datetime.utcnow().replace(microsecond=0),
                lower_limit + datetime.timedelta(self.batch_size)
            )
            if lower_limit > upper_limit:
                break

def run ( self , start_date = None , end_date = None , update_bookmark = True ) : if not Index ( self . event_index , using = self . client ) . exists ( ) : return lower_limit = start_date or self . get_bookmark ( ) if lower_limit is None : return upper_limit = min ( end_date or datetime . datetime . max , datetime . datetime . utcnow ( ) . replace ( microsecond = 0 ) , datetime . datetime . combine ( lower_limit + datetime . timedelta ( self . batch_size ) , datetime . datetime . min . time ( ) ) ) while upper_limit <= datetime . datetime . utcnow ( ) : self . indices = set ( ) self . new_bookmark = upper_limit . strftime ( self . doc_id_suffix ) bulk ( self . client , self . agg_iter ( lower_limit , upper_limit ) , stats_only = True , chunk_size = 50 ) current_search_client . indices . flush ( index = ',' . join ( self . indices ) , wait_if_ongoing = True ) if update_bookmark : self . set_bookmark ( ) self . indices = set ( ) lower_limit = lower_limit + datetime . timedelta ( self . batch_size ) upper_limit = min ( end_date or datetime . datetime . max , datetime . datetime . utcnow ( ) . replace ( microsecond = 0 ) , lower_limit + datetime . timedelta ( self . batch_size ) ) if lower_limit > upper_limit : break


def list_bookmarks(self, start_date=None, end_date=None, limit=None):
        """List the aggregation's bookmarks."""
        query = Search(
            using=self.client,
            index=self.aggregation_alias,
            doc_type=self.bookmark_doc_type
        ).sort({'date': {'order': 'desc'}})

        range_args = {}
        if start_date:
            range_args['gte'] = self._format_range_dt(
                start_date.replace(microsecond=0))
        if end_date:
            range_args['lte'] = self._format_range_dt(
                end_date.replace(microsecond=0))
        if range_args:
            query = query.filter('range', date=range_args)

        return query[0:limit].execute() if limit else query.scan()

def list_bookmarks ( self , start_date = None , end_date = None , limit = None ) : query = Search ( using = self . client , index = self . aggregation_alias , doc_type = self . bookmark_doc_type ) . sort ( { 'date' : { 'order' : 'desc' } } ) range_args = { } if start_date : range_args [ 'gte' ] = self . _format_range_dt ( start_date . replace ( microsecond = 0 ) ) if end_date : range_args [ 'lte' ] = self . _format_range_dt ( end_date . replace ( microsecond = 0 ) ) if range_args : query = query . filter ( 'range' , date = range_args ) return query [ 0 : limit ] . execute ( ) if limit else query . scan ( )


def delete(self, start_date=None, end_date=None):
        """Delete aggregation documents."""
        aggs_query = Search(
            using=self.client,
            index=self.aggregation_alias,
            doc_type=self.aggregation_doc_type
        ).extra(_source=False)

        range_args = {}
        if start_date:
            range_args['gte'] = self._format_range_dt(
                start_date.replace(microsecond=0))
        if end_date:
            range_args['lte'] = self._format_range_dt(
                end_date.replace(microsecond=0))
        if range_args:
            aggs_query = aggs_query.filter('range', timestamp=range_args)

        bookmarks_query = Search(
            using=self.client,
            index=self.aggregation_alias,
            doc_type=self.bookmark_doc_type
        ).sort({'date': {'order': 'desc'}})

        if range_args:
            bookmarks_query = bookmarks_query.filter('range', date=range_args)

        def _delete_actions():
            for query in (aggs_query, bookmarks_query):
                affected_indices = set()
                for doc in query.scan():
                    affected_indices.add(doc.meta.index)
                    yield dict(_index=doc.meta.index,
                               _op_type='delete',
                               _id=doc.meta.id,
                               _type=doc.meta.doc_type)
                current_search_client.indices.flush(
                    index=','.join(affected_indices), wait_if_ongoing=True)
        bulk(self.client, _delete_actions(), refresh=True)

def delete ( self , start_date = None , end_date = None ) : aggs_query = Search ( using = self . client , index = self . aggregation_alias , doc_type = self . aggregation_doc_type ) . extra ( _source = False ) range_args = { } if start_date : range_args [ 'gte' ] = self . _format_range_dt ( start_date . replace ( microsecond = 0 ) ) if end_date : range_args [ 'lte' ] = self . _format_range_dt ( end_date . replace ( microsecond = 0 ) ) if range_args : aggs_query = aggs_query . filter ( 'range' , timestamp = range_args ) bookmarks_query = Search ( using = self . client , index = self . aggregation_alias , doc_type = self . bookmark_doc_type ) . sort ( { 'date' : { 'order' : 'desc' } } ) if range_args : bookmarks_query = bookmarks_query . filter ( 'range' , date = range_args ) def _delete_actions ( ) : for query in ( aggs_query , bookmarks_query ) : affected_indices = set ( ) for doc in query . scan ( ) : affected_indices . add ( doc . meta . index ) yield dict ( _index = doc . meta . index , _op_type = 'delete' , _id = doc . meta . id , _type = doc . meta . doc_type ) current_search_client . indices . flush ( index = ',' . join ( affected_indices ) , wait_if_ongoing = True ) bulk ( self . client , _delete_actions ( ) , refresh = True )


def get(self, timeout=None):
        """
        Return value on success, or raise exception on failure.
        """
        result = None
        try:
            result = self._result.get(True, timeout=timeout)
        except Empty:
            raise Timeout()

        if isinstance(result, Failure):
            six.reraise(*result.exc_info)
        else:
            return result

def get ( self , timeout = None ) : result = None try : result = self . _result . get ( True , timeout = timeout ) except Empty : raise Timeout ( ) if isinstance ( result , Failure ) : six . reraise ( * result . exc_info ) else : return result

Return value on success or raise exception on failure .


def _events_process(event_types=None, eager=False):
    """Process stats events."""
    event_types = event_types or list(current_stats.enabled_events)
    if eager:
        process_events.apply((event_types,), throw=True)
        click.secho('Events processed successfully.', fg='green')
    else:
        process_events.delay(event_types)
        click.secho('Events processing task sent...', fg='yellow')

def
_events_process
(
event_types
=
,
eager
=
False
)
:
event_types
=
event_types
or
list
(
current_stats
.
enabled_events
)
if
eager
:
process_events
.
apply
(
(
event_types
,
)
,
throw
=
True
)
click
.
secho
(
'Events processed successfully.'
,
fg
=
'green'
)
else
:
process_events
.
delay
(
event_types
)
click
.
secho
(
'Events processing task sent...'
,
fg
=
'yellow'
)


def _aggregations_process(aggregation_types=None,
                          start_date=None, end_date=None,
                          update_bookmark=False, eager=False):
    """Process stats aggregations."""
    aggregation_types = (aggregation_types or
                         list(current_stats.enabled_aggregations))
    if eager:
        aggregate_events.apply(
            (aggregation_types,),
            dict(start_date=start_date, end_date=end_date,
                 update_bookmark=update_bookmark),
            throw=True)
        click.secho('Aggregations processed successfully.', fg='green')
    else:
        aggregate_events.delay(
            aggregation_types, start_date=start_date, end_date=end_date)
        click.secho('Aggregations processing task sent...', fg='yellow')

def
_aggregations_process
(
aggregation_types
=
,
start_date
=
,
end_date
=
,
update_bookmark
=
False
,
eager
=
False
)
:
aggregation_types
=
(
aggregation_types
or
list
(
current_stats
.
enabled_aggregations
)
)
if
eager
:
aggregate_events
.
apply
(
(
aggregation_types
,
)
,
dict
(
start_date
=
start_date
,
end_date
=
end_date
,
update_bookmark
=
update_bookmark
)
,
throw
=
True
)
click
.
secho
(
'Aggregations processed successfully.'
,
fg
=
'green'
)
else
:
aggregate_events
.
delay
(
aggregation_types
,
start_date
=
start_date
,
end_date
=
end_date
)
click
.
secho
(
'Aggregations processing task sent...'
,
fg
=
'yellow'
)


def _aggregations_delete(aggregation_types=None,
                         start_date=None, end_date=None):
    """Delete computed aggregations."""
    aggregation_types = (aggregation_types or
                         list(current_stats.enabled_aggregations))
    for a in aggregation_types:
        aggr_cfg = current_stats.aggregations[a]
        aggregator = aggr_cfg.aggregator_class(
            name=aggr_cfg.name, **aggr_cfg.aggregator_config)
        aggregator.delete(start_date, end_date)

def _aggregations_delete ( aggregation_types = None , start_date = None , end_date = None ) : aggregation_types = ( aggregation_types or list ( current_stats . enabled_aggregations ) ) for a in aggregation_types : aggr_cfg = current_stats . aggregations [ a ] aggregator = aggr_cfg . aggregator_class ( name = aggr_cfg . name , ** aggr_cfg . aggregator_config ) aggregator . delete ( start_date , end_date )


def _aggregations_list_bookmarks(aggregation_types=None,
                                 start_date=None, end_date=None, limit=None):
    """List aggregation bookmarks."""
    aggregation_types = (aggregation_types or
                         list(current_stats.enabled_aggregations))
    for a in aggregation_types:
        aggr_cfg = current_stats.aggregations[a]
        aggregator = aggr_cfg.aggregator_class(
            name=aggr_cfg.name, **aggr_cfg.aggregator_config)
        bookmarks = aggregator.list_bookmarks(start_date, end_date, limit)
        click.echo('{}:'.format(a))
        for b in bookmarks:
            click.echo(' - {}'.format(b.date))

def
_aggregations_list_bookmarks
(
aggregation_types
=
,
start_date
=
,
end_date
=
,
limit
=
)
:
aggregation_types
=
(
aggregation_types
or
list
(
current_stats
.
enabled_aggregations
)
)
for
a
in
aggregation_types
:
aggr_cfg
=
current_stats
.
aggregations
[
a
]
aggregator
=
aggr_cfg
.
aggregator_class
(
name
=
aggr_cfg
.
name
,
**
aggr_cfg
.
aggregator_config
)
bookmarks
=
aggregator
.
list_bookmarks
(
start_date
,
end_date
,
limit
)
click
.
echo
(
'{}:'
.
format
(
a
)
)
for
b
in
bookmarks
:
click
.
echo
(
' - {}'
.
format
(
b
.
date
)
)


def _events_config(self):
        """Load events configuration."""
        # import iter_entry_points here so that it can be mocked in tests
        result = {}
        for ep in iter_entry_points(
                group=self.entry_point_group_events):
            for cfg in ep.load()():
                if cfg['event_type'] not in self.enabled_events:
                    continue
                elif cfg['event_type'] in result:
                    raise DuplicateEventError(
                        'Duplicate event {0} in entry point '
                        '{1}'.format(cfg['event_type'], ep.name))
                # Update the default configuration with env/overlay config.
                cfg.update(
                    self.enabled_events[cfg['event_type']] or {}
                )
                result[cfg['event_type']] = cfg
        return result

def
_events_config
(
self
)
:
result
=
{
}
for
ep
in
iter_entry_points
(
group
=
self
.
entry_point_group_events
)
:
for
cfg
in
ep
.
load
(
)
(
)
:
if
cfg
[
'event_type'
]
not
in
self
.
enabled_events
:
continue
elif
cfg
[
'event_type'
]
in
result
:
raise
DuplicateEventError
(
'Duplicate event {0} in entry point '
'{1}'
.
format
(
cfg
[
'event_type'
]
,
ep
.
name
)
)
cfg
.
update
(
self
.
enabled_events
[
cfg
[
'event_type'
]
]
or
{
}
)
result
[
cfg
[
'event_type'
]
]
=
cfg
return
result


def _aggregations_config(self):
        """Load aggregation configurations."""
        result = {}
        for ep in iter_entry_points(
                group=self.entry_point_group_aggs):
            for cfg in ep.load()():
                if cfg['aggregation_name'] not in self.enabled_aggregations:
                    continue
                elif cfg['aggregation_name'] in result:
                    raise DuplicateAggregationError(
                        'Duplicate aggregation {0} in entry point '
                        '{1}'.format(cfg['event_type'], ep.name))
                # Update the default configuration with env/overlay config.
                cfg.update(
                    self.enabled_aggregations[cfg['aggregation_name']] or {}
                )
                result[cfg['aggregation_name']] = cfg
        return result


def _queries_config(self):
        """Load queries configuration."""
        result = {}
        for ep in iter_entry_points(group=self.entry_point_group_queries):
            for cfg in ep.load()():
                if cfg['query_name'] not in self.enabled_queries:
                    continue
                elif cfg['query_name'] in result:
                    raise DuplicateQueryError(
                        'Duplicate query {0} in entry point '
                        '{1}'.format(cfg['query'], ep.name))
                # Update the default configuration with env/overlay config.
                cfg.update(
                    self.enabled_queries[cfg['query_name']] or {}
                )
                result[cfg['query_name']] = cfg
        return result

def
_queries_config
(
self
)
:
result
=
{
}
for
ep
in
iter_entry_points
(
group
=
self
.
entry_point_group_queries
)
:
for
cfg
in
ep
.
load
(
)
(
)
:
if
cfg
[
'query_name'
]
not
in
self
.
enabled_queries
:
continue
elif
cfg
[
'query_name'
]
in
result
:
raise
DuplicateQueryError
(
'Duplicate query {0} in entry point '
'{1}'
.
format
(
cfg
[
'query'
]
,
ep
.
name
)
)
cfg
.
update
(
self
.
enabled_queries
[
cfg
[
'query_name'
]
]
or
{
}
)
result
[
cfg
[
'query_name'
]
]
=
cfg
return
result


def publish(self, event_type, events):
        """Publish events."""
        assert event_type in self.events
        current_queues.queues['stats-{}'.format(event_type)].publish(events)

def publish ( self , event_type , events ) : assert event_type in self . events current_queues . queues [ 'stats-{}' . format ( event_type ) ] . publish ( events )


def consume(self, event_type, no_ack=True, payload=True):
        """Comsume all pending events."""
        assert event_type in self.events
        return current_queues.queues['stats-{}'.format(event_type)].consume(
            payload=payload)

def consume ( self , event_type , no_ack = True , payload = True ) : assert event_type in self . events return current_queues . queues [ 'stats-{}' . format ( event_type ) ] . consume ( payload = payload )


def tell(self, message, sender=no_sender):
        """ Send a message to this actor. Asynchronous fire-and-forget.

        :param message: The message to send.
        :type message: Any

        :param sender: The sender of the message. If provided it will be made
            available to the receiving actor via the :attr:`Actor.sender` attribute.
        :type sender: :class:`Actor`
        """
        if sender is not no_sender and not isinstance(sender, ActorRef):
            raise ValueError("Sender must be actor reference")

        self._cell.send_message(message, sender)

def
tell
(
self
,
message
,
sender
=
no_sender
)
:
if
sender
is
not
no_sender
and
not
isinstance
(
sender
,
ActorRef
)
:
raise
ValueError
(
"Sender must be actor reference"
)
self
.
_cell
.
send_message
(
message
,
sender
)

Send a message to this actor . Asynchronous fire - and - forget .


def get_anonymization_salt(ts):
    """Get the anonymization salt based on the event timestamp's day."""
    salt_key = 'stats:salt:{}'.format(ts.date().isoformat())
    salt = current_cache.get(salt_key)
    if not salt:
        salt_bytes = os.urandom(32)
        salt = b64encode(salt_bytes).decode('utf-8')
        current_cache.set(salt_key, salt, timeout=60 * 60 * 24)
    return salt

def get_anonymization_salt ( ts ) : salt_key = 'stats:salt:{}' . format ( ts . date ( ) . isoformat ( ) ) salt = current_cache . get ( salt_key ) if not salt : salt_bytes = os . urandom ( 32 ) salt = b64encode ( salt_bytes ) . decode ( 'utf-8' ) current_cache . set ( salt_key , salt , timeout = 60 * 60 * 24 ) return salt

Get the anonymization salt based on the event timestamp s day .


def get_geoip(ip):
    """Lookup country for IP address."""
    reader = geolite2.reader()
    ip_data = reader.get(ip) or {}
    return ip_data.get('country', {}).get('iso_code')

def get_geoip ( ip ) : reader = geolite2 . reader ( ) ip_data = reader . get ( ip ) or { } return ip_data . get ( 'country' , { } ) . get ( 'iso_code' )


def get_user():
    """User information.

    .. note::

       **Privacy note** A users IP address, user agent string, and user id
       (if logged in) is sent to a message queue, where it is stored for about
       5 minutes. The information is used to:

       - Detect robot visits from the user agent string.
       - Generate an anonymized visitor id (using a random salt per day).
       - Detect the users host contry based on the IP address.

       The information is then discarded.
    """
    return dict(
        ip_address=request.remote_addr,
        user_agent=request.user_agent.string,
        user_id=(
            current_user.get_id() if current_user.is_authenticated else None
        ),
        session_id=session.get('sid_s')
    )

def get_user ( ) : return dict ( ip_address = request . remote_addr , user_agent = request . user_agent . string , user_id = ( current_user . get_id ( ) if current_user . is_authenticated else None ) , session_id = session . get ( 'sid_s' ) )


def default_permission_factory(query_name, params):
    """Default permission factory.

    It enables by default the statistics if they don't have a dedicated
    permission factory.
    """
    from invenio_stats import current_stats
    if current_stats.queries[query_name].permission_factory is None:
        return AllowAllPermission
    else:
        return current_stats.queries[query_name].permission_factory(
            query_name, params
        )

def default_permission_factory ( query_name , params ) : from invenio_stats import current_stats if current_stats . queries [ query_name ] . permission_factory is None : return AllowAllPermission else : return current_stats . queries [ query_name ] . permission_factory ( query_name , params )


def register_templates():
    """Register elasticsearch templates for events."""
    event_templates = [current_stats._events_config[e]
                       ['templates']
                       for e in
                       current_stats._events_config]
    aggregation_templates = [current_stats._aggregations_config[a]
                             ['templates']
                             for a in
                             current_stats._aggregations_config]
    return event_templates + aggregation_templates

def register_templates ( ) : event_templates = [ current_stats . _events_config [ e ] [ 'templates' ] for e in current_stats . _events_config ] aggregation_templates = [ current_stats . _aggregations_config [ a ] [ 'templates' ] for a in current_stats . _aggregations_config ] return event_templates + aggregation_templates

Register elasticsearch templates for events .


def aggregate_events(aggregations, start_date=None, end_date=None,
                     update_bookmark=True):
    """Aggregate indexed events."""
    start_date = dateutil_parse(start_date) if start_date else None
    end_date = dateutil_parse(end_date) if end_date else None
    results = []
    for a in aggregations:
        aggr_cfg = current_stats.aggregations[a]
        aggregator = aggr_cfg.aggregator_class(
            name=aggr_cfg.name, **aggr_cfg.aggregator_config)
        results.append(aggregator.run(start_date, end_date, update_bookmark))
    return results

def aggregate_events ( aggregations , start_date = None , end_date = None , update_bookmark = True ) : start_date = dateutil_parse ( start_date ) if start_date else None end_date = dateutil_parse ( end_date ) if end_date else None results = [ ] for a in aggregations : aggr_cfg = current_stats . aggregations [ a ] aggregator = aggr_cfg . aggregator_class ( name = aggr_cfg . name , ** aggr_cfg . aggregator_config ) results . append ( aggregator . run ( start_date , end_date , update_bookmark ) ) return results


def _handle_request(self, scheme, netloc, path, headers, body=None, method="GET"):
        """
        Run the actual request
        """
        backend_url = "{}://{}{}".format(scheme, netloc, path)
        try:
            response = self.http_request.request(backend_url, method=method, body=body, headers=dict(headers))
            self._return_response(response)
        except Exception as e:
            body = "Invalid response from backend: '{}' Server might be busy".format(e.message)
            logging.debug(body)
            self.send_error(httplib.SERVICE_UNAVAILABLE, body)


def anonymize_user(doc):
    """Preprocess an event by anonymizing user information.

    The anonymization is done by removing fields that can uniquely identify a
    user, such as the user's ID, session ID, IP address and User Agent, and
    hashing them to produce a ``visitor_id`` and ``unique_session_id``. To
    further secure the method, a randomly generated 32-byte salt is used, that
    expires after 24 hours and is discarded. The salt values are stored in
    Redis (or whichever backend Invenio-Cache uses). The ``unique_session_id``
    is calculated in the same way as the ``visitor_id``, with the only
    difference that it also takes into account the hour of the event . All of
    these rules effectively mean that a user can have a unique ``visitor_id``
    for each day and unique ``unique_session_id`` for each hour of a day.

    This session ID generation process was designed according to the `Project
    COUNTER Code of Practice <https://www.projectcounter.org/code-of-
    practice-sections/general-information/>`_.

    In addition to that the country of the user is extracted from the IP
    address as a ISO 3166-1 alpha-2 two-letter country code (e.g. "CH" for
    Switzerland).
    """
    ip = doc.pop('ip_address', None)
    if ip:
        doc.update({'country': get_geoip(ip)})

    user_id = doc.pop('user_id', '')
    session_id = doc.pop('session_id', '')
    user_agent = doc.pop('user_agent', '')

    # A 'User Session' is defined as activity by a user in a period of
    # one hour. timeslice represents the hour of the day in which
    # the event has been generated and together with user info it determines
    # the 'User Session'
    timestamp = arrow.get(doc.get('timestamp'))
    timeslice = timestamp.strftime('%Y%m%d%H')
    salt = get_anonymization_salt(timestamp)

    visitor_id = hashlib.sha224(salt.encode('utf-8'))
    # TODO: include random salt here, that changes once a day.
    # m.update(random_salt)
    if user_id:
        visitor_id.update(user_id.encode('utf-8'))
    elif session_id:
        visitor_id.update(session_id.encode('utf-8'))
    elif ip and user_agent:
        vid = '{}|{}|{}'.format(ip, user_agent, timeslice)
        visitor_id.update(vid.encode('utf-8'))
    else:
        # TODO: add random data?
        pass

    unique_session_id = hashlib.sha224(salt.encode('utf-8'))
    if user_id:
        sid = '{}|{}'.format(user_id, timeslice)
        unique_session_id.update(sid.encode('utf-8'))
    elif session_id:
        sid = '{}|{}'.format(session_id, timeslice)
        unique_session_id.update(sid.encode('utf-8'))
    elif ip and user_agent:
        sid = '{}|{}|{}'.format(ip, user_agent, timeslice)
        unique_session_id.update(sid.encode('utf-8'))

    doc.update(dict(
        visitor_id=visitor_id.hexdigest(),
        unique_session_id=unique_session_id.hexdigest()
    ))

    return doc

def anonymize_user ( doc ) : ip = doc . pop ( 'ip_address' , None ) if ip : doc . update ( { 'country' : get_geoip ( ip ) } ) user_id = doc . pop ( 'user_id' , '' ) session_id = doc . pop ( 'session_id' , '' ) user_agent = doc . pop ( 'user_agent' , '' ) timestamp = arrow . get ( doc . get ( 'timestamp' ) ) timeslice = timestamp . strftime ( '%Y%m%d%H' ) salt = get_anonymization_salt ( timestamp ) visitor_id = hashlib . sha224 ( salt . encode ( 'utf-8' ) ) if user_id : visitor_id . update ( user_id . encode ( 'utf-8' ) ) elif session_id : visitor_id . update ( session_id . encode ( 'utf-8' ) ) elif ip and user_agent : vid = '{}|{}|{}' . format ( ip , user_agent , timeslice ) visitor_id . update ( vid . encode ( 'utf-8' ) ) else : pass unique_session_id = hashlib . sha224 ( salt . encode ( 'utf-8' ) ) if user_id : sid = '{}|{}' . format ( user_id , timeslice ) unique_session_id . update ( sid . encode ( 'utf-8' ) ) elif session_id : sid = '{}|{}' . format ( session_id , timeslice ) unique_session_id . update ( sid . encode ( 'utf-8' ) ) elif ip and user_agent : sid = '{}|{}|{}' . format ( ip , user_agent , timeslice ) unique_session_id . update ( sid . encode ( 'utf-8' ) ) doc . update ( dict ( visitor_id = visitor_id . hexdigest ( ) , unique_session_id = unique_session_id . hexdigest ( ) ) ) return doc

Preprocess an event by anonymizing user information .


def hash_id(iso_timestamp, msg):
    """Generate event id, optimized for ES."""
    return '{0}-{1}'.format(iso_timestamp,
                            hashlib.sha1(
                                msg.get('unique_id').encode('utf-8') +
                                str(msg.get('visitor_id')).
                                encode('utf-8')).
                            hexdigest())

def hash_id ( iso_timestamp , msg ) : return '{0}-{1}' . format ( iso_timestamp , hashlib . sha1 ( msg . get ( 'unique_id' ) . encode ( 'utf-8' ) + str ( msg . get ( 'visitor_id' ) ) . encode ( 'utf-8' ) ) . hexdigest ( ) )


def run(self):
        """Process events queue."""
        return elasticsearch.helpers.bulk(
            self.client,
            self.actionsiter(),
            stats_only=True,
            chunk_size=50
        )

def run ( self ) : return elasticsearch . helpers . bulk ( self . client , self . actionsiter ( ) , stats_only = True , chunk_size = 50 )


def register_events():
    """Register sample events."""
    return [
        dict(
            event_type='file-download',
            templates='invenio_stats.contrib.file_download',
            processor_class=EventsIndexer,
            processor_config=dict(
                preprocessors=[
                    flag_robots,
                    anonymize_user,
                    build_file_unique_id
                ])),
        dict(
            event_type='record-view',
            templates='invenio_stats.contrib.record_view',
            processor_class=EventsIndexer,
            processor_config=dict(
                preprocessors=[
                    flag_robots,
                    anonymize_user,
                    build_record_unique_id
                ]))
    ]

def register_events ( ) : return [ dict ( event_type = 'file-download' , templates = 'invenio_stats.contrib.file_download' , processor_class = EventsIndexer , processor_config = dict ( preprocessors = [ flag_robots , anonymize_user , build_file_unique_id ] ) ) , dict ( event_type = 'record-view' , templates = 'invenio_stats.contrib.record_view' , processor_class = EventsIndexer , processor_config = dict ( preprocessors = [ flag_robots , anonymize_user , build_record_unique_id ] ) ) ]


def register_aggregations():
    """Register sample aggregations."""
    return [dict(
        aggregation_name='file-download-agg',
        templates='invenio_stats.contrib.aggregations.aggr_file_download',
        aggregator_class=StatAggregator,
        aggregator_config=dict(
            client=current_search_client,
            event='file-download',
            aggregation_field='unique_id',
            aggregation_interval='day',
            copy_fields=dict(
                file_key='file_key',
                bucket_id='bucket_id',
                file_id='file_id',
            ),
            metric_aggregation_fields={
                'unique_count': ('cardinality', 'unique_session_id',
                                 {'precision_threshold': 1000}),
                'volume': ('sum', 'size', {}),
            },
        )), dict(
        aggregation_name='record-view-agg',
        templates='invenio_stats.contrib.aggregations.aggr_record_view',
        aggregator_class=StatAggregator,
        aggregator_config=dict(
            client=current_search_client,
            event='record-view',
            aggregation_field='unique_id',
            aggregation_interval='day',
            copy_fields=dict(
                record_id='record_id',
                pid_type='pid_type',
                pid_value='pid_value',
            ),
            metric_aggregation_fields={
                'unique_count': ('cardinality', 'unique_session_id',
                                 {'precision_threshold': 1000}),
            },
        ))]


def register_queries():
    """Register queries."""
    return [
        dict(
            query_name='bucket-file-download-histogram',
            query_class=ESDateHistogramQuery,
            query_config=dict(
                index='stats-file-download',
                doc_type='file-download-day-aggregation',
                copy_fields=dict(
                    bucket_id='bucket_id',
                    file_key='file_key',
                ),
                required_filters=dict(
                    bucket_id='bucket_id',
                    file_key='file_key',
                )
            )
        ),
        dict(
            query_name='bucket-file-download-total',
            query_class=ESTermsQuery,
            query_config=dict(
                index='stats-file-download',
                doc_type='file-download-day-aggregation',
                copy_fields=dict(
                    # bucket_id='bucket_id',
                ),
                required_filters=dict(
                    bucket_id='bucket_id',
                ),
                aggregated_fields=['file_key']
            )
        ),
    ]

def register_queries ( ) : return [ dict ( query_name = 'bucket-file-download-histogram' , query_class = ESDateHistogramQuery , query_config = dict ( index = 'stats-file-download' , doc_type = 'file-download-day-aggregation' , copy_fields = dict ( bucket_id = 'bucket_id' , file_key = 'file_key' , ) , required_filters = dict ( bucket_id = 'bucket_id' , file_key = 'file_key' , ) ) ) , dict ( query_name = 'bucket-file-download-total' , query_class = ESTermsQuery , query_config = dict ( index = 'stats-file-download' , doc_type = 'file-download-day-aggregation' , copy_fields = dict ( ) , required_filters = dict ( bucket_id = 'bucket_id' , ) , aggregated_fields = [ 'file_key' ] ) ) , ]


def _parse_time(self, tokens):
        """
        Parse the date range for the query

        E.g. WHERE time > now() - 48h AND time < now() - 24h
        would result in DateRange(datetime_start, datetime_end)
        where
        datetime_start would be parsed from now() - 48h
        and
        datetime_end would be parsed from now() - 24h

        :param tokens:
        :return:
        """
        return self.time_parser.parse(self.parse_keyword(Keyword.WHERE, tokens))

def _parse_time ( self , tokens ) : return self . time_parser . parse ( self . parse_keyword ( Keyword . WHERE , tokens ) )


def extract_date(self, date):
        """Extract date from string if necessary.

        :returns: the extracted date.
        """
        if isinstance(date, six.string_types):
            try:
                date = dateutil.parser.parse(date)
            except ValueError:
                raise ValueError(
                    'Invalid date format for statistic {}.'
                ).format(self.query_name)
        if not isinstance(date, datetime):
            raise TypeError(
                'Invalid date type for statistic {}.'
            ).format(self.query_name)
        return date

def
extract_date
(
self
,
date
)
:
if
isinstance
(
date
,
six
.
string_types
)
:
try
:
date
=
dateutil
.
parser
.
parse
(
date
)
except
ValueError
:
raise
ValueError
(
'Invalid date format for statistic {}.'
)
.
format
(
self
.
query_name
)
if
not
isinstance
(
date
,
datetime
)
:
raise
TypeError
(
'Invalid date type for statistic {}.'
)
.
format
(
self
.
query_name
)
return
date


def run(self, start_date=None, end_date=None, **kwargs):
        """Run the query."""
        start_date = self.extract_date(start_date) if start_date else None
        end_date = self.extract_date(end_date) if end_date else None
        self.validate_arguments(start_date, end_date, **kwargs)

        agg_query = self.build_query(start_date, end_date, **kwargs)
        query_result = agg_query.execute().to_dict()
        res = self.process_query_result(query_result, start_date, end_date)
        return res

def run ( self , start_date = None , end_date = None , ** kwargs ) : start_date = self . extract_date ( start_date ) if start_date else None end_date = self . extract_date ( end_date ) if end_date else None self . validate_arguments ( start_date , end_date , ** kwargs ) agg_query = self . build_query ( start_date , end_date , ** kwargs ) query_result = agg_query . execute ( ) . to_dict ( ) res = self . process_query_result ( query_result , start_date , end_date ) return res


def file_download_event_builder(event, sender_app, obj=None, **kwargs):
    """Build a file-download event."""
    event.update(dict(
        # When:
        timestamp=datetime.datetime.utcnow().isoformat(),
        # What:
        bucket_id=str(obj.bucket_id),
        file_id=str(obj.file_id),
        file_key=obj.key,
        size=obj.file.size,
        referrer=request.referrer,
        # Who:
        **get_user()
    ))
    return event

def file_download_event_builder ( event , sender_app , obj = None , ** kwargs ) : event . update ( dict ( timestamp = datetime . datetime . utcnow ( ) . isoformat ( ) , bucket_id = str ( obj . bucket_id ) , file_id = str ( obj . file_id ) , file_key = obj . key , size = obj . file . size , referrer = request . referrer , ** get_user ( ) ) ) return event


def record_view_event_builder(event, sender_app, pid=None, record=None,
                              **kwargs):
    """Build a record-view event."""
    event.update(dict(
        # When:
        timestamp=datetime.datetime.utcnow().isoformat(),
        # What:
        record_id=str(record.id),
        pid_type=pid.pid_type,
        pid_value=str(pid.pid_value),
        referrer=request.referrer,
        # Who:
        **get_user()
    ))
    return event

def record_view_event_builder ( event , sender_app , pid = None , record = None , ** kwargs ) : event . update ( dict ( timestamp = datetime . datetime . utcnow ( ) . isoformat ( ) , record_id = str ( record . id ) , pid_type = pid . pid_type , pid_value = str ( pid . pid_value ) , referrer = request . referrer , ** get_user ( ) ) ) return event


def check_write_permissions(file):
    """
    Check if we can write to the given file

    Otherwise since we might detach the process to run in the background
    we might never find out that writing failed and get an ugly
    exit message on startup. For example:
    ERROR: Child exited immediately with non-zero exit code 127

    So we catch this error upfront and print a nicer error message
    with a hint on how to fix it.
    """
    try:
        open(file, 'a')
    except IOError:
        print("Can't open file {}. "
              "Please grant write permissions or change the path in your config".format(file))
        sys.exit(1)


def _is_root():
    """Checks if the user is rooted."""
    import os
    import ctypes
    try:
        return os.geteuid() == 0
    except AttributeError:
        return ctypes.windll.shell32.IsUserAnAdmin() != 0
    return False

def _is_root ( ) : import os import ctypes try : return os . geteuid ( ) == 0 except AttributeError : return ctypes . windll . shell32 . IsUserAnAdmin ( ) != 0 return False