def convert(in_file, out_file, in_fmt="", out_fmt=""):
    """
    Converts in_file to out_file, guessing datatype in the absence of
    in_fmt and out_fmt.

    Arguments:
        in_file:    The name of the (existing) datafile to read
        out_file:   The name of the file to create with converted data
        in_fmt:     Optional. The format of incoming data, if not guessable
        out_fmt:    Optional. The format of outgoing data, if not guessable

    Returns:
        String. Output filename
    """
    # First verify that in_file exists and out_file doesn't.
    in_file = os.path.expanduser(in_file)
    out_file = os.path.expanduser(out_file)

    if not os.path.exists(in_file):
        raise IOError("Input file {0} does not exist, stopping..."
                      .format(in_file))

    # Get formats, either by explicitly naming them or by guessing.
    # TODO: It'd be neat to check here if an explicit fmt matches the guess.
    in_fmt = in_fmt.lower() or _guess_format_from_extension(
        in_file.split('.')[-1].lower())
    out_fmt = out_fmt.lower() or _guess_format_from_extension(
        out_file.split('.')[-1].lower())

    if not in_fmt or not out_fmt:
        raise ValueError("Cannot determine conversion formats.")
        return False

    if in_fmt is out_fmt:
        # This is the case when this module (intended for LONI) is used
        # indescriminately to 'funnel' data into one format.
        shutil.copyfileobj(in_file, out_file)
        return out_file

    # Import
    if in_fmt == 'hdf5':
        from . import hdf5
        data = hdf5.load(in_file)
    elif in_fmt == 'tiff':
        from . import tiff
        data = tiff.load(in_file)
    elif in_fmt == 'png':
        from . import png
        data = png.load(in_file)
    else:
        return _fail_pair_conversion(in_fmt, out_fmt)

    # Export
    if out_fmt == 'hdf5':
        from . import hdf5
        return hdf5.save(out_file, data)
    elif out_fmt == 'tiff':
        from . import tiff
        return tiff.save(out_file, data)
    elif out_fmt == 'png':
        from . import png
        return png.export_png(out_file, data)
    else:
        return _fail_pair_conversion(in_fmt, out_fmt)

    return _fail_pair_conversion(in_fmt, out_fmt)

Converts in_file to out_file guessing datatype in the absence of in_fmt and out_fmt .


def build_graph(self, project, site, subject, session, scan,
                    size, email=None, invariants=Invariants.ALL,
                    fiber_file=DEFAULT_FIBER_FILE, atlas_file=None,
                    use_threads=False, callback=None):
        """
        Builds a graph using the graph-services endpoint.

        Arguments:
            project (str): The project to use
            site (str): The site in question
            subject (str): The subject's identifier
            session (str): The session (per subject)
            scan (str): The scan identifier
            size (str): Whether to return a big (grute.BIG) or small
                (grute.SMALL) graph. For a better explanation, see m2g.io.
            email (str : self.email)*: An email to notify
            invariants (str[]: Invariants.ALL)*: An array of invariants to
                compute. You can use the grute.Invariants class to construct a
                list, or simply pass grute.Invariants.ALL to compute them all.
            fiber_file (str: DEFAULT_FIBER_FILE)*: A local filename of an
                MRI Studio .dat file
            atlas_file (str: None)*: A local atlas file, in NIFTI .nii format.
                If none is specified, the Desikan atlas is used by default.
            use_threads (bool: False)*: Whether to run the download in a Python
                thread. If set to True, the call to `build_graph` will end
                quickly, and the `callback` will be called with the returned
                status-code of the restful call as its only argument.
            callback (function: None)*: The function to run upon completion of
                the call, if using threads. (Will not be called if use_threads
                is set to False.)

        Returns:
            HTTP Response if use_threads is False. Otherwise, None

        Raises:
            ValueError: When the supplied values are invalid (contain invalid
                characters, bad email address supplied, etc.)
            RemoteDataNotFoundError: When the data cannot be processed due to
                a server error.
        """
        if email is None:
            email = self.email

        if not set(invariants) <= set(Invariants.ALL):
            raise ValueError("Invariants must be a subset of Invariants.ALL.")

        if use_threads and callback is not None:
            if not hasattr(callback, '__call__'):
                raise ValueError("callback must be a function.")
            if len(inspect.getargspec(callback).args) != 1:
                raise ValueError("callback must take exactly 1 argument.")

        # Once we get here, we know the callback is
        if size not in [self.BIG, self.SMALL]:
            raise ValueError("size must be either grute.BIG or grute.SMALL.")

        url = "buildgraph/{}/{}/{}/{}/{}/{}/{}/{}/".format(
            project,
            site,
            subject,
            session,
            scan,
            size,
            email,
            "/".join(invariants)
        )

        if " " in url:
            raise ValueError("Arguments must not contain spaces.")

        if use_threads:
            # Run in the background.
            download_thread = threading.Thread(
                target=self._run_build_graph,
                args=[url, fiber_file, atlas_file, callback]
            )
            download_thread.start()
        else:
            # Run in the foreground.
            return self._run_build_graph(url, fiber_file, atlas_file)
        return

Builds a graph using the graph - services endpoint .


def compute_invariants(self, graph_file, input_format,
                           invariants=Invariants.ALL, email=None,
                           use_threads=False, callback=None):
        """
        Compute invariants from an existing GraphML file using the remote
        grute graph services.

        Arguments:
            graph_file (str): The filename of the graphml file
            input_format (str): One of grute.GraphFormats
            invariants (str[]: Invariants.ALL)*: An array of grute.Invariants
                to compute on the graph
            email (str: self.email)*: The email to notify upon completion
            use_threads (bool: False)*: Whether to use Python threads to run
                computation in the background when waiting for the server to
                return the invariants
            callback (function: None)*: The function to run upon completion of
                the call, if using threads. (Will not be called if use_threads
                is set to False.)

        Returns:
            HTTP Response if use_threads is False. Otherwise, None

        Raises:
            ValueError: If the graph file does not exist, or if there are
                issues with the passed arguments
            RemoteDataUploadError: If there is an issue packing the file
            RemoteError: If the server experiences difficulty computing invs
        """
        if email is None:
            email = self.email

        if input_format not in GraphFormats._any:
            raise ValueError("Invalid input format, {}.".format(input_format))

        if not set(invariants) <= set(Invariants.ALL):
            raise ValueError("Invariants must be a subset of Invariants.ALL.")

        if use_threads and callback is not None:
            if not hasattr(callback, '__call__'):
                raise ValueError("callback must be a function.")
            if len(inspect.getargspec(callback).args) != 1:
                raise ValueError("callback must take exactly 1 argument.")

        url = "graphupload/{}/{}/{}/".format(
            email,
            input_format,
            "/".join(invariants)
        )

        if " " in url:
            raise ValueError("Arguments cannot have spaces in them.")

        if not (os.path.exists(graph_file)):
            raise ValueError("File {} does not exist.".format(graph_file))

        if use_threads:
            # Run in the background.
            upload_thread = threading.Thread(
                target=self._run_compute_invariants,
                args=[url, graph_file, callback]
            )
            upload_thread.start()

        else:
            # Run in the foreground.
            return self._run_compute_invariants(url, graph_file)
        return

Compute invariants from an existing GraphML file using the remote grute graph services .


def convert_graph(self, graph_file, input_format, output_formats,
                      email=None, use_threads=False, callback=None):
        """
        Convert a graph from one GraphFormat to another.

        Arguments:
            graph_file (str): Filename of the file to convert
            input_format (str): A grute.GraphFormats
            output_formats (str[]): A grute.GraphFormats
            email (str: self.email)*: The email to notify
            use_threads (bool: False)*: Whether to use Python threads to run
                computation in the background when waiting for the server
            callback (function: None)*: The function to run upon completion of
                the call, if using threads. (Will not be called if use_threads
                is set to False.)

        Returns:
            HTTP Response if use_threads=False. Else, no return value.

        Raises:
            RemoteDataUploadError: If there's an issue uploading the data
            RemoteError: If there's a server-side issue
            ValueError: If there's a problem with the supplied arguments
        """
        if email is None:
            email = self.email

        if input_format not in GraphFormats._any:
            raise ValueError("Invalid input format {}.".format(input_format))

        if not set(output_formats) <= set(GraphFormats._any):
            raise ValueError("Output formats must be a GraphFormats.")

        if use_threads and callback is not None:
            if not hasattr(callback, '__call__'):
                raise ValueError("callback must be a function.")
            if len(inspect.getargspec(callback).args) != 1:
                raise ValueError("callback must take exactly 1 argument.")

        if not (os.path.exists(graph_file)):
            raise ValueError("No such file, {}!".format(graph_file))

        url = "convert/{}/{}/{}/l".format(
            email,
            input_format,
            ','.join(output_formats)
        )

        if " " in url:
            raise ValueError("Spaces are not permitted in arguments.")

        if use_threads:
            # Run in the background.
            convert_thread = threading.Thread(
                target=self._run_convert_graph,
                args=[url, graph_file, callback]
            )
            convert_thread.start()
        else:
            # Run in the foreground.
            return self._run_convert_graph(url, graph_file)
        return

Convert a graph from one GraphFormat to another .


def to_dict(ramons, flatten=False):
    """
    Converts a RAMON object list to a JSON-style dictionary. Useful for going
    from an array of RAMONs to a dictionary, indexed by ID.

    Arguments:
        ramons (RAMON[]): A list of RAMON objects
        flatten (boolean: False): Not implemented

    Returns:
        dict: A python dictionary of RAMON objects.
    """
    if type(ramons) is not list:
        ramons = [ramons]

    out_ramons = {}
    for r in ramons:
        out_ramons[r.id] = {
            "id": r.id,
            "type": _reverse_ramon_types[type(r)],
            "metadata": vars(r)
        }
    return out_ramons

def to_dict ( ramons , flatten = False ) : if type ( ramons ) is not list : ramons = [ ramons ] out_ramons = { } for r in ramons : out_ramons [ r . id ] = { "id" : r . id , "type" : _reverse_ramon_types [ type ( r ) ] , "metadata" : vars ( r ) } return out_ramons

Converts a RAMON object list to a JSON - style dictionary . Useful for going from an array of RAMONs to a dictionary indexed by ID .


def RAMON(typ):
        """
        Takes str or int, returns class type
        """
        if six.PY2:
            lookup = [str, unicode]
        elif six.PY3:
            lookup = [str]

        if type(typ) is int:
            return _ramon_types[typ]
        elif type(typ) in lookup:
            return _ramon_types[_types[typ]]

def RAMON ( typ ) : if six . PY2 : lookup = [ str , unicode ] elif six . PY3 : lookup = [ str ] if type ( typ ) is int : return _ramon_types [ typ ] elif type ( typ ) in lookup : return _ramon_types [ _types [ typ ] ]


def delete_channel(self, channel_name, project_name, dataset_name):
        """
        Deletes a channel given its name, name of its project
        , and name of its dataset.

        Arguments:
            channel_name (str): Channel name
            project_name (str): Project name
            dataset_name (str): Dataset name

        Returns:
            bool: True if channel deleted, False if not
        """
        return self.resources.delete_channel(channel_name, project_name,
                                             dataset_name)

def delete_channel ( self , channel_name , project_name , dataset_name ) : return self . resources . delete_channel ( channel_name , project_name , dataset_name )

Deletes a channel given its name name of its project and name of its dataset .


def add_dataset(self, dataset_name, imagesize, voxelres, offset=None,
                    timerange=None, scalinglevels=None, scaling=None):
        """
        Add a new dataset to the ingest.

        Arguments:
            dataset_name (str): Dataset Name is the overarching name of the
                research effort. Standard naming convention is to do
                LabNamePublicationYear or LeadResearcherCurrentYear.
            imagesize (int, int, int): Image size is the pixel count
                dimensions of the data. For example is the data is stored
                as a series of 100 slices each 2100x2000 pixel TIFF images,
                the X,Y,Z dimensions are (2100, 2000, 100).
            voxelres (float, float, float): Voxel Resolution is the number
                of voxels per unit pixel. We store X,Y,Z voxel resolution
                separately.
            offset (int, int, int): If your data is not well aligned and
                there is "excess" image data you do not wish to examine, but
                are present in your images, offset is how you specify where
                your actual image starts. Offset is provided a pixel
                coordinate offset from origin which specifies the "actual"
                origin of the image. The offset is for X,Y,Z dimensions.
            timerange (int, int): Time Range is a parameter to support
                storage of Time Series data, so the value of the tuple is a
                0 to X range of how many images over time were taken. It
                takes 2 inputs timeStepStart and timeStepStop.
            scalinglevels (int): Scaling levels is the number of levels the
                data is scalable to (how many zoom levels are present in the
                data). The highest resolution of the data is at scaling level
                0, and for each level up the data is down sampled by 2x2
                (per slice). To learn more about the sampling service used,
                visit the the propagation service page.
            scaling (int): Scaling is the scaling method of the data being
                stored. 0 corresponds to a Z-slice orientation (as in a
                collection of tiff images in which each tiff is a slice on
                the z plane) where data will be scaled only on the xy plane,
                not the z plane. 1 corresponds to an isotropic orientation
                (in which each tiff is a slice on the y plane) where data
                is scaled along all axis.

        Returns:
            None
        """
        self.dataset = (dataset_name.strip().replace(" ", ""), imagesize,
                        voxelres, offset, timerange, scalinglevels, scaling)

def
add_dataset
(
self
,
dataset_name
,
imagesize
,
voxelres
,
offset
=
,
timerange
=
,
scalinglevels
=
,
scaling
=
)
:
self
.
dataset
=
(
dataset_name
.
strip
(
)
.
replace
(
" "
,
""
)
,
imagesize
,
voxelres
,
offset
,
timerange
,
scalinglevels
,
scaling
)


def nd_json(self, dataset, project, channel_list, metadata):
        """
        Genarate ND json object.
        """
        nd_dict = {}
        nd_dict['dataset'] = self.dataset_dict(*dataset)
        nd_dict['project'] = self.project_dict(*project)
        nd_dict['metadata'] = metadata
        nd_dict['channels'] = {}
        for channel_name, value in channel_list.items():
            nd_dict['channels'][channel_name] = self.channel_dict(*value)

        return json.dumps(nd_dict, sort_keys=True, indent=4)

def nd_json ( self , dataset , project , channel_list , metadata ) : nd_dict = { } nd_dict [ 'dataset' ] = self . dataset_dict ( * dataset ) nd_dict [ 'project' ] = self . project_dict ( * project ) nd_dict [ 'metadata' ] = metadata nd_dict [ 'channels' ] = { } for channel_name , value in channel_list . items ( ) : nd_dict [ 'channels' ] [ channel_name ] = self . channel_dict ( * value ) return json . dumps ( nd_dict , sort_keys = True , indent = 4 )


def dataset_dict(
        self, dataset_name, imagesize, voxelres,
            offset, timerange, scalinglevels, scaling):
        """Generate the dataset dictionary"""
        dataset_dict = {}
        dataset_dict['dataset_name'] = dataset_name
        dataset_dict['imagesize'] = imagesize
        dataset_dict['voxelres'] = voxelres
        if offset is not None:
            dataset_dict['offset'] = offset
        if timerange is not None:
            dataset_dict['timerange'] = timerange
        if scalinglevels is not None:
            dataset_dict['scalinglevels'] = scalinglevels
        if scaling is not None:
            dataset_dict['scaling'] = scaling
        return dataset_dict

def dataset_dict ( self , dataset_name , imagesize , voxelres , offset , timerange , scalinglevels , scaling ) : dataset_dict = { } dataset_dict [ 'dataset_name' ] = dataset_name dataset_dict [ 'imagesize' ] = imagesize dataset_dict [ 'voxelres' ] = voxelres if offset is not None : dataset_dict [ 'offset' ] = offset if timerange is not None : dataset_dict [ 'timerange' ] = timerange if scalinglevels is not None : dataset_dict [ 'scalinglevels' ] = scalinglevels if scaling is not None : dataset_dict [ 'scaling' ] = scaling return dataset_dict


def channel_dict(self, channel_name, datatype, channel_type, data_url,
                     file_format, file_type, exceptions, resolution,
                     windowrange, readonly):
        """
        Generate the project dictionary.
        """
        channel_dict = {}
        channel_dict['channel_name'] = channel_name
        channel_dict['datatype'] = datatype
        channel_dict['channel_type'] = channel_type
        if exceptions is not None:
            channel_dict['exceptions'] = exceptions
        if resolution is not None:
            channel_dict['resolution'] = resolution
        if windowrange is not None:
            channel_dict['windowrange'] = windowrange
        if readonly is not None:
            channel_dict['readonly'] = readonly
        channel_dict['data_url'] = data_url
        channel_dict['file_format'] = file_format
        channel_dict['file_type'] = file_type
        return channel_dict

def channel_dict ( self , channel_name , datatype , channel_type , data_url , file_format , file_type , exceptions , resolution , windowrange , readonly ) : channel_dict = { } channel_dict [ 'channel_name' ] = channel_name channel_dict [ 'datatype' ] = datatype channel_dict [ 'channel_type' ] = channel_type if exceptions is not None : channel_dict [ 'exceptions' ] = exceptions if resolution is not None : channel_dict [ 'resolution' ] = resolution if windowrange is not None : channel_dict [ 'windowrange' ] = windowrange if readonly is not None : channel_dict [ 'readonly' ] = readonly channel_dict [ 'data_url' ] = data_url channel_dict [ 'file_format' ] = file_format channel_dict [ 'file_type' ] = file_type return channel_dict


def project_dict(self, project_name, token_name, public):
        """
        Genarate the project dictionary.
        """
        project_dict = {}
        project_dict['project_name'] = project_name
        if token_name is not None:
            if token_name == '':
                project_dict['token_name'] = project_name
            else:
                project_dict['token_name'] = token_name
        else:
            project_dict['token_name'] = project_name

        if public is not None:
            project_dict['public'] = public
        return project_dict

def project_dict ( self , project_name , token_name , public ) : project_dict = { } project_dict [ 'project_name' ] = project_name if token_name is not None : if token_name == '' : project_dict [ 'token_name' ] = project_name else : project_dict [ 'token_name' ] = token_name else : project_dict [ 'token_name' ] = project_name if public is not None : project_dict [ 'public' ] = public return project_dict


def identify_imagesize(self, image_type, image_path='/tmp/img.'):
        """
        Identify the image size using the data location and other parameters
        """
        dims = ()
        try:
            if (image_type.lower() == 'png'):
                dims = np.shape(ndpng.load('{}{}'.format(
                    image_path, image_type
                )))
            elif (image_type.lower() == 'tif' or image_type.lower() == 'tiff'):
                dims = np.shape(ndtiff.load('{}{}'.format(
                    image_path, image_type
                )))
            else:
                raise ValueError("Unsupported image type.")
        except:
            raise OSError('The file was not accessible at {}{}'.format(
                image_path,
                image_type
            ))
        return dims[::-1]

def
identify_imagesize
(
self
,
image_type
,
image_path
=
'/tmp/img.'
)
:
dims
=
(
)
try
:
if
(
image_type
.
lower
(
)
==
'png'
)
:
dims
=
np
.
shape
(
ndpng
.
load
(
'{}{}'
.
format
(
image_path
,
image_type
)
)
)
elif
(
image_type
.
lower
(
)
==
'tif'
or
image_type
.
lower
(
)
==
'tiff'
)
:
dims
=
np
.
shape
(
ndtiff
.
load
(
'{}{}'
.
format
(
image_path
,
image_type
)
)
)
else
:
raise
ValueError
(
"Unsupported image type."
)
except
:
raise
OSError
(
'The file was not accessible at {}{}'
.
format
(
image_path
,
image_type
)
)
return
dims
[
:
:
1
]


def put_data(self, data):
        """
        Try to post data to the server.
        """
        URLPath = self.oo.url("autoIngest/")
        # URLPath = 'https://{}/ca/autoIngest/'.format(self.oo.site_host)
        try:
            response = requests.post(URLPath, data=json.dumps(data),
                                     verify=False)
            assert(response.status_code == 200)
            print("From ndio: {}".format(response.content))
        except:
            raise OSError("Error in posting JSON file {}\
".format(response.status_code))

def
put_data
(
self
,
data
)
:
URLPath
=
self
.
oo
.
url
(
"autoIngest/"
)
try
:
response
=
requests
.
post
(
URLPath
,
data
=
json
.
dumps
(
data
)
,
verify
=
False
)
assert
(
response
.
status_code
==
200
)
print
(
"From ndio: {}"
.
format
(
response
.
content
)
)
except
:
raise
OSError
(
"Error in posting JSON file {}\"
.
format
(
response
.
status_code
)
)


def find_path(name, config, wsonly=False):
    """Find path for given workspace and|or repository."""
    workspace = Workspace(config)
    config = config["workspaces"]

    path_list = {}

    if name.find('/') != -1:
        wsonly = False
        try:
            ws, repo = name.split('/')
        except ValueError:
            raise ValueError("There is too many / in `name` argument. "
                             "Argument syntax: `workspace/repository`.")
        if (workspace.exists(ws)):
            if (repo in config[ws]["repositories"]):
                path_name = "%s/%s" % (ws, repo)
                path_list[path_name] = config[ws]["repositories"][repo]

    for ws_name, ws in sorted(config.items()):
        if (name == ws_name):
            if wsonly is True:
                return {ws_name: ws["path"]}
            repositories = sorted(config[ws_name]["repositories"].items())
            for name, path in repositories:
                path_list["%s/%s" % (ws_name, name)] = path
            break

        for repo_name, repo_path in sorted(ws["repositories"].items()):
            if (repo_name == name):
                path_list["%s/%s" % (ws_name, repo_name)] = repo_path

    return path_list

Find path for given workspace and|or repository .


def get_public_tokens(self):
        """
        Get a list of public tokens available on this server.

        Arguments:
            None

        Returns:
            str[]: list of public tokens
        """
        r = self.remote_utils.get_url(self.url() + "public_tokens/")
        return r.json()

def get_public_tokens ( self ) : r = self . remote_utils . get_url ( self . url ( ) + "public_tokens/" ) return r . json ( )

Get a list of public tokens available on this server .


def get_proj_info(self, token):
        """
        Return the project info for a given token.

        Arguments:
            token (str): Token to return information for

        Returns:
            JSON: representation of proj_info
        """
        r = self.remote_utils.get_url(self.url() + "{}/info/".format(token))
        return r.json()

def get_proj_info ( self , token ) : r = self . remote_utils . get_url ( self . url ( ) + "{}/info/" . format ( token ) ) return r . json ( )

Return the project info for a given token .


def set_metadata(self, token, data):
        """
        Insert new metadata into the OCP metadata database.

        Arguments:
            token (str): Token of the datum to set
            data (str): A dictionary to insert as metadata. Include `secret`.

        Returns:
            json: Info of the inserted ID (convenience) or an error message.

        Throws:
            RemoteDataUploadError: If the token is already populated, or if
                there is an issue with your specified `secret` key.
        """
        req = requests.post(self.meta_url("metadata/ocp/set/" + token),
                            json=data, verify=False)

        if req.status_code != 200:
            raise RemoteDataUploadError(
                "Could not upload metadata: " + req.json()['message']
            )
        return req.json()

def
set_metadata
(
self
,
token
,
data
)
:
req
=
requests
.
post
(
self
.
meta_url
(
"metadata/ocp/set/"
+
token
)
,
json
=
data
,
verify
=
False
)
if
req
.
status_code
!=
200
:
raise
RemoteDataUploadError
(
"Could not upload metadata: "
+
req
.
json
(
)
[
'message'
]
)
return
req
.
json
(
)

Insert new metadata into the OCP metadata database .


def get_url(self, url):
        """
        Get a response object for a given url.

        Arguments:
            url (str): The url make a get to
            token (str): The authentication token

        Returns:
            obj: The response object
        """
        try:
            req = requests.get(url, headers={
                'Authorization': 'Token {}'.format(self._user_token)
            }, verify=False)
            if req.status_code is 403:
                raise ValueError("Access Denied")
            else:
                return req
        except requests.exceptions.ConnectionError as e:
            if str(e) == '403 Client Error: Forbidden':
                raise ValueError('Access Denied')
            else:
                raise e

def
get_url
(
self
,
url
)
:
try
:
req
=
requests
.
get
(
url
,
headers
=
{
'Authorization'
:
'Token {}'
.
format
(
self
.
_user_token
)
}
,
verify
=
False
)
if
req
.
status_code
is
403
:
raise
ValueError
(
"Access Denied"
)
else
:
return
req
except
requests
.
exceptions
.
ConnectionError
as
e
:
if
str
(
e
)
==
'403 Client Error: Forbidden'
:
raise
ValueError
(
'Access Denied'
)
else
:
raise
e


def post_url(self, url, token='', json=None, data=None, headers=None):
        """
        Returns a post resquest object taking in a url, user token, and
        possible json information.

        Arguments:
            url (str): The url to make post to
            token (str): The authentication token
            json (dict): json info to send

        Returns:
            obj: Post request object
        """
        if (token == ''):
            token = self._user_token

        if headers:
            headers.update({'Authorization': 'Token {}'.format(token)})
        else:
            headers = {'Authorization': 'Token {}'.format(token)}

        if json:
            return requests.post(url,
                                 headers=headers,
                                 json=json,
                                 verify=False)
        if data:
            return requests.post(url,
                                 headers=headers,
                                 data=data,
                                 verify=False)

        return requests.post(url,
                             headers=headers,
                             verify=False)

def
post_url
(
self
,
url
,
token
=
''
,
json
=
,
data
=
,
headers
=
)
:
if
(
token
==
''
)
:
token
=
self
.
_user_token
if
headers
:
headers
.
update
(
{
'Authorization'
:
'Token {}'
.
format
(
token
)
}
)
else
:
headers
=
{
'Authorization'
:
'Token {}'
.
format
(
token
)
}
if
json
:
return
requests
.
post
(
url
,
headers
=
headers
,
json
=
json
,
verify
=
False
)
if
data
:
return
requests
.
post
(
url
,
headers
=
headers
,
data
=
data
,
verify
=
False
)
return
requests
.
post
(
url
,
headers
=
headers
,
verify
=
False
)

Returns a post resquest object taking in a url user token and possible json information .


def delete_url(self, url, token=''):
        """
        Returns a delete resquest object taking in a url and user token.

        Arguments:
            url (str): The url to make post to
            token (str): The authentication token

        Returns:
            obj: Delete request object
        """
        if (token == ''):
            token = self._user_token

        return requests.delete(url,
                               headers={
                                   'Authorization': 'Token {}'.format(token)},
                               verify=False,)

def
delete_url
(
self
,
url
,
token
=
''
)
:
if
(
token
==
''
)
:
token
=
self
.
_user_token
return
requests
.
delete
(
url
,
headers
=
{
'Authorization'
:
'Token {}'
.
format
(
token
)
}
,
verify
=
False
,
)

Returns a delete resquest object taking in a url and user token .


def load(hdf5_filename):
    """
    Import a HDF5 file into a numpy array.

    Arguments:
        hdf5_filename:  A string filename of a HDF5 datafile

    Returns:
        A numpy array with data from the HDF5 file
    """
    # Expand filename to be absolute
    hdf5_filename = os.path.expanduser(hdf5_filename)

    try:
        f = h5py.File(hdf5_filename, "r")
        # neurodata stores data inside the 'cutout' h5 dataset
        data_layers = f.get('image').get('CUTOUT')
    except Exception as e:
        raise ValueError("Could not load file {0} for conversion. {}".format(
                         hdf5_filename, e))
        raise

    return numpy.array(data_layers)


def save(hdf5_filename, array):
    """
    Export a numpy array to a HDF5 file.

    Arguments:
        hdf5_filename (str): A filename to which to save the HDF5 data
        array (numpy.ndarray): The numpy array to save to HDF5

    Returns:
        String. The expanded filename that now holds the HDF5 data
    """
    # Expand filename to be absolute
    hdf5_filename = os.path.expanduser(hdf5_filename)

    try:
        h = h5py.File(hdf5_filename, "w")
        h.create_dataset('CUTOUT', data=array)
        h.close()
    except Exception as e:
        raise ValueError("Could not save HDF5 file {0}.".format(hdf5_filename))

    return hdf5_filename

def
save
(
hdf5_filename
,
array
)
:
hdf5_filename
=
os
.
path
.
expanduser
(
hdf5_filename
)
try
:
h
=
h5py
.
File
(
hdf5_filename
,
"w"
)
h
.
create_dataset
(
'CUTOUT'
,
data
=
array
)
h
.
close
(
)
except
Exception
as
e
:
raise
ValueError
(
"Could not save HDF5 file {0}."
.
format
(
hdf5_filename
)
)
return
hdf5_filename


def infer_gaps_in_tree(df_seq, tree, id_col='id', sequence_col='sequence'):
    """Adds a character matrix to DendroPy tree and infers gaps using
    Fitch's algorithm.

    Infer gaps in sequences at ancestral nodes.
    """
    taxa = tree.taxon_namespace

    # Get alignment as fasta
    alignment = df_seq.phylo.to_fasta(id_col=id_col, id_only=True,
                                      sequence_col=sequence_col)

    # Build a Sequence data matrix from Dendropy
    data = dendropy.ProteinCharacterMatrix.get(
        data=alignment,
        schema="fasta",
        taxon_namespace=taxa)

    # Construct a map object between sequence data and tree data.
    taxon_state_sets_map = data.taxon_state_sets_map(gaps_as_missing=False)

    # Fitch algorithm to determine placement of gaps
    dendropy.model.parsimony.fitch_down_pass(tree.postorder_node_iter(),
            taxon_state_sets_map=taxon_state_sets_map)
    dendropy.model.parsimony.fitch_up_pass(tree.preorder_node_iter())
    return tree

def infer_gaps_in_tree ( df_seq , tree , id_col = 'id' , sequence_col = 'sequence' ) : taxa = tree . taxon_namespace alignment = df_seq . phylo . to_fasta ( id_col = id_col , id_only = True , sequence_col = sequence_col ) data = dendropy . ProteinCharacterMatrix . get ( data = alignment , schema = "fasta" , taxon_namespace = taxa ) taxon_state_sets_map = data . taxon_state_sets_map ( gaps_as_missing = False ) dendropy . model . parsimony . fitch_down_pass ( tree . postorder_node_iter ( ) , taxon_state_sets_map = taxon_state_sets_map ) dendropy . model . parsimony . fitch_up_pass ( tree . preorder_node_iter ( ) ) return tree

Adds a character matrix to DendroPy tree and infers gaps using Fitch s algorithm .


def nvim_io_recover(self, io: NvimIORecover[A]) -> NvimIO[B]:
        '''calls `map` to shift the recover execution to flat_map_nvim_io
        '''
        return eval_step(self.vim)(io.map(lambda a: a))

def nvim_io_recover ( self , io : NvimIORecover [ A ] ) -> NvimIO [ B ] : return eval_step ( self . vim ) ( io . map ( lambda a : a ) )

calls map to shift the recover execution to flat_map_nvim_io


def install_jinja_translations():
    """
    Install our gettext and ngettext functions into Jinja2's environment.
    """
    class Translation(object):
        """
        We pass this object to jinja so it can find our gettext implementation.
        If we pass the GNUTranslation object directly, it won't have our
        context and whitespace stripping action.
        """
        ugettext = staticmethod(ugettext)
        ungettext = staticmethod(ungettext)

    import jingo
    jingo.env.install_gettext_translations(Translation)

def install_jinja_translations ( ) : class Translation ( object ) : ugettext = staticmethod ( ugettext ) ungettext = staticmethod ( ungettext ) import jingo jingo . env . install_gettext_translations ( Translation )

Install our gettext and ngettext functions into Jinja2 s environment .


def exclusive_ns(guard: StateGuard[A], desc: str, thunk: Callable[..., NS[A, B]], *a: Any) -> Do:
    '''this is the central unsafe function, using a lock and updating the state in `guard` in-place.
    '''
    yield guard.acquire()
    log.debug2(lambda: f'exclusive: {desc}')
    state, response = yield N.ensure_failure(thunk(*a).run(guard.state), guard.release)
    yield N.delay(lambda v: unsafe_update_state(guard, state))
    yield guard.release()
    log.debug2(lambda: f'release: {desc}')
    yield N.pure(response)

def
exclusive_ns
(
guard
:
StateGuard
[
A
]
,
desc
:
str
,
thunk
:
Callable
[
...
,
NS
[
A
,
B
]
]
,
*
a
:
Any
)
->
Do
:
yield
guard
.
acquire
(
)
log
.
debug2
(
lambda
:
f'exclusive: {desc}'
)
state
,
response
=
yield
N
.
ensure_failure
(
thunk
(
*
a
)
.
run
(
guard
.
state
)
,
guard
.
release
)
yield
N
.
delay
(
lambda
v
:
unsafe_update_state
(
guard
,
state
)
)
yield
guard
.
release
(
)
log
.
debug2
(
lambda
:
f'release: {desc}'
)
yield
N
.
pure
(
response
)

this is the central unsafe function using a lock and updating the state in guard in - place .


def _percent(data, part, total):
    """
    Calculate a percentage.
    """
    try:
        return round(100 * float(data[part]) / float(data[total]), 1)
    except ZeroDivisionError:
        return 0

def _percent ( data , part , total ) : try : return round ( 100 * float ( data [ part ] ) / float ( data [ total ] ) , 1 ) except ZeroDivisionError : return 0


def _get_cache_stats(server_name=None):
    """
    Get stats info.
    """
    server_info = {}
    for svr in mc_client.get_stats():
        svr_info = svr[0].split(' ')
        svr_name = svr_info[0]
        svr_stats = svr[1]
        svr_stats['bytes_percent'] = _percent(svr_stats, 'bytes', 'limit_maxbytes')
        svr_stats['get_hit_rate'] = _percent(svr_stats, 'get_hits', 'cmd_get')
        svr_stats['get_miss_rate'] = _percent(svr_stats, 'get_misses', 'cmd_get')
        if server_name and server_name == svr_name:
            return svr_stats
        server_info[svr_name] = svr_stats
    return server_info


def _get_cache_slabs(server_name=None):
    """
    Get slabs info.
    """
    server_info = {}
    for svr in mc_client.get_slabs():
        svr_info = svr[0].split(' ')
        svr_name = svr_info[0]
        if server_name and server_name == svr_name:
            return svr[1]
        server_info[svr_name] = svr[1]
    return server_info

def
_get_cache_slabs
(
server_name
=
)
:
server_info
=
{
}
for
svr
in
mc_client
.
get_slabs
(
)
:
svr_info
=
svr
[
0
]
.
split
(
' '
)
svr_name
=
svr_info
[
0
]
if
server_name
and
server_name
==
svr_name
:
return
svr
[
1
]
server_info
[
svr_name
]
=
svr
[
1
]
return
server_info


def _context_data(data, request=None):
    """
    Add admin global context, for compatibility with Django 1.7
    """
    try:
        return dict(site.each_context(request).items() + data.items())
    except AttributeError:
        return data

def _context_data ( data , request = None ) : try : return dict ( site . each_context ( request ) . items ( ) + data . items ( ) ) except AttributeError : return data

Add admin global context for compatibility with Django 1 . 7


def server_status(request):
    """
    Return the status of all servers.
    """
    data = {
        'cache_stats': _get_cache_stats(),
        'can_get_slabs': hasattr(mc_client, 'get_slabs'),
    }
    return render_to_response('memcache_admin/server_status.html', data, RequestContext(request))

def server_status ( request ) : data = { 'cache_stats' : _get_cache_stats ( ) , 'can_get_slabs' : hasattr ( mc_client , 'get_slabs' ) , } return render_to_response ( 'memcache_admin/server_status.html' , data , RequestContext ( request ) )


def dashboard(request):
    """
    Show the dashboard.
    """
    # mc_client will be a dict if memcached is not configured
    if not isinstance(mc_client, dict):
        cache_stats = _get_cache_stats()
    else:
        cache_stats = None
    if cache_stats:
        data = _context_data({
            'title': _('Memcache Dashboard'),
            'cache_stats': cache_stats,
            'can_get_slabs': hasattr(mc_client, 'get_slabs'),
            'REFRESH_RATE': SETTINGS['REFRESH_RATE'],
        },
            request)
        template = 'memcache_admin/dashboard.html'
    else:
        data = _context_data({
            'title': _('Memcache Dashboard - Error'),
            'error_message': _('Unable to connect to a memcache server.'),
        },
            request)
        template = 'memcache_admin/dashboard_error.html'
    return render_to_response(template, data, RequestContext(request))


def stats(request, server_name):
    """
    Show server statistics.
    """
    server_name = server_name.strip('/')
    data = _context_data({
        'title': _('Memcache Statistics for %s') % server_name,
        'cache_stats': _get_cache_stats(server_name),
    },
        request)
    return render_to_response('memcache_admin/stats.html', data, RequestContext(request))

def
stats
(
request
,
server_name
)
:
server_name
=
server_name
.
strip
(
'/'
)
data
=
_context_data
(
{
'title'
:
_
(
'Memcache Statistics for %s'
)
%
server_name
,
'cache_stats'
:
_get_cache_stats
(
server_name
)
,
}
,
request
)
return
render_to_response
(
'memcache_admin/stats.html'
,
data
,
RequestContext
(
request
)
)


def slabs(request, server_name):
    """
    Show server slabs.
    """
    data = _context_data({
        'title': _('Memcache Slabs for %s') % server_name,
        'cache_slabs': _get_cache_slabs(server_name),
    },
        request)
    return render_to_response('memcache_admin/slabs.html', data, RequestContext(request))

def
slabs
(
request
,
server_name
)
:
data
=
_context_data
(
{
'title'
:
_
(
'Memcache Slabs for %s'
)
%
server_name
,
'cache_slabs'
:
_get_cache_slabs
(
server_name
)
,
}
,
request
)
return
render_to_response
(
'memcache_admin/slabs.html'
,
data
,
RequestContext
(
request
)
)


def human_bytes(value):
    """
    Convert a byte value into a human-readable format.
    """
    value = float(value)
    if value >= 1073741824:
        gigabytes = value / 1073741824
        size = '%.2f GB' % gigabytes
    elif value >= 1048576:
        megabytes = value / 1048576
        size = '%.2f MB' % megabytes
    elif value >= 1024:
        kilobytes = value / 1024
        size = '%.2f KB' % kilobytes
    else:
        size = '%.2f B' % value
    return size

def
human_bytes
(
value
)
:
value
=
float
(
value
)
if
value
>=
1073741824
:
gigabytes
=
value
/
1073741824
size
=
'%.2f GB'
%
gigabytes
elif
value
>=
1048576
:
megabytes
=
value
/
1048576
size
=
'%.2f MB'
%
megabytes
elif
value
>=
1024
:
kilobytes
=
value
/
1024
size
=
'%.2f KB'
%
kilobytes
else
:
size
=
'%.2f B'
%
value
return
size

Convert a byte value into a human - readable format .


def find_config(self, children):
        """
        Find a config in our children so we can fill in variables in our other
        children with its data.
        """
        named_config = None
        found_config = None

        # first see if we got a kwarg named 'config', as this guy is special
        if 'config' in children:
            if type(children['config']) == str:
                children['config'] = ConfigFile(children['config'])
            elif isinstance(children['config'], Config):
                children['config'] = children['config']
            elif type(children['config']) == dict:
                children['config'] = Config(data=children['config'])
            else:
                raise TypeError("Don't know how to turn {} into a Config".format(type(children['config'])))

            named_config = children['config']

        # next check the other kwargs
        for k in children:
            if isinstance(children[k], Config):
                found_config = children[k]

        # if we still don't have a config, see if there's a directory with one
        for k in children:
            if isinstance(children[k], Directory):
                for j in children[k]._children:
                    if j == 'config' and not named_config:
                        named_config = children[k]._children[j]
                    if isinstance(children[k]._children[j], Config):
                        found_config = children[k]._children[j]

        if named_config:
            return named_config
        else:
            return found_config

Find a config in our children so we can fill in variables in our other children with its data .


def add(self, **kwargs):
        """
        Add objects to the environment.
        """
        for key in kwargs:
            if type(kwargs[key]) == str:
                self._children[key] = Directory(kwargs[key])
            else:
                self._children[key] = kwargs[key]
            self._children[key]._env = self
            self._children[key].apply_config(ConfigApplicator(self.config))
            self._children[key].prepare()

def add ( self , ** kwargs ) : for key in kwargs : if type ( kwargs [ key ] ) == str : self . _children [ key ] = Directory ( kwargs [ key ] ) else : self . _children [ key ] = kwargs [ key ] self . _children [ key ] . _env = self self . _children [ key ] . apply_config ( ConfigApplicator ( self . config ) ) self . _children [ key ] . prepare ( )


def apply_config(self, applicator):
        """
        Replace any config tokens in the file's path with values from the config.
        """
        if type(self._fpath) == str:
            self._fpath = applicator.apply(self._fpath)

def apply_config ( self , applicator ) : if type ( self . _fpath ) == str : self . _fpath = applicator . apply ( self . _fpath )

Replace any config tokens in the file s path with values from the config .


def path(self):
        """
        Get the path to the file relative to its parent.
        """
        if self._parent:
            return os.path.join(self._parent.path, self._fpath)
        else:
            return self._fpath

def path ( self ) : if self . _parent : return os . path . join ( self . _parent . path , self . _fpath ) else : return self . _fpath

Get the path to the file relative to its parent .


def read(self):
        """
        Read and return the contents of the file.
        """
        with open(self.path) as f:
            d = f.read()
        return d

def read ( self ) : with open ( self . path ) as f : d = f . read ( ) return d

Read and return the contents of the file .


def write(self, data, mode='w'):
        """
        Write data to the file.

        `data` is the data to write
        `mode` is the mode argument to pass to `open()`
        """
        with open(self.path, mode) as f:
            f.write(data)

def write ( self , data , mode = 'w' ) : with open ( self . path , mode ) as f : f . write ( data )


def configure(self):
        """
        Configure the Python logging module for this file.
        """
        # build a file handler for this file
        handler = logging.FileHandler(self.path, delay=True)

        # if we got a format string, create a formatter with it
        if self._format:
            handler.setFormatter(logging.Formatter(self._format))

        # if we got a string for the formatter, assume it's the name of a
        # formatter in the environment's config
        if type(self._formatter) == str:
            if self._env and self._env.config.logging.dict_config.formatters[self._formatter]:
                d = self._env.config.logging.dict_config.formatters[self._formatter].to_dict()
                handler.setFormatter(logging.Formatter(**d))
        elif type(self._formatter) == dict:
            # if it's a dict it must be the actual formatter params
            handler.setFormatter(logging.Formatter(**self._formatter))

        # add the file handler to whatever loggers were specified
        if len(self._loggers):
            for name in self._loggers:
                logging.getLogger(name).addHandler(handler)
        else:
            # none specified, just add it to the root logger
            logging.getLogger().addHandler(handler)

def configure ( self ) : handler = logging . FileHandler ( self . path , delay = True ) if self . _format : handler . setFormatter ( logging . Formatter ( self . _format ) ) if type ( self . _formatter ) == str : if self . _env and self . _env . config . logging . dict_config . formatters [ self . _formatter ] : d = self . _env . config . logging . dict_config . formatters [ self . _formatter ] . to_dict ( ) handler . setFormatter ( logging . Formatter ( ** d ) ) elif type ( self . _formatter ) == dict : handler . setFormatter ( logging . Formatter ( ** self . _formatter ) ) if len ( self . _loggers ) : for name in self . _loggers : logging . getLogger ( name ) . addHandler ( handler ) else : logging . getLogger ( ) . addHandler ( handler )

Configure the Python logging module for this file .


def create(self):
        """
        Create the file.

        If the file already exists an exception will be raised
        """
        if not os.path.exists(self.path):
            open(self.path, 'a').close()
        else:
            raise Exception("File exists: {}".format(self.path))

def
create
(
self
)
:
if
not
os
.
path
.
exists
(
self
.
path
)
:
open
(
self
.
path
,
'a'
)
.
close
(
)
else
:
raise
Exception
(
"File exists: {}"
.
format
(
self
.
path
)
)


def apply_config(self, applicator):
        """
        Replace any config tokens with values from the config.
        """
        if type(self._path) == str:
            self._path = applicator.apply(self._path)

        for key in self._children:
            self._children[key].apply_config(applicator)

def apply_config ( self , applicator ) : if type ( self . _path ) == str : self . _path = applicator . apply ( self . _path ) for key in self . _children : self . _children [ key ] . apply_config ( applicator )

Replace any config tokens with values from the config .


def path(self):
        """
        Return the path to this directory.
        """
        p = ''

        if self._parent and self._parent.path:
            p = os.path.join(p, self._parent.path)
        if self._base:
            p = os.path.join(p, self._base)
        if self._path:
            p = os.path.join(p, self._path)

        return p

def path ( self ) : p = '' if self . _parent and self . _parent . path : p = os . path . join ( p , self . _parent . path ) if self . _base : p = os . path . join ( p , self . _base ) if self . _path : p = os . path . join ( p , self . _path ) return p


def remove(self, recursive=True, ignore_error=True):
        """
        Remove the directory.
        """
        try:
            if recursive or self._cleanup == 'recursive':
                shutil.rmtree(self.path)
            else:
                os.rmdir(self.path)
        except Exception as e:
            if not ignore_error:
                raise e

def remove ( self , recursive = True , ignore_error = True ) : try : if recursive or self . _cleanup == 'recursive' : shutil . rmtree ( self . path ) else : os . rmdir ( self . path ) except Exception as e : if not ignore_error : raise e


def prepare(self):
        """
        Prepare the Directory for use in an Environment.

        This will create the directory if the create flag is set.
        """
        if self._create:
            self.create()
        for k in self._children:
            self._children[k]._env = self._env
            self._children[k].prepare()

def prepare ( self ) : if self . _create : self . create ( ) for k in self . _children : self . _children [ k ] . _env = self . _env self . _children [ k ] . prepare ( )

Prepare the Directory for use in an Environment .


def cleanup(self):
        """
        Clean up children and remove the directory.

        Directory will only be removed if the cleanup flag is set.
        """
        for k in self._children:
            self._children[k].cleanup()

        if self._cleanup:
            self.remove(True)

def cleanup ( self ) : for k in self . _children : self . _children [ k ] . cleanup ( ) if self . _cleanup : self . remove ( True )

Clean up children and remove the directory .


def path_to(self, path):
        """
        Find the path to something inside this directory.
        """
        return os.path.join(self.path, str(path))

def path_to ( self , path ) : return os . path . join ( self . path , str ( path ) )

Find the path to something inside this directory .


def list(self):
        """
        List the contents of the directory.
        """
        return [File(f, parent=self) for f in os.listdir(self.path)]

def list ( self ) : return [ File ( f , parent = self ) for f in os . listdir ( self . path ) ]


def write(self, filename, data, mode='w'):
        """
        Write to a file in the directory.
        """
        with open(self.path_to(str(filename)), mode) as f:
            f.write(data)

def write ( self , filename , data , mode = 'w' ) : with open ( self . path_to ( str ( filename ) ) , mode ) as f : f . write ( data )


def read(self, filename):
        """
        Read a file from the directory.
        """
        with open(self.path_to(str(filename))) as f:
            d = f.read()
        return d

def read ( self , filename ) : with open ( self . path_to ( str ( filename ) ) ) as f : d = f . read ( ) return d


def add(self, *args, **kwargs):
        """
        Add objects to the directory.
        """
        for key in kwargs:
            if isinstance(kwargs[key], str):
                self._children[key] = File(kwargs[key])
            else:
                self._children[key] = kwargs[key]
            self._children[key]._parent = self
            self._children[key]._env = self._env

        added = []
        for arg in args:
            if isinstance(arg, File):
                self._children[arg.name] = arg
                self._children[arg.name]._parent = self
                self._children[arg.name]._env = self._env
            elif isinstance(arg, str):
                f = File(arg)
                added.append(f)
                self._children[arg] = f
                self._children[arg]._parent = self
                self._children[arg]._env = self._env
            else:
                raise TypeError(type(arg))

        # if we were passed a single file/filename, return the File object for convenience
        if len(added) == 1:
            return added[0]
        if len(args) == 1:
            return args[0]

def add ( self , * args , ** kwargs ) : for key in kwargs : if isinstance ( kwargs [ key ] , str ) : self . _children [ key ] = File ( kwargs [ key ] ) else : self . _children [ key ] = kwargs [ key ] self . _children [ key ] . _parent = self self . _children [ key ] . _env = self . _env added = [ ] for arg in args : if isinstance ( arg , File ) : self . _children [ arg . name ] = arg self . _children [ arg . name ] . _parent = self self . _children [ arg . name ] . _env = self . _env elif isinstance ( arg , str ) : f = File ( arg ) added . append ( f ) self . _children [ arg ] = f self . _children [ arg ] . _parent = self self . _children [ arg ] . _env = self . _env else : raise TypeError ( type ( arg ) ) if len ( added ) == 1 : return added [ 0 ] if len ( args ) == 1 : return args [ 0 ]


def save(self):
        """
        Save the state to a file.
        """
        with open(self.path, 'w') as f:
            f.write(yaml.dump(dict(self.d)))

def save ( self ) : with open ( self . path , 'w' ) as f : f . write ( yaml . dump ( dict ( self . d ) ) )


def load(self):
        """
        Load a saved state file.
        """
        if os.path.exists(self.path):
            with open(self.path, 'r') as f:
                self.d = yaml.safe_load(f.read().replace('\t', ' '*4))

def
load
(
self
)
:
if
os
.
path
.
exists
(
self
.
path
)
:
with
open
(
self
.
path
,
'r'
)
as
f
:
self
.
d
=
yaml
.
safe_load
(
f
.
read
(
)
.
replace
(
'\t'
,
' '
*
4
)
)


def cleanup(self):
        """
        Clean up the saved state.
        """
        if os.path.exists(self.path):
            os.remove(self.path)

def cleanup ( self ) : if os . path . exists ( self . path ) : os . remove ( self . path )


def load_plugins(self, directory):
        """
        Loads plugins from the specified directory.

        `directory` is the full path to a directory containing python modules
        which each contain a subclass of the Plugin class.

        There is no criteria for a valid plugin at this level - any python
        module found in the directory will be loaded. Only modules that
        implement a subclass of the Plugin class above will be collected.

        The directory will be traversed recursively.
        """
        # walk directory
        for filename in os.listdir(directory):
            # path to file
            filepath = os.path.join(directory, filename)

            # if it's a file, load it
            modname, ext = os.path.splitext(filename)
            if os.path.isfile(filepath) and ext == '.py':
                file, path, descr = imp.find_module(modname, [directory])
                if file:
                    mod = imp.load_module(modname, file, path, descr)

            # if it's a directory, recurse into it
            if os.path.isdir(filepath):
                self.load_plugins(filepath)

def load_plugins ( self , directory ) : for filename in os . listdir ( directory ) : filepath = os . path . join ( directory , filename ) modname , ext = os . path . splitext ( filename ) if os . path . isfile ( filepath ) and ext == '.py' : file , path , descr = imp . find_module ( modname , [ directory ] ) if file : mod = imp . load_module ( modname , file , path , descr ) if os . path . isdir ( filepath ) : self . load_plugins ( filepath )

Loads plugins from the specified directory .


def update_dict(target, source):
    """
    Recursively merge values from a nested dictionary into another nested
    dictionary.

    For example:

    >>> target = {
    ...     'thing': 123,
    ...     'thang': {
    ...         'a': 1,
    ...         'b': 2
    ...     }
    ... }
    >>> source = {
    ...     'thang': {
    ...         'a': 666,
    ...         'c': 777
    ...     }
    ... }
    >>> update_dict(target, source)
    >>> target
    {
        'thing': 123,
        'thang': {
            'a': 666,
            'b': 2,
            'c': 777
        }
    }
    """
    for k,v in source.items():
        if isinstance(v, dict) and k in target and isinstance(source[k], dict):
            update_dict(target[k], v)
        else:
            target[k] = v

def update_dict ( target , source ) : for k , v in source . items ( ) : if isinstance ( v , dict ) and k in target and isinstance ( source [ k ] , dict ) : update_dict ( target [ k ] , v ) else : target [ k ] = v

Recursively merge values from a nested dictionary into another nested dictionary .


def _child(self, path):
        """
        Return a ConfigNode object representing a child node with the specified
        relative path.
        """
        if self._path:
            path = '{}.{}'.format(self._path, path)
        return ConfigNode(root=self._root, path=path)

def _child ( self , path ) : if self . _path : path = '{}.{}' . format ( self . _path , path ) return ConfigNode ( root = self . _root , path = path )

Return a ConfigNode object representing a child node with the specified relative path .


def _resolve_path(self, create=False):
        """
        Returns a tuple of a reference to the last container in the path, and
        the last component in the key path.

        For example, with a self._value like this:

        {
            'thing': {
                'another': {
                    'some_leaf': 5,
                    'one_more': {
                        'other_leaf': 'x'
                    }
                }
            }
        }

        And a self._path of: 'thing.another.some_leaf'

        This will return a tuple of a reference to the 'another' dict, and
        'some_leaf', allowing the setter and casting methods to directly access
        the item referred to by the key path.
        """
        # Split up the key path
        if type(self._path) == str:
            key_path = self._path.split('.')
        else:
            key_path = [self._path]

        # Start at the root node
        node = self._root._data
        nodes = [self._root._data]

        # Traverse along key path
        while len(key_path):
            # Get the next key in the key path
            key = key_path.pop(0)

            # See if the test could be an int for array access, if so assume it is
            try:
                key = int(key)
            except:
                pass

            # If the next level doesn't exist, create it
            if create:
                if type(node) == dict and key not in node:
                    node[key] = {}
                elif type(node) == list and type(key) == int and len(node) < key:
                    node.append([None for i in range(key-len(node))])

            # Store the last node and traverse down the hierarchy
            nodes.append(node)
            try:
                node = node[key]
            except TypeError:
                if type(key) == int:
                    raise IndexError(key)
                else:
                    raise KeyError(key)

        return (nodes[-1], key)

def _resolve_path ( self , create = False ) : if type ( self . _path ) == str : key_path = self . _path . split ( '.' ) else : key_path = [ self . _path ] node = self . _root . _data nodes = [ self . _root . _data ] while len ( key_path ) : key = key_path . pop ( 0 ) try : key = int ( key ) except : pass if create : if type ( node ) == dict and key not in node : node [ key ] = { } elif type ( node ) == list and type ( key ) == int and len ( node ) < key : node . append ( [ None for i in range ( key - len ( node ) ) ] ) nodes . append ( node ) try : node = node [ key ] except TypeError : if type ( key ) == int : raise IndexError ( key ) else : raise KeyError ( key ) return ( nodes [ - 1 ] , key )

Returns a tuple of a reference to the last container in the path and the last component in the key path .


def _get_value(self):
        """
        Get the value represented by this node.
        """
        if self._path:
            try:
                container, last = self._resolve_path()
                return container[last]
            except KeyError:
                return None
            except IndexError:
                return None
        else:
            return self._data

def _get_value ( self ) : if self . _path : try : container , last = self . _resolve_path ( ) return container [ last ] except KeyError : return None except IndexError : return None else : return self . _data

Get the value represented by this node .


def update(self, data={}, options={}):
        """
        Update the configuration with new data.

        This can be passed either or both `data` and `options`.

        `options` is a dict of keypath/value pairs like this (similar to
        CherryPy's config mechanism:

            >>> c.update(options={
            ...     'server.port': 8080,
            ...     'server.host': 'localhost',
            ...     'admin.email': 'admin@lol'
            ... })

        `data` is a dict of actual config data, like this:

            >>> c.update(data={
            ...     'server': {
            ...         'port': 8080,
            ...         'host': 'localhost'
            ...     },
            ...     'admin': {
            ...         'email': 'admin@lol'
            ...     }
            ... })
        """
        # Handle an update with a set of options like CherryPy does
        for key in options:
            self[key] = options[key]

        # Merge in any data in `data`
        if isinstance(data, ConfigNode):
            data = data._get_value()
        update_dict(self._get_value(), data)

def update ( self , data = { } , options = { } ) : for key in options : self [ key ] = options [ key ] if isinstance ( data , ConfigNode ) : data = data . _get_value ( ) update_dict ( self . _get_value ( ) , data )

Update the configuration with new data .


def load(self, reload=False):
        """
        Load the config and defaults from files.
        """
        if reload or not self._loaded:
            # load defaults
            if self._defaults_file and type(self._defaults_file) == str:
                self._defaults_file = File(self._defaults_file, parent=self._parent)
            defaults = {}
            if self._defaults_file:
                defaults = yaml.safe_load(self._defaults_file.read().replace('\t', '    '))

            # load data
            data = {}
            if self.exists:
                data = yaml.safe_load(self.read().replace('\t', '    '))

            # initialise with the loaded data
            self._defaults = defaults
            self._data = copy.deepcopy(self._defaults)
            self.update(data=data)

            # if specified, apply environment variables
            if self._apply_env:
                self.update(ConfigEnv(self._env_prefix))

            self._loaded = True

        return self

def
load
(
self
,
reload
=
False
)
:
if
reload
or
not
self
.
_loaded
:
if
self
.
_defaults_file
and
type
(
self
.
_defaults_file
)
==
str
:
self
.
_defaults_file
=
File
(
self
.
_defaults_file
,
parent
=
self
.
_parent
)
defaults
=
{
}
if
self
.
_defaults_file
:
defaults
=
yaml
.
safe_load
(
self
.
_defaults_file
.
read
(
)
.
replace
(
'\t'
,
'    '
)
)
data
=
{
}
if
self
.
exists
:
data
=
yaml
.
safe_load
(
self
.
read
(
)
.
replace
(
'\t'
,
'    '
)
)
self
.
_defaults
=
defaults
self
.
_data
=
copy
.
deepcopy
(
self
.
_defaults
)
self
.
update
(
data
=
data
)
if
self
.
_apply_env
:
self
.
update
(
ConfigEnv
(
self
.
_env_prefix
)
)
self
.
_loaded
=
True
return
self

Load the config and defaults from files .


def apply_to_str(self, obj):
        """
        Apply the config to a string.
        """
        toks = re.split('({config:|})', obj)
        newtoks = []
        try:
            while len(toks):
                tok = toks.pop(0)
                if tok == '{config:':
                    # pop the config variable, look it up
                    var = toks.pop(0)
                    val = self.config[var]

                    # if we got an empty node, then it didn't exist
                    if type(val) == ConfigNode and val == None:
                        raise KeyError("No such config variable '{}'".format(var))

                    # add the value to the list
                    newtoks.append(str(val))

                    # pop the '}'
                    toks.pop(0)
                else:
                    # not the start of a config block, just append it to the list
                    newtoks.append(tok)
            return ''.join(newtoks)
        except IndexError:
            pass

        return obj

def
apply_to_str
(
self
,
obj
)
:
toks
=
re
.
split
(
'({config:|})'
,
obj
)
newtoks
=
[
]
try
:
while
len
(
toks
)
:
tok
=
toks
.
pop
(
0
)
if
tok
==
'{config:'
:
var
=
toks
.
pop
(
0
)
val
=
self
.
config
[
var
]
if
type
(
val
)
==
ConfigNode
and
val
==
:
raise
KeyError
(
"No such config variable '{}'"
.
format
(
var
)
)
newtoks
.
append
(
str
(
val
)
)
toks
.
pop
(
0
)
else
:
newtoks
.
append
(
tok
)
return
''
.
join
(
newtoks
)
except
IndexError
:
pass
return
obj


def build_callback_url(request, urlname, message):
    """
    Build Twilio callback url for confirming message delivery status

    :type message: OutgoingSMS
    """
    location = reverse(urlname, kwargs={"pk": message.pk})

    callback_domain = getattr(settings, "TWILIO_CALLBACK_DOMAIN", None)
    if callback_domain:
        url = "{}://{}{}".format(
            "https" if getattr(settings, "TWILIO_CALLBACK_USE_HTTPS", False) else "http",
            callback_domain,
            location
        )

    elif request is not None:
        url = request.build_absolute_uri(location)
    else:
        raise ValueError(
            "Unable to build callback url. Configure TWILIO_CALLBACK_DOMAIN "
            "or pass request object to function call"
        )
    return url


def process_input(self):
        """Called when socket is read-ready"""
        try:
            pyngus.read_socket_input(self.connection, self.socket)
        except Exception as e:
            LOG.error("Exception on socket read: %s", str(e))
            self.connection.close_input()
            self.connection.close()
        self.connection.process(time.time())

def
process_input
(
self
)
:
try
:
pyngus
.
read_socket_input
(
self
.
connection
,
self
.
socket
)
except
Exception
as
e
:
LOG
.
error
(
"Exception on socket read: %s"
,
str
(
e
)
)
self
.
connection
.
close_input
(
)
self
.
connection
.
close
(
)
self
.
connection
.
process
(
time
.
time
(
)
)


def send_output(self):
        """Called when socket is write-ready"""
        try:
            pyngus.write_socket_output(self.connection,
                                       self.socket)
        except Exception as e:
            LOG.error("Exception on socket write: %s", str(e))
            self.connection.close_output()
            self.connection.close()
        self.connection.process(time.time())

def
send_output
(
self
)
:
try
:
pyngus
.
write_socket_output
(
self
.
connection
,
self
.
socket
)
except
Exception
as
e
:
LOG
.
error
(
"Exception on socket write: %s"
,
str
(
e
)
)
self
.
connection
.
close_output
(
)
self
.
connection
.
close
(
)
self
.
connection
.
process
(
time
.
time
(
)
)


def _send_request(self):
        """Send a message containing the RPC method call
        """
        msg = Message()
        msg.subject = "An RPC call!"
        msg.address = self._to
        msg.reply_to = self._reply_to
        msg.body = self._method
        msg.correlation_id = 5  # whatever...

        print("sending RPC call request: %s" % str(self._method))
        # @todo send timeout self._sender.send(msg, self, None, time.time() +
        # 10)
        self._sender.send(msg, self)

def
_send_request
(
self
)
:
msg
=
Message
(
)
msg
.
subject
=
"An RPC call!"
msg
.
address
=
self
.
_to
msg
.
reply_to
=
self
.
_reply_to
msg
.
body
=
self
.
_method
msg
.
correlation_id
=
5
print
(
"sending RPC call request: %s"
%
str
(
self
.
_method
)
)
self
.
_sender
.
send
(
msg
,
self
)


def read_socket_input(connection, socket_obj):
    """Read from the network layer and processes all data read.  Can
    support both blocking and non-blocking sockets.
    Returns the number of input bytes processed, or EOS if input processing
    is done.  Any exceptions raised by the socket are re-raised.
    """
    count = connection.needs_input
    if count <= 0:
        return count  # 0 or EOS

    while True:
        try:
            sock_data = socket_obj.recv(count)
            break
        except socket.timeout as e:
            LOG.debug("Socket timeout exception %s", str(e))
            raise  # caller must handle
        except socket.error as e:
            err = e.errno
            if err in [errno.EAGAIN,
                       errno.EWOULDBLOCK,
                       errno.EINTR]:
                # try again later
                return 0
            # otherwise, unrecoverable, caller must handle
            LOG.debug("Socket error exception %s", str(e))
            raise
        except Exception as e:  # beats me... assume fatal
            LOG.debug("unknown socket exception %s", str(e))
            raise  # caller must handle

    if len(sock_data) > 0:
        count = connection.process_input(sock_data)
    else:
        LOG.debug("Socket closed")
        count = Connection.EOS
        connection.close_input()
        connection.close_output()
    return count

def
read_socket_input
(
connection
,
socket_obj
)
:
count
=
connection
.
needs_input
if
count
<=
0
:
return
count
while
True
:
try
:
sock_data
=
socket_obj
.
recv
(
count
)
break
except
socket
.
timeout
as
e
:
LOG
.
debug
(
"Socket timeout exception %s"
,
str
(
e
)
)
raise
except
socket
.
error
as
e
:
err
=
e
.
errno
if
err
in
[
errno
.
EAGAIN
,
errno
.
EWOULDBLOCK
,
errno
.
EINTR
]
:
return
0
LOG
.
debug
(
"Socket error exception %s"
,
str
(
e
)
)
raise
except
Exception
as
e
:
LOG
.
debug
(
"unknown socket exception %s"
,
str
(
e
)
)
raise
if
len
(
sock_data
)
>
0
:
count
=
connection
.
process_input
(
sock_data
)
else
:
LOG
.
debug
(
"Socket closed"
)
count
=
Connection
.
EOS
connection
.
close_input
(
)
connection
.
close_output
(
)
return
count

Read from the network layer and processes all data read . Can support both blocking and non - blocking sockets . Returns the number of input bytes processed or EOS if input processing is done . Any exceptions raised by the socket are re - raised .


def write_socket_output(connection, socket_obj):
    """Write data to the network layer.  Can support both blocking and
    non-blocking sockets.
    Returns the number of output bytes sent, or EOS if output processing
    is done.  Any exceptions raised by the socket are re-raised.
    """
    count = connection.has_output
    if count <= 0:
        return count  # 0 or EOS

    data = connection.output_data()
    if not data:
        # error - has_output > 0, but no data?
        return Connection.EOS

    while True:
        try:
            count = socket_obj.send(data)
            break
        except socket.timeout as e:
            LOG.debug("Socket timeout exception %s", str(e))
            raise  # caller must handle
        except socket.error as e:
            err = e.errno
            if err in [errno.EAGAIN,
                       errno.EWOULDBLOCK,
                       errno.EINTR]:
                # try again later
                return 0
            # else assume fatal let caller handle it:
            LOG.debug("Socket error exception %s", str(e))
            raise
        except Exception as e:  # beats me... assume fatal
            LOG.debug("unknown socket exception %s", str(e))
            raise

    if count > 0:
        connection.output_written(count)
    elif data:
        LOG.debug("Socket closed")
        count = Connection.EOS
        connection.close_output()
        connection.close_input()
    return count

def
write_socket_output
(
connection
,
socket_obj
)
:
count
=
connection
.
has_output
if
count
<=
0
:
return
count
data
=
connection
.
output_data
(
)
if
not
data
:
return
Connection
.
EOS
while
True
:
try
:
count
=
socket_obj
.
send
(
data
)
break
except
socket
.
timeout
as
e
:
LOG
.
debug
(
"Socket timeout exception %s"
,
str
(
e
)
)
raise
except
socket
.
error
as
e
:
err
=
e
.
errno
if
err
in
[
errno
.
EAGAIN
,
errno
.
EWOULDBLOCK
,
errno
.
EINTR
]
:
return
0
LOG
.
debug
(
"Socket error exception %s"
,
str
(
e
)
)
raise
except
Exception
as
e
:
LOG
.
debug
(
"unknown socket exception %s"
,
str
(
e
)
)
raise
if
count
>
0
:
connection
.
output_written
(
count
)
elif
data
:
LOG
.
debug
(
"Socket closed"
)
count
=
Connection
.
EOS
connection
.
close_output
(
)
connection
.
close_input
(
)
return
count

Write data to the network layer . Can support both blocking and non - blocking sockets . Returns the number of output bytes sent or EOS if output processing is done . Any exceptions raised by the socket are re - raised .


def _not_reentrant(func):
    """Decorator that prevents callbacks from calling into link methods that
    are not reentrant """
    def wrap(*args, **kws):
        link = args[0]
        if link._callback_lock.in_callback:
            m = "Link %s cannot be invoked from a callback!" % func
            raise RuntimeError(m)
        return func(*args, **kws)
    return wrap


def _get_remote_settle_modes(pn_link):
    """Return a map containing the settle modes as provided by the remote.
    Skip any default value.
    """
    modes = {}
    snd = pn_link.remote_snd_settle_mode
    if snd == proton.Link.SND_UNSETTLED:
        modes['snd-settle-mode'] = 'unsettled'
    elif snd == proton.Link.SND_SETTLED:
        modes['snd-settle-mode'] = 'settled'
    if pn_link.remote_rcv_settle_mode == proton.Link.RCV_SECOND:
        modes['rcv-settle-mode'] = 'second'
    return modes

def _get_remote_settle_modes ( pn_link ) : modes = { } snd = pn_link . remote_snd_settle_mode if snd == proton . Link . SND_UNSETTLED : modes [ 'snd-settle-mode' ] = 'unsettled' elif snd == proton . Link . SND_SETTLED : modes [ 'snd-settle-mode' ] = 'settled' if pn_link . remote_rcv_settle_mode == proton . Link . RCV_SECOND : modes [ 'rcv-settle-mode' ] = 'second' return modes

Return a map containing the settle modes as provided by the remote . Skip any default value .


def configure(self, target_address, source_address, handler, properties):
        """Assign addresses, properties, etc."""
        self._handler = handler
        self._properties = properties

        dynamic_props = None
        if properties:
            dynamic_props = properties.get("dynamic-node-properties")
            mode = _dist_modes.get(properties.get("distribution-mode"))
            if mode is not None:
                self._pn_link.source.distribution_mode = mode
            mode = _snd_settle_modes.get(properties.get("snd-settle-mode"))
            if mode is not None:
                self._pn_link.snd_settle_mode = mode
            mode = _rcv_settle_modes.get(properties.get("rcv-settle-mode"))
            if mode is not None:
                self._pn_link.rcv_settle_mode = mode

        if target_address is None:
            if not self._pn_link.is_sender:
                raise Exception("Dynamic target not allowed")
            self._pn_link.target.dynamic = True
            if dynamic_props:
                self._pn_link.target.properties.clear()
                self._pn_link.target.properties.put_dict(dynamic_props)
        elif target_address:
            self._pn_link.target.address = target_address

        if source_address is None:
            if not self._pn_link.is_receiver:
                raise Exception("Dynamic source not allowed")
            self._pn_link.source.dynamic = True
            if dynamic_props:
                self._pn_link.source.properties.clear()
                self._pn_link.source.properties.put_dict(dynamic_props)
        elif source_address:
            self._pn_link.source.address = source_address

def
configure
(
self
,
target_address
,
source_address
,
handler
,
properties
)
:
self
.
_handler
=
handler
self
.
_properties
=
properties
dynamic_props
=
if
properties
:
dynamic_props
=
properties
.
get
(
"dynamic-node-properties"
)
mode
=
_dist_modes
.
get
(
properties
.
get
(
"distribution-mode"
)
)
if
mode
is
not
:
self
.
_pn_link
.
source
.
distribution_mode
=
mode
mode
=
_snd_settle_modes
.
get
(
properties
.
get
(
"snd-settle-mode"
)
)
if
mode
is
not
:
self
.
_pn_link
.
snd_settle_mode
=
mode
mode
=
_rcv_settle_modes
.
get
(
properties
.
get
(
"rcv-settle-mode"
)
)
if
mode
is
not
:
self
.
_pn_link
.
rcv_settle_mode
=
mode
if
target_address
is
:
if
not
self
.
_pn_link
.
is_sender
:
raise
Exception
(
"Dynamic target not allowed"
)
self
.
_pn_link
.
target
.
dynamic
=
True
if
dynamic_props
:
self
.
_pn_link
.
target
.
properties
.
clear
(
)
self
.
_pn_link
.
target
.
properties
.
put_dict
(
dynamic_props
)
elif
target_address
:
self
.
_pn_link
.
target
.
address
=
target_address
if
source_address
is
:
if
not
self
.
_pn_link
.
is_receiver
:
raise
Exception
(
"Dynamic source not allowed"
)
self
.
_pn_link
.
source
.
dynamic
=
True
if
dynamic_props
:
self
.
_pn_link
.
source
.
properties
.
clear
(
)
self
.
_pn_link
.
source
.
properties
.
put_dict
(
dynamic_props
)
elif
source_address
:
self
.
_pn_link
.
source
.
address
=
source_address


def source_address(self):
        """Return the authorative source of the link."""
        # If link is a sender, source is determined by the local
        # value, else use the remote.
        if self._pn_link.is_sender:
            return self._pn_link.source.address
        else:
            return self._pn_link.remote_source.address

def source_address ( self ) : if self . _pn_link . is_sender : return self . _pn_link . source . address else : return self . _pn_link . remote_source . address

Return the authorative source of the link .


def target_address(self):
        """Return the authorative target of the link."""
        # If link is a receiver, target is determined by the local
        # value, else use the remote.
        if self._pn_link.is_receiver:
            return self._pn_link.target.address
        else:
            return self._pn_link.remote_target.address

def target_address ( self ) : if self . _pn_link . is_receiver : return self . _pn_link . target . address else : return self . _pn_link . remote_target . address

Return the authorative target of the link .


def _session_closed(self):
        """Remote has closed the session used by this link."""
        # if link not already closed:
        if self._endpoint_state & proton.Endpoint.REMOTE_ACTIVE:
            # simulate close received
            self._process_remote_state()
        elif self._endpoint_state & proton.Endpoint.REMOTE_UNINIT:
            # locally created link, will never come up
            self._failed = True
            self._link_failed("Parent session closed.")

def
_session_closed
(
self
)
:
if
self
.
_endpoint_state
&
proton
.
Endpoint
.
REMOTE_ACTIVE
:
self
.
_process_remote_state
(
)
elif
self
.
_endpoint_state
&
proton
.
Endpoint
.
REMOTE_UNINIT
:
self
.
_failed
=
True
self
.
_link_failed
(
"Parent session closed."
)

Remote has closed the session used by this link .


def new_sender(self, name):
        """Create a new sender link."""
        pn_link = self._pn_session.sender(name)
        return self.request_sender(pn_link)

def new_sender ( self , name ) : pn_link = self . _pn_session . sender ( name ) return self . request_sender ( pn_link )


def request_sender(self, pn_link):
        """Create link from request for a sender."""
        sl = SenderLink(self._connection, pn_link)
        self._links.add(sl)
        return sl

def request_sender ( self , pn_link ) : sl = SenderLink ( self . _connection , pn_link ) self . _links . add ( sl ) return sl


def new_receiver(self, name):
        """Create a new receiver link."""
        pn_link = self._pn_session.receiver(name)
        return self.request_receiver(pn_link)

def new_receiver ( self , name ) : pn_link = self . _pn_session . receiver ( name ) return self . request_receiver ( pn_link )


def request_receiver(self, pn_link):
        """Create link from request for a receiver."""
        rl = ReceiverLink(self._connection, pn_link)
        self._links.add(rl)
        return rl

def request_receiver ( self , pn_link ) : rl = ReceiverLink ( self . _connection , pn_link ) self . _links . add ( rl ) return rl

Create link from request for a receiver .


def link_destroyed(self, link):
        """Link has been destroyed."""
        self._links.discard(link)
        if not self._links:
            # no more links
            LOG.debug("destroying unneeded session")
            self._pn_session.close()
            self._pn_session.free()
            self._pn_session = None
            self._connection = None

def
link_destroyed
(
self
,
link
)
:
self
.
_links
.
discard
(
link
)
if
not
self
.
_links
:
LOG
.
debug
(
"destroying unneeded session"
)
self
.
_pn_session
.
close
(
)
self
.
_pn_session
.
free
(
)
self
.
_pn_session
=
self
.
_connection
=


def _ep_need_close(self):
        """Peer has closed its end of the session."""
        LOG.debug("Session %s close requested - closing...",
                  self._name)
        links = self._links.copy()  # may modify _links
        for link in links:
            link._session_closed()

Peer has closed its end of the session .


def _process_endpoint_event(self, event):
        """Called when the Proton Engine generates an endpoint state change
        event.
        """
        state_fsm = Endpoint._FSM[self._state]
        entry = state_fsm.get(event)
        if not entry:
            # protocol error: invalid event for current state
            old_state = self._state
            self._state = Endpoint.STATE_ERROR
            self._ep_error("invalid event=%s in state=%s" %
                           (Endpoint.EVENT_NAMES[event],
                            Endpoint.STATE_NAMES[old_state]))
            return

        self._state = entry[0]
        if entry[1]:
            entry[1](self)

def
_process_endpoint_event
(
self
,
event
)
:
state_fsm
=
Endpoint
.
_FSM
[
self
.
_state
]
entry
=
state_fsm
.
get
(
event
)
if
not
entry
:
old_state
=
self
.
_state
self
.
_state
=
Endpoint
.
STATE_ERROR
self
.
_ep_error
(
"invalid event=%s in state=%s"
%
(
Endpoint
.
EVENT_NAMES
[
event
]
,
Endpoint
.
STATE_NAMES
[
old_state
]
)
)
return
self
.
_state
=
entry
[
0
]
if
entry
[
1
]
:
entry
[
1
]
(
self
)

Called when the Proton Engine generates an endpoint state change event .


def extendMarkdown(self, md, md_globals):
        """Modifies inline patterns."""
        mark_tag = SimpleTagPattern(MARK_RE, 'mark')
        md.inlinePatterns.add('mark', mark_tag, '_begin')

def extendMarkdown ( self , md , md_globals ) : mark_tag = SimpleTagPattern ( MARK_RE , 'mark' ) md . inlinePatterns . add ( 'mark' , mark_tag , '_begin' )


def receiver_remote_closed(self, receiver_link, pn_condition):
        """Peer has closed its end of the link."""
        LOG.debug("receiver_remote_closed condition=%s", pn_condition)
        receiver_link.close()
        self.done = True

def
receiver_remote_closed
(
self
,
receiver_link
,
pn_condition
)
:
LOG
.
debug
(
"receiver_remote_closed condition=%s"
,
pn_condition
)
receiver_link
.
close
(
)
self
.
done
=
True


def receiver_failed(self, receiver_link, error):
        """Protocol error occurred."""
        LOG.warn("receiver_failed error=%s", error)
        receiver_link.close()
        self.done = True

def
receiver_failed
(
self
,
receiver_link
,
error
)
:
LOG
.
warn
(
"receiver_failed error=%s"
,
error
)
receiver_link
.
close
(
)
self
.
done
=
True


def get_host_port(server_address):
    """Parse the hostname and port out of the server_address."""
    regex = re.compile(r"^amqp://([a-zA-Z0-9.]+)(:([\d]+))?$")
    x = regex.match(server_address)
    if not x:
        raise Exception("Bad address syntax: %s" % server_address)
    matches = x.groups()
    host = matches[0]
    port = int(matches[2]) if matches[2] else None
    return host, port

def
get_host_port
(
server_address
)
:
regex
=
re
.
compile
(
r"^amqp://([a-zA-Z0-9.]+)(:([\d]+))?$"
)
x
=
regex
.
match
(
server_address
)
if
not
x
:
raise
Exception
(
"Bad address syntax: %s"
%
server_address
)
matches
=
x
.
groups
(
)
host
=
matches
[
0
]
port
=
int
(
matches
[
2
]
)
if
matches
[
2
]
else
return
host
,
port

Parse the hostname and port out of the server_address .


def connect_socket(host, port, blocking=True):
    """Create a TCP connection to the server."""
    addr = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)
    if not addr:
        raise Exception("Could not translate address '%s:%s'"
                        % (host, str(port)))
    my_socket = socket.socket(addr[0][0], addr[0][1], addr[0][2])
    if not blocking:
        my_socket.setblocking(0)
    try:
        my_socket.connect(addr[0][4])
    except socket.error as e:
        if e.errno != errno.EINPROGRESS:
            raise
    return my_socket

def
connect_socket
(
host
,
port
,
blocking
=
True
)
:
addr
=
socket
.
getaddrinfo
(
host
,
port
,
socket
.
AF_INET
,
socket
.
SOCK_STREAM
)
if
not
addr
:
raise
Exception
(
"Could not translate address '%s:%s'"
%
(
host
,
str
(
port
)
)
)
my_socket
=
socket
.
socket
(
addr
[
0
]
[
0
]
,
addr
[
0
]
[
1
]
,
addr
[
0
]
[
2
]
)
if
not
blocking
:
my_socket
.
setblocking
(
0
)
try
:
my_socket
.
connect
(
addr
[
0
]
[
4
]
)
except
socket
.
error
as
e
:
if
e
.
errno
!=
errno
.
EINPROGRESS
:
raise
return
my_socket


def server_socket(host, port, backlog=10):
    """Create a TCP listening socket for a server."""
    addr = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)
    if not addr:
        raise Exception("Could not translate address '%s:%s'"
                        % (host, str(port)))
    my_socket = socket.socket(addr[0][0], addr[0][1], addr[0][2])
    my_socket.setblocking(0)  # 0=non-blocking
    try:
        my_socket.bind(addr[0][4])
        my_socket.listen(backlog)
    except socket.error as e:
        if e.errno != errno.EINPROGRESS:
            raise
    return my_socket

def
server_socket
(
host
,
port
,
backlog
=
10
)
:
addr
=
socket
.
getaddrinfo
(
host
,
port
,
socket
.
AF_INET
,
socket
.
SOCK_STREAM
)
if
not
addr
:
raise
Exception
(
"Could not translate address '%s:%s'"
%
(
host
,
str
(
port
)
)
)
my_socket
=
socket
.
socket
(
addr
[
0
]
[
0
]
,
addr
[
0
]
[
1
]
,
addr
[
0
]
[
2
]
)
my_socket
.
setblocking
(
0
)
try
:
my_socket
.
bind
(
addr
[
0
]
[
4
]
)
my_socket
.
listen
(
backlog
)
except
socket
.
error
as
e
:
if
e
.
errno
!=
errno
.
EINPROGRESS
:
raise
return
my_socket

Create a TCP listening socket for a server .


def need_processing(self):
        """A utility to help determine which connections need
        processing. Returns a triple of lists containing those connections that
        0) need to read from the network, 1) need to write to the network, 2)
        waiting for pending timers to expire.  The timer list is sorted with
        the connection next expiring at index 0.
        """
        readers = []
        writers = []
        timer_heap = []
        for c in iter(self._connections.values()):
            if c.needs_input > 0:
                readers.append(c)
            if c.has_output > 0:
                writers.append(c)
            if c.deadline:
                heapq.heappush(timer_heap, (c.next_tick, c))
        timers = []
        while timer_heap:
            x = heapq.heappop(timer_heap)
            timers.append(x[1])

        return (readers, writers, timers)

def need_processing ( self ) : readers = [ ] writers = [ ] timer_heap = [ ] for c in iter ( self . _connections . values ( ) ) : if c . needs_input > 0 : readers . append ( c ) if c . has_output > 0 : writers . append ( c ) if c . deadline : heapq . heappush ( timer_heap , ( c . next_tick , c ) ) timers = [ ] while timer_heap : x = heapq . heappop ( timer_heap ) timers . append ( x [ 1 ] ) return ( readers , writers , timers )

A utility to help determine which connections need processing . Returns a triple of lists containing those connections that 0 ) need to read from the network 1 ) need to write to the network 2 ) waiting for pending timers to expire . The timer list is sorted with the connection next expiring at index 0 .


def _not_reentrant(func):
        """Decorator that prevents callbacks from calling into methods that are
        not reentrant
        """
        def wrap(self, *args, **kws):
            if self._callback_lock and self._callback_lock.in_callback:
                m = "Connection %s cannot be invoked from a callback!" % func
                raise RuntimeError(m)
            return func(self, *args, **kws)
        return wrap


def process(self, now):
        """Perform connection state processing."""
        if self._pn_connection is None:
            LOG.error("Connection.process() called on destroyed connection!")
            return 0

        # do nothing until the connection has been opened
        if self._pn_connection.state & proton.Endpoint.LOCAL_UNINIT:
            return 0

        if self._pn_sasl and not self._sasl_done:
            # wait until SASL has authenticated
            if (_PROTON_VERSION < (0, 10)):
                if self._pn_sasl.state not in (proton.SASL.STATE_PASS,
                                               proton.SASL.STATE_FAIL):
                    LOG.debug("SASL in progress. State=%s",
                              str(self._pn_sasl.state))
                    if self._handler:
                        with self._callback_lock:
                            self._handler.sasl_step(self, self._pn_sasl)
                    return self._next_deadline

                self._sasl_done = True
                if self._handler:
                    with self._callback_lock:
                        self._handler.sasl_done(self, self._pn_sasl,
                                                self._pn_sasl.outcome)
            else:
                if self._pn_sasl.outcome is not None:
                    self._sasl_done = True
                    if self._handler:
                        with self._callback_lock:
                            self._handler.sasl_done(self, self._pn_sasl,
                                                    self._pn_sasl.outcome)

        # process timer events:
        timer_deadline = self._expire_timers(now)
        transport_deadline = self._pn_transport.tick(now)
        if timer_deadline and transport_deadline:
            self._next_deadline = min(timer_deadline, transport_deadline)
        else:
            self._next_deadline = timer_deadline or transport_deadline

        # process events from proton:
        pn_event = self._pn_collector.peek()
        while pn_event:
            # LOG.debug("pn_event: %s received", pn_event.type)
            if _Link._handle_proton_event(pn_event, self):
                pass
            elif self._handle_proton_event(pn_event):
                pass
            elif _SessionProxy._handle_proton_event(pn_event, self):
                pass
            self._pn_collector.pop()
            pn_event = self._pn_collector.peek()

        # check for connection failure after processing all pending
        # engine events:
        if self._error:
            if self._handler:
                # nag application until connection is destroyed
                self._next_deadline = now
                with self._callback_lock:
                    self._handler.connection_failed(self, self._error)
        elif (self._endpoint_state == self._CLOSED and
              self._read_done and self._write_done):
            # invoke closed callback after endpoint has fully closed and
            # all pending I/O has completed:
            if self._handler:
                with self._callback_lock:
                    self._handler.connection_closed(self)

        return self._next_deadline


def output_data(self):
        """Get a buffer of data that needs to be written to the network.
        """
        c = self.has_output
        if c <= 0:
            return None
        try:
            buf = self._pn_transport.peek(c)
        except Exception as e:
            self._connection_failed(str(e))
            return None
        return buf

def output_data ( self ) : c = self . has_output if c <= 0 : return None try : buf = self . _pn_transport . peek ( c ) except Exception as e : self . _connection_failed ( str ( e ) ) return None return buf

Get a buffer of data that needs to be written to the network .


def create_sender(self, source_address, target_address=None,
                      event_handler=None, name=None, properties=None):
        """Factory method for Sender links."""
        ident = name or str(source_address)
        if ident in self._sender_links:
            raise KeyError("Sender %s already exists!" % ident)

        session = _SessionProxy("session-%s" % ident, self)
        session.open()
        sl = session.new_sender(ident)
        sl.configure(target_address, source_address, event_handler, properties)
        self._sender_links[ident] = sl
        return sl

def
create_sender
(
self
,
source_address
,
target_address
=
,
event_handler
=
,
name
=
,
properties
=
)
:
ident
=
name
or
str
(
source_address
)
if
ident
in
self
.
_sender_links
:
raise
KeyError
(
"Sender %s already exists!"
%
ident
)
session
=
_SessionProxy
(
"session-%s"
%
ident
,
self
)
session
.
open
(
)
sl
=
session
.
new_sender
(
ident
)
sl
.
configure
(
target_address
,
source_address
,
event_handler
,
properties
)
self
.
_sender_links
[
ident
]
=
sl
return
sl


def reject_sender(self, link_handle, pn_condition=None):
        """Rejects the SenderLink, and destroys the handle."""
        link = self._sender_links.get(link_handle)
        if not link:
            raise Exception("Invalid link_handle: %s" % link_handle)
        link.reject(pn_condition)
        # note: normally, link.destroy() cannot be called from a callback,
        # but this link was never made available to the application so this
        # link is only referenced by the connection
        link.destroy()

def
reject_sender
(
self
,
link_handle
,
pn_condition
=
)
:
link
=
self
.
_sender_links
.
get
(
link_handle
)
if
not
link
:
raise
Exception
(
"Invalid link_handle: %s"
%
link_handle
)
link
.
reject
(
pn_condition
)
link
.
destroy
(
)

Rejects the SenderLink and destroys the handle .


def create_receiver(self, target_address, source_address=None,
                        event_handler=None, name=None, properties=None):
        """Factory method for creating Receive links."""
        ident = name or str(target_address)
        if ident in self._receiver_links:
            raise KeyError("Receiver %s already exists!" % ident)

        session = _SessionProxy("session-%s" % ident, self)
        session.open()
        rl = session.new_receiver(ident)
        rl.configure(target_address, source_address, event_handler, properties)
        self._receiver_links[ident] = rl
        return rl

def
create_receiver
(
self
,
target_address
,
source_address
=
,
event_handler
=
,
name
=
,
properties
=
)
:
ident
=
name
or
str
(
target_address
)
if
ident
in
self
.
_receiver_links
:
raise
KeyError
(
"Receiver %s already exists!"
%
ident
)
session
=
_SessionProxy
(
"session-%s"
%
ident
,
self
)
session
.
open
(
)
rl
=
session
.
new_receiver
(
ident
)
rl
.
configure
(
target_address
,
source_address
,
event_handler
,
properties
)
self
.
_receiver_links
[
ident
]
=
rl
return
rl

Factory method for creating Receive links .


def _connection_failed(self, error="Error not specified!"):
        """Clean up after connection failure detected."""
        if not self._error:
            LOG.error("Connection failed: %s", str(error))
            self._error = error

def
_connection_failed
(
self
,
error
=
"Error not specified!"
)
:
if
not
self
.
_error
:
LOG
.
error
(
"Connection failed: %s"
,
str
(
error
)
)
self
.
_error
=
error

Clean up after connection failure detected .


def _ep_active(self):
        """Both ends of the Endpoint have become active."""
        LOG.debug("Connection is up")
        if self._handler:
            with self._callback_lock:
                self._handler.connection_active(self)

def
_ep_active
(
self
)
:
LOG
.
debug
(
"Connection is up"
)
if
self
.
_handler
:
with
self
.
_callback_lock
:
self
.
_handler
.
connection_active
(
self
)

Both ends of the Endpoint have become active .


def _ep_need_close(self):
        """The remote has closed its end of the endpoint."""
        LOG.debug("Connection remotely closed")
        if self._handler:
            cond = self._pn_connection.remote_condition
            with self._callback_lock:
                self._handler.connection_remote_closed(self, cond)

def
_ep_need_close
(
self
)
:
LOG
.
debug
(
"Connection remotely closed"
)
if
self
.
_handler
:
cond
=
self
.
_pn_connection
.
remote_condition
with
self
.
_callback_lock
:
self
.
_handler
.
connection_remote_closed
(
self
,
cond
)

The remote has closed its end of the endpoint .


def _ep_error(self, error):
        """The endpoint state machine failed due to protocol error."""
        super(Connection, self)._ep_error(error)
        self._connection_failed("Protocol error occurred.")

def
_ep_error
(
self
,
error
)
:
super
(
Connection
,
self
)
.
_ep_error
(
error
)
self
.
_connection_failed
(
"Protocol error occurred."
)

The endpoint state machine failed due to protocol error .