def main(sample_id, fastq_pair, trim_range, trim_opts, phred, adapters_file,
         clear):
    """ Main executor of the trimmomatic template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    fastq_pair : list
        Two element list containing the paired FastQ files.
    trim_range : list
        Two element list containing the trimming range.
    trim_opts : list
        Four element list containing several trimmomatic options:
        [*SLIDINGWINDOW*; *LEADING*; *TRAILING*; *MINLEN*]
    phred : int
        Guessed phred score for the sample. The phred score is a generated
        output from :py:class:`templates.integrity_coverage`.
    adapters_file : str
        Path to adapters file. If not provided, or the path is not available,
        it will use the default adapters from Trimmomatic will be used
    clear : str
        Can be either 'true' or 'false'. If 'true', the input fastq files will
        be removed at the end of the run, IF they are in the working directory
    """

    logger.info("Starting trimmomatic")

    # Create base CLI
    cli = [
        "java",
        "-Xmx{}".format("$task.memory"[:-1].lower().replace(" ", "")),
        "-jar",
        TRIM_PATH.strip(),
        "PE",
        "-threads",
        "$task.cpus"
    ]

    # If the phred encoding was detected, provide it
    try:
        # Check if the provided PHRED can be converted to int
        phred = int(phred)
        phred_flag = "-phred{}".format(str(phred))
        cli += [phred_flag]
    # Could not detect phred encoding. Do not add explicit encoding to
    # trimmomatic and let it guess
    except ValueError:
        pass

    # Add input samples to CLI
    cli += fastq_pair

    # Add output file names
    output_names = []
    for i in range(len(fastq_pair)):
        output_names.append("{}_{}_trim.fastq.gz".format(
            SAMPLE_ID, str(i + 1)))
        output_names.append("{}_{}_U.fastq.gz".format(
            SAMPLE_ID, str(i + 1)))
    cli += output_names

    if trim_range != ["None"]:
        cli += [
            "CROP:{}".format(trim_range[1]),
            "HEADCROP:{}".format(trim_range[0]),
        ]

    if os.path.exists(adapters_file):
        logger.debug("Using the provided adapters file '{}'".format(
            adapters_file))
    else:
        logger.debug("Adapters file '{}' not provided or does not exist. Using"
                     " default adapters".format(adapters_file))
        adapters_file = merge_default_adapters()

    cli += [
        "ILLUMINACLIP:{}:3:30:10:6:true".format(adapters_file)
    ]

    #create log file im temporary dir to avoid issues when running on a docker container in macOS
    logfile = os.path.join(tempfile.mkdtemp(prefix='tmp'), "{}_trimlog.txt".format(sample_id))

    # Add trimmomatic options
    cli += [
        "SLIDINGWINDOW:{}".format(trim_opts[0]),
        "LEADING:{}".format(trim_opts[1]),
        "TRAILING:{}".format(trim_opts[2]),
        "MINLEN:{}".format(trim_opts[3]),
        "TOPHRED33",
        "-trimlog",
        logfile
    ]

    logger.debug("Running trimmomatic subprocess with command: {}".format(cli))

    p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE)
    stdout, stderr = p.communicate()

    # Attempt to decode STDERR output from bytes. If unsuccessful, coerce to
    # string
    try:
        stderr = stderr.decode("utf8")
    except (UnicodeDecodeError, AttributeError):
        stderr = str(stderr)

    logger.info("Finished trimmomatic subprocess with STDOUT:\\n"
                "======================================\\n{}".format(stdout))
    logger.info("Finished trimmomatic subprocesswith STDERR:\\n"
                "======================================\\n{}".format(stderr))
    logger.info("Finished trimmomatic with return code: {}".format(
        p.returncode))

    trimmomatic_log(logfile, sample_id)

    if p.returncode == 0 and os.path.exists("{}_1_trim.fastq.gz".format(
            SAMPLE_ID)):
        clean_up(fastq_pair, clear)

    # Check if trimmomatic ran successfully. If not, write the error message
    # to the status channel and exit.
    with open(".status", "w") as status_fh:
        if p.returncode != 0:
            status_fh.write("fail")
            return
        else:
            status_fh.write("pass")

Main executor of the trimmomatic template .


def depth_file_reader(depth_file):
    """
    Function that parse samtools depth file and creates 3 dictionaries that
    will be useful to make the outputs of this script, both the tabular file
    and the json file that may be imported by pATLAS

    Parameters
    ----------
    depth_file: textIO
        the path to depth file for each sample

    Returns
    -------
    depth_dic_coverage: dict
            dictionary with the coverage per position for each plasmid
    """

    # dict to store the mean coverage for each reference
    depth_dic_coverage = {}

    for line in depth_file:
        tab_split = line.split()  # split by any white space
        reference = "_".join(tab_split[0].strip().split("_")[0:3])  # store
        # only the gi for the reference
        position = tab_split[1]
        num_reads_align = float(tab_split[2].rstrip())

        if reference not in depth_dic_coverage:
            depth_dic_coverage[reference] = {}

        depth_dic_coverage[reference][position] = num_reads_align

    logger.info("Finished parsing depth file.")
    depth_file.close()

    logger.debug("Size of dict_cov: {} kb".format(
        asizeof(depth_dic_coverage)/1024))

    return depth_dic_coverage

def
depth_file_reader
(
depth_file
)
:
depth_dic_coverage
=
{
}
for
line
in
depth_file
:
tab_split
=
line
.
split
(
)
reference
=
"_"
.
join
(
tab_split
[
0
]
.
strip
(
)
.
split
(
"_"
)
[
0
:
3
]
)
position
=
tab_split
[
1
]
num_reads_align
=
float
(
tab_split
[
2
]
.
rstrip
(
)
)
if
reference
not
in
depth_dic_coverage
:
depth_dic_coverage
[
reference
]
=
{
}
depth_dic_coverage
[
reference
]
[
position
]
=
num_reads_align
logger
.
info
(
"Finished parsing depth file."
)
depth_file
.
close
(
)
logger
.
debug
(
"Size of dict_cov: {} kb"
.
format
(
asizeof
(
depth_dic_coverage
)
/
1024
)
)
return
depth_dic_coverage

Function that parse samtools depth file and creates 3 dictionaries that will be useful to make the outputs of this script both the tabular file and the json file that may be imported by pATLAS


def main(depth_file, json_dict, cutoff, sample_id):
    """
    Function that handles the inputs required to parse depth files from bowtie
    and dumps a dict to a json file that can be imported into pATLAS.

    Parameters
    ----------
    depth_file: str
         the path to depth file for each sample
    json_dict: str
        the file that contains the dictionary with keys and values for
        accessions
        and their respective lengths
    cutoff: str
        the cutoff used to trim the unwanted matches for the minimum coverage
        results from mapping. This value may range between 0 and 1.
    sample_id: str
        the id of the sample being parsed

    """

    # check for the appropriate value for the cutoff value for coverage results
    logger.debug("Cutoff value: {}. Type: {}".format(cutoff, type(cutoff)))
    try:
        cutoff_val = float(cutoff)
        if cutoff_val < 0.4:
            logger.warning("This cutoff value will generate a high volume of "
                           "plot data. Therefore '.report.json' can be too big")
    except ValueError:
        logger.error("Cutoff value should be a string such as: '0.6'. "
                     "The outputted value: {}. Make sure to provide an "
                     "appropriate value for --cov_cutoff".format(cutoff))
        sys.exit(1)

    # loads dict from file, this file is provided in docker image

    plasmid_length = json.load(open(json_dict))
    if plasmid_length:
        logger.info("Loaded dictionary of plasmid lengths")
    else:
        logger.error("Something went wrong and plasmid lengths dictionary"
                     "could not be loaded. Check if process received this"
                     "param successfully.")
        sys.exit(1)

    # read depth file
    depth_file_in = open(depth_file)

    # first reads the depth file and generates dictionaries to handle the input
    # to a simpler format
    logger.info("Reading depth file and creating dictionary to dump.")
    depth_dic_coverage = depth_file_reader(depth_file_in)
    percentage_bases_covered, dict_cov = generate_jsons(depth_dic_coverage,
                                                        plasmid_length,
                                                        cutoff_val)

    if percentage_bases_covered and dict_cov:
        logger.info("percentage_bases_covered length: {}".format(
            str(len(percentage_bases_covered))))
        logger.info("dict_cov length: {}".format(str(len(dict_cov))))
    else:
        logger.error("Both dicts that dump to JSON file or .report.json are "
                     "empty.")

    # then dump do file
    logger.info("Dumping to {}".format("{}_mapping.json".format(depth_file)))
    with open("{}_mapping.json".format(depth_file), "w") as output_json:
        output_json.write(json.dumps(percentage_bases_covered))

    json_dic = {
        "tableRow": [{
            "sample": sample_id,
            "data": [{
                "header": "Mapping",
                "table": "plasmids",
                "patlas_mapping": percentage_bases_covered,
                "value": len(percentage_bases_covered)
            }]
        }],
        "sample": sample_id,
        "patlas_mapping": percentage_bases_covered,
        "plotData": [{
            "sample": sample_id,
            "data": {
                "patlasMappingSliding": dict_cov
            },
        }]
    }

    logger.debug("Size of dict_cov: {} kb".format(asizeof(json_dic)/1024))
    logger.info("Writing to .report.json")
    with open(".report.json", "w") as json_report:
        json_report.write(json.dumps(json_dic, separators=(",", ":")))

Function that handles the inputs required to parse depth files from bowtie and dumps a dict to a json file that can be imported into pATLAS .


def _set_template(self, template):
        """Sets the path to the appropriate jinja template file

        When a Process instance is initialized, this method will fetch
        the location of the appropriate template file, based on the
        ``template`` argument. It will raise an exception is the template
        file is not found. Otherwise, it will set the
        :py:attr:`Process.template_path` attribute.
        """

        # Set template directory
        tpl_dir = join(dirname(abspath(__file__)), "templates")

        # Set template file path
        tpl_path = join(tpl_dir, template + ".nf")

        if not os.path.exists(tpl_path):
            raise eh.ProcessError(
                "Template {} does not exist".format(tpl_path))

        self._template_path = join(tpl_dir, template + ".nf")

def
_set_template
(
self
,
template
)
:
tpl_dir
=
join
(
dirname
(
abspath
(
__file__
)
)
,
"templates"
)
tpl_path
=
join
(
tpl_dir
,
template
+
".nf"
)
if
not
os
.
path
.
exists
(
tpl_path
)
:
raise
eh
.
ProcessError
(
"Template {} does not exist"
.
format
(
tpl_path
)
)
self
.
_template_path
=
join
(
tpl_dir
,
template
+
".nf"
)


def set_main_channel_names(self, input_suffix, output_suffix, lane):
        """Sets the main channel names based on the provide input and
        output channel suffixes. This is performed when connecting processes.

        Parameters
        ----------
        input_suffix : str
            Suffix added to the input channel. Should be based on the lane
            and an arbitrary unique id
        output_suffix : str
            Suffix added to the output channel. Should be based on the lane
            and an arbitrary unique id
        lane : int
            Sets the lane of the process.
        """

        self.input_channel = "{}_in_{}".format(self.template, input_suffix)
        self.output_channel = "{}_out_{}".format(self.template, output_suffix)
        self.lane = lane

def set_main_channel_names ( self , input_suffix , output_suffix , lane ) : self . input_channel = "{}_in_{}" . format ( self . template , input_suffix ) self . output_channel = "{}_out_{}" . format ( self . template , output_suffix ) self . lane = lane

Sets the main channel names based on the provide input and output channel suffixes . This is performed when connecting processes .


def get_user_channel(self, input_channel, input_type=None):
        """Returns the main raw channel for the process

        Provided with at least a channel name, this method returns the raw
        channel name and specification (the nextflow string definition)
        for the process. By default, it will fork from the raw input of
        the process' :attr:`~Process.input_type` attribute. However, this
        behaviour can be overridden by providing the ``input_type`` argument.

        If the specified or inferred input type exists in the
        :attr:`~Process.RAW_MAPPING` dictionary, the channel info dictionary
        will be retrieved along with the specified input channel. Otherwise,
        it will return None.

        An example of the returned dictionary is::

             {"input_channel": "myChannel",
             "params": "fastq",
             "channel": "IN_fastq_raw",
             "channel_str":"IN_fastq_raw = Channel.fromFilePairs(params.fastq)"
            }

        Returns
        -------
        dict or None
            Dictionary with the complete raw channel info. None if no
            channel is found.
        """

        res = {"input_channel": input_channel}

        itype = input_type if input_type else self.input_type

        if itype in self.RAW_MAPPING:

            channel_info = self.RAW_MAPPING[itype]

            return {**res, **channel_info}

def get_user_channel ( self , input_channel , input_type = None ) : res = { "input_channel" : input_channel } itype = input_type if input_type else self . input_type if itype in self . RAW_MAPPING : channel_info = self . RAW_MAPPING [ itype ] return { ** res , ** channel_info }


def render(template, context):
        """Wrapper to the jinja2 render method from a template file

        Parameters
        ----------
        template : str
            Path to template file.
        context : dict
            Dictionary with kwargs context to populate the template
        """

        path, filename = os.path.split(template)

        return jinja2.Environment(
            loader=jinja2.FileSystemLoader(path or './')
        ).get_template(filename).render(context)

def render ( template , context ) : path , filename = os . path . split ( template ) return jinja2 . Environment ( loader = jinja2 . FileSystemLoader ( path or './' ) ) . get_template ( filename ) . render ( context )


def template_str(self):
        """Class property that returns a populated template string

        This property allows the template of a particular process to be
        dynamically generated and returned when doing ``Process.template_str``.

        Returns
        -------
        x : str
            String with the complete and populated process template

        """

        if not self._context:
            raise eh.ProcessError("Channels must be setup first using the "
                                  "set_channels method")

        logger.debug("Setting context for template {}: {}".format(
            self.template, self._context
        ))

        x = self.render(self._template_path, self._context)
        return x


def set_channels(self, **kwargs):
        """ General purpose method that sets the main channels

        This method will take a variable number of keyword arguments to
        set the :py:attr:`Process._context` attribute with the information
        on the main channels for the process. This is done by appending
        the process ID (:py:attr:`Process.pid`) attribute to the input,
        output and status channel prefix strings. In the output channel,
        the process ID is incremented by 1 to allow the connection with the
        channel in the next process.

        The ``**kwargs`` system for setting the :py:attr:`Process._context`
        attribute also provides additional flexibility. In this way,
        individual processes can provide additional information not covered
        in this method, without changing it.

        Parameters
        ----------
        kwargs : dict
            Dictionary with the keyword arguments for setting up the template
            context
        """

        if not self.pid:
            self.pid = "{}_{}".format(self.lane, kwargs.get("pid"))

        for i in self.status_channels:
            if i.startswith("STATUS_"):
                self.status_strs.append("{}_{}".format(i, self.pid))
            else:
                self.status_strs.append("STATUS_{}_{}".format(i, self.pid))

        if self.main_forks:
            logger.debug("Setting main fork channels: {}".format(
                self.main_forks))
            operator = "set" if len(self.main_forks) == 1 else "into"
            self.forks = ["\n{}.{}{{ {} }}\n".format(
                self.output_channel, operator, ";".join(self.main_forks))]

        self._context = {**kwargs, **{"input_channel": self.input_channel,
                                      "output_channel": self.output_channel,
                                      "template": self.template,
                                      "forks": "\n".join(self.forks),
                                      "pid": self.pid}}

def
set_channels
(
self
,
**
kwargs
)
:
if
not
self
.
pid
:
self
.
pid
=
"{}_{}"
.
format
(
self
.
lane
,
kwargs
.
get
(
"pid"
)
)
for
i
in
self
.
status_channels
:
if
i
.
startswith
(
"STATUS_"
)
:
self
.
status_strs
.
append
(
"{}_{}"
.
format
(
i
,
self
.
pid
)
)
else
:
self
.
status_strs
.
append
(
"STATUS_{}_{}"
.
format
(
i
,
self
.
pid
)
)
if
self
.
main_forks
:
logger
.
debug
(
"Setting main fork channels: {}"
.
format
(
self
.
main_forks
)
)
operator
=
"set"
if
len
(
self
.
main_forks
)
==
1
else
"into"
self
.
forks
=
[
"\n{}.{}{{ {} }}\n"
.
format
(
self
.
output_channel
,
operator
,
";"
.
join
(
self
.
main_forks
)
)
]
self
.
_context
=
{
**
kwargs
,
**
{
"input_channel"
:
self
.
input_channel
,
"output_channel"
:
self
.
output_channel
,
"template"
:
self
.
template
,
"forks"
:
"\n"
.
join
(
self
.
forks
)
,
"pid"
:
self
.
pid
}
}


def update_main_forks(self, sink):
        """Updates the forks attribute with the sink channel destination

        Parameters
        ----------
        sink : str
            Channel onto which the main input will be forked to

        """

        if not self.main_forks:
            self.main_forks = [self.output_channel]
            self.output_channel = "_{}".format(self.output_channel)
        self.main_forks.append(sink)

        # fork_lst = self.forks + self.main_forks
        operator = "set" if len(self.main_forks) == 1 else "into"
        self.forks = ["\n{}.{}{{ {} }}\n".format(
            self.output_channel, operator, ";".join(self.main_forks))]

        self._context = {**self._context,
                         **{"forks": "".join(self.forks),
                            "output_channel": self.output_channel}}

def
update_main_forks
(
self
,
sink
)
:
if
not
self
.
main_forks
:
self
.
main_forks
=
[
self
.
output_channel
]
self
.
output_channel
=
"_{}"
.
format
(
self
.
output_channel
)
self
.
main_forks
.
append
(
sink
)
operator
=
"set"
if
len
(
self
.
main_forks
)
==
1
else
"into"
self
.
forks
=
[
"\n{}.{}{{ {} }}\n"
.
format
(
self
.
output_channel
,
operator
,
";"
.
join
(
self
.
main_forks
)
)
]
self
.
_context
=
{
**
self
.
_context
,
**
{
"forks"
:
""
.
join
(
self
.
forks
)
,
"output_channel"
:
self
.
output_channel
}
}


def set_secondary_channel(self, source, channel_list):
        """ General purpose method for setting a secondary channel

        This method allows a given source channel to be forked into one or
        more channels and sets those forks in the :py:attr:`Process.forks`
        attribute. Both the source and the channels in the ``channel_list``
        argument must be the final channel strings,  which means that this
        method should be called only after setting the main channels.

        If the source is not a main channel, this will simply create a fork
        or set for every channel in the ``channel_list`` argument list::

            SOURCE_CHANNEL_1.into{SINK_1;SINK_2}

        If the source is a main channel, this will apply some changes to
        the output channel of the process, to avoid overlapping main output
        channels.  For instance, forking the main output channel for process
        2 would create a ``MAIN_2.into{...}``. The issue here is that the
        ``MAIN_2`` channel is expected as the input of the next process, but
        now is being used to create the fork. To solve this issue, the output
        channel is modified into ``_MAIN_2``, and the fork is set to
        the channels provided channels plus the ``MAIN_2`` channel::

            _MAIN_2.into{MAIN_2;MAIN_5;...}

        Parameters
        ----------
        source : str
            String with the name of the source channel
        channel_list : list
            List of channels that will receive a fork of the secondary
            channel
        """

        logger.debug("Setting secondary channel for source '{}': {}".format(
            source, channel_list))

        source = "{}_{}".format(source, self.pid)

        # Removes possible duplicate channels, when the fork is terminal
        channel_list = sorted(list(set(channel_list)))

        # When there is only one channel to fork into, use the 'set' operator
        # instead of 'into'
        op = "set" if len(channel_list) == 1 else "into"
        self.forks.append("\n{}.{}{{ {} }}\n".format(
            source, op, ";".join(channel_list)))

        logger.debug("Setting forks attribute to: {}".format(self.forks))
        self._context = {**self._context, **{"forks": "\n".join(self.forks)}}


def update_attributes(self, attr_dict):
        """Updates the directives attribute from a dictionary object.

        This will only update the directives for processes that have been
        defined in the subclass.

        Parameters
        ----------
        attr_dict : dict
            Dictionary containing the attributes that will be used to update
            the process attributes and/or directives.

        """

        # Update directives
        # Allowed attributes to write
        valid_directives = ["pid", "ignore_type", "ignore_pid", "extra_input",
                            "group", "input_type"]

        for attribute, val in attr_dict.items():

            # If the attribute has a valid directive key, update that
            # directive
            if attribute in valid_directives and hasattr(self, attribute):
                setattr(self, attribute, val)

            # The params attribute is special, in the sense that it provides
            # information for the self.params attribute.
            elif attribute == "params":
                for name, value in val.items():
                    if name in self.params:
                        self.params[name]["default"] = value
                    else:
                        raise eh.ProcessError(
                            "The parameter name '{}' does not exist for "
                            "component '{}'".format(name, self.template))

            else:
                for p in self.directives:
                    self.directives[p][attribute] = val

Updates the directives attribute from a dictionary object .


def set_compiler_channels(self, channel_list, operator="mix"):
        """General method for setting the input channels for the status process

        Given a list of status channels that are gathered during the pipeline
        construction, this method will automatically set the input channel
        for the status process. This makes use of the ``mix`` channel operator
        of nextflow for multiple channels::

            STATUS_1.mix(STATUS_2,STATUS_3,...)

        This will set the ``status_channels`` key for the ``_context``
        attribute of the process.

        Parameters
        ----------
        channel_list : list
            List of strings with the final name of the status channels
        operator : str
            Specifies the operator used to join the compiler channels.
            Available options are 'mix'and 'join'.
        """

        if not channel_list:
            raise eh.ProcessError("At least one status channel must be "
                                  "provided to include this process in the "
                                  "pipeline")

        if len(channel_list) == 1:
            logger.debug("Setting only one status channel: {}".format(
                channel_list[0]))
            self._context = {"compile_channels": channel_list[0]}

        else:

            first_status = channel_list[0]

            if operator == "mix":
                lst = ",".join(channel_list[1:])

                s = "{}.mix({})".format(first_status, lst)

            elif operator == "join":

                s = first_status
                for ch in channel_list[1:]:
                    s += ".join({})".format(ch)

                s += ".map{ ot -> [ ot[0], ot[1..-1] ] }"

            logger.debug("Status channel string: {}".format(s))

            self._context = {"compile_channels": s}


def set_raw_inputs(self, raw_input):
        """Sets the main input channels of the pipeline and their forks.

        The ``raw_input`` dictionary input should contain one entry for each
        input type (fastq, fasta, etc). The corresponding value should be a
        dictionary/json with the following key:values:

        - ``channel``: Name of the raw input channel (e.g.: channel1)
        - ``channel_str``: The nextflow definition of the channel and
           eventual checks (e.g.: channel1 = Channel.fromPath(param))
        - ``raw_forks``: A list of channels to which the channel name will
          for to.

        Each new type of input parameter is automatically added to the
        :attr:`params` attribute, so that they are automatically collected
        for the pipeline description and help.

        Parameters
        ----------
        raw_input : dict
            Contains an entry for each input type with the channel name,
            channel string and forks.
        """

        logger.debug("Setting raw inputs using raw input dict: {}".format(
            raw_input))

        primary_inputs = []

        for input_type, el in raw_input.items():

            primary_inputs.append(el["channel_str"])

            # Update the process' parameters with the raw input
            raw_channel = self.RAW_MAPPING[input_type]
            self.params[input_type] = {
                "default": raw_channel["default_value"],
                "description": raw_channel["description"]
            }

            op = "set" if len(el["raw_forks"]) == 1 else "into"

            self.forks.append("\n{}.{}{{ {} }}\n".format(
                el["channel"], op, ";".join(el["raw_forks"])
            ))

        logger.debug("Setting raw inputs: {}".format(primary_inputs))
        logger.debug("Setting forks attribute to: {}".format(self.forks))
        self._context = {**self._context,
                         **{"forks": "\n".join(self.forks),
                            "main_inputs": "\n".join(primary_inputs)}}

Sets the main input channels of the pipeline and their forks .


def set_secondary_inputs(self, channel_dict):
        """ Adds secondary inputs to the start of the pipeline.

        This channels are inserted into the pipeline file as they are
        provided in the values of the argument.

        Parameters
        ----------
        channel_dict : dict
            Each entry should be <parameter>: <channel string>.
        """

        logger.debug("Setting secondary inputs: {}".format(channel_dict))

        secondary_input_str = "\n".join(list(channel_dict.values()))
        self._context = {**self._context,
                         **{"secondary_inputs": secondary_input_str}}

def
set_secondary_inputs
(
self
,
channel_dict
)
:
logger
.
debug
(
"Setting secondary inputs: {}"
.
format
(
channel_dict
)
)
secondary_input_str
=
"\n"
.
join
(
list
(
channel_dict
.
values
(
)
)
)
self
.
_context
=
{
**
self
.
_context
,
**
{
"secondary_inputs"
:
secondary_input_str
}
}

Adds secondary inputs to the start of the pipeline .


def set_extra_inputs(self, channel_dict):
        """Sets the initial definition of the extra input channels.

        The ``channel_dict`` argument should contain the input type and
        destination channel of each parameter (which is the key)::

            channel_dict = {
                "param1": {
                    "input_type": "fasta"
                    "channels": ["abricate_2_3", "chewbbaca_3_4"]
                }
            }

        Parameters
        ----------
        channel_dict : dict
            Dictionary with the extra_input parameter as key, and a dictionary
            as a value with the input_type and destination channels
        """

        extra_inputs = []

        for param, info in channel_dict.items():

            # Update the process' parameters with the raw input
            raw_channel = self.RAW_MAPPING[info["input_type"]]
            self.params[param] = {
                "default": raw_channel["default_value"],
                "description": raw_channel["description"]
            }

            channel_name = "IN_{}_extraInput".format(param)
            channel_str = self.RAW_MAPPING[info["input_type"]]["channel_str"]
            extra_inputs.append("{} = {}".format(channel_name,
                                                 channel_str.format(param)))

            op = "set" if len(info["channels"]) == 1 else "into"
            extra_inputs.append("{}.{}{{ {} }}".format(
                channel_name, op, ";".join(info["channels"])))

        self._context = {
            **self._context,
            **{"extra_inputs": "\n".join(extra_inputs)}
        }

def
set_extra_inputs
(
self
,
channel_dict
)
:
extra_inputs
=
[
]
for
param
,
info
in
channel_dict
.
items
(
)
:
raw_channel
=
self
.
RAW_MAPPING
[
info
[
"input_type"
]
]
self
.
params
[
param
]
=
{
"default"
:
raw_channel
[
"default_value"
]
,
"description"
:
raw_channel
[
"description"
]
}
channel_name
=
"IN_{}_extraInput"
.
format
(
param
)
channel_str
=
self
.
RAW_MAPPING
[
info
[
"input_type"
]
]
[
"channel_str"
]
extra_inputs
.
append
(
"{} = {}"
.
format
(
channel_name
,
channel_str
.
format
(
param
)
)
)
op
=
"set"
if
len
(
info
[
"channels"
]
)
==
1
else
"into"
extra_inputs
.
append
(
"{}.{}{{ {} }}"
.
format
(
channel_name
,
op
,
";"
.
join
(
info
[
"channels"
]
)
)
)
self
.
_context
=
{
**
self
.
_context
,
**
{
"extra_inputs"
:
"\n"
.
join
(
extra_inputs
)
}
}

Sets the initial definition of the extra input channels .


def _parse_coverage(header_str):
        """Attempts to retrieve the coverage value from the header string.

        It splits the header by "_" and then screens the list backwards in
        search of the first float value. This will be interpreted as the
        coverage value. If it cannot find a float value, it returns None.
        This search methodology is based on the strings of assemblers
        like spades and skesa that put the mean kmer coverage for each
        contig in its corresponding fasta header.

        Parameters
        ----------
        header_str : str
            String

        Returns
        -------
        float or None
            The coverage value for the contig. None if it cannot find the
            value in the provide string.
        """

        cov = None
        for i in header_str.split("_")[::-1]:
            try:
                cov = float(i)
                break
            except ValueError:
                continue

        return cov

def _parse_coverage ( header_str ) : cov = None for i in header_str . split ( "_" ) [ : : - 1 ] : try : cov = float ( i ) break except ValueError : continue return cov

Attempts to retrieve the coverage value from the header string .


def _parse_assembly(self, assembly_file):
        """Parse an assembly fasta file.

        This is a Fasta parsing method that populates the
        :py:attr:`~Assembly.contigs` attribute with data for each contig in the
        assembly.

        The insertion of data on the self.contigs is done by the
        :py:meth:`Assembly._populate_contigs` method, which also calculates
        GC content and proportions.

        Parameters
        ----------
        assembly_file : str
            Path to the assembly fasta file.

        """

        # Temporary storage of sequence data
        seq_temp = []
        # Id counter for contig that will serve as key in self.contigs
        contig_id = 0
        # Initialize kmer coverage and header
        cov, header = None, None

        with open(assembly_file) as fh:

            logger.debug("Starting iteration of assembly file: {}".format(
                assembly_file))
            for line in fh:
                # Skip empty lines
                if not line.strip():
                    continue
                else:
                    # Remove whitespace surrounding line for further processing
                    line = line.strip()

                if line.startswith(">"):
                    # If a sequence has already been populated, save the
                    # previous contig information
                    if seq_temp:
                        # Use join() to convert string list into the full
                        # contig string. This is generally much more efficient
                        # than successively concatenating strings.
                        seq = "".join(seq_temp)

                        logger.debug("Populating contig with contig_id '{}', "
                                     "header '{}' and cov '{}'".format(
                                        contig_id, header, cov))
                        self._populate_contigs(contig_id, header, cov, seq)

                        # Reset temporary sequence storage
                        seq_temp = []
                        contig_id += 1

                    header = line[1:]
                    cov = self._parse_coverage(line)

                else:
                    seq_temp.append(line)

            # Populate last contig entry
            logger.debug("Populating contig with contig_id '{}', "
                         "header '{}' and cov '{}'".format(
                            contig_id, header, cov))
            seq = "".join(seq_temp)
            self._populate_contigs(contig_id, header, cov, seq)


def _get_gc_content(sequence, length):
        """Get GC content and proportions.

        Parameters
        ----------
        sequence : str
            The complete sequence of the contig.
        length : int
            The length of the sequence contig.

        Returns
        -------
        x : dict
            Dictionary with the at/gc/n counts and proportions

        """

        # Get AT/GC/N counts
        at = sum(map(sequence.count, ["A", "T"]))
        gc = sum(map(sequence.count, ["G", "C"]))
        n = length - (at + gc)

        # Get AT/GC/N proportions
        at_prop = at / length
        gc_prop = gc / length
        n_prop = n / length

        return {"at": at, "gc": gc, "n": n,
                "at_prop": at_prop, "gc_prop": gc_prop, "n_prop": n_prop}

def _get_gc_content ( sequence , length ) : at = sum ( map ( sequence . count , [ "A" , "T" ] ) ) gc = sum ( map ( sequence . count , [ "G" , "C" ] ) ) n = length - ( at + gc ) at_prop = at / length gc_prop = gc / length n_prop = n / length return { "at" : at , "gc" : gc , "n" : n , "at_prop" : at_prop , "gc_prop" : gc_prop , "n_prop" : n_prop }


def filter_contigs(self, *comparisons):
        """Filters the contigs of the assembly according to user provided\
        comparisons.

        The comparisons must be a list of three elements with the
        :py:attr:`~Assembly.contigs` key, operator and test value. For
        example, to filter contigs with a minimum length of 250, a comparison
        would be::

            self.filter_contigs(["length", ">=", 250])

        The filtered contig ids will be stored in the
        :py:attr:`~Assembly.filtered_ids` list.

        The result of the test for all contigs will be stored in the
        :py:attr:`~Assembly.report` dictionary.

        Parameters
        ----------
        comparisons : list
            List with contig key, operator and value to test.

        """

        # Reset list of filtered ids
        self.filtered_ids = []
        self.report = {}

        gc_filters = [
            ["gc_prop", ">=", self.min_gc],
            ["gc_prop", "<=", 1 - self.min_gc]
        ]

        self.filters = list(comparisons) + gc_filters

        logger.debug("Filtering contigs using filters: {}".format(
            self.filters))

        for contig_id, contig in self.contigs.items():
            for key, op, value in list(comparisons) + gc_filters:
                if not self._test_truth(contig[key], op, value):
                    self.filtered_ids.append(contig_id)
                    self.report[contig_id] = "{}/{}/{}".format(key,
                                                               contig[key],
                                                               value)
                    break
                else:
                    self.report[contig_id] = "pass"

def
filter_contigs
(
self
,
*
comparisons
)
:
self
.
filtered_ids
=
[
]
self
.
report
=
{
}
gc_filters
=
[
[
"gc_prop"
,
">="
,
self
.
min_gc
]
,
[
"gc_prop"
,
"<="
,
1
self
.
min_gc
]
]
self
.
filters
=
list
(
comparisons
)
+
gc_filters
logger
.
debug
(
"Filtering contigs using filters: {}"
.
format
(
self
.
filters
)
)
for
contig_id
,
contig
in
self
.
contigs
.
items
(
)
:
for
key
,
op
,
value
in
list
(
comparisons
)
+
gc_filters
:
if
not
self
.
_test_truth
(
contig
[
key
]
,
op
,
value
)
:
self
.
filtered_ids
.
append
(
contig_id
)
self
.
report
[
contig_id
]
=
"{}/{}/{}"
.
format
(
key
,
contig
[
key
]
,
value
)
break
else
:
self
.
report
[
contig_id
]
=
"pass"

Filters the contigs of the assembly according to user provided \ comparisons .


def get_assembly_length(self):
        """Returns the length of the assembly, without the filtered contigs.

        Returns
        -------
        x : int
            Total length of the assembly.

        """

        return sum(
            [vals["length"] for contig_id, vals in self.contigs.items()
             if contig_id not in self.filtered_ids])

def get_assembly_length ( self ) : return sum ( [ vals [ "length" ] for contig_id , vals in self . contigs . items ( ) if contig_id not in self . filtered_ids ] )

Returns the length of the assembly without the filtered contigs .


def write_assembly(self, output_file, filtered=True):
        """Writes the assembly to a new file.

        The ``filtered`` option controls whether the new assembly will be
        filtered or not.

        Parameters
        ----------
        output_file : str
            Name of the output assembly file.
        filtered : bool
            If ``True``, does not include filtered ids.
        """

        logger.debug("Writing the filtered assembly into: {}".format(
            output_file))
        with open(output_file, "w") as fh:

            for contig_id, contig in self.contigs.items():
                if contig_id not in self.filtered_ids and filtered:
                    fh.write(">{}_{}\\n{}\\n".format(self.sample,
                                                     contig["header"],
                                                     contig["sequence"]))


def write_report(self, output_file):
        """Writes a report with the test results for the current assembly

        Parameters
        ----------
        output_file : str
            Name of the output assembly file.

        """

        logger.debug("Writing the assembly report into: {}".format(
            output_file))
        with open(output_file, "w") as fh:

            for contig_id, vals in self.report.items():
                fh.write("{}, {}\\n".format(contig_id, vals))

def
write_report
(
self
,
output_file
)
:
logger
.
debug
(
"Writing the assembly report into: {}"
.
format
(
output_file
)
)
with
open
(
output_file
,
"w"
)
as
fh
:
for
contig_id
,
vals
in
self
.
report
.
items
(
)
:
fh
.
write
(
"{}, {}\\n"
.
format
(
contig_id
,
vals
)
)


def remove_inner_forks(text):
    """Recursively removes nested brackets

    This function is used to remove nested brackets from fork strings using
    regular expressions

    Parameters
    ----------
    text: str
        The string that contains brackets with inner forks to be removed

    Returns
    -------
    text: str
        the string with only the processes that are not in inner forks, thus
        the processes that belong to a given fork.

    """

    n = 1  # run at least once for one level of fork
    # Then this loop assures that all brackets will get removed in a nested
    # structure
    while n:
        # this removes non-nested brackets
        text, n = re.subn(r'\([^()]*\)', '', text)

    return text

def remove_inner_forks ( text ) : n = 1 while n : text , n = re . subn ( r'\([^()]*\)' , '' , text ) return text


def inner_fork_insanity_checks(pipeline_string):
    """
    This function performs two sanity checks in the pipeline string. The first
    check, assures that each fork contains a lane token '|', while the second
    check looks for duplicated processes within the same fork.

    Parameters
    ----------
    pipeline_string: str
         String with the definition of the pipeline, e.g.::
             'processA processB processC(ProcessD | ProcessE)'

    """

    # first lets get all forks to a list.
    list_of_forks = []  # stores forks
    left_indexes = []  # stores indexes of left brackets

    # iterate through the string looking for '(' and ')'.
    for pos, char in enumerate(pipeline_string):
        if char == FORK_TOKEN:
            # saves pos to left_indexes list
            left_indexes.append(pos)
        elif char == CLOSE_TOKEN and len(left_indexes) > 0:
            # saves fork to list_of_forks
            list_of_forks.append(pipeline_string[left_indexes[-1] + 1: pos])
            # removes last bracket from left_indexes list
            left_indexes = left_indexes[:-1]

    # sort list in descending order of number of forks
    list_of_forks.sort(key=lambda x: x.count(FORK_TOKEN), reverse=True)

    # Now, we can iterate through list_of_forks and check for errors in each
    # fork
    for fork in list_of_forks:
        # remove inner forks for these checks since each fork has its own entry
        # in list_of_forks. Note that each fork is now sorted in descending
        # order which enables to remove sequentially the string for the fork
        # potentially with more inner forks
        for subfork in list_of_forks:
            # checks if subfork is contained in fork and if they are different,
            # avoiding to remove itself
            if subfork in list_of_forks and subfork != fork:
                # removes inner forks. Note that string has no spaces
                fork_simplified = fork.replace("({})".format(subfork), "")
            else:
                fork_simplified = fork

        # Checks if there is no fork separator character '|' within each fork
        if not len(fork_simplified.split(LANE_TOKEN)) > 1:
            raise SanityError("One of the forks doesn't have '|' "
                              "separator between the processes to fork. This is"
                              " the prime suspect: '({})'".format(fork))

This function performs two sanity checks in the pipeline string . The first check assures that each fork contains a lane token | while the second check looks for duplicated processes within the same fork .


def insanity_checks(pipeline_str):
    """Wrapper that performs all sanity checks on the pipeline string

    Parameters
    ----------
    pipeline_str : str
        String with the pipeline definition
    """

    # Gets rid of all spaces in string
    p_string = pipeline_str.replace(" ", "").strip()

    # some of the check functions use the pipeline_str as the user provided but
    # the majority uses the parsed p_string.
    checks = [
        [p_string, [
            empty_tasks,
            brackets_but_no_lanes,
            brackets_insanity_check,
            lane_char_insanity_check,
            final_char_insanity_check,
            fork_procs_insanity_check,
            start_proc_insanity_check,
            late_proc_insanity_check
        ]],
        [pipeline_str, [
            inner_fork_insanity_checks
        ]]
    ]

    # executes sanity checks in pipeline string before parsing it.
    for param, func_list in checks:
        for func in func_list:
            func(param)

def
insanity_checks
(
pipeline_str
)
:
p_string
=
pipeline_str
.
replace
(
" "
,
""
)
.
strip
(
)
checks
=
[
[
p_string
,
[
empty_tasks
,
brackets_but_no_lanes
,
brackets_insanity_check
,
lane_char_insanity_check
,
final_char_insanity_check
,
fork_procs_insanity_check
,
start_proc_insanity_check
,
late_proc_insanity_check
]
]
,
[
pipeline_str
,
[
inner_fork_insanity_checks
]
]
]
for
param
,
func_list
in
checks
:
for
func
in
func_list
:
func
(
param
)


def parse_pipeline(pipeline_str):
    """Parses a pipeline string into a list of dictionaries with the connections
     between processes

    Parameters
    ----------
    pipeline_str : str
        String with the definition of the pipeline, e.g.::
            'processA processB processC(ProcessD | ProcessE)'

    Returns
    -------
    pipeline_links : list

    """

    if os.path.exists(pipeline_str):
        logger.debug("Found pipeline file: {}".format(pipeline_str))
        with open(pipeline_str) as fh:
            pipeline_str = "".join([x.strip() for x in fh.readlines()])

    logger.info(colored_print("Resulting pipeline string:\n"))
    logger.info(colored_print(pipeline_str + "\n"))

    # Perform pipeline insanity checks
    insanity_checks(pipeline_str)

    logger.debug("Parsing pipeline string: {}".format(pipeline_str))

    pipeline_links = []
    lane = 1

    # Add unique identifiers to each process to allow a correct connection
    # between forks with same processes
    pipeline_str_modified, identifiers_to_tags = add_unique_identifiers(
        pipeline_str)

    # Get number of forks in the pipeline
    nforks = pipeline_str_modified.count(FORK_TOKEN)
    logger.debug("Found {} fork(s)".format(nforks))

    # If there are no forks, connect the pipeline as purely linear
    if not nforks:
        logger.debug("Detected linear pipeline string : {}".format(
            pipeline_str))
        linear_pipeline = ["__init__"] + pipeline_str_modified.split()
        pipeline_links.extend(linear_connection(linear_pipeline, lane))
        # Removes unique identifiers used for correctly assign fork parents with
        #  a possible same process name
        pipeline_links = remove_unique_identifiers(identifiers_to_tags,
                                                   pipeline_links)
        return pipeline_links

    for i in range(nforks):

        logger.debug("Processing fork {} in lane {}".format(i, lane))
        # Split the pipeline at each fork start position. fields[-1] will
        # hold the process after the fork. fields[-2] will hold the processes
        # before the fork.
        fields = pipeline_str_modified.split(FORK_TOKEN, i + 1)

        # Get the processes before the fork. This may be empty when the
        # fork is at the beginning of the pipeline.
        previous_process = fields[-2].split(LANE_TOKEN)[-1].split()
        logger.debug("Previous processes string: {}".format(fields[-2]))
        logger.debug("Previous processes list: {}".format(previous_process))
        # Get lanes after the fork
        next_lanes = get_lanes(fields[-1])
        logger.debug("Next lanes object: {}".format(next_lanes))
        # Get the immediate targets of the fork
        fork_sink = [x[0] for x in next_lanes]
        logger.debug("The fork sinks into the processes: {}".format(fork_sink))

        # The first fork is a special case, where the processes before AND
        # after the fork (until the start of another fork) are added to
        # the ``pipeline_links`` variable. Otherwise, only the processes
        # after the fork will be added
        if i == 0:
            # If there are no previous process, the fork is at the beginning
            # of the pipeline string. In this case, inject the special
            # "init" process.
            if not previous_process:
                previous_process = ["__init__"]
                lane = 0
            else:
                previous_process = ["__init__"] + previous_process

            # Add the linear modules before the fork
            pipeline_links.extend(
                linear_connection(previous_process, lane))

        fork_source = previous_process[-1]
        logger.debug("Fork source is set to: {}".format(fork_source))
        fork_lane = get_source_lane(previous_process, pipeline_links)
        logger.debug("Fork lane is set to: {}".format(fork_lane))
        # Add the forking modules
        pipeline_links.extend(
            fork_connection(fork_source, fork_sink, fork_lane, lane))
        # Add the linear connections in the subsequent lanes
        pipeline_links.extend(
            linear_lane_connection(next_lanes, lane))

        lane += len(fork_sink)

    pipeline_links = remove_unique_identifiers(identifiers_to_tags,
                                               pipeline_links)
    return pipeline_links

def
parse_pipeline
(
pipeline_str
)
:
if
os
.
path
.
exists
(
pipeline_str
)
:
logger
.
debug
(
"Found pipeline file: {}"
.
format
(
pipeline_str
)
)
with
open
(
pipeline_str
)
as
fh
:
pipeline_str
=
""
.
join
(
[
x
.
strip
(
)
for
x
in
fh
.
readlines
(
)
]
)
logger
.
info
(
colored_print
(
"Resulting pipeline string:\n"
)
)
logger
.
info
(
colored_print
(
pipeline_str
+
"\n"
)
)
insanity_checks
(
pipeline_str
)
logger
.
debug
(
"Parsing pipeline string: {}"
.
format
(
pipeline_str
)
)
pipeline_links
=
[
]
lane
=
1
pipeline_str_modified
,
identifiers_to_tags
=
add_unique_identifiers
(
pipeline_str
)
nforks
=
pipeline_str_modified
.
count
(
FORK_TOKEN
)
logger
.
debug
(
"Found {} fork(s)"
.
format
(
nforks
)
)
if
not
nforks
:
logger
.
debug
(
"Detected linear pipeline string : {}"
.
format
(
pipeline_str
)
)
linear_pipeline
=
[
"__init__"
]
+
pipeline_str_modified
.
split
(
)
pipeline_links
.
extend
(
linear_connection
(
linear_pipeline
,
lane
)
)
pipeline_links
=
remove_unique_identifiers
(
identifiers_to_tags
,
pipeline_links
)
return
pipeline_links
for
i
in
range
(
nforks
)
:
logger
.
debug
(
"Processing fork {} in lane {}"
.
format
(
i
,
lane
)
)
fields
=
pipeline_str_modified
.
split
(
FORK_TOKEN
,
i
+
1
)
previous_process
=
fields
[
2
]
.
split
(
LANE_TOKEN
)
[
1
]
.
split
(
)
logger
.
debug
(
"Previous processes string: {}"
.
format
(
fields
[
2
]
)
)
logger
.
debug
(
"Previous processes list: {}"
.
format
(
previous_process
)
)
next_lanes
=
get_lanes
(
fields
[
1
]
)
logger
.
debug
(
"Next lanes object: {}"
.
format
(
next_lanes
)
)
fork_sink
=
[
x
[
0
]
for
x
in
next_lanes
]
logger
.
debug
(
"The fork sinks into the processes: {}"
.
format
(
fork_sink
)
)
if
i
==
0
:
if
not
previous_process
:
previous_process
=
[
"__init__"
]
lane
=
0
else
:
previous_process
=
[
"__init__"
]
+
previous_process
pipeline_links
.
extend
(
linear_connection
(
previous_process
,
lane
)
)
fork_source
=
previous_process
[
1
]
logger
.
debug
(
"Fork source is set to: {}"
.
format
(
fork_source
)
)
fork_lane
=
get_source_lane
(
previous_process
,
pipeline_links
)
logger
.
debug
(
"Fork lane is set to: {}"
.
format
(
fork_lane
)
)
pipeline_links
.
extend
(
fork_connection
(
fork_source
,
fork_sink
,
fork_lane
,
lane
)
)
pipeline_links
.
extend
(
linear_lane_connection
(
next_lanes
,
lane
)
)
lane
+=
len
(
fork_sink
)
pipeline_links
=
remove_unique_identifiers
(
identifiers_to_tags
,
pipeline_links
)
return
pipeline_links


def get_source_lane(fork_process, pipeline_list):
    """Returns the lane of the last process that matches fork_process

    Parameters
    ----------
    fork_process : list
        List of processes before the fork.
    pipeline_list : list
        List with the pipeline connection dictionaries.

    Returns
    -------
    int
        Lane of the last process that matches fork_process
    """

    fork_source = fork_process[-1]
    fork_sig = [x for x in fork_process if x != "__init__"]

    for position, p in enumerate(pipeline_list[::-1]):

        if p["output"]["process"] == fork_source:

            lane = p["output"]["lane"]
            logger.debug("Possible source match found in position {} in lane"
                         " {}".format(position, lane))
            lane_sequence = [x["output"]["process"] for x in pipeline_list
                             if x["output"]["lane"] == lane]
            logger.debug("Testing lane sequence '{}' against fork signature"
                         " '{}'".format(lane_sequence, fork_sig))
            if lane_sequence == fork_sig:
                return p["output"]["lane"]

    return 0


def get_lanes(lanes_str):
    """From a raw pipeline string, get a list of lanes from the start
    of the current fork.

    When the pipeline is being parsed, it will be split at every fork
    position. The string at the right of the fork position will be provided
    to this function. It's job is to retrieve the lanes that result
    from that fork, ignoring any nested forks.

    Parameters
    ----------
    lanes_str : str
        Pipeline string after a fork split

    Returns
    -------
    lanes : list
        List of lists, with the list of processes for each lane

    """

    logger.debug("Parsing lanes from raw string: {}".format(lanes_str))

    # Temporarily stores the lanes string after removal of nested forks
    parsed_lanes = ""
    # Flag used to determined whether the cursor is inside or outside the
    # right fork
    infork = 0
    for i in lanes_str:

        # Nested fork started
        if i == FORK_TOKEN:
            infork += 1
        # Nested fork stopped
        if i == CLOSE_TOKEN:
            infork -= 1

        if infork < 0:
            break

        # Save only when in the right fork
        if infork == 0:
            # Ignore forking syntax tokens
            if i not in [FORK_TOKEN, CLOSE_TOKEN]:
                parsed_lanes += i

    return [x.split() for x in parsed_lanes.split(LANE_TOKEN)]

def
get_lanes
(
lanes_str
)
:
logger
.
debug
(
"Parsing lanes from raw string: {}"
.
format
(
lanes_str
)
)
parsed_lanes
=
""
infork
=
0
for
i
in
lanes_str
:
if
i
==
FORK_TOKEN
:
infork
+=
1
if
i
==
CLOSE_TOKEN
:
infork
-=
1
if
infork
<
0
:
break
if
infork
==
0
:
if
i
not
in
[
FORK_TOKEN
,
CLOSE_TOKEN
]
:
parsed_lanes
+=
i
return
[
x
.
split
(
)
for
x
in
parsed_lanes
.
split
(
LANE_TOKEN
)
]

From a raw pipeline string get a list of lanes from the start of the current fork .


def linear_connection(plist, lane):
    """Connects a linear list of processes into a list of dictionaries

    Parameters
    ----------
    plist : list
        List with process names. This list should contain at least two entries.
    lane : int
        Corresponding lane of the processes

    Returns
    -------
    res : list
        List of dictionaries with the links between processes
    """

    logger.debug(
        "Establishing linear connection with processes: {}".format(plist))

    res = []
    previous = None

    for p in plist:
        # Skip first process
        if not previous:
            previous = p
            continue

        res.append({
            "input": {
                "process": previous,
                "lane": lane
            },
            "output": {
                "process": p,
                "lane": lane
            }
        })
        previous = p

    return res


def fork_connection(source, sink, source_lane, lane):
    """Makes the connection between a process and the first processes in the
    lanes to which it forks.

    The ``lane`` argument should correspond to the lane of the source process.
    For each lane in ``sink``, the lane counter will increase.

    Parameters
    ----------
    source : str
        Name of the process that is forking
    sink : list
        List of the processes where the source will fork to. Each element
        corresponds to the start of a lane.
    source_lane : int
        Lane of the forking process
    lane : int
        Lane of the source process

    Returns
    -------
    res : list
        List of dictionaries with the links between processes
    """

    logger.debug("Establishing forking of source '{}' into processes"
                 " '{}'. Source lane set to '{}' and lane set to '{}'".format(
                    source, sink, source_lane, lane))

    res = []
    # Increase the lane counter for the first lane
    lane_counter = lane + 1

    for p in sink:
        res.append({
            "input": {
                "process": source,
                "lane": source_lane
            },
            "output": {
                "process": p,
                "lane": lane_counter
            }
        })
        lane_counter += 1

    return res

Makes the connection between a process and the first processes in the lanes to which it forks .


def add_unique_identifiers(pipeline_str):
    """Returns the pipeline string with unique identifiers and a dictionary with
     references between the unique keys and the original values

    Parameters
    ----------
    pipeline_str : str
        Pipeline string

    Returns
    -------
    str
        Pipeline string with unique identifiers
    dict
        Match between process unique values and original names
    """

    # Add space at beginning and end of pipeline to allow regex mapping of final
    # process in linear pipelines
    pipeline_str_modified = " {} ".format(pipeline_str)

    # Regex to get all process names. Catch all words without spaces and that
    # are not fork tokens or pipes
    reg_find_proc = r"[^\s{}{}{}]+".format(LANE_TOKEN, FORK_TOKEN, CLOSE_TOKEN)
    process_names = re.findall(reg_find_proc, pipeline_str_modified)

    identifiers_to_tags = {}
    """
    dict: Matches new process names (identifiers) with original process
    names
    """

    new_process_names = []
    """
    list: New process names used to replace in the pipeline string
    """

    # Assigns the new process names by appending a numeric id at the end of
    # the process name
    for index, val in enumerate(process_names):
        if "=" in val:
            parts = val.split("=")
            new_id = "{}_{}={}".format(parts[0], index, parts[1])
        else:
            new_id = "{}_{}".format(val, index)

        # add new process with id
        new_process_names.append(new_id)
        # makes a match between new process name and original process name
        identifiers_to_tags[new_id] = val

    # Add space between forks, pipes and the process names for the replace
    # regex to work
    match_result = lambda match: " {} ".format(match.group())

    # force to add a space between each token so that regex modification can
    # be applied
    find = r'[{}{}{}]+'.format(FORK_TOKEN, LANE_TOKEN, CLOSE_TOKEN)
    pipeline_str_modified = re.sub(find, match_result, pipeline_str_modified)

    # Replace original process names by the unique identifiers
    for index, val in enumerate(process_names):
        # regex to replace process names with non assigned process ids
        # escape characters are required to match to the dict keys
        # (identifiers_to_tags), since python keys with escape characters
        # must be escaped
        find = r'{}[^_]'.format(val).replace("\\", "\\\\")
        pipeline_str_modified = re.sub(find, new_process_names[index] + " ",
                                       pipeline_str_modified, 1)

    return pipeline_str_modified, identifiers_to_tags

def
add_unique_identifiers
(
pipeline_str
)
:
pipeline_str_modified
=
" {} "
.
format
(
pipeline_str
)
reg_find_proc
=
r"[^\s{}{}{}]+"
.
format
(
LANE_TOKEN
,
FORK_TOKEN
,
CLOSE_TOKEN
)
process_names
=
re
.
findall
(
reg_find_proc
,
pipeline_str_modified
)
identifiers_to_tags
=
{
}
new_process_names
=
[
]
for
index
,
val
in
enumerate
(
process_names
)
:
if
"="
in
val
:
parts
=
val
.
split
(
"="
)
new_id
=
"{}_{}={}"
.
format
(
parts
[
0
]
,
index
,
parts
[
1
]
)
else
:
new_id
=
"{}_{}"
.
format
(
val
,
index
)
new_process_names
.
append
(
new_id
)
identifiers_to_tags
[
new_id
]
=
val
match_result
=
lambda
match
:
" {} "
.
format
(
match
.
group
(
)
)
find
=
r'[{}{}{}]+'
.
format
(
FORK_TOKEN
,
LANE_TOKEN
,
CLOSE_TOKEN
)
pipeline_str_modified
=
re
.
sub
(
find
,
match_result
,
pipeline_str_modified
)
for
index
,
val
in
enumerate
(
process_names
)
:
find
=
r'{}[^_]'
.
format
(
val
)
.
replace
(
"\\"
,
"\\\\"
)
pipeline_str_modified
=
re
.
sub
(
find
,
new_process_names
[
index
]
+
" "
,
pipeline_str_modified
,
1
)
return
pipeline_str_modified
,
identifiers_to_tags


def remove_unique_identifiers(identifiers_to_tags, pipeline_links):
    """Removes unique identifiers and add the original process names to the
    already parsed pipelines

    Parameters
    ----------
    identifiers_to_tags : dict
        Match between unique process identifiers and process names
    pipeline_links: list
        Parsed pipeline list with unique identifiers

    Returns
    -------
    list
        Pipeline list with original identifiers
    """

    # Replaces the unique identifiers by the original process names
    for index, val in enumerate(pipeline_links):
        if val["input"]["process"] != "__init__":
            val["input"]["process"] = identifiers_to_tags[
                val["input"]["process"]]
        if val["output"]["process"] != "__init__":
            val["output"]["process"] = identifiers_to_tags[
                val["output"]["process"]]

    return pipeline_links

def remove_unique_identifiers ( identifiers_to_tags , pipeline_links ) : for index , val in enumerate ( pipeline_links ) : if val [ "input" ] [ "process" ] != "__init__" : val [ "input" ] [ "process" ] = identifiers_to_tags [ val [ "input" ] [ "process" ] ] if val [ "output" ] [ "process" ] != "__init__" : val [ "output" ] [ "process" ] = identifiers_to_tags [ val [ "output" ] [ "process" ] ] return pipeline_links


def _check_required_files(self):
        """Checks whetner the trace and log files are available
        """

        if not os.path.exists(self.trace_file):
            raise eh.InspectionError("The provided trace file could not be "
                                     "opened: {}".format(self.trace_file))

        if not os.path.exists(self.log_file):
            raise eh.InspectionError("The .nextflow.log files could not be "
                                     "opened. Are you sure you are in a "
                                     "nextflow project directory?")

def
_check_required_files
(
self
)
:
if
not
os
.
path
.
exists
(
self
.
trace_file
)
:
raise
eh
.
InspectionError
(
"The provided trace file could not be "
"opened: {}"
.
format
(
self
.
trace_file
)
)
if
not
os
.
path
.
exists
(
self
.
log_file
)
:
raise
eh
.
InspectionError
(
"The .nextflow.log files could not be "
"opened. Are you sure you are in a "
"nextflow project directory?"
)


def _header_mapping(header):
        """Parses the trace file header and retrieves the positions of each
        column key.

        Parameters
        ----------
        header : str
            The header line of nextflow's trace file

        Returns
        -------
        dict
            Mapping the column ID to its position (e.g.: {"tag":2})
        """

        return dict(
            (x.strip(), pos) for pos, x in enumerate(header.split("\t"))
        )

def _header_mapping ( header ) : return dict ( ( x . strip ( ) , pos ) for pos , x in enumerate ( header . split ( "\t" ) ) )

Parses the trace file header and retrieves the positions of each column key .


def _hms(s):
        """Converts a hms string into seconds.

        Parameters
        ----------
        s : str
            The hms string can be something like '20s', '1m30s' or '300ms'.

        Returns
        -------
        float
            Time in seconds.

        """

        if s == "-":
            return 0

        if s.endswith("ms"):
            return float(s.rstrip("ms")) / 1000

        fields = list(map(float, re.split("[dhms]", s)[:-1]))
        if len(fields) == 4:
            return fields[0] * 24 * 3600 + fields[1] * 3600 + fields[2] * 60 +\
                fields[3]
        if len(fields) == 3:
            return fields[0] * 3600 + fields[1] * 60 + fields[2]
        elif len(fields) == 2:
            return fields[0] * 60 + fields[1]
        else:
            return fields[0]

def _hms ( s ) : if s == "-" : return 0 if s . endswith ( "ms" ) : return float ( s . rstrip ( "ms" ) ) / 1000 fields = list ( map ( float , re . split ( "[dhms]" , s ) [ : - 1 ] ) ) if len ( fields ) == 4 : return fields [ 0 ] * 24 * 3600 + fields [ 1 ] * 3600 + fields [ 2 ] * 60 + fields [ 3 ] if len ( fields ) == 3 : return fields [ 0 ] * 3600 + fields [ 1 ] * 60 + fields [ 2 ] elif len ( fields ) == 2 : return fields [ 0 ] * 60 + fields [ 1 ] else : return fields [ 0 ]


def _size_coverter(s):
        """Converts size string into megabytes

        Parameters
        ----------
        s : str
            The size string can be '30KB', '20MB' or '1GB'

        Returns
        -------
        float
            With the size in bytes

        """

        if s.upper().endswith("KB"):
            return float(s.rstrip("KB")) / 1024

        elif s.upper().endswith(" B"):
            return float(s.rstrip("B")) / 1024 / 1024

        elif s.upper().endswith("MB"):
            return float(s.rstrip("MB"))

        elif s.upper().endswith("GB"):
            return float(s.rstrip("GB")) * 1024

        elif s.upper().endswith("TB"):
            return float(s.rstrip("TB")) * 1024 * 1024

        else:
            return float(s)

def
_size_coverter
(
s
)
:
if
s
.
upper
(
)
.
endswith
(
"KB"
)
:
return
float
(
s
.
rstrip
(
"KB"
)
)
/
1024
elif
s
.
upper
(
)
.
endswith
(
" B"
)
:
return
float
(
s
.
rstrip
(
"B"
)
)
/
1024
/
1024
elif
s
.
upper
(
)
.
endswith
(
"MB"
)
:
return
float
(
s
.
rstrip
(
"MB"
)
)
elif
s
.
upper
(
)
.
endswith
(
"GB"
)
:
return
float
(
s
.
rstrip
(
"GB"
)
)
*
1024
elif
s
.
upper
(
)
.
endswith
(
"TB"
)
:
return
float
(
s
.
rstrip
(
"TB"
)
)
*
1024
*
1024
else
:
return
float
(
s
)


def _get_pipeline_processes(self):
        """Parses the .nextflow.log file and retrieves the complete list
        of processes

        This method searches for specific signatures at the beginning of the
        .nextflow.log file::

             Apr-19 19:07:32.660 [main] DEBUG nextflow.processor
             TaskProcessor - Creating operator > report_corrupt_1_1 --
             maxForks: 4

        When a line with the .*Creating operator.* signature is found, the
        process name is retrieved and populates the :attr:`processes` attribute
        """

        with open(self.log_file) as fh:

            for line in fh:
                if re.match(".*Creating operator.*", line):
                    # Retrieves the process name from the string
                    match = re.match(".*Creating operator > (.*) --", line)
                    process = match.group(1)

                    if any([process.startswith(x) for x in self._blacklist]):
                        continue

                    if process not in self.skip_processes:
                        self.processes[match.group(1)] = {
                            "barrier": "W",
                            "submitted": set(),
                            "finished": set(),
                            "failed": set(),
                            "retry": set(),
                            "cpus": None,
                            "memory": None
                        }
                        self.process_tags[process] = {}

                # Retrieves the pipeline name from the string
                if re.match(".*Launching `.*` \[.*\] ", line):
                    tag_match = re.match(".*Launching `.*` \[(.*)\] ", line)
                    self.pipeline_tag = tag_match.group(1) if tag_match else \
                        "?"
                    name_match = re.match(".*Launching `(.*)` \[.*\] ", line)
                    self.pipeline_name = name_match.group(1) if name_match \
                        else "?"

        self.content_lines = len(self.processes)

def
_get_pipeline_processes
(
self
)
:
with
open
(
self
.
log_file
)
as
fh
:
for
line
in
fh
:
if
re
.
match
(
".*Creating operator.*"
,
line
)
:
match
=
re
.
match
(
".*Creating operator > (.*) --"
,
line
)
process
=
match
.
group
(
1
)
if
any
(
[
process
.
startswith
(
x
)
for
x
in
self
.
_blacklist
]
)
:
continue
if
process
not
in
self
.
skip_processes
:
self
.
processes
[
match
.
group
(
1
)
]
=
{
"barrier"
:
"W"
,
"submitted"
:
set
(
)
,
"finished"
:
set
(
)
,
"failed"
:
set
(
)
,
"retry"
:
set
(
)
,
"cpus"
:
,
"memory"
:
}
self
.
process_tags
[
process
]
=
{
}
if
re
.
match
(
".*Launching `.*` \[.*\] "
,
line
)
:
tag_match
=
re
.
match
(
".*Launching `.*` \[(.*)\] "
,
line
)
self
.
pipeline_tag
=
tag_match
.
group
(
1
)
if
tag_match
else
"?"
name_match
=
re
.
match
(
".*Launching `(.*)` \[.*\] "
,
line
)
self
.
pipeline_name
=
name_match
.
group
(
1
)
if
name_match
else
"?"
self
.
content_lines
=
len
(
self
.
processes
)

Parses the . nextflow . log file and retrieves the complete list of processes


def _clear_inspect(self):
        """Clears inspect attributes when re-executing a pipeline"""

        self.trace_info = defaultdict(list)
        self.process_tags = {}
        self.process_stats = {}
        self.samples = []
        self.stored_ids = []
        self.stored_log_ids = []
        self.time_start = None
        self.time_stop = None
        self.execution_command = None
        self.nextflow_version = None
        self.abort_cause = None
        self._c = 0
        # Clean up of tag running status
        for p in self.processes.values():
            p["barrier"] = "W"
            for i in ["submitted", "finished", "failed", "retry"]:
                p[i] = set()

def _clear_inspect ( self ) : self . trace_info = defaultdict ( list ) self . process_tags = { } self . process_stats = { } self . samples = [ ] self . stored_ids = [ ] self . stored_log_ids = [ ] self . time_start = None self . time_stop = None self . execution_command = None self . nextflow_version = None self . abort_cause = None self . _c = 0 for p in self . processes . values ( ) : p [ "barrier" ] = "W" for i in [ "submitted" , "finished" , "failed" , "retry" ] : p [ i ] = set ( )

Clears inspect attributes when re - executing a pipeline


def _update_barrier_status(self):
        """Checks whether the channels to each process have been closed.
        """

        with open(self.log_file) as fh:

            for line in fh:

                # Exit barrier update after session abort signal
                if "Session aborted" in line:
                    return

                if "<<< barrier arrive" in line:
                    # Retrieve process name from string
                    process_m = re.match(".*process: (.*)\)", line)
                    if process_m:
                        process = process_m.group(1)
                        # Updates process channel to complete
                        if process in self.processes:
                            self.processes[process]["barrier"] = "C"

def
_update_barrier_status
(
self
)
:
with
open
(
self
.
log_file
)
as
fh
:
for
line
in
fh
:
if
"Session aborted"
in
line
:
return
if
"<<< barrier arrive"
in
line
:
process_m
=
re
.
match
(
".*process: (.*)\)"
,
line
)
if
process_m
:
process
=
process_m
.
group
(
1
)
if
process
in
self
.
processes
:
self
.
processes
[
process
]
[
"barrier"
]
=
"C"

Checks whether the channels to each process have been closed .


def _retrieve_log(path):
        """Method used to retrieve the contents of a log file into a list.

        Parameters
        ----------
        path

        Returns
        -------
        list or None
            Contents of the provided file, each line as a list entry
        """

        if not os.path.exists(path):
            return None

        with open(path) as fh:
            return fh.readlines()

def _retrieve_log ( path ) : if not os . path . exists ( path ) : return None with open ( path ) as fh : return fh . readlines ( )

Method used to retrieve the contents of a log file into a list .


def _assess_resource_warnings(self, process, vals):
        """Assess whether the cpu load or memory usage is above the allocation

        Parameters
        ----------
        process : str
            Process name
        vals : vals
            List of trace information for each tag of that process

        Returns
        -------
        cpu_warnings : dict
            Keys are tags and values are the excessive cpu load
        mem_warnings : dict
            Keys are tags and values are the excessive rss
        """

        cpu_warnings = {}
        mem_warnings = {}

        for i in vals:
            try:
                expected_load = float(i["cpus"]) * 100
                cpu_load = float(i["%cpu"].replace(",", ".").replace("%", ""))

                if expected_load * 0.9 > cpu_load > expected_load * 1.10:
                    cpu_warnings[i["tag"]] = {
                        "expected":  expected_load,
                        "value": cpu_load
                    }
            except (ValueError, KeyError):
                pass

            try:
                rss = self._size_coverter(i["rss"])
                mem_allocated = self._size_coverter(i["memory"])

                if rss > mem_allocated * 1.10:
                    mem_warnings[i["tag"]] = {
                        "expected": mem_allocated,
                        "value": rss
                    }
            except (ValueError, KeyError):
                pass

        return cpu_warnings, mem_warnings

def _assess_resource_warnings ( self , process , vals ) : cpu_warnings = { } mem_warnings = { } for i in vals : try : expected_load = float ( i [ "cpus" ] ) * 100 cpu_load = float ( i [ "%cpu" ] . replace ( "," , "." ) . replace ( "%" , "" ) ) if expected_load * 0.9 > cpu_load > expected_load * 1.10 : cpu_warnings [ i [ "tag" ] ] = { "expected" : expected_load , "value" : cpu_load } except ( ValueError , KeyError ) : pass try : rss = self . _size_coverter ( i [ "rss" ] ) mem_allocated = self . _size_coverter ( i [ "memory" ] ) if rss > mem_allocated * 1.10 : mem_warnings [ i [ "tag" ] ] = { "expected" : mem_allocated , "value" : rss } except ( ValueError , KeyError ) : pass return cpu_warnings , mem_warnings


def _update_process_stats(self):
        """Updates the process stats with the information from the processes

        This method is called at the end of each static parsing of the nextflow
        trace file. It re-populates the :attr:`process_stats` dictionary
        with the new stat metrics.
        """

        good_status = ["COMPLETED", "CACHED"]

        for process, vals in self.trace_info.items():

            # Update submission status of tags for each process
            vals = self._update_tag_status(process, vals)

            # Update process resources
            self._update_process_resources(process, vals)

            self.process_stats[process] = {}

            inst = self.process_stats[process]

            # Get number of completed samples
            inst["completed"] = "{}".format(
                len([x for x in vals if x["status"] in good_status]))

            # Get average time
            try:
                time_array = [self._hms(x["realtime"]) for x in vals]
                mean_time = round(sum(time_array) / len(time_array), 1)
                mean_time_str = strftime('%H:%M:%S', gmtime(mean_time))
                inst["realtime"] = mean_time_str
            # When the realtime column is not present
            except KeyError:
                inst["realtime"] = "-"

            # Get cumulative cpu/hours
            try:
                cpu_hours = [self._cpu_load_parser(
                    x["cpus"], x["%cpu"], x["realtime"]) for x in vals]
                inst["cpuhour"] = round(sum(cpu_hours), 2)
            # When the realtime, cpus or %cpus column are not present
            except KeyError:
                inst["cpuhour"] = "-"

            # Assess resource warnings
            inst["cpu_warnings"], inst["mem_warnings"] = \
                self._assess_resource_warnings(process, vals)

            # Get maximum memory
            try:
                rss_values = [self._size_coverter(x["rss"]) for x in vals
                              if x["rss"] != "-"]
                if rss_values:
                    max_rss = round(max(rss_values))
                    rss_str = self._size_compress(max_rss)
                else:
                    rss_str = "-"
                inst["maxmem"] = rss_str
            except KeyError:
                inst["maxmem"] = "-"

            # Get read size
            try:
                rchar_values = [self._size_coverter(x["rchar"]) for x in vals
                                if x["rchar"] != "-"]
                if rchar_values:
                    avg_rchar = round(sum(rchar_values) / len(rchar_values))
                    rchar_str = self._size_compress(avg_rchar)
                else:
                    rchar_str = "-"
            except KeyError:
                rchar_str = "-"
            inst["avgread"] = rchar_str

            # Get write size
            try:
                wchar_values = [self._size_coverter(x["wchar"]) for x in vals
                                if x["wchar"] != "-"]
                if wchar_values:
                    avg_wchar = round(sum(wchar_values) / len(wchar_values))
                    wchar_str = self._size_compress(avg_wchar)
                else:
                    wchar_str = "-"
            except KeyError:
                wchar_str = "-"
            inst["avgwrite"] = wchar_str

def _update_process_stats ( self ) : good_status = [ "COMPLETED" , "CACHED" ] for process , vals in self . trace_info . items ( ) : vals = self . _update_tag_status ( process , vals ) self . _update_process_resources ( process , vals ) self . process_stats [ process ] = { } inst = self . process_stats [ process ] inst [ "completed" ] = "{}" . format ( len ( [ x for x in vals if x [ "status" ] in good_status ] ) ) try : time_array = [ self . _hms ( x [ "realtime" ] ) for x in vals ] mean_time = round ( sum ( time_array ) / len ( time_array ) , 1 ) mean_time_str = strftime ( '%H:%M:%S' , gmtime ( mean_time ) ) inst [ "realtime" ] = mean_time_str except KeyError : inst [ "realtime" ] = "-" try : cpu_hours = [ self . _cpu_load_parser ( x [ "cpus" ] , x [ "%cpu" ] , x [ "realtime" ] ) for x in vals ] inst [ "cpuhour" ] = round ( sum ( cpu_hours ) , 2 ) except KeyError : inst [ "cpuhour" ] = "-" inst [ "cpu_warnings" ] , inst [ "mem_warnings" ] = self . _assess_resource_warnings ( process , vals ) try : rss_values = [ self . _size_coverter ( x [ "rss" ] ) for x in vals if x [ "rss" ] != "-" ] if rss_values : max_rss = round ( max ( rss_values ) ) rss_str = self . _size_compress ( max_rss ) else : rss_str = "-" inst [ "maxmem" ] = rss_str except KeyError : inst [ "maxmem" ] = "-" try : rchar_values = [ self . _size_coverter ( x [ "rchar" ] ) for x in vals if x [ "rchar" ] != "-" ] if rchar_values : avg_rchar = round ( sum ( rchar_values ) / len ( rchar_values ) ) rchar_str = self . _size_compress ( avg_rchar ) else : rchar_str = "-" except KeyError : rchar_str = "-" inst [ "avgread" ] = rchar_str try : wchar_values = [ self . _size_coverter ( x [ "wchar" ] ) for x in vals if x [ "wchar" ] != "-" ] if wchar_values : avg_wchar = round ( sum ( wchar_values ) / len ( wchar_values ) ) wchar_str = self . _size_compress ( avg_wchar ) else : wchar_str = "-" except KeyError : wchar_str = "-" inst [ "avgwrite" ] = wchar_str


def log_parser(self):
        """Method that parses the nextflow log file once and updates the
        submitted number of samples for each process
        """

        # Check the timestamp of the log file. Only proceed with the parsing
        # if it changed from the previous time.
        size_stamp = os.path.getsize(self.log_file)
        self.log_retry = 0
        if size_stamp and size_stamp == self.log_sizestamp:
            return
        else:
            logger.debug("Updating log size stamp to: {}".format(size_stamp))
            self.log_sizestamp = size_stamp

        # Regular expression to catch four groups:
        # 1. Start timestamp
        # 2. Work directory hash
        # 3. Process name
        # 4. Tag name
        r = ".* (.*) \[.*\].*\[(.*)\].*process > (.*) \((.*)\).*"

        with open(self.log_file) as fh:

            for line in fh:
                if "Submitted process >" in line or \
                        "Re-submitted process >" in line or \
                        "Cached process >" in line:
                    m = re.match(r, line)
                    if not m:
                        continue

                    time_start = m.group(1)
                    workdir = m.group(2)
                    process = m.group(3)
                    tag = m.group(4)

                    # Skip if this line has already been parsed
                    if time_start + tag not in self.stored_log_ids:
                        self.stored_log_ids.append(time_start + tag)
                    else:
                        continue

                    # For first time processes
                    if process not in self.processes:
                        continue
                    p = self.processes[process]

                    # Skip is process/tag combination has finished or is retrying
                    if tag in list(p["finished"]) + list(p["retry"]):
                        continue

                    # Update failed process/tags when they have been re-submitted
                    if tag in list(p["failed"]) and \
                            "Re-submitted process >" in line:
                        p["retry"].add(tag)
                        self.send = True
                        continue

                    # Set process barrier to running. Check for barrier status
                    # are performed at the end of the trace parsing in the
                    # _update_barrier_status method.
                    p["barrier"] = "R"
                    if tag not in p["submitted"]:
                        p["submitted"].add(tag)
                        # Update the process_tags attribute with the new tag.
                        # Update only when the tag does not exist. This may rarely
                        # occur when the tag is parsed first in the trace file
                        if tag not in self.process_tags[process]:
                            self.process_tags[process][tag] = {
                                "workdir": self._expand_path(workdir),
                                "start": time_start
                            }
                            self.send = True
                        # When the tag is filled in the trace file parsing,
                        # the timestamp may not be present in the trace. In
                        # those cases, fill that information here.
                        elif not self.process_tags[process][tag]["start"]:
                            self.process_tags[process][tag]["start"] = time_start
                            self.send = True

        self._update_pipeline_status()


def update_inspection(self):
        """Wrapper method that calls the appropriate main updating methods of
        the inspection.

        It is meant to be used inside a loop (like while), so that it can
        continuously update the class attributes from the trace and log files.
        It already implements checks to parse these files only when they
        change, and they ignore entries that have been previously processes.
        """

        try:
            self.log_parser()
        except (FileNotFoundError, StopIteration) as e:
            logger.debug("ERROR: " + str(sys.exc_info()[0]))
            self.log_retry += 1
            if self.log_retry == self.MAX_RETRIES:
                raise e
        try:
            self.trace_parser()
        except (FileNotFoundError, StopIteration) as e:
            logger.debug("ERROR: " + str(sys.exc_info()[0]))
            self.trace_retry += 1
            if self.trace_retry == self.MAX_RETRIES:
                raise e

def
update_inspection
(
self
)
:
try
:
self
.
log_parser
(
)
except
(
FileNotFoundError
,
StopIteration
)
as
e
:
logger
.
debug
(
"ERROR: "
+
str
(
sys
.
exc_info
(
)
[
0
]
)
)
self
.
log_retry
+=
1
if
self
.
log_retry
==
self
.
MAX_RETRIES
:
raise
e
try
:
self
.
trace_parser
(
)
except
(
FileNotFoundError
,
StopIteration
)
as
e
:
logger
.
debug
(
"ERROR: "
+
str
(
sys
.
exc_info
(
)
[
0
]
)
)
self
.
trace_retry
+=
1
if
self
.
trace_retry
==
self
.
MAX_RETRIES
:
raise
e

Wrapper method that calls the appropriate main updating methods of the inspection .


def display_overview(self):
        """Displays the default pipeline inspection overview
        """

        stay_alive = True

        self.screen = curses.initscr()

        self.screen.keypad(True)
        self.screen.nodelay(-1)
        curses.cbreak()
        curses.noecho()
        curses.start_color()

        self.screen_lines = self.screen.getmaxyx()[0]
        # self.screen_width = self.screen.getmaxyx()[1]

        try:
            while stay_alive:

                # Provide functionality to certain keybindings
                self._curses_keybindings()
                # Updates main inspector attributes
                self.update_inspection()
                # Display curses interface
                self.flush_overview()

                sleep(self.refresh_rate)
        except FileNotFoundError:
            sys.stderr.write(colored_print(
                "ERROR: nextflow log and/or trace files are no longer "
                "reachable!", "red_bold"))
        except Exception as e:
            sys.stderr.write(str(e))
        finally:
            curses.nocbreak()
            self.screen.keypad(0)
            curses.echo()
            curses.endwin()


def _updown(self, direction):
        """Provides curses scroll functionality.
        """

        if direction == "up" and self.top_line != 0:
            self.top_line -= 1
        elif direction == "down" and \
                self.screen.getmaxyx()[0] + self.top_line\
                <= self.content_lines + 3:
            self.top_line += 1

def _updown ( self , direction ) : if direction == "up" and self . top_line != 0 : self . top_line -= 1 elif direction == "down" and self . screen . getmaxyx ( ) [ 0 ] + self . top_line <= self . content_lines + 3 : self . top_line += 1


def _rightleft(self, direction):
        """Provides curses horizontal padding"""

        if direction == "left" and self.padding != 0:
            self.padding -= 1

        if direction == "right" and \
                self.screen.getmaxyx()[1] + self.padding < self.max_width:
            self.padding += 1

def _rightleft ( self , direction ) : if direction == "left" and self . padding != 0 : self . padding -= 1 if direction == "right" and self . screen . getmaxyx ( ) [ 1 ] + self . padding < self . max_width : self . padding += 1


def _get_log_lines(self, n=300):
        """Returns a list with the last ``n`` lines of the nextflow log file

        Parameters
        ----------
        n : int
            Number of last lines from the log file

        Returns
        -------
        list
            List of strings with the nextflow log
        """

        with open(self.log_file) as fh:
            last_lines = fh.readlines()[-n:]

        return last_lines

def _get_log_lines ( self , n = 300 ) : with open ( self . log_file ) as fh : last_lines = fh . readlines ( ) [ - n : ] return last_lines

Returns a list with the last n lines of the nextflow log file


def _prepare_static_info(self):
        """Prepares the first batch of information, containing static
        information such as the pipeline file, and configuration files

        Returns
        -------
        dict
            Dict with the static information for the first POST request
        """

        pipeline_files = {}

        with open(join(self.workdir, self.pipeline_name)) as fh:
            pipeline_files["pipelineFile"] = fh.readlines()

        nf_config = join(self.workdir, "nextflow.config")
        if os.path.exists(nf_config):
            with open(nf_config) as fh:
                pipeline_files["configFile"] = fh.readlines()

        # Check for specific flowcraft configurations files
        configs = {
            "params.config": "paramsFile",
            "resources.config": "resourcesFile",
            "containers.config": "containersFile",
            "user.config": "userFile",
        }
        for config, key in configs.items():
            cfile = join(self.workdir, config)
            if os.path.exists(cfile):
                with open(cfile) as fh:
                    pipeline_files[key] = fh.readlines()

        return pipeline_files

def _prepare_static_info ( self ) : pipeline_files = { } with open ( join ( self . workdir , self . pipeline_name ) ) as fh : pipeline_files [ "pipelineFile" ] = fh . readlines ( ) nf_config = join ( self . workdir , "nextflow.config" ) if os . path . exists ( nf_config ) : with open ( nf_config ) as fh : pipeline_files [ "configFile" ] = fh . readlines ( ) configs = { "params.config" : "paramsFile" , "resources.config" : "resourcesFile" , "containers.config" : "containersFile" , "user.config" : "userFile" , } for config , key in configs . items ( ) : cfile = join ( self . workdir , config ) if os . path . exists ( cfile ) : with open ( cfile ) as fh : pipeline_files [ key ] = fh . readlines ( ) return pipeline_files

Prepares the first batch of information containing static information such as the pipeline file and configuration files


def _dag_file_to_dict(self):
        """Function that opens the dotfile named .treeDag.json in the current
        working directory

        Returns
        -------
        Returns a dictionary with the dag object to be used in the post
        instance available through the method _establish_connection

        """
        try:
            dag_file = open(os.path.join(self.workdir, ".treeDag.json"))
            dag_json = json.load(dag_file)
        except (FileNotFoundError, json.decoder.JSONDecodeError):
            logger.warning(colored_print(
                "WARNING: dotfile named .treeDag.json not found or corrupted",
                "red_bold"))
            dag_json = {}

        return dag_json

Function that opens the dotfile named . treeDag . json in the current working directory


def _get_run_hash(self):
        """Gets the hash of the nextflow file"""

        # Get name and path of the pipeline from the log file
        pipeline_path = get_nextflow_filepath(self.log_file)

        # Get hash from the entire pipeline file
        pipeline_hash = hashlib.md5()
        with open(pipeline_path, "rb") as fh:
            for chunk in iter(lambda: fh.read(4096), b""):
                pipeline_hash.update(chunk)
        # Get hash from the current working dir and hostname
        workdir = self.workdir.encode("utf8")
        hostname = socket.gethostname().encode("utf8")
        hardware_addr = str(uuid.getnode()).encode("utf8")
        dir_hash = hashlib.md5(workdir + hostname + hardware_addr)

        return pipeline_hash.hexdigest() + dir_hash.hexdigest()

def _get_run_hash ( self ) : pipeline_path = get_nextflow_filepath ( self . log_file ) pipeline_hash = hashlib . md5 ( ) with open ( pipeline_path , "rb" ) as fh : for chunk in iter ( lambda : fh . read ( 4096 ) , b"" ) : pipeline_hash . update ( chunk ) workdir = self . workdir . encode ( "utf8" ) hostname = socket . gethostname ( ) . encode ( "utf8" ) hardware_addr = str ( uuid . getnode ( ) ) . encode ( "utf8" ) dir_hash = hashlib . md5 ( workdir + hostname + hardware_addr ) return pipeline_hash . hexdigest ( ) + dir_hash . hexdigest ( )


def get_nextflow_filepath(log_file):
    """Gets the nextflow file path from the nextflow log file. It searches for
    the nextflow run command throughout the file.

    Parameters
    ----------
    log_file : str
        Path for the .nextflow.log file

    Returns
    -------
    str
        Path for the nextflow file
    """

    with open(log_file) as fh:
        # Searches for the first occurence of the nextflow pipeline
        # file name in the .nextflow.log file
        while 1:
            line = fh.readline()
            if not line:
                # file is empty
                raise eh.LogError("Nextflow command path could not be found - Is "
                                 ".nextflow.log empty?")
            try:
                # Regex supports absolute paths and relative paths
                pipeline_path = re.match(".*\s(.*.nf).*", line) \
                    .group(1)
                return pipeline_path
            except AttributeError:
                continue

Gets the nextflow file path from the nextflow log file . It searches for the nextflow run command throughout the file .


def main(sample_id, assembly, min_size):
    """Main executor of the split_fasta template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    assembly : list
        Assembly file.
    min_size : int
        Minimum contig size.

    """

    logger.info("Starting script")

    f_open = open(assembly, "rU")

    entry = (x[1] for x in groupby(f_open, lambda line: line[0] == ">"))

    success = 0

    for header in entry:
        headerStr = header.__next__()[1:].strip()
        seq = "".join(s.strip() for s in entry.__next__())
        if len(seq) >= min_size:
            with open(sample_id + '_' + headerStr.replace(" ","_").replace("=","_") + '.fasta', "w") as output_file:
                output_file.write(">" + sample_id + "_" + headerStr.replace(" ","_").replace("=","_") + "\\n" + seq + "\\n")
                success += 1

    f_open.close()

    logger.info("{} sequences sucessfully splitted.".format(success))

def
main
(
sample_id
,
assembly
,
min_size
)
:
logger
.
info
(
"Starting script"
)
f_open
=
open
(
assembly
,
"rU"
)
entry
=
(
x
[
1
]
for
x
in
groupby
(
f_open
,
lambda
line
:
line
[
0
]
==
">"
)
)
success
=
0
for
header
in
entry
:
headerStr
=
header
.
__next__
(
)
[
1
:
]
.
strip
(
)
seq
=
""
.
join
(
s
.
strip
(
)
for
s
in
entry
.
__next__
(
)
)
if
len
(
seq
)
>=
min_size
:
with
open
(
sample_id
+
'_'
+
headerStr
.
replace
(
" "
,
"_"
)
.
replace
(
"="
,
"_"
)
+
'.fasta'
,
"w"
)
as
output_file
:
output_file
.
write
(
">"
+
sample_id
+
"_"
+
headerStr
.
replace
(
" "
,
"_"
)
.
replace
(
"="
,
"_"
)
+
"\\n"
+
seq
+
"\\n"
)
success
+=
1
f_open
.
close
(
)
logger
.
info
(
"{} sequences sucessfully splitted."
.
format
(
success
)
)

Main executor of the split_fasta template .


def main(sample_id, trace_file, workdir):
    """
    Parses a nextflow trace file, searches for processes with a specific tag
    and sends a JSON report with the relevant information

    The expected fields for the trace file are::

        0. task_id
        1. process
        2. tag
        3. status
        4. exit code
        5. start timestamp
        6. container
        7. cpus
        8. duration
        9. realtime
        10. queue
        11. cpu percentage
        12. memory percentage
        13. real memory size of the process
        14. virtual memory size of the process

    Parameters
    ----------
    trace_file : str
        Path to the nextflow trace file
    """

    # Determine the path of the stored JSON for the sample_id
    stats_suffix = ".stats.json"
    stats_path = join(workdir, sample_id + stats_suffix)
    trace_path = join(workdir, trace_file)

    logger.info("Starting pipeline status routine")

    logger.debug("Checking for previous pipeline status data")
    stats_array = get_previous_stats(stats_path)
    logger.info("Stats JSON object set to : {}".format(stats_array))

    # Search for this substring in the tags field. Only lines with this
    # tag will be processed for the reports
    tag = " getStats"
    logger.debug("Tag variable set to: {}".format(tag))

    logger.info("Starting parsing of trace file: {}".format(trace_path))
    with open(trace_path) as fh:

        header = next(fh).strip().split()
        logger.debug("Header set to: {}".format(header))

        for line in fh:
            fields = line.strip().split("\t")
            # Check if tag substring is in the tag field of the nextflow trace
            if tag in fields[2] and fields[3] == "COMPLETED":
                logger.debug(
                    "Parsing trace line with COMPLETED status: {}".format(
                        line))
                current_json = get_json_info(fields, header)

                stats_array[fields[0]] = current_json
            else:
                logger.debug(
                    "Ignoring trace line without COMPLETED status"
                    " or stats specific tag: {}".format(
                        line))

    with open(join(stats_path), "w") as fh, open(".report.json", "w") as rfh:
        fh.write(json.dumps(stats_array, separators=(",", ":")))
        rfh.write(json.dumps(stats_array, separators=(",", ":")))

Parses a nextflow trace file searches for processes with a specific tag and sends a JSON report with the relevant information