def check_insert_size(data, sample):
    """
    check mean insert size for this sample and update
    hackersonly.max_inner_mate_distance if need be. This value controls how
    far apart mate pairs can be to still be considered for bedtools merging
    downstream.
    """

    ## pipe stats output to grep
    cmd1 = [ipyrad.bins.samtools, "stats", sample.files.mapped_reads]
    cmd2 = ["grep", "SN"]
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)

    ## get piped result
    res = proc2.communicate()[0]

    ## raise exception on failure and do cleanup
    if proc2.returncode:
        raise IPyradWarningExit("error in %s: %s", cmd2, res)

    ## starting vals
    avg_insert = 0
    stdv_insert = 0
    avg_len = 0

    ## iterate over results
    for line in res.split("\n"):
        if "insert size average" in line:
            avg_insert = float(line.split(":")[-1].strip())

        elif "insert size standard deviation" in line:
            ## hack to fix sim data when stdv is 0.0. Shouldn't
            ## impact real data bcz stdv gets rounded up below
            stdv_insert = float(line.split(":")[-1].strip()) + 0.1

        elif "average length" in line:
            avg_len = float(line.split(":")[-1].strip())

    LOGGER.debug("avg {} stdv {} avg_len {}"\
                 .format(avg_insert, stdv_insert, avg_len))

    ## If all values return successfully set the max inner mate distance.
    ## This is tricky. avg_insert is the average length of R1+R2+inner mate
    ## distance. avg_len is the average length of a read. If there are lots
    ## of reads that overlap then avg_insert will be close to but bigger than
    ## avg_len. We are looking for the right value for `bedtools merge -d`
    ## which wants to know the max distance between reads.
    if all([avg_insert, stdv_insert, avg_len]):
        ## If 2 * the average length of a read is less than the average
        ## insert size then most reads DO NOT overlap
        if stdv_insert < 5:
            stdv_insert = 5.
        if (2 * avg_len) < avg_insert:
            hack = avg_insert + (3 * np.math.ceil(stdv_insert)) - (2 * avg_len)

        ## If it is > than the average insert size then most reads DO
        ## overlap, so we have to calculate inner mate distance a little
        ## differently.
        else:
            hack = (avg_insert - avg_len) + (3 * np.math.ceil(stdv_insert))

        ## set the hackerdict value
        LOGGER.info("stdv: hacked insert size is %s", hack)
        data._hackersonly["max_inner_mate_distance"] = int(np.math.ceil(hack))

    else:
        ## If something fsck then set a relatively conservative distance
        data._hackersonly["max_inner_mate_distance"] = 300
        LOGGER.debug("inner mate distance for {} - {}".format(sample.name,\
                    data._hackersonly["max_inner_mate_distance"]))

def
check_insert_size
(
data
,
sample
)
:
cmd1
=
[
ipyrad
.
bins
.
samtools
,
"stats"
,
sample
.
files
.
mapped_reads
]
cmd2
=
[
"grep"
,
"SN"
]
proc1
=
sps
.
Popen
(
cmd1
,
stderr
=
sps
.
STDOUT
,
stdout
=
sps
.
PIPE
)
proc2
=
sps
.
Popen
(
cmd2
,
stderr
=
sps
.
STDOUT
,
stdout
=
sps
.
PIPE
,
stdin
=
proc1
.
stdout
)
res
=
proc2
.
communicate
(
)
[
0
]
if
proc2
.
returncode
:
raise
IPyradWarningExit
(
"error in %s: %s"
,
cmd2
,
res
)
avg_insert
=
0
stdv_insert
=
0
avg_len
=
0
for
line
in
res
.
split
(
"\n"
)
:
if
"insert size average"
in
line
:
avg_insert
=
float
(
line
.
split
(
":"
)
[
1
]
.
strip
(
)
)
elif
"insert size standard deviation"
in
line
:
stdv_insert
=
float
(
line
.
split
(
":"
)
[
1
]
.
strip
(
)
)
+
0.1
elif
"average length"
in
line
:
avg_len
=
float
(
line
.
split
(
":"
)
[
1
]
.
strip
(
)
)
LOGGER
.
debug
(
"avg {} stdv {} avg_len {}"
.
format
(
avg_insert
,
stdv_insert
,
avg_len
)
)
if
all
(
[
avg_insert
,
stdv_insert
,
avg_len
]
)
:
if
stdv_insert
<
5
:
stdv_insert
=
5.
if
(
2
*
avg_len
)
<
avg_insert
:
hack
=
avg_insert
+
(
3
*
np
.
math
.
ceil
(
stdv_insert
)
)
(
2
*
avg_len
)
else
:
hack
=
(
avg_insert
avg_len
)
+
(
3
*
np
.
math
.
ceil
(
stdv_insert
)
)
LOGGER
.
info
(
"stdv: hacked insert size is %s"
,
hack
)
data
.
_hackersonly
[
"max_inner_mate_distance"
]
=
int
(
np
.
math
.
ceil
(
hack
)
)
else
:
data
.
_hackersonly
[
"max_inner_mate_distance"
]
=
300
LOGGER
.
debug
(
"inner mate distance for {} - {}"
.
format
(
sample
.
name
,
data
.
_hackersonly
[
"max_inner_mate_distance"
]
)
)

check mean insert size for this sample and update hackersonly . max_inner_mate_distance if need be . This value controls how far apart mate pairs can be to still be considered for bedtools merging downstream .


def bedtools_merge(data, sample):
    """
    Get all contiguous genomic regions with one or more overlapping
    reads. This is the shell command we'll eventually run

    bedtools bamtobed -i 1A_0.sorted.bam | bedtools merge [-d 100]
        -i <input_bam>  :   specifies the input file to bed'ize
        -d <int>        :   For PE set max distance between reads
    """
    LOGGER.info("Entering bedtools_merge: %s", sample.name)
    mappedreads = os.path.join(data.dirs.refmapping,
                               sample.name+"-mapped-sorted.bam")

    ## command to call `bedtools bamtobed`, and pipe output to stdout
    ## Usage:   bedtools bamtobed [OPTIONS] -i <bam>
    ## Usage:   bedtools merge [OPTIONS] -i <bam>
    cmd1 = [ipyrad.bins.bedtools, "bamtobed", "-i", mappedreads]
    cmd2 = [ipyrad.bins.bedtools, "merge", "-i", "-"]

    ## If PE the -d flag to tell bedtools how far apart to allow mate pairs.
    ## If SE the -d flag is negative, specifying that SE reads need to
    ## overlap by at least a specific number of bp. This prevents the
    ## stairstep syndrome when a + and - read are both extending from
    ## the same cutsite. Passing a negative number to `merge -d` gets this done.
    if 'pair' in data.paramsdict["datatype"]:
        check_insert_size(data, sample)
        #cmd2.insert(2, str(data._hackersonly["max_inner_mate_distance"]))
        cmd2.insert(2, str(data._hackersonly["max_inner_mate_distance"]))
        cmd2.insert(2, "-d")
    else:
        cmd2.insert(2, str(-1 * data._hackersonly["min_SE_refmap_overlap"]))
        cmd2.insert(2, "-d")

    ## pipe output from bamtobed into merge
    LOGGER.info("stdv: bedtools merge cmds: %s %s", cmd1, cmd2)
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)
    result = proc2.communicate()[0]
    proc1.stdout.close()

    ## check for errors and do cleanup
    if proc2.returncode:
        raise IPyradWarningExit("error in %s: %s", cmd2, result)

    ## Write the bedfile out, because it's useful sometimes.
    if os.path.exists(ipyrad.__debugflag__):
        with open(os.path.join(data.dirs.refmapping, sample.name + ".bed"), 'w') as outfile:
            outfile.write(result)

    ## Report the number of regions we're returning
    nregions = len(result.strip().split("\n"))
    LOGGER.info("bedtools_merge: Got # regions: %s", nregions)
    return result

def
bedtools_merge
(
data
,
sample
)
:
LOGGER
.
info
(
"Entering bedtools_merge: %s"
,
sample
.
name
)
mappedreads
=
os
.
path
.
join
(
data
.
dirs
.
refmapping
,
sample
.
name
+
"-mapped-sorted.bam"
)
cmd1
=
[
ipyrad
.
bins
.
bedtools
,
"bamtobed"
,
"-i"
,
mappedreads
]
cmd2
=
[
ipyrad
.
bins
.
bedtools
,
"merge"
,
"-i"
,
"-"
]
if
'pair'
in
data
.
paramsdict
[
"datatype"
]
:
check_insert_size
(
data
,
sample
)
cmd2
.
insert
(
2
,
str
(
data
.
_hackersonly
[
"max_inner_mate_distance"
]
)
)
cmd2
.
insert
(
2
,
"-d"
)
else
:
cmd2
.
insert
(
2
,
str
(
1
*
data
.
_hackersonly
[
"min_SE_refmap_overlap"
]
)
)
cmd2
.
insert
(
2
,
"-d"
)
LOGGER
.
info
(
"stdv: bedtools merge cmds: %s %s"
,
cmd1
,
cmd2
)
proc1
=
sps
.
Popen
(
cmd1
,
stderr
=
sps
.
STDOUT
,
stdout
=
sps
.
PIPE
)
proc2
=
sps
.
Popen
(
cmd2
,
stderr
=
sps
.
STDOUT
,
stdout
=
sps
.
PIPE
,
stdin
=
proc1
.
stdout
)
result
=
proc2
.
communicate
(
)
[
0
]
proc1
.
stdout
.
close
(
)
if
proc2
.
returncode
:
raise
IPyradWarningExit
(
"error in %s: %s"
,
cmd2
,
result
)
if
os
.
path
.
exists
(
ipyrad
.
__debugflag__
)
:
with
open
(
os
.
path
.
join
(
data
.
dirs
.
refmapping
,
sample
.
name
+
".bed"
)
,
'w'
)
as
outfile
:
outfile
.
write
(
result
)
nregions
=
len
(
result
.
strip
(
)
.
split
(
"\n"
)
)
LOGGER
.
info
(
"bedtools_merge: Got # regions: %s"
,
nregions
)
return
result

Get all contiguous genomic regions with one or more overlapping reads . This is the shell command we ll eventually run


def refmap_stats(data, sample):
    """
    Get the number of mapped and unmapped reads for a sample
    and update sample.stats
    """
    ## shorter names
    mapf = os.path.join(data.dirs.refmapping, sample.name+"-mapped-sorted.bam")
    umapf = os.path.join(data.dirs.refmapping, sample.name+"-unmapped.bam")

    ## get from unmapped
    cmd1 = [ipyrad.bins.samtools, "flagstat", umapf]
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    result1 = proc1.communicate()[0]

    ## get from mapped
    cmd2 = [ipyrad.bins.samtools, "flagstat", mapf]
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)
    result2 = proc2.communicate()[0]

    ## store results
    ## If PE, samtools reports the _actual_ number of reads mapped, both
    ## R1 and R2, so here if PE divide the results by 2 to stay consistent
    ## with how we've been reporting R1 and R2 as one "read pair"
    if "pair" in data.paramsdict["datatype"]:
        sample.stats["refseq_unmapped_reads"] = int(result1.split()[0]) / 2
        sample.stats["refseq_mapped_reads"] = int(result2.split()[0]) / 2
    else:
        sample.stats["refseq_unmapped_reads"] = int(result1.split()[0])
        sample.stats["refseq_mapped_reads"] = int(result2.split()[0])

    sample_cleanup(data, sample)

def refmap_stats ( data , sample ) : mapf = os . path . join ( data . dirs . refmapping , sample . name + "-mapped-sorted.bam" ) umapf = os . path . join ( data . dirs . refmapping , sample . name + "-unmapped.bam" ) cmd1 = [ ipyrad . bins . samtools , "flagstat" , umapf ] proc1 = sps . Popen ( cmd1 , stderr = sps . STDOUT , stdout = sps . PIPE ) result1 = proc1 . communicate ( ) [ 0 ] cmd2 = [ ipyrad . bins . samtools , "flagstat" , mapf ] proc2 = sps . Popen ( cmd2 , stderr = sps . STDOUT , stdout = sps . PIPE ) result2 = proc2 . communicate ( ) [ 0 ] if "pair" in data . paramsdict [ "datatype" ] : sample . stats [ "refseq_unmapped_reads" ] = int ( result1 . split ( ) [ 0 ] ) / 2 sample . stats [ "refseq_mapped_reads" ] = int ( result2 . split ( ) [ 0 ] ) / 2 else : sample . stats [ "refseq_unmapped_reads" ] = int ( result1 . split ( ) [ 0 ] ) sample . stats [ "refseq_mapped_reads" ] = int ( result2 . split ( ) [ 0 ] ) sample_cleanup ( data , sample )

Get the number of mapped and unmapped reads for a sample and update sample . stats


def refmap_init(data, sample, force):
    """ create some file handles for refmapping """
    ## make some persistent file handles for the refmap reads files
    sample.files.unmapped_reads = os.path.join(data.dirs.edits,
                                  "{}-refmap_derep.fastq".format(sample.name))
    sample.files.mapped_reads = os.path.join(data.dirs.refmapping,
                                  "{}-mapped-sorted.bam".format(sample.name))

def refmap_init ( data , sample , force ) : sample . files . unmapped_reads = os . path . join ( data . dirs . edits , "{}-refmap_derep.fastq" . format ( sample . name ) ) sample . files . mapped_reads = os . path . join ( data . dirs . refmapping , "{}-mapped-sorted.bam" . format ( sample . name ) )


def _subsample(self):
        """ returns a subsample of unlinked snp sites """
        spans = self.maparr
        samp = np.zeros(spans.shape[0], dtype=np.uint64)
        for i in xrange(spans.shape[0]):
            samp[i] = np.random.randint(spans[i, 0], spans[i, 1], 1)
        return samp

def _subsample ( self ) : spans = self . maparr samp = np . zeros ( spans . shape [ 0 ] , dtype = np . uint64 ) for i in xrange ( spans . shape [ 0 ] ) : samp [ i ] = np . random . randint ( spans [ i , 0 ] , spans [ i , 1 ] , 1 ) return samp


def draw(self, axes):
        """
        Returns a treemix plot on a toyplot.axes object.
        """
        ## create a toytree object from the treemix tree result
        tre = toytree.tree(newick=self.results.tree)
        tre.draw(
            axes=axes,
            use_edge_lengths=True,
            tree_style='c',
            tip_labels_align=True,
            edge_align_style={"stroke-width": 1}
        );

        ## get coords
        for admix in self.results.admixture:
            ## parse admix event
            pidx, pdist, cidx, cdist, weight = admix
            a = _get_admix_point(tre, pidx, pdist)
            b = _get_admix_point(tre, cidx, cdist)

            ## add line for admixture edge
            mark = axes.plot(
                a = (a[0], b[0]),
                b = (a[1], b[1]),
                style={"stroke-width": 10*weight,
                       "stroke-opacity": 0.95,
                       "stroke-linecap": "round"}
            )

            ## add points at admixture sink
            axes.scatterplot(
                a = (b[0]),
                b = (b[1]),
                size=8,
                title="weight: {}".format(weight),
            )

        ## add scale bar for edge lengths
        axes.y.show=False
        axes.x.ticks.show=True
        axes.x.label.text = "Drift parameter"
        return axes

def
draw
(
self
,
axes
)
:
tre
=
toytree
.
tree
(
newick
=
self
.
results
.
tree
)
tre
.
draw
(
axes
=
axes
,
use_edge_lengths
=
True
,
tree_style
=
'c'
,
tip_labels_align
=
True
,
edge_align_style
=
{
"stroke-width"
:
1
}
)
for
admix
in
self
.
results
.
admixture
:
pidx
,
pdist
,
cidx
,
cdist
,
weight
=
admix
a
=
_get_admix_point
(
tre
,
pidx
,
pdist
)
b
=
_get_admix_point
(
tre
,
cidx
,
cdist
)
mark
=
axes
.
plot
(
a
=
(
a
[
0
]
,
b
[
0
]
)
,
b
=
(
a
[
1
]
,
b
[
1
]
)
,
style
=
{
"stroke-width"
:
10
*
weight
,
"stroke-opacity"
:
0.95
,
"stroke-linecap"
:
"round"
}
)
axes
.
scatterplot
(
a
=
(
b
[
0
]
)
,
b
=
(
b
[
1
]
)
,
size
=
8
,
title
=
"weight: {}"
.
format
(
weight
)
,
)
axes
.
y
.
show
=
False
axes
.
x
.
ticks
.
show
=
True
axes
.
x
.
label
.
text
=
"Drift parameter"
return
axes

Returns a treemix plot on a toyplot . axes object .


def _resolveambig(subseq):
    """
    Randomly resolves iupac hetero codes. This is a shortcut
    for now, we could instead use the phased alleles in RAD loci.
    """
    N = []
    for col in subseq:
        rand = np.random.binomial(1, 0.5)
        N.append([_AMBIGS[i][rand] for i in col])
    return np.array(N)

def _resolveambig ( subseq ) : N = [ ] for col in subseq : rand = np . random . binomial ( 1 , 0.5 ) N . append ( [ _AMBIGS [ i ] [ rand ] for i in col ] ) return np . array ( N )

Randomly resolves iupac hetero codes . This is a shortcut for now we could instead use the phased alleles in RAD loci .


def _count_PIS(seqsamp, N):
    """ filters for loci with >= N PIS """
    counts = [Counter(col) for col in seqsamp.T if not ("-" in col or "N" in col)]
    pis = [i.most_common(2)[1][1] > 1 for i in counts if len(i.most_common(2))>1]
    if sum(pis) >= N:
        return sum(pis)
    else:
        return 0

def _count_PIS ( seqsamp , N ) : counts = [ Counter ( col ) for col in seqsamp . T if not ( "-" in col or "N" in col ) ] pis = [ i . most_common ( 2 ) [ 1 ] [ 1 ] > 1 for i in counts if len ( i . most_common ( 2 ) ) > 1 ] if sum ( pis ) >= N : return sum ( pis ) else : return 0


def _write_nex(self, mdict, nlocus):
        """
        function that takes a dictionary mapping names to sequences,
        and a locus number, and writes it as a NEXUS file with a mrbayes
        analysis block given a set of mcmc arguments.
        """

        ## create matrix as a string
        max_name_len = max([len(i) for i in mdict])
        namestring = "{:<" + str(max_name_len+1) + "} {}\n"
        matrix = ""
        for i in mdict.items():
            matrix += namestring.format(i[0], i[1])

        ## ensure dir
        minidir = os.path.realpath(os.path.join(self.workdir, self.name))
        if not os.path.exists(minidir):
            os.makedirs(minidir)

        ## write nexus block
        handle = os.path.join(minidir, "{}.nex".format(nlocus))
        with open(handle, 'w') as outnex:
            outnex.write(NEXBLOCK.format(**{
                "ntax": len(mdict),
                "nchar": len(mdict.values()[0]),
                "matrix": matrix,
                "ngen": self.params.mb_mcmc_ngen,
                "sfreq": self.params.mb_mcmc_sample_freq,
                "burnin": self.params.mb_mcmc_burnin,
                }))

def
_write_nex
(
self
,
mdict
,
nlocus
)
:
max_name_len
=
max
(
[
len
(
i
)
for
i
in
mdict
]
)
namestring
=
"{:<"
+
str
(
max_name_len
+
1
)
+
"} {}\n"
matrix
=
""
for
i
in
mdict
.
items
(
)
:
matrix
+=
namestring
.
format
(
i
[
0
]
,
i
[
1
]
)
minidir
=
os
.
path
.
realpath
(
os
.
path
.
join
(
self
.
workdir
,
self
.
name
)
)
if
not
os
.
path
.
exists
(
minidir
)
:
os
.
makedirs
(
minidir
)
handle
=
os
.
path
.
join
(
minidir
,
"{}.nex"
.
format
(
nlocus
)
)
with
open
(
handle
,
'w'
)
as
outnex
:
outnex
.
write
(
NEXBLOCK
.
format
(
**
{
"ntax"
:
len
(
mdict
)
,
"nchar"
:
len
(
mdict
.
values
(
)
[
0
]
)
,
"matrix"
:
matrix
,
"ngen"
:
self
.
params
.
mb_mcmc_ngen
,
"sfreq"
:
self
.
params
.
mb_mcmc_sample_freq
,
"burnin"
:
self
.
params
.
mb_mcmc_burnin
,
}
)
)

function that takes a dictionary mapping names to sequences and a locus number and writes it as a NEXUS file with a mrbayes analysis block given a set of mcmc arguments .


def _read_sample_names(fname):
    """ Read in sample names from a plain text file. This is a convenience
    function for branching so if you have tons of sample names you can
    pass in a file rather than having to set all the names at the command
    line.
    """
    try:
        with open(fname, 'r') as infile:
            subsamples = [x.split()[0] for x in infile.readlines() if x.strip()]

    except Exception as inst:
        print("Failed to read input file with sample names.\n{}".format(inst))
        raise inst

    return subsamples

Read in sample names from a plain text file . This is a convenience function for branching so if you have tons of sample names you can pass in a file rather than having to set all the names at the command line .


def _bufcountlines(filename, gzipped):
    """
    fast line counter. Used to quickly sum number of input reads when running
    link_fastqs to append files. """
    if gzipped:
        fin = gzip.open(filename)
    else:
        fin = open(filename)
    nlines = 0
    buf_size = 1024 * 1024
    read_f = fin.read # loop optimization
    buf = read_f(buf_size)
    while buf:
        nlines += buf.count('\n')
        buf = read_f(buf_size)
    fin.close()
    return nlines

def _bufcountlines ( filename , gzipped ) : if gzipped : fin = gzip . open ( filename ) else : fin = open ( filename ) nlines = 0 buf_size = 1024 * 1024 read_f = fin . read buf = read_f ( buf_size ) while buf : nlines += buf . count ( '\n' ) buf = read_f ( buf_size ) fin . close ( ) return nlines

fast line counter . Used to quickly sum number of input reads when running link_fastqs to append files .


def _zbufcountlines(filename, gzipped):
    """ faster line counter """
    if gzipped:
        cmd1 = ["gunzip", "-c", filename]
    else:
        cmd1 = ["cat", filename]
    cmd2 = ["wc"]

    proc1 = sps.Popen(cmd1, stdout=sps.PIPE, stderr=sps.PIPE)
    proc2 = sps.Popen(cmd2, stdin=proc1.stdout, stdout=sps.PIPE, stderr=sps.PIPE)
    res = proc2.communicate()[0]
    if proc2.returncode:
        raise IPyradWarningExit("error zbufcountlines {}:".format(res))
    LOGGER.info(res)
    nlines = int(res.split()[0])
    return nlines

def
_zbufcountlines
(
filename
,
gzipped
)
:
if
gzipped
:
cmd1
=
[
"gunzip"
,
"-c"
,
filename
]
else
:
cmd1
=
[
"cat"
,
filename
]
cmd2
=
[
"wc"
]
proc1
=
sps
.
Popen
(
cmd1
,
stdout
=
sps
.
PIPE
,
stderr
=
sps
.
PIPE
)
proc2
=
sps
.
Popen
(
cmd2
,
stdin
=
proc1
.
stdout
,
stdout
=
sps
.
PIPE
,
stderr
=
sps
.
PIPE
)
res
=
proc2
.
communicate
(
)
[
0
]
if
proc2
.
returncode
:
raise
IPyradWarningExit
(
"error zbufcountlines {}:"
.
format
(
res
)
)
LOGGER
.
info
(
res
)
nlines
=
int
(
res
.
split
(
)
[
0
]
)
return
nlines


def _tuplecheck(newvalue, dtype=str):
    """
    Takes a string argument and returns value as a tuple.
    Needed for paramfile conversion from CLI to set_params args
    """

    if isinstance(newvalue, list):
        newvalue = tuple(newvalue)

    if isinstance(newvalue, str):
        newvalue = newvalue.rstrip(")").strip("(")
        try:
            newvalue = tuple([dtype(i.strip()) for i in newvalue.split(",")])

        ## Type error is thrown by tuple if it's applied to a non-iterable.
        except TypeError:
            newvalue = tuple(dtype(newvalue))

        ## If dtype fails to cast any element of newvalue
        except ValueError:
            LOGGER.info("Assembly.tuplecheck() failed to cast to {} - {}"\
                        .format(dtype, newvalue))
            raise

        except Exception as inst:
            LOGGER.info(inst)
            raise SystemExit(\
            "\nError: Param`{}` is not formatted correctly.\n({})\n"\
                 .format(newvalue, inst))

    return newvalue

Takes a string argument and returns value as a tuple . Needed for paramfile conversion from CLI to set_params args


def stats(self):
        """ Returns a data frame with Sample data and state. """
        nameordered = self.samples.keys()
        nameordered.sort()

        ## Set pandas to display all samples instead of truncating
        pd.options.display.max_rows = len(self.samples)
        statdat = pd.DataFrame([self.samples[i].stats for i in nameordered],
                      index=nameordered).dropna(axis=1, how='all')
        # ensure non h,e columns print as ints
        for column in statdat:
            if column not in ["hetero_est", "error_est"]:
                statdat[column] = np.nan_to_num(statdat[column]).astype(int)
        return statdat

def stats ( self ) : nameordered = self . samples . keys ( ) nameordered . sort ( ) pd . options . display . max_rows = len ( self . samples ) statdat = pd . DataFrame ( [ self . samples [ i ] . stats for i in nameordered ] , index = nameordered ) . dropna ( axis = 1 , how = 'all' ) for column in statdat : if column not in [ "hetero_est" , "error_est" ] : statdat [ column ] = np . nan_to_num ( statdat [ column ] ) . astype ( int ) return statdat

Returns a data frame with Sample data and state .


def files(self):
        """ Returns a data frame with Sample files. Not very readable... """
        nameordered = self.samples.keys()
        nameordered.sort()
        ## replace curdir with . for shorter printing
        #fullcurdir = os.path.realpath(os.path.curdir)
        return pd.DataFrame([self.samples[i].files for i in nameordered],
                      index=nameordered).dropna(axis=1, how='all')

def files ( self ) : nameordered = self . samples . keys ( ) nameordered . sort ( ) return pd . DataFrame ( [ self . samples [ i ] . files for i in nameordered ] , index = nameordered ) . dropna ( axis = 1 , how = 'all' )

Returns a data frame with Sample files . Not very readable ...


def _build_stat(self, idx):
        """ Returns a data frame with Sample stats for each step """
        nameordered = self.samples.keys()
        nameordered.sort()
        newdat = pd.DataFrame([self.samples[i].stats_dfs[idx] \
                               for i in nameordered], index=nameordered)\
                               .dropna(axis=1, how='all')
        return newdat

def _build_stat ( self , idx ) : nameordered = self . samples . keys ( ) nameordered . sort ( ) newdat = pd . DataFrame ( [ self . samples [ i ] . stats_dfs [ idx ] for i in nameordered ] , index = nameordered ) . dropna ( axis = 1 , how = 'all' ) return newdat


def get_params(self, param=""):
        """ pretty prints params if called as a function """
        fullcurdir = os.path.realpath(os.path.curdir)
        if not param:
            for index, (key, value) in enumerate(self.paramsdict.items()):
                if isinstance(value, str):
                    value = value.replace(fullcurdir+"/", "./")
                sys.stdout.write("{}{:<4}{:<28}{:<45}\n"\
                    .format(self._spacer, index, key, value))
        else:
            try:
                if int(param):
                    #sys.stdout.write(self.paramsdict.values()[int(param)-1])
                    return self.paramsdict.values()[int(param)]
            except (ValueError, TypeError, NameError, IndexError):
                try:
                    return self.paramsdict[param]
                except KeyError:
                    return 'key not recognized'

def
get_params
(
self
,
param
=
""
)
:
fullcurdir
=
os
.
path
.
realpath
(
os
.
path
.
curdir
)
if
not
param
:
for
index
,
(
key
,
value
)
in
enumerate
(
self
.
paramsdict
.
items
(
)
)
:
if
isinstance
(
value
,
str
)
:
value
=
value
.
replace
(
fullcurdir
+
"/"
,
"./"
)
sys
.
stdout
.
write
(
"{}{:<4}{:<28}{:<45}\n"
.
format
(
self
.
_spacer
,
index
,
key
,
value
)
)
else
:
try
:
if
int
(
param
)
:
return
self
.
paramsdict
.
values
(
)
[
int
(
param
)
]
except
(
ValueError
,
TypeError
,
NameError
,
IndexError
)
:
try
:
return
self
.
paramsdict
[
param
]
except
KeyError
:
return
'key not recognized'


def set_params(self, param, newvalue):
        """
        Set a parameter to a new value. Raises error if newvalue is wrong type.

        Note
        ----
        Use [Assembly].get_params() to see the parameter values currently
        linked to the Assembly object.

        Parameters
        ----------
        param : int or str
            The index (e.g., 1) or string name (e.g., "project_dir")
            for the parameter that will be changed.

        newvalue : int, str, or tuple
            The new value for the parameter selected for `param`. Use
            `ipyrad.get_params_info()` to get further information about
            a given parameter. If the wrong type is entered for newvalue
            (e.g., a str when it should be an int), an error will be raised.
            Further information about each parameter is also available
            in the documentation.

        Examples
        --------
        ## param 'project_dir' takes only a str as input
        [Assembly].set_params('project_dir', 'new_directory')

        ## param 'restriction_overhang' must be a tuple or str, if str it is
        ## converted to a tuple with the second entry empty.
        [Assembly].set_params('restriction_overhang', ('CTGCAG', 'CCGG')

        ## param 'max_shared_Hs_locus' can be an int or a float:
        [Assembly].set_params('max_shared_Hs_locus', 0.25)

        """

        ## this includes current params and some legacy params for conversion
        legacy_params = ["edit_cutsites", "trim_overhang"]
        current_params = self.paramsdict.keys()
        allowed_params = current_params + legacy_params

        ## require parameter recognition
        #if not ((param in range(50)) or \
        #        (param in [str(i) for i in range(50)]) or \
        #        (param in allowed_params)):
        if not param in allowed_params:
            raise IPyradParamsError("Parameter key not recognized: {}"\
                                    .format(param))

        ## make string
        param = str(param)
        ## get index if param is keyword arg (this index is now zero based!)
        if len(param) < 3:
            param = self.paramsdict.keys()[int(param)]

        ## run assertions on new param
        try:
            self = _paramschecker(self, param, newvalue)

        except Exception as inst:
            raise IPyradWarningExit(BAD_PARAMETER\
                                    .format(param, inst, newvalue))

def
set_params
(
self
,
param
,
newvalue
)
:
legacy_params
=
[
"edit_cutsites"
,
"trim_overhang"
]
current_params
=
self
.
paramsdict
.
keys
(
)
allowed_params
=
current_params
+
legacy_params
if
not
param
in
allowed_params
:
raise
IPyradParamsError
(
"Parameter key not recognized: {}"
.
format
(
param
)
)
param
=
str
(
param
)
if
len
(
param
)
<
3
:
param
=
self
.
paramsdict
.
keys
(
)
[
int
(
param
)
]
try
:
self
=
_paramschecker
(
self
,
param
,
newvalue
)
except
Exception
as
inst
:
raise
IPyradWarningExit
(
BAD_PARAMETER
.
format
(
param
,
inst
,
newvalue
)
)

Set a parameter to a new value . Raises error if newvalue is wrong type .


def branch(self, newname, subsamples=None, infile=None):
        """
        Returns a copy of the Assembly object. Does not allow Assembly
        object names to be replicated in namespace or path.
        """
        ## subsample by removal or keeping.
        remove = 0

        ## is there a better way to ask if it already exists?
        if (newname == self.name or os.path.exists(
                                    os.path.join(self.paramsdict["project_dir"],
                                    newname+".assembly"))):
            print("{}Assembly object named {} already exists"\
                  .format(self._spacer, newname))

        else:
            ## Make sure the new name doesn't have any wacky characters
            self._check_name(newname)

            ## Bozo-check. Carve off 'params-' if it's in the new name.
            if newname.startswith("params-"):
                newname = newname.split("params-")[1]

            ## create a copy of the Assembly obj
            newobj = copy.deepcopy(self)
            newobj.name = newname
            newobj.paramsdict["assembly_name"] = newname

            if subsamples and infile:
                print(BRANCH_NAMES_AND_INPUT)

            if infile:
                if infile[0] == "-":
                    remove = 1
                    infile = infile[1:]
                if os.path.exists(infile):
                    subsamples = _read_sample_names(infile)

            ## if remove then swap the samples
            if remove:
                subsamples = list(set(self.samples.keys()) - set(subsamples))

            ## create copies of each subsampled Sample obj
            if subsamples:
                for sname in subsamples:
                    if sname in self.samples:
                        newobj.samples[sname] = copy.deepcopy(self.samples[sname])
                    else:
                        print("Sample name not found: {}".format(sname))

                ## reload sample dict w/o non subsamples
                newobj.samples = {name:sample for name, sample in \
                           newobj.samples.items() if name in subsamples}

            ## create copies of each subsampled Sample obj
            else:
                for sample in self.samples:
                    newobj.samples[sample] = copy.deepcopy(self.samples[sample])

            ## save json of new obj and return object
            newobj.save()
            return newobj

Returns a copy of the Assembly object . Does not allow Assembly object names to be replicated in namespace or path .


def _step1func(self, force, ipyclient):
        """ hidden wrapped function to start step 1 """

        ## check input data files
        sfiles = self.paramsdict["sorted_fastq_path"]
        rfiles = self.paramsdict["raw_fastq_path"]

        ## do not allow both a sorted_fastq_path and a raw_fastq
        if sfiles and rfiles:
            raise IPyradWarningExit(NOT_TWO_PATHS)

        ## but also require that at least one exists
        if not (sfiles or rfiles):
            raise IPyradWarningExit(NO_SEQ_PATH_FOUND)

        ## print headers
        if self._headers:
            if sfiles:
                print("\n{}Step 1: Loading sorted fastq data to Samples"\
                      .format(self._spacer))
            else:
                print("\n{}Step 1: Demultiplexing fastq data to Samples"\
                    .format(self._spacer))

        ## if Samples already exist then no demultiplexing
        if self.samples:
            if not force:
                print(SAMPLES_EXIST.format(len(self.samples), self.name))
            else:
                ## overwrite existing data else do demux
                if glob.glob(sfiles):
                    self._link_fastqs(ipyclient=ipyclient, force=force)
                else:
                    assemble.demultiplex.run2(self, ipyclient, force)

        ## Creating new Samples
        else:
            ## first check if demultiplexed files exist in sorted path
            if glob.glob(sfiles):
                self._link_fastqs(ipyclient=ipyclient)

            ## otherwise do the demultiplexing
            else:
                assemble.demultiplex.run2(self, ipyclient, force)


def _step2func(self, samples, force, ipyclient):
        """ hidden wrapped function to start step 2"""

        ## print header
        if self._headers:
            print("\n  Step 2: Filtering reads ")

        ## If no samples in this assembly then it means you skipped step1,
        if not self.samples.keys():
            raise IPyradWarningExit(FIRST_RUN_1)

        ## Get sample objects from list of strings, if API.
        samples = _get_samples(self, samples)

        if not force:
            ## print warning and skip if all are finished
            if all([i.stats.state >= 2 for i in samples]):
                print(EDITS_EXIST.format(len(samples)))
                return

        ## Run samples through rawedit
        assemble.rawedit.run2(self, samples, force, ipyclient)

def
_step2func
(
self
,
samples
,
force
,
ipyclient
)
:
if
self
.
_headers
:
print
(
"\n  Step 2: Filtering reads "
)
if
not
self
.
samples
.
keys
(
)
:
raise
IPyradWarningExit
(
FIRST_RUN_1
)
samples
=
_get_samples
(
self
,
samples
)
if
not
force
:
if
all
(
[
i
.
stats
.
state
>=
2
for
i
in
samples
]
)
:
print
(
EDITS_EXIST
.
format
(
len
(
samples
)
)
)
return
assemble
.
rawedit
.
run2
(
self
,
samples
,
force
,
ipyclient
)


def _step4func(self, samples, force, ipyclient):
        """ hidden wrapped function to start step 4 """

        if self._headers:
            print("\n  Step 4: Joint estimation of error rate and heterozygosity")

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## Check if all/none in the right state
        if not self._samples_precheck(samples, 4, force):
            raise IPyradError(FIRST_RUN_3)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 4 for i in samples]):
                print(JOINTS_EXIST.format(len(samples)))
                return

        ## send to function
        assemble.jointestimate.run(self, samples, force, ipyclient)


def _step5func(self, samples, force, ipyclient):
        """ hidden wrapped function to start step 5 """
        ## print header
        if self._headers:
            print("\n  Step 5: Consensus base calling ")

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## Check if all/none in the right state
        if not self._samples_precheck(samples, 5, force):
            raise IPyradError(FIRST_RUN_4)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 5 for i in samples]):
                print(CONSENS_EXIST.format(len(samples)))
                return
        ## pass samples to rawedit
        assemble.consens_se.run(self, samples, force, ipyclient)

def
_step5func
(
self
,
samples
,
force
,
ipyclient
)
:
if
self
.
_headers
:
print
(
"\n  Step 5: Consensus base calling "
)
samples
=
_get_samples
(
self
,
samples
)
if
not
self
.
_samples_precheck
(
samples
,
5
,
force
)
:
raise
IPyradError
(
FIRST_RUN_4
)
elif
not
force
:
if
all
(
[
i
.
stats
.
state
>=
5
for
i
in
samples
]
)
:
print
(
CONSENS_EXIST
.
format
(
len
(
samples
)
)
)
return
assemble
.
consens_se
.
run
(
self
,
samples
,
force
,
ipyclient
)


def _step6func(self,
        samples,
        noreverse,
        force,
        randomseed,
        ipyclient,
        **kwargs):
        """
        Hidden function to start Step 6.
        """

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## remove samples that aren't ready
        csamples = self._samples_precheck(samples, 6, force)

        ## print CLI header
        if self._headers:
            print("\n  Step 6: Clustering at {} similarity across {} samples".\
                  format(self.paramsdict["clust_threshold"], len(csamples)))

        ## Check if all/none in the right state
        if not csamples:
            raise IPyradError(FIRST_RUN_5)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 6 for i in csamples]):
                print(DATABASE_EXISTS.format(len(samples)))
                return

        ## run if this point is reached. We no longer check for existing
        ## h5 file, since checking Sample states should suffice.
        assemble.cluster_across.run(
            self,
            csamples,
            noreverse,
            force,
            randomseed,
            ipyclient,
            **kwargs)


def _samples_precheck(self, samples, mystep, force):
        """ Return a list of samples that are actually ready for the next step.
            Each step runs this prior to calling run, makes it easier to
            centralize and normalize how each step is checking sample states.
            mystep is the state produced by the current step.
        """
        subsample = []
        ## filter by state
        for sample in samples:
            if sample.stats.state < mystep - 1:
                LOGGER.debug("Sample {} not in proper state."\
                             .format(sample.name))
            else:
                subsample.append(sample)
        return subsample

def
_samples_precheck
(
self
,
samples
,
mystep
,
force
)
:
subsample
=
[
]
for
sample
in
samples
:
if
sample
.
stats
.
state
<
mystep
1
:
LOGGER
.
debug
(
"Sample {} not in proper state."
.
format
(
sample
.
name
)
)
else
:
subsample
.
append
(
sample
)
return
subsample

Return a list of samples that are actually ready for the next step . Each step runs this prior to calling run makes it easier to centralize and normalize how each step is checking sample states . mystep is the state produced by the current step .


def combinefiles(filepath):
    """ Joins first and second read file names """
    ## unpack seq files in filepath
    fastqs = glob.glob(filepath)
    firsts = [i for i in fastqs if "_R1_" in i]

    ## check names
    if not firsts:
        raise IPyradWarningExit("First read files names must contain '_R1_'.")

    ## get paired reads
    seconds = [ff.replace("_R1_", "_R2_") for ff in firsts]
    return zip(firsts, seconds)


def get_barcode_func(data, longbar):
    """ returns the fastest func given data & longbar"""
    ## build func for finding barcode
    if longbar[1] == 'same':
        if data.paramsdict["datatype"] == '2brad':
            def getbarcode(cutters, read1, longbar):
                """ find barcode for 2bRAD data """
                return read1[1][:-(len(cutters[0][0]) + 1)][-longbar[0]:]

        else:
            def getbarcode(_, read1, longbar):
                """ finds barcode for invariable length barcode data """
                return read1[1][:longbar[0]]
    else:
        def getbarcode(cutters, read1, longbar):
            """ finds barcode for variable barcode lengths"""
            return findbcode(cutters, longbar, read1)
    return getbarcode

def get_barcode_func ( data , longbar ) : if longbar [ 1 ] == 'same' : if data . paramsdict [ "datatype" ] == '2brad' : def getbarcode ( cutters , read1 , longbar ) : return read1 [ 1 ] [ : - ( len ( cutters [ 0 ] [ 0 ] ) + 1 ) ] [ - longbar [ 0 ] : ] else : def getbarcode ( _ , read1 , longbar ) : return read1 [ 1 ] [ : longbar [ 0 ] ] else : def getbarcode ( cutters , read1 , longbar ) : return findbcode ( cutters , longbar , read1 ) return getbarcode


def get_quart_iter(tups):
    """ returns an iterator to grab four lines at a time """

    if tups[0].endswith(".gz"):
        ofunc = gzip.open
    else:
        ofunc = open

    ## create iterators
    ofile1 = ofunc(tups[0], 'r')
    fr1 = iter(ofile1)
    quart1 = itertools.izip(fr1, fr1, fr1, fr1)
    if tups[1]:
        ofile2 = ofunc(tups[1], 'r')
        fr2 = iter(ofile2)
        quart2 = itertools.izip(fr2, fr2, fr2, fr2)
        quarts = itertools.izip(quart1, quart2)
    else:
        ofile2 = 0
        quarts = itertools.izip(quart1, iter(int, 1))

    ## make a generator
    def feedme(quarts):
        for quart in quarts:
            yield quart
    genquarts = feedme(quarts)

    ## return generator and handles
    return genquarts, ofile1, ofile2

def get_quart_iter ( tups ) : if tups [ 0 ] . endswith ( ".gz" ) : ofunc = gzip . open else : ofunc = open ofile1 = ofunc ( tups [ 0 ] , 'r' ) fr1 = iter ( ofile1 ) quart1 = itertools . izip ( fr1 , fr1 , fr1 , fr1 ) if tups [ 1 ] : ofile2 = ofunc ( tups [ 1 ] , 'r' ) fr2 = iter ( ofile2 ) quart2 = itertools . izip ( fr2 , fr2 , fr2 , fr2 ) quarts = itertools . izip ( quart1 , quart2 ) else : ofile2 = 0 quarts = itertools . izip ( quart1 , iter ( int , 1 ) ) def feedme ( quarts ) : for quart in quarts : yield quart genquarts = feedme ( quarts ) return genquarts , ofile1 , ofile2


def writetofastq(data, dsort, read):
    """
    Writes sorted data 'dsort dict' to a tmp files
    """
    if read == 1:
        rrr = "R1"
    else:
        rrr = "R2"

    for sname in dsort:
        ## skip writing if empty. Write to tmpname
        handle = os.path.join(data.dirs.fastqs,
                "{}_{}_.fastq".format(sname, rrr))
        with open(handle, 'a') as out:
            out.write("".join(dsort[sname]))

def writetofastq ( data , dsort , read ) : if read == 1 : rrr = "R1" else : rrr = "R2" for sname in dsort : handle = os . path . join ( data . dirs . fastqs , "{}_{}_.fastq" . format ( sname , rrr ) ) with open ( handle , 'a' ) as out : out . write ( "" . join ( dsort [ sname ] ) )

Writes sorted data dsort dict to a tmp files


def collate_files(data, sname, tmp1s, tmp2s):
    """
    Collate temp fastq files in tmp-dir into 1 gzipped sample.
    """
    ## out handle
    out1 = os.path.join(data.dirs.fastqs, "{}_R1_.fastq.gz".format(sname))
    out = io.BufferedWriter(gzip.open(out1, 'w'))

    ## build cmd
    cmd1 = ['cat']
    for tmpfile in tmp1s:
        cmd1 += [tmpfile]

    ## compression function
    proc = sps.Popen(['which', 'pigz'], stderr=sps.PIPE, stdout=sps.PIPE).communicate()
    if proc[0].strip():
        compress = ["pigz"]
    else:
        compress = ["gzip"]

    ## call cmd
    proc1 = sps.Popen(cmd1, stderr=sps.PIPE, stdout=sps.PIPE)
    proc2 = sps.Popen(compress, stdin=proc1.stdout, stderr=sps.PIPE, stdout=out)
    err = proc2.communicate()
    if proc2.returncode:
        raise IPyradWarningExit("error in collate_files R1 %s", err)
    proc1.stdout.close()
    out.close()

    ## then cleanup
    for tmpfile in tmp1s:
        os.remove(tmpfile)

    if 'pair' in data.paramsdict["datatype"]:
        ## out handle
        out2 = os.path.join(data.dirs.fastqs, "{}_R2_.fastq.gz".format(sname))
        out = io.BufferedWriter(gzip.open(out2, 'w'))

        ## build cmd
        cmd1 = ['cat']
        for tmpfile in tmp2s:
            cmd1 += [tmpfile]

        ## call cmd
        proc1 = sps.Popen(cmd1, stderr=sps.PIPE, stdout=sps.PIPE)
        proc2 = sps.Popen(compress, stdin=proc1.stdout, stderr=sps.PIPE, stdout=out)
        err = proc2.communicate()
        if proc2.returncode:
            raise IPyradWarningExit("error in collate_files R2 %s", err)
        proc1.stdout.close()
        out.close()

        ## then cleanup
        for tmpfile in tmp2s:
            os.remove(tmpfile)

def
collate_files
(
data
,
sname
,
tmp1s
,
tmp2s
)
:
out1
=
os
.
path
.
join
(
data
.
dirs
.
fastqs
,
"{}_R1_.fastq.gz"
.
format
(
sname
)
)
out
=
io
.
BufferedWriter
(
gzip
.
open
(
out1
,
'w'
)
)
cmd1
=
[
'cat'
]
for
tmpfile
in
tmp1s
:
cmd1
+=
[
tmpfile
]
proc
=
sps
.
Popen
(
[
'which'
,
'pigz'
]
,
stderr
=
sps
.
PIPE
,
stdout
=
sps
.
PIPE
)
.
communicate
(
)
if
proc
[
0
]
.
strip
(
)
:
compress
=
[
"pigz"
]
else
:
compress
=
[
"gzip"
]
proc1
=
sps
.
Popen
(
cmd1
,
stderr
=
sps
.
PIPE
,
stdout
=
sps
.
PIPE
)
proc2
=
sps
.
Popen
(
compress
,
stdin
=
proc1
.
stdout
,
stderr
=
sps
.
PIPE
,
stdout
=
out
)
err
=
proc2
.
communicate
(
)
if
proc2
.
returncode
:
raise
IPyradWarningExit
(
"error in collate_files R1 %s"
,
err
)
proc1
.
stdout
.
close
(
)
out
.
close
(
)
for
tmpfile
in
tmp1s
:
os
.
remove
(
tmpfile
)
if
'pair'
in
data
.
paramsdict
[
"datatype"
]
:
out2
=
os
.
path
.
join
(
data
.
dirs
.
fastqs
,
"{}_R2_.fastq.gz"
.
format
(
sname
)
)
out
=
io
.
BufferedWriter
(
gzip
.
open
(
out2
,
'w'
)
)
cmd1
=
[
'cat'
]
for
tmpfile
in
tmp2s
:
cmd1
+=
[
tmpfile
]
proc1
=
sps
.
Popen
(
cmd1
,
stderr
=
sps
.
PIPE
,
stdout
=
sps
.
PIPE
)
proc2
=
sps
.
Popen
(
compress
,
stdin
=
proc1
.
stdout
,
stderr
=
sps
.
PIPE
,
stdout
=
out
)
err
=
proc2
.
communicate
(
)
if
proc2
.
returncode
:
raise
IPyradWarningExit
(
"error in collate_files R2 %s"
,
err
)
proc1
.
stdout
.
close
(
)
out
.
close
(
)
for
tmpfile
in
tmp2s
:
os
.
remove
(
tmpfile
)

Collate temp fastq files in tmp - dir into 1 gzipped sample .


def estimate_optim(data, testfile, ipyclient):
    """
    Estimate a reasonable optim value by grabbing a chunk of sequences,
    decompressing and counting them, to estimate the full file size.
    """
    ## count the len of one file and assume all others are similar len
    insize = os.path.getsize(testfile)
    tmp_file_name = os.path.join(data.paramsdict["project_dir"], "tmp-step1-count.fq")
    if testfile.endswith(".gz"):
        infile = gzip.open(testfile)
        outfile = gzip.open(tmp_file_name, 'wb', compresslevel=5)
    else:
        infile = open(testfile)
        outfile = open(tmp_file_name, 'w')

    ## We'll take the average of the size of a file based on the
    ## first 10000 reads to approximate number of reads in the main file
    outfile.write("".join(itertools.islice(infile, 40000)))
    outfile.close()
    infile.close()

    ## Get the size of the tmp file
    tmp_size = os.path.getsize(tmp_file_name)

    ## divide by the tmp file size and multiply by 10000 to approximate
    ## the size of the input .fq files
    inputreads = int(insize / tmp_size) * 10000
    os.remove(tmp_file_name)

    return inputreads

def estimate_optim ( data , testfile , ipyclient ) : insize = os . path . getsize ( testfile ) tmp_file_name = os . path . join ( data . paramsdict [ "project_dir" ] , "tmp-step1-count.fq" ) if testfile . endswith ( ".gz" ) : infile = gzip . open ( testfile ) outfile = gzip . open ( tmp_file_name , 'wb' , compresslevel = 5 ) else : infile = open ( testfile ) outfile = open ( tmp_file_name , 'w' ) outfile . write ( "" . join ( itertools . islice ( infile , 40000 ) ) ) outfile . close ( ) infile . close ( ) tmp_size = os . path . getsize ( tmp_file_name ) inputreads = int ( insize / tmp_size ) * 10000 os . remove ( tmp_file_name ) return inputreads

Estimate a reasonable optim value by grabbing a chunk of sequences decompressing and counting them to estimate the full file size .


def _cleanup_and_die(data):
    """ cleanup func for step 1 """
    tmpfiles = glob.glob(os.path.join(data.dirs.fastqs, "tmp_*_R*.fastq"))
    tmpfiles += glob.glob(os.path.join(data.dirs.fastqs, "tmp_*.p"))
    for tmpf in tmpfiles:
        os.remove(tmpf)

def _cleanup_and_die ( data ) : tmpfiles = glob . glob ( os . path . join ( data . dirs . fastqs , "tmp_*_R*.fastq" ) ) tmpfiles += glob . glob ( os . path . join ( data . dirs . fastqs , "tmp_*.p" ) ) for tmpf in tmpfiles : os . remove ( tmpf )


def splitfiles(data, raws, ipyclient):
    """ sends raws to be chunked"""

    ## create a tmpdir for chunked_files and a chunk optimizer
    tmpdir = os.path.join(data.paramsdict["project_dir"], "tmp-chunks-"+data.name)
    if os.path.exists(tmpdir):
        shutil.rmtree(tmpdir)
    os.makedirs(tmpdir)

    ## chunk into 8M reads
    totalreads = estimate_optim(data, raws[0][0], ipyclient)
    optim = int(8e6)
    njobs = int(totalreads/(optim/4.)) * len(raws)

    ## if more files than cpus: no chunking
    nosplit = 0
    if (len(raws) > len(ipyclient)) or (totalreads < optim):
        nosplit = 1

    ## send slices N at a time. The dict chunkfiles stores a tuple of rawpairs
    ## dictionary to store asyncresults for sorting jobs
    start = time.time()
    chunkfiles = {}
    for fidx, tups in enumerate(raws):
        handle = os.path.splitext(os.path.basename(tups[0]))[0]
        ## if number of lines is > 20M then just submit it
        if nosplit:
            chunkfiles[handle] = [tups]
        else:
            ## chunk the file using zcat_make_temps
            chunklist = zcat_make_temps(data, tups, fidx, tmpdir, optim, njobs, start)
            chunkfiles[handle] = chunklist
    if not nosplit:
        print("")

    return chunkfiles

def splitfiles ( data , raws , ipyclient ) : tmpdir = os . path . join ( data . paramsdict [ "project_dir" ] , "tmp-chunks-" + data . name ) if os . path . exists ( tmpdir ) : shutil . rmtree ( tmpdir ) os . makedirs ( tmpdir ) totalreads = estimate_optim ( data , raws [ 0 ] [ 0 ] , ipyclient ) optim = int ( 8e6 ) njobs = int ( totalreads / ( optim / 4. ) ) * len ( raws ) nosplit = 0 if ( len ( raws ) > len ( ipyclient ) ) or ( totalreads < optim ) : nosplit = 1 start = time . time ( ) chunkfiles = { } for fidx , tups in enumerate ( raws ) : handle = os . path . splitext ( os . path . basename ( tups [ 0 ] ) ) [ 0 ] if nosplit : chunkfiles [ handle ] = [ tups ] else : chunklist = zcat_make_temps ( data , tups , fidx , tmpdir , optim , njobs , start ) chunkfiles [ handle ] = chunklist if not nosplit : print ( "" ) return chunkfiles


def putstats(pfile, handle, statdicts):
    """ puts stats from pickles into a dictionary """

    ## load in stats
    with open(pfile, 'r') as infile:
        filestats, samplestats = pickle.load(infile)

    ## get dicts from statdicts tuple
    perfile, fsamplehits, fbarhits, fmisses, fdbars = statdicts

    ## pull new stats
    #handle = os.path.splitext(os.path.basename(handle))[0]
    perfile[handle] += filestats

    ## update sample stats
    samplehits, barhits, misses, dbars = samplestats
    fsamplehits.update(samplehits)
    fbarhits.update(barhits)
    fmisses.update(misses)
    fdbars.update(dbars)

    ## repack the tuple and return
    statdicts = perfile, fsamplehits, fbarhits, fmisses, fdbars
    return statdicts

def putstats ( pfile , handle , statdicts ) : with open ( pfile , 'r' ) as infile : filestats , samplestats = pickle . load ( infile ) perfile , fsamplehits , fbarhits , fmisses , fdbars = statdicts perfile [ handle ] += filestats samplehits , barhits , misses , dbars = samplestats fsamplehits . update ( samplehits ) fbarhits . update ( barhits ) fmisses . update ( misses ) fdbars . update ( dbars ) statdicts = perfile , fsamplehits , fbarhits , fmisses , fdbars return statdicts


def _countmatrix(lxs):
    """ fill a matrix with pairwise data sharing """

    ## an empty matrix
    share = np.zeros((lxs.shape[0], lxs.shape[0]))

    ## fill above
    names = range(lxs.shape[0])
    for row in lxs:
        for samp1, samp2 in itertools.combinations(names, 2):
            shared = lxs[samp1, lxs[samp2] > 0].sum()
            share[samp1, samp2] = shared

    ## mirror below
    ##share[]

    ## fill diagonal with total sample coverage
    for row in xrange(len(names)):
        share[row, row] = lxs[row].sum()

    return share

def _countmatrix ( lxs ) : share = np . zeros ( ( lxs . shape [ 0 ] , lxs . shape [ 0 ] ) ) names = range ( lxs . shape [ 0 ] ) for row in lxs : for samp1 , samp2 in itertools . combinations ( names , 2 ) : shared = lxs [ samp1 , lxs [ samp2 ] > 0 ] . sum ( ) share [ samp1 , samp2 ] = shared for row in xrange ( len ( names ) ) : share [ row , row ] = lxs [ row ] . sum ( ) return share


def paramname(param=""):
    """ Get the param name from the dict index value.
    """

    try:
        name = pinfo[str(param)][0].strip().split(" ")[1]
    except (KeyError, ValueError) as err:
        ## TODO: paramsinfo get description by param string not working.
        ## It would be cool to have an assembly object bcz then you could
        ## just do this:
        ##
        ## print(pinfo[data.paramsinfo.keys().index(param)])
        print("\tKey name/number not recognized - ".format(param), err)
        raise

    return name

def
paramname
(
param
=
""
)
:
try
:
name
=
pinfo
[
str
(
param
)
]
[
0
]
.
strip
(
)
.
split
(
" "
)
[
1
]
except
(
KeyError
,
ValueError
)
as
err
:
print
(
"\tKey name/number not recognized - "
.
format
(
param
)
,
err
)
raise
return
name

Get the param name from the dict index value .


def save_json2(data):
    """ save to json."""

    ## convert everything to dicts
    ## skip _ipcluster cuz it's made new.
    datadict = OrderedDict([
        ("outfiles", data.__dict__["outfiles"]),
        ("stats_files", dict(data.__dict__["stats_files"])),
        ("stats_dfs", data.__dict__["stats_dfs"])
        ])

def save_json2 ( data ) : datadict = OrderedDict ( [ ( "outfiles" , data . __dict__ [ "outfiles" ] ) , ( "stats_files" , dict ( data . __dict__ [ "stats_files" ] ) ) , ( "stats_dfs" , data . __dict__ [ "stats_dfs" ] ) ] )


def save_json(data):
    """ Save assembly and samples as json """

    ## data as dict
    #### skip _ipcluster because it's made new
    #### skip _headers because it's loaded new
    #### statsfiles save only keys
    #### samples save only keys
    datadict = OrderedDict([
        ("_version", data.__dict__["_version"]),
        ("_checkpoint", data.__dict__["_checkpoint"]),
        ("name", data.__dict__["name"]),
        ("dirs", data.__dict__["dirs"]),
        ("paramsdict", data.__dict__["paramsdict"]),
        ("samples", data.__dict__["samples"].keys()),
        ("populations", data.__dict__["populations"]),
        ("database", data.__dict__["database"]),
        ("clust_database", data.__dict__["clust_database"]),
        ("outfiles", data.__dict__["outfiles"]),
        ("barcodes", data.__dict__["barcodes"]),
        ("stats_files", data.__dict__["stats_files"]),
        ("_hackersonly", data.__dict__["_hackersonly"]),
        ])

    ## sample dict
    sampledict = OrderedDict([])
    for key, sample in data.samples.iteritems():
        sampledict[key] = sample._to_fulldict()

    ## json format it using cumstom Encoder class
    fulldumps = json.dumps({
        "assembly": datadict,
        "samples": sampledict
        },
        cls=Encoder,
        sort_keys=False, indent=4, separators=(",", ":"),
        )

    ## save to file
    assemblypath = os.path.join(data.dirs.project, data.name+".json")
    if not os.path.exists(data.dirs.project):
        os.mkdir(data.dirs.project)

    ## protect save from interruption
    done = 0
    while not done:
        try:
            with open(assemblypath, 'w') as jout:
                jout.write(fulldumps)
            done = 1
        except (KeyboardInterrupt, SystemExit):
            print('.')
            continue

def save_json ( data ) : datadict = OrderedDict ( [ ( "_version" , data . __dict__ [ "_version" ] ) , ( "_checkpoint" , data . __dict__ [ "_checkpoint" ] ) , ( "name" , data . __dict__ [ "name" ] ) , ( "dirs" , data . __dict__ [ "dirs" ] ) , ( "paramsdict" , data . __dict__ [ "paramsdict" ] ) , ( "samples" , data . __dict__ [ "samples" ] . keys ( ) ) , ( "populations" , data . __dict__ [ "populations" ] ) , ( "database" , data . __dict__ [ "database" ] ) , ( "clust_database" , data . __dict__ [ "clust_database" ] ) , ( "outfiles" , data . __dict__ [ "outfiles" ] ) , ( "barcodes" , data . __dict__ [ "barcodes" ] ) , ( "stats_files" , data . __dict__ [ "stats_files" ] ) , ( "_hackersonly" , data . __dict__ [ "_hackersonly" ] ) , ] ) sampledict = OrderedDict ( [ ] ) for key , sample in data . samples . iteritems ( ) : sampledict [ key ] = sample . _to_fulldict ( ) fulldumps = json . dumps ( { "assembly" : datadict , "samples" : sampledict } , cls = Encoder , sort_keys = False , indent = 4 , separators = ( "," , ":" ) , ) assemblypath = os . path . join ( data . dirs . project , data . name + ".json" ) if not os . path . exists ( data . dirs . project ) : os . mkdir ( data . dirs . project ) done = 0 while not done : try : with open ( assemblypath , 'w' ) as jout : jout . write ( fulldumps ) done = 1 except ( KeyboardInterrupt , SystemExit ) : print ( '.' ) continue


def encode(self, obj):
        """ function to encode json string"""
        def hint_tuples(item):
            """ embeds __tuple__ hinter in json strings """
            if isinstance(item, tuple):
                return {'__tuple__': True, 'items': item}
            if isinstance(item, list):
                return [hint_tuples(e) for e in item]
            if isinstance(item, dict):
                return {
                    key: hint_tuples(val) for key, val in item.iteritems()
                    }
            else:
                return item

        return super(Encoder, self).encode(hint_tuples(obj))

def encode ( self , obj ) : def hint_tuples ( item ) : if isinstance ( item , tuple ) : return { '__tuple__' : True , 'items' : item } if isinstance ( item , list ) : return [ hint_tuples ( e ) for e in item ] if isinstance ( item , dict ) : return { key : hint_tuples ( val ) for key , val in item . iteritems ( ) } else : return item return super ( Encoder , self ) . encode ( hint_tuples ( obj ) )


def depthplot(data, samples=None, dims=(None,None), canvas=(None,None),
              xmax=50, log=False, outprefix=None, use_maxdepth=False):
    """ plots histogram of coverages across clusters"""

    ## select samples to be plotted, requires depths info
    if not samples:
        samples = data.samples.keys()
        samples.sort()
    subsamples = OrderedDict([(i, data.samples[i]) for i in samples])

    ## get canvas dimensions based on n-samples
    if any(dims):
        ## user-supplied dimensions (...)
        print("userdims")
    else:
        if len(subsamples) <= 4:
            ## set dimension to N samples
            dims = (1, len(subsamples))
        else:
            dims = (len(subsamples)/4, 4)

    ## create canvas
    if any(canvas):
        print("usercanvas")
        canvas = toyplot.Canvas(width=canvas[0], height=canvas[1])
    else:
        canvas = toyplot.Canvas(width=200*dims[1], height=150*dims[0])

    ## get all of the data arrays
    for panel, sample in enumerate(subsamples):
        ## statistical called bins
        statdat = subsamples[sample].depths
        statdat = statdat[statdat >= data.paramsdict["mindepth_statistical"]]
        if use_maxdepth:
            statdat = {i:j for (i, j) in statdat if \
                                        i < data.paramsdict["maxdepth"]}
        sdat = np.histogram(statdat, range(50))

        ## majrule called bins
        statdat = subsamples[sample].depths
        statdat = statdat[statdat < data.paramsdict["mindepth_statistical"]]
        statdat = statdat[statdat >= data.paramsdict["mindepth_majrule"]]
        if use_maxdepth:
            statdat = statdat[statdat < data.paramsdict["maxdepth"]]
        mdat = np.histogram(statdat, range(50))

        ## excluded bins
        tots = data.samples[sample].depths
        tots = tots[tots < data.paramsdict["mindepth_majrule"]]
        if use_maxdepth:
            tots = tots[tots < data.paramsdict["maxdepth"]]
        edat = np.histogram(tots, range(50))

        ## fill in each panel of canvas with a sample
        axes = canvas.cartesian(grid=(dims[0], dims[1], panel), gutter=25)
        axes.x.domain.xmax = xmax
        axes.label.text = sample
        if log:
            axes.y.scale = "log"

        # heights = np.column_stack((sdat,mdat,edat))
        axes.bars(sdat)
        axes.bars(edat)
        axes.bars(mdat)

    ## return objects to be saved...
    if outprefix:
        toyplot.html.render(canvas, fobj=outprefix+".html")
        toyplot.svg.render(canvas, fobj=outprefix+".svg")

def depthplot ( data , samples = None , dims = ( None , None ) , canvas = ( None , None ) , xmax = 50 , log = False , outprefix = None , use_maxdepth = False ) : if not samples : samples = data . samples . keys ( ) samples . sort ( ) subsamples = OrderedDict ( [ ( i , data . samples [ i ] ) for i in samples ] ) if any ( dims ) : print ( "userdims" ) else : if len ( subsamples ) <= 4 : dims = ( 1 , len ( subsamples ) ) else : dims = ( len ( subsamples ) / 4 , 4 ) if any ( canvas ) : print ( "usercanvas" ) canvas = toyplot . Canvas ( width = canvas [ 0 ] , height = canvas [ 1 ] ) else : canvas = toyplot . Canvas ( width = 200 * dims [ 1 ] , height = 150 * dims [ 0 ] ) for panel , sample in enumerate ( subsamples ) : statdat = subsamples [ sample ] . depths statdat = statdat [ statdat >= data . paramsdict [ "mindepth_statistical" ] ] if use_maxdepth : statdat = { i : j for ( i , j ) in statdat if i < data . paramsdict [ "maxdepth" ] } sdat = np . histogram ( statdat , range ( 50 ) ) statdat = subsamples [ sample ] . depths statdat = statdat [ statdat < data . paramsdict [ "mindepth_statistical" ] ] statdat = statdat [ statdat >= data . paramsdict [ "mindepth_majrule" ] ] if use_maxdepth : statdat = statdat [ statdat < data . paramsdict [ "maxdepth" ] ] mdat = np . histogram ( statdat , range ( 50 ) ) tots = data . samples [ sample ] . depths tots = tots [ tots < data . paramsdict [ "mindepth_majrule" ] ] if use_maxdepth : tots = tots [ tots < data . paramsdict [ "maxdepth" ] ] edat = np . histogram ( tots , range ( 50 ) ) axes = canvas . cartesian ( grid = ( dims [ 0 ] , dims [ 1 ] , panel ) , gutter = 25 ) axes . x . domain . xmax = xmax axes . label . text = sample if log : axes . y . scale = "log" axes . bars ( sdat ) axes . bars ( edat ) axes . bars ( mdat ) if outprefix : toyplot . html . render ( canvas , fobj = outprefix + ".html" ) toyplot . svg . render ( canvas , fobj = outprefix + ".svg" )


def _parse_00(ofile):
    """
    return 00 outfile as a pandas DataFrame
    """
    with open(ofile) as infile:
        ## read in the results summary from the end of the outfile
        arr = np.array(
            [" "] + infile.read().split("Summary of MCMC results\n\n\n")[1:][0]\
            .strip().split())

        ## reshape array
        rows = 12
        cols = (arr.shape[0] + 1) / rows
        arr = arr.reshape(rows, cols)

        ## make into labeled data frame
        df = pd.DataFrame(
            data=arr[1:, 1:],
            columns=arr[0, 1:],
            index=arr[1:, 0],
            ).T
        return df

def
_parse_00
(
ofile
)
:
with
open
(
ofile
)
as
infile
:
arr
=
np
.
array
(
[
" "
]
+
infile
.
read
(
)
.
split
(
"Summary of MCMC results\n\n\n"
)
[
1
:
]
[
0
]
.
strip
(
)
.
split
(
)
)
rows
=
12
cols
=
(
arr
.
shape
[
0
]
+
1
)
/
rows
arr
=
arr
.
reshape
(
rows
,
cols
)
df
=
pd
.
DataFrame
(
data
=
arr
[
1
:
,
1
:
]
,
columns
=
arr
[
0
,
1
:
]
,
index
=
arr
[
1
:
,
0
]
,
)
.
T
return
df


def _parse_01(ofiles, individual=False):
    """
    a subfunction for summarizing results
    """

    ## parse results from outfiles
    cols = []
    dats = []
    for ofile in ofiles:

        ## parse file
        with open(ofile) as infile:
            dat  = infile.read()
        lastbits = dat.split(".mcmc.txt\n\n")[1:]
        results = lastbits[0].split("\n\n")[0].split()

        ## get shape from ...
        shape = (((len(results) - 3) / 4), 4)
        dat = np.array(results[3:]).reshape(shape)
        cols.append(dat[:, 3].astype(float))

    if not individual:
        ## get mean results across reps
        cols = np.array(cols)
        cols = cols.sum(axis=0) / len(ofiles) #10.
        dat[:, 3] = cols.astype(str)

        ## format as a DF
        df = pd.DataFrame(dat[:, 1:])
        df.columns = ["delim", "prior", "posterior"]
        nspecies = 1 + np.array([list(i) for i in dat[:, 1]], dtype=int).sum(axis=1)
        df["nspecies"] = nspecies
        return df

    else:
        ## get mean results across reps
        #return cols
        res = []
        for i in xrange(len(cols)):
            x = dat
            x[:, 3] = cols[i].astype(str)
            x = pd.DataFrame(x[:, 1:])
            x.columns = ['delim', 'prior', 'posterior']
            nspecies = 1 + np.array([list(i) for i in dat[:, 1]], dtype=int).sum(axis=1)
            x["nspecies"] = nspecies
            res.append(x)
        return res

def _parse_01 ( ofiles , individual = False ) : cols = [ ] dats = [ ] for ofile in ofiles : with open ( ofile ) as infile : dat = infile . read ( ) lastbits = dat . split ( ".mcmc.txt\n\n" ) [ 1 : ] results = lastbits [ 0 ] . split ( "\n\n" ) [ 0 ] . split ( ) shape = ( ( ( len ( results ) - 3 ) / 4 ) , 4 ) dat = np . array ( results [ 3 : ] ) . reshape ( shape ) cols . append ( dat [ : , 3 ] . astype ( float ) ) if not individual : cols = np . array ( cols ) cols = cols . sum ( axis = 0 ) / len ( ofiles ) dat [ : , 3 ] = cols . astype ( str ) df = pd . DataFrame ( dat [ : , 1 : ] ) df . columns = [ "delim" , "prior" , "posterior" ] nspecies = 1 + np . array ( [ list ( i ) for i in dat [ : , 1 ] ] , dtype = int ) . sum ( axis = 1 ) df [ "nspecies" ] = nspecies return df else : res = [ ] for i in xrange ( len ( cols ) ) : x = dat x [ : , 3 ] = cols [ i ] . astype ( str ) x = pd . DataFrame ( x [ : , 1 : ] ) x . columns = [ 'delim' , 'prior' , 'posterior' ] nspecies = 1 + np . array ( [ list ( i ) for i in dat [ : , 1 ] ] , dtype = int ) . sum ( axis = 1 ) x [ "nspecies" ] = nspecies res . append ( x ) return res


def _load_existing_results(self, name, workdir):
        """
        Load existing results files for an object with this workdir and name.
        This does NOT reload the parameter settings for the object...
        """
        ## get mcmcs
        path = os.path.realpath(os.path.join(self.workdir, self.name))
        mcmcs = glob.glob(path+"_r*.mcmc.txt")
        outs = glob.glob(path+"_r*.out.txt")
        trees = glob.glob(path+"_r*.tre")

        for mcmcfile in mcmcs:
            if mcmcfile not in self.files.mcmcfiles:
                self.files.mcmcfiles.append(mcmcfile)
        for outfile in outs:
            if outfile not in self.files.outfiles:
                self.files.outfiles.append(outfile)
        for tree in trees:
            if tree not in self.files.treefiles:
                self.files.treefiles.append(tree)

def _load_existing_results ( self , name , workdir ) : path = os . path . realpath ( os . path . join ( self . workdir , self . name ) ) mcmcs = glob . glob ( path + "_r*.mcmc.txt" ) outs = glob . glob ( path + "_r*.out.txt" ) trees = glob . glob ( path + "_r*.tre" ) for mcmcfile in mcmcs : if mcmcfile not in self . files . mcmcfiles : self . files . mcmcfiles . append ( mcmcfile ) for outfile in outs : if outfile not in self . files . outfiles : self . files . outfiles . append ( outfile ) for tree in trees : if tree not in self . files . treefiles : self . files . treefiles . append ( tree )

Load existing results files for an object with this workdir and name . This does NOT reload the parameter settings for the object ...


def summarize_results(self, individual_results=False):
        """
        Prints a summarized table of results from replicate runs, or,
        if individual_result=True, then returns a list of separate
        dataframes for each replicate run.
        """

        ## return results depending on algorithm

        ## algorithm 00
        if (not self.params.infer_delimit) & (not self.params.infer_sptree):
            if individual_results:
                ## return a list of parsed CSV results
                return [_parse_00(i) for i in self.files.outfiles]
            else:
                ## concatenate each CSV and then get stats w/ describe
                return pd.concat(
                    [pd.read_csv(i, sep='\t', index_col=0) \
                    for i in self.files.mcmcfiles]).describe().T

        ## algorithm 01
        if self.params.infer_delimit & (not self.params.infer_sptree):
            return _parse_01(self.files.outfiles, individual=individual_results)

        ## others
        else:
            return "summary function not yet ready for this type of result"

Prints a summarized table of results from replicate runs or if individual_result = True then returns a list of separate dataframes for each replicate run .


def multi_muscle_align(data, samples, ipyclient):
    """
    Sends the cluster bits to nprocessors for muscle alignment. They return
    with indel.h5 handles to be concatenated into a joint h5.
    """
    LOGGER.info("starting alignments")

    ## get client
    lbview = ipyclient.load_balanced_view()
    start = time.time()
    printstr = " aligning clusters     | {} | s6 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## submit clustbits as jobs to engines. The chunkfiles are removed when they
    ## are finished so this job can even be restarted if it was half finished,
    ## though that is probably rare.
    path = os.path.join(data.tmpdir, data.name + ".chunk_*")
    clustbits = glob.glob(path)
    jobs = {}
    for idx in xrange(len(clustbits)):
        args = [data, samples, clustbits[idx]]
        jobs[idx] = lbview.apply(persistent_popen_align3, *args)
    allwait = len(jobs)
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## print progress while bits are aligning
    while 1:
        finished = [i.ready() for i in jobs.values()]
        fwait = sum(finished)
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(allwait, fwait, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if all(finished):
            break

    ## check for errors in muscle_align_across
    keys = jobs.keys()
    for idx in keys:
        if not jobs[idx].successful():
            LOGGER.error("error in persistent_popen_align %s", jobs[idx].exception())
            raise IPyradWarningExit("error in step 6 {}".format(jobs[idx].exception()))
        del jobs[idx]
    print("")

def
multi_muscle_align
(
data
,
samples
,
ipyclient
)
:
LOGGER
.
info
(
"starting alignments"
)
lbview
=
ipyclient
.
load_balanced_view
(
)
start
=
time
.
time
(
)
printstr
=
" aligning clusters     | {} | s6 |"
elapsed
=
datetime
.
timedelta
(
seconds
=
int
(
time
.
time
(
)
start
)
)
progressbar
(
20
,
0
,
printstr
.
format
(
elapsed
)
,
spacer
=
data
.
_spacer
)
path
=
os
.
path
.
join
(
data
.
tmpdir
,
data
.
name
+
".chunk_*"
)
clustbits
=
glob
.
glob
(
path
)
jobs
=
{
}
for
idx
in
xrange
(
len
(
clustbits
)
)
:
args
=
[
data
,
samples
,
clustbits
[
idx
]
]
jobs
[
idx
]
=
lbview
.
apply
(
persistent_popen_align3
,
*
args
)
allwait
=
len
(
jobs
)
elapsed
=
datetime
.
timedelta
(
seconds
=
int
(
time
.
time
(
)
start
)
)
progressbar
(
20
,
0
,
printstr
.
format
(
elapsed
)
,
spacer
=
data
.
_spacer
)
while
1
:
finished
=
[
i
.
ready
(
)
for
i
in
jobs
.
values
(
)
]
fwait
=
sum
(
finished
)
elapsed
=
datetime
.
timedelta
(
seconds
=
int
(
time
.
time
(
)
start
)
)
progressbar
(
allwait
,
fwait
,
printstr
.
format
(
elapsed
)
,
spacer
=
data
.
_spacer
)
time
.
sleep
(
0.1
)
if
all
(
finished
)
:
break
keys
=
jobs
.
keys
(
)
for
idx
in
keys
:
if
not
jobs
[
idx
]
.
successful
(
)
:
LOGGER
.
error
(
"error in persistent_popen_align %s"
,
jobs
[
idx
]
.
exception
(
)
)
raise
IPyradWarningExit
(
"error in step 6 {}"
.
format
(
jobs
[
idx
]
.
exception
(
)
)
)
del
jobs
[
idx
]
print
(
""
)

Sends the cluster bits to nprocessors for muscle alignment . They return with indel . h5 handles to be concatenated into a joint h5 .


def concatclusts(outhandle, alignbits):
    """ concatenates sorted aligned cluster tmpfiles and removes them."""
    with gzip.open(outhandle, 'wb') as out:
        for fname in alignbits:
            with open(fname) as infile:
                out.write(infile.read()+"//\n//\n")

def concatclusts ( outhandle , alignbits ) : with gzip . open ( outhandle , 'wb' ) as out : for fname in alignbits : with open ( fname ) as infile : out . write ( infile . read ( ) + "//\n//\n" )

concatenates sorted aligned cluster tmpfiles and removes them .


def fill_dups_arr(data):
    """
    fills the duplicates array from the multi_muscle_align tmp files
    """
    ## build the duplicates array
    duplefiles = glob.glob(os.path.join(data.tmpdir, "duples_*.tmp.npy"))
    duplefiles.sort(key=lambda x: int(x.rsplit("_", 1)[-1][:-8]))

    ## enter the duplicates filter into super h5 array
    io5 = h5py.File(data.clust_database, 'r+')
    dfilter = io5["duplicates"]

    ## enter all duple arrays into full duplicates array
    init = 0
    for dupf in duplefiles:
        end = int(dupf.rsplit("_", 1)[-1][:-8])
        inarr = np.load(dupf)
        dfilter[init:end] = inarr
        init += end-init
        #os.remove(dupf)
    #del inarr

    ## continued progress bar
    LOGGER.info("all duplicates: %s", dfilter[:].sum())
    io5.close()

def
fill_dups_arr
(
data
)
:
duplefiles
=
glob
.
glob
(
os
.
path
.
join
(
data
.
tmpdir
,
"duples_*.tmp.npy"
)
)
duplefiles
.
sort
(
key
=
lambda
x
:
int
(
x
.
rsplit
(
"_"
,
1
)
[
1
]
[
:
8
]
)
)
io5
=
h5py
.
File
(
data
.
clust_database
,
'r+'
)
dfilter
=
io5
[
"duplicates"
]
init
=
0
for
dupf
in
duplefiles
:
end
=
int
(
dupf
.
rsplit
(
"_"
,
1
)
[
1
]
[
:
8
]
)
inarr
=
np
.
load
(
dupf
)
dfilter
[
init
:
end
]
=
inarr
init
+=
end
init
LOGGER
.
info
(
"all duplicates: %s"
,
dfilter
[
:
]
.
sum
(
)
)
io5
.
close
(
)


def build_tmp_h5(data, samples):
    """ build tmp h5 arrays that can return quick access for nloci"""
    ## get samples and names, sorted
    snames = [i.name for i in samples]
    snames.sort()

    ## Build an array for quickly indexing consens reads from catg files.
    ## save as a npy int binary file.
    uhandle = os.path.join(data.dirs.across, data.name+".utemp.sort")
    bseeds = os.path.join(data.dirs.across, data.name+".tmparrs.h5")

    ## send as first async1 job
    get_seeds_and_hits(uhandle, bseeds, snames)

def build_tmp_h5 ( data , samples ) : snames = [ i . name for i in samples ] snames . sort ( ) uhandle = os . path . join ( data . dirs . across , data . name + ".utemp.sort" ) bseeds = os . path . join ( data . dirs . across , data . name + ".tmparrs.h5" ) get_seeds_and_hits ( uhandle , bseeds , snames )


def get_nloci(data):
    """ return nloci from the tmp h5 arr"""
    bseeds = os.path.join(data.dirs.across, data.name+".tmparrs.h5")
    with h5py.File(bseeds) as io5:
        return io5["seedsarr"].shape[0]

def get_nloci ( data ) : bseeds = os . path . join ( data . dirs . across , data . name + ".tmparrs.h5" ) with h5py . File ( bseeds ) as io5 : return io5 [ "seedsarr" ] . shape [ 0 ]


def singlecat(data, sample, bseeds, sidx, nloci):
    """
    Orders catg data for each sample into the final locus order. This allows
    all of the individual catgs to simply be combined later. They are also in
    the same order as the indels array, so indels are inserted from the indel
    array that is passed in.
    """

    LOGGER.info("in single cat here")
    ## enter ref data?
    isref = 'reference' in data.paramsdict["assembly_method"]

    ## grab seeds and hits info for this sample
    with h5py.File(bseeds, 'r') as io5:
        ## get hits just for this sample and sort them by sample order index
        hits = io5["uarr"][:]
        hits = hits[hits[:, 1] == sidx, :]
        #hits = hits[hits[:, 2].argsort()]
        ## get seeds just for this sample and sort them by sample order index
        seeds = io5["seedsarr"][:]
        seeds = seeds[seeds[:, 1] == sidx, :]
        #seeds = seeds[seeds[:, 2].argsort()]
        full = np.concatenate((seeds, hits))
        full = full[full[:, 0].argsort()]

    ## still using max+20 len limit, rare longer merged reads get trimmed
    ## we need to allow room for indels to be added too
    maxlen = data._hackersonly["max_fragment_length"] + 20

    ## we'll fill a new catg and alleles arr for this sample in locus order,
    ## which is known from seeds and hits
    ocatg = np.zeros((nloci, maxlen, 4), dtype=np.uint32)
    onall = np.zeros(nloci, dtype=np.uint8)
    ochrom = np.zeros((nloci, 3), dtype=np.int64)

    ## grab the sample's data and write to ocatg and onall
    if not sample.files.database:
        raise IPyradWarningExit("missing catg file - {}".format(sample.name))

    with h5py.File(sample.files.database, 'r') as io5:
        ## get it and delete it
        catarr = io5["catg"][:]
        tmp = catarr[full[:, 2], :maxlen, :]
        del catarr
        ocatg[full[:, 0], :tmp.shape[1], :] = tmp
        del tmp

        ## get it and delete it
        nall = io5["nalleles"][:]
        onall[full[:, 0]] = nall[full[:, 2]]
        del nall

        ## fill the reference data
        if isref:
            chrom = io5["chroms"][:]
            ochrom[full[:, 0]] = chrom[full[:, 2]]
            del chrom

    ## get indel locations for this sample
    ipath = os.path.join(data.dirs.across, data.name+".tmp.indels.hdf5")
    with h5py.File(ipath, 'r') as ih5:
        indels = ih5["indels"][sidx, :, :maxlen]

    ## insert indels into ocatg
    newcatg = inserted_indels(indels, ocatg)
    del ocatg, indels

    ## save individual tmp h5 data
    smpio = os.path.join(data.dirs.across, sample.name+'.tmp.h5')
    with h5py.File(smpio, 'w') as oh5:
        oh5.create_dataset("icatg", data=newcatg, dtype=np.uint32)
        oh5.create_dataset("inall", data=onall, dtype=np.uint8)
        if isref:
            oh5.create_dataset("ichrom", data=ochrom, dtype=np.int64)

def
singlecat
(
data
,
sample
,
bseeds
,
sidx
,
nloci
)
:
LOGGER
.
info
(
"in single cat here"
)
isref
=
'reference'
in
data
.
paramsdict
[
"assembly_method"
]
with
h5py
.
File
(
bseeds
,
'r'
)
as
io5
:
hits
=
io5
[
"uarr"
]
[
:
]
hits
=
hits
[
hits
[
:
,
1
]
==
sidx
,
:
]
seeds
=
io5
[
"seedsarr"
]
[
:
]
seeds
=
seeds
[
seeds
[
:
,
1
]
==
sidx
,
:
]
full
=
np
.
concatenate
(
(
seeds
,
hits
)
)
full
=
full
[
full
[
:
,
0
]
.
argsort
(
)
]
maxlen
=
data
.
_hackersonly
[
"max_fragment_length"
]
+
20
ocatg
=
np
.
zeros
(
(
nloci
,
maxlen
,
4
)
,
dtype
=
np
.
uint32
)
onall
=
np
.
zeros
(
nloci
,
dtype
=
np
.
uint8
)
ochrom
=
np
.
zeros
(
(
nloci
,
3
)
,
dtype
=
np
.
int64
)
if
not
sample
.
files
.
database
:
raise
IPyradWarningExit
(
"missing catg file - {}"
.
format
(
sample
.
name
)
)
with
h5py
.
File
(
sample
.
files
.
database
,
'r'
)
as
io5
:
catarr
=
io5
[
"catg"
]
[
:
]
tmp
=
catarr
[
full
[
:
,
2
]
,
:
maxlen
,
:
]
del
catarr
ocatg
[
full
[
:
,
0
]
,
:
tmp
.
shape
[
1
]
,
:
]
=
tmp
del
tmp
nall
=
io5
[
"nalleles"
]
[
:
]
onall
[
full
[
:
,
0
]
]
=
nall
[
full
[
:
,
2
]
]
del
nall
if
isref
:
chrom
=
io5
[
"chroms"
]
[
:
]
ochrom
[
full
[
:
,
0
]
]
=
chrom
[
full
[
:
,
2
]
]
del
chrom
ipath
=
os
.
path
.
join
(
data
.
dirs
.
across
,
data
.
name
+
".tmp.indels.hdf5"
)
with
h5py
.
File
(
ipath
,
'r'
)
as
ih5
:
indels
=
ih5
[
"indels"
]
[
sidx
,
:
,
:
maxlen
]
newcatg
=
inserted_indels
(
indels
,
ocatg
)
del
ocatg
,
indels
smpio
=
os
.
path
.
join
(
data
.
dirs
.
across
,
sample
.
name
+
'.tmp.h5'
)
with
h5py
.
File
(
smpio
,
'w'
)
as
oh5
:
oh5
.
create_dataset
(
"icatg"
,
data
=
newcatg
,
dtype
=
np
.
uint32
)
oh5
.
create_dataset
(
"inall"
,
data
=
onall
,
dtype
=
np
.
uint8
)
if
isref
:
oh5
.
create_dataset
(
"ichrom"
,
data
=
ochrom
,
dtype
=
np
.
int64
)

Orders catg data for each sample into the final locus order . This allows all of the individual catgs to simply be combined later . They are also in the same order as the indels array so indels are inserted from the indel array that is passed in .


def write_to_fullarr(data, sample, sidx):
    """ writes arrays to h5 disk """

    ## enter ref data?
    #isref = 'reference' in data.paramsdict["assembly_method"]
    LOGGER.info("writing fullarr %s %s", sample.name, sidx)

    ## save big arrays to disk temporarily
    with h5py.File(data.clust_database, 'r+') as io5:
        ## open views into the arrays we plan to fill
        chunk = io5["catgs"].attrs["chunksize"][0]
        catg = io5["catgs"]
        nall = io5["nalleles"]

        ## adding an axis to newcatg makes it write about 1000X faster.
        smpio = os.path.join(data.dirs.across, sample.name+'.tmp.h5')
        with h5py.File(smpio) as indat:

            ## grab all of the data from this sample's arrays
            newcatg = indat["icatg"] #[:]
            onall = indat["inall"]   #[:]

            ## enter it into the full array one chunk at a time
            for cidx in xrange(0, catg.shape[0], chunk):
                end = cidx + chunk
                catg[cidx:end, sidx:sidx+1, :] = np.expand_dims(newcatg[cidx:end, :], axis=1)
                nall[:, sidx:sidx+1] = np.expand_dims(onall, axis=1)

def
write_to_fullarr
(
data
,
sample
,
sidx
)
:
LOGGER
.
info
(
"writing fullarr %s %s"
,
sample
.
name
,
sidx
)
with
h5py
.
File
(
data
.
clust_database
,
'r+'
)
as
io5
:
chunk
=
io5
[
"catgs"
]
.
attrs
[
"chunksize"
]
[
0
]
catg
=
io5
[
"catgs"
]
nall
=
io5
[
"nalleles"
]
smpio
=
os
.
path
.
join
(
data
.
dirs
.
across
,
sample
.
name
+
'.tmp.h5'
)
with
h5py
.
File
(
smpio
)
as
indat
:
newcatg
=
indat
[
"icatg"
]
onall
=
indat
[
"inall"
]
for
cidx
in
xrange
(
0
,
catg
.
shape
[
0
]
,
chunk
)
:
end
=
cidx
+
chunk
catg
[
cidx
:
end
,
sidx
:
sidx
+
1
,
:
]
=
np
.
expand_dims
(
newcatg
[
cidx
:
end
,
:
]
,
axis
=
1
)
nall
[
:
,
sidx
:
sidx
+
1
]
=
np
.
expand_dims
(
onall
,
axis
=
1
)


def dask_chroms(data, samples):
    """
    A dask relay function to fill chroms for all samples
    """

    ## example concatenating with dask
    h5s = [os.path.join(data.dirs.across, s.name+".tmp.h5") for s in samples]
    handles = [h5py.File(i) for i in h5s]
    dsets = [i['/ichrom'] for i in handles]
    arrays = [da.from_array(dset, chunks=(10000, 3)) for dset in dsets]
    stack = da.stack(arrays, axis=2)

    ## max chrom (should we check for variable hits? if so, things can get wonk)
    maxchrom = da.max(stack, axis=2)[:, 0]

    ## max pos
    maxpos = da.max(stack, axis=2)[:, 2]

    ## min pos
    mask = stack == 0
    stack[mask] = 9223372036854775807  ## max int64 value
    minpos = da.min(stack, axis=2)[:, 1]
    final = da.stack([maxchrom, minpos, maxpos], axis=1)
    final.to_hdf5(data.clust_database, "/chroms")

    ## close the h5 handles
    _ = [i.close() for i in handles]

def dask_chroms ( data , samples ) : h5s = [ os . path . join ( data . dirs . across , s . name + ".tmp.h5" ) for s in samples ] handles = [ h5py . File ( i ) for i in h5s ] dsets = [ i [ '/ichrom' ] for i in handles ] arrays = [ da . from_array ( dset , chunks = ( 10000 , 3 ) ) for dset in dsets ] stack = da . stack ( arrays , axis = 2 ) maxchrom = da . max ( stack , axis = 2 ) [ : , 0 ] maxpos = da . max ( stack , axis = 2 ) [ : , 2 ] mask = stack == 0 stack [ mask ] = 9223372036854775807 minpos = da . min ( stack , axis = 2 ) [ : , 1 ] final = da . stack ( [ maxchrom , minpos , maxpos ] , axis = 1 ) final . to_hdf5 ( data . clust_database , "/chroms" ) _ = [ i . close ( ) for i in handles ]


def inserted_indels(indels, ocatg):
    """
    inserts indels into the catg array
    """
    ## return copy with indels inserted
    newcatg = np.zeros(ocatg.shape, dtype=np.uint32)

    ## iterate over loci and make extensions for indels
    for iloc in xrange(ocatg.shape[0]):
        ## get indels indices
        indidx = np.where(indels[iloc, :])[0]
        if np.any(indidx):
            ## which new (empty) rows will be added
            allrows = np.arange(ocatg.shape[1])
            mask = np.ones(allrows.shape[0], dtype=np.bool_)
            for idx in indidx:
                mask[idx] = False
            not_idx = allrows[mask == 1]

            ## fill in new data into all other spots
            newcatg[iloc][not_idx] = ocatg[iloc, :not_idx.shape[0]]
        else:
            newcatg[iloc] = ocatg[iloc]
    return newcatg

def inserted_indels ( indels , ocatg ) : newcatg = np . zeros ( ocatg . shape , dtype = np . uint32 ) for iloc in xrange ( ocatg . shape [ 0 ] ) : indidx = np . where ( indels [ iloc , : ] ) [ 0 ] if np . any ( indidx ) : allrows = np . arange ( ocatg . shape [ 1 ] ) mask = np . ones ( allrows . shape [ 0 ] , dtype = np . bool_ ) for idx in indidx : mask [ idx ] = False not_idx = allrows [ mask == 1 ] newcatg [ iloc ] [ not_idx ] = ocatg [ iloc , : not_idx . shape [ 0 ] ] else : newcatg [ iloc ] = ocatg [ iloc ] return newcatg


def count_seeds(usort):
    """
    uses bash commands to quickly count N seeds from utemp file
    """
    with open(usort, 'r') as insort:
        cmd1 = ["cut", "-f", "2"]
        cmd2 = ["uniq"]
        cmd3 = ["wc"]
        proc1 = sps.Popen(cmd1, stdin=insort, stdout=sps.PIPE, close_fds=True)
        proc2 = sps.Popen(cmd2, stdin=proc1.stdout, stdout=sps.PIPE, close_fds=True)
        proc3 = sps.Popen(cmd3, stdin=proc2.stdout, stdout=sps.PIPE, close_fds=True)
        res = proc3.communicate()
        nseeds = int(res[0].split()[0])
        proc1.stdout.close()
        proc2.stdout.close()
        proc3.stdout.close()
    return nseeds

def count_seeds ( usort ) : with open ( usort , 'r' ) as insort : cmd1 = [ "cut" , "-f" , "2" ] cmd2 = [ "uniq" ] cmd3 = [ "wc" ] proc1 = sps . Popen ( cmd1 , stdin = insort , stdout = sps . PIPE , close_fds = True ) proc2 = sps . Popen ( cmd2 , stdin = proc1 . stdout , stdout = sps . PIPE , close_fds = True ) proc3 = sps . Popen ( cmd3 , stdin = proc2 . stdout , stdout = sps . PIPE , close_fds = True ) res = proc3 . communicate ( ) nseeds = int ( res [ 0 ] . split ( ) [ 0 ] ) proc1 . stdout . close ( ) proc2 . stdout . close ( ) proc3 . stdout . close ( ) return nseeds


def sort_seeds(uhandle, usort):
    """ sort seeds from cluster results"""
    cmd = ["sort", "-k", "2", uhandle, "-o", usort]
    proc = sps.Popen(cmd, close_fds=True)
    proc.communicate()

def sort_seeds ( uhandle , usort ) : cmd = [ "sort" , "-k" , "2" , uhandle , "-o" , usort ] proc = sps . Popen ( cmd , close_fds = True ) proc . communicate ( )


def build_clustbits(data, ipyclient, force):
    """
    Reconstitutes clusters from .utemp and htemp files and writes them
    to chunked files for aligning in muscle.
    """

    ## If you run this step then we clear all tmp .fa and .indel.h5 files
    if os.path.exists(data.tmpdir):
        shutil.rmtree(data.tmpdir)
        os.mkdir(data.tmpdir)

    ## parallel client
    lbview = ipyclient.load_balanced_view()
    start = time.time()
    printstr = " building clusters     | {} | s6 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(3, 0, printstr.format(elapsed), spacer=data._spacer)

    uhandle = os.path.join(data.dirs.across, data.name+".utemp")
    usort = os.path.join(data.dirs.across, data.name+".utemp.sort")

    async1 = ""
    ## skip usorting if not force and already exists
    if not os.path.exists(usort) or force:

        ## send sort job to engines. Sorted seeds allows us to work through
        ## the utemp file one locus at a time instead of reading all into mem.
        LOGGER.info("building reads file -- loading utemp file into mem")
        async1 = lbview.apply(sort_seeds, *(uhandle, usort))
        while 1:
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(3, 0, printstr.format(elapsed), spacer=data._spacer)
            if async1.ready():
                break
            else:
                time.sleep(0.1)

    ## send count seeds job to engines.
    async2 = lbview.apply(count_seeds, usort)
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(3, 1, printstr.format(elapsed), spacer=data._spacer)
        if async2.ready():
            break
        else:
            time.sleep(0.1)

    ## wait for both to finish while printing progress timer
    nseeds = async2.result()

    ## send the clust bit building job to work and track progress
    async3 = lbview.apply(sub_build_clustbits, *(data, usort, nseeds))
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(3, 2, printstr.format(elapsed), spacer=data._spacer)
        if async3.ready():
            break
        else:
            time.sleep(0.1)
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(3, 3, printstr.format(elapsed), spacer=data._spacer)
    print("")

    ## check for errors
    for job in [async1, async2, async3]:
        try:
            if not job.successful():
                raise IPyradWarningExit(job.result())
        except AttributeError:
            ## If we skip usorting then async1 == "" so the call to
            ## successful() raises, but we can ignore it.
            pass

Reconstitutes clusters from . utemp and htemp files and writes them to chunked files for aligning in muscle .


def sub_build_clustbits(data, usort, nseeds):
    """
    A subfunction of build_clustbits to allow progress tracking. This func
    splits the unaligned clusters into bits for aligning on separate cores.
    """

    ## load FULL concat fasta file into a dict. This could cause RAM issues.
    ## this file has iupac codes in it, not ambigs resolved, and is gzipped.
    LOGGER.info("loading full _catcons file into memory")
    allcons = {}
    conshandle = os.path.join(data.dirs.across, data.name+"_catcons.tmp")
    with gzip.open(conshandle, 'rb') as iocons:
        cons = itertools.izip(*[iter(iocons)]*2)
        for namestr, seq in cons:
            nnn, sss = [i.strip() for i in namestr, seq]
            allcons[nnn[1:]] = sss

    ## set optim to approximately 4 chunks per core. Smaller allows for a bit
    ## cleaner looking progress bar. 40 cores will make 160 files.
    optim = ((nseeds // (data.cpus*4)) + (nseeds % (data.cpus*4)))
    LOGGER.info("building clustbits, optim=%s, nseeds=%s, cpus=%s",
                optim, nseeds, data.cpus)

    ## iterate through usort grabbing seeds and matches
    with open(usort, 'rb') as insort:
        ## iterator, seed null, and seqlist null
        isort = iter(insort)
        loci = 0
        lastseed = 0
        fseqs = []
        seqlist = []
        seqsize = 0

        while 1:
            ## grab the next line
            try:
                hit, seed, ori = isort.next().strip().split()
            except StopIteration:
                break

            try:
                ## if same seed, append match
                if seed != lastseed:
                    ## store the last fseq, count it, and clear it
                    if fseqs:
                        seqlist.append("\n".join(fseqs))
                        seqsize += 1
                        fseqs = []
                    ## occasionally write to file
                    if seqsize >= optim:
                        if seqlist:
                            loci += seqsize
                            with open(os.path.join(data.tmpdir,
                                data.name+".chunk_{}".format(loci)), 'w') as clustsout:
                                LOGGER.debug("writing chunk - seqsize {} loci {} {}".format(seqsize, loci, clustsout.name))
                                clustsout.write("\n//\n//\n".join(seqlist)+"\n//\n//\n")
                            ## reset list and counter
                            seqlist = []
                            seqsize = 0

                    ## store the new seed on top of fseq
                    fseqs.append(">{}\n{}".format(seed, allcons[seed]))
                    lastseed = seed

                ## add match to the seed
                seq = allcons[hit]
                ## revcomp if orientation is reversed
                if ori == "-":
                    seq = fullcomp(seq)[::-1]
                fseqs.append(">{}\n{}".format(hit, seq))
            except KeyError as inst:
                ## Caught bad seed or hit? Log and continue.
                LOGGER.error("Bad Seed/Hit: seqsize {}\tloci {}\tseed {}\thit {}".format(seqsize, loci, seed, hit))

    ## write whatever is left over to the clusts file
    if fseqs:
        seqlist.append("\n".join(fseqs))
        seqsize += 1
        loci += seqsize
    if seqlist:
        with open(os.path.join(data.tmpdir,
            data.name+".chunk_{}".format(loci)), 'w') as clustsout:
            clustsout.write("\n//\n//\n".join(seqlist)+"\n//\n//\n")

    ## final progress and cleanup
    del allcons
    clustbits = glob.glob(os.path.join(data.tmpdir, data.name+".chunk_*"))

    ## return stuff
    return clustbits, loci

A subfunction of build_clustbits to allow progress tracking . This func splits the unaligned clusters into bits for aligning on separate cores .


def cleanup_tempfiles(data):
    """
    Function to remove older files. This is called either in substep 1 or after
    the final substep so that tempfiles are retained for restarting interrupted
    jobs until we're sure they're no longer needed.
    """

    ## remove align-related tmp files
    tmps1 = glob.glob(os.path.join(data.tmpdir, "*.fa"))
    tmps2 = glob.glob(os.path.join(data.tmpdir, "*.npy"))
    for tmp in tmps1 + tmps2:
        if os.path.exists(tmp):
            os.remove(tmp)

    ## remove cluster related files
    removal = [
        os.path.join(data.dirs.across, data.name+".utemp"),
        os.path.join(data.dirs.across, data.name+".htemp"),
        os.path.join(data.dirs.across, data.name+"_catcons.tmp"),
        os.path.join(data.dirs.across, data.name+"_cathaps.tmp"),
        os.path.join(data.dirs.across, data.name+"_catshuf.tmp"),
        os.path.join(data.dirs.across, data.name+"_catsort.tmp"),
        os.path.join(data.dirs.across, data.name+".tmparrs.h5"),
        os.path.join(data.dirs.across, data.name+".tmp.indels.hdf5"),
        ]
    for rfile in removal:
        if os.path.exists(rfile):
            os.remove(rfile)

    ## remove singlecat related h5 files
    smpios = glob.glob(os.path.join(data.dirs.across, '*.tmp.h5'))
    for smpio in smpios:
        if os.path.exists(smpio):
            os.remove(smpio)

def cleanup_tempfiles ( data ) : tmps1 = glob . glob ( os . path . join ( data . tmpdir , "*.fa" ) ) tmps2 = glob . glob ( os . path . join ( data . tmpdir , "*.npy" ) ) for tmp in tmps1 + tmps2 : if os . path . exists ( tmp ) : os . remove ( tmp ) removal = [ os . path . join ( data . dirs . across , data . name + ".utemp" ) , os . path . join ( data . dirs . across , data . name + ".htemp" ) , os . path . join ( data . dirs . across , data . name + "_catcons.tmp" ) , os . path . join ( data . dirs . across , data . name + "_cathaps.tmp" ) , os . path . join ( data . dirs . across , data . name + "_catshuf.tmp" ) , os . path . join ( data . dirs . across , data . name + "_catsort.tmp" ) , os . path . join ( data . dirs . across , data . name + ".tmparrs.h5" ) , os . path . join ( data . dirs . across , data . name + ".tmp.indels.hdf5" ) , ] for rfile in removal : if os . path . exists ( rfile ) : os . remove ( rfile ) smpios = glob . glob ( os . path . join ( data . dirs . across , '*.tmp.h5' ) ) for smpio in smpios : if os . path . exists ( smpio ) : os . remove ( smpio )

Function to remove older files . This is called either in substep 1 or after the final substep so that tempfiles are retained for restarting interrupted jobs until we re sure they re no longer needed .


def assembly_cleanup(data):
    """ cleanup for assembly object """

    ## build s2 results data frame
    data.stats_dfs.s2 = data._build_stat("s2")
    data.stats_files.s2 = os.path.join(data.dirs.edits, 's2_rawedit_stats.txt')

    ## write stats for all samples
    with io.open(data.stats_files.s2, 'w', encoding='utf-8') as outfile:
        data.stats_dfs.s2.fillna(value=0).astype(np.int).to_string(outfile)

def assembly_cleanup ( data ) : data . stats_dfs . s2 = data . _build_stat ( "s2" ) data . stats_files . s2 = os . path . join ( data . dirs . edits , 's2_rawedit_stats.txt' ) with io . open ( data . stats_files . s2 , 'w' , encoding = 'utf-8' ) as outfile : data . stats_dfs . s2 . fillna ( value = 0 ) . astype ( np . int ) . to_string ( outfile )


def parse_single_results(data, sample, res1):
    """ parse results from cutadapt into sample data"""

    ## set default values
    #sample.stats_dfs.s2["reads_raw"] = 0
    sample.stats_dfs.s2["trim_adapter_bp_read1"] = 0
    sample.stats_dfs.s2["trim_quality_bp_read1"] = 0
    sample.stats_dfs.s2["reads_filtered_by_Ns"] = 0
    sample.stats_dfs.s2["reads_filtered_by_minlen"] = 0
    sample.stats_dfs.s2["reads_passed_filter"] = 0

    ## parse new values from cutadapt results output
    lines = res1.strip().split("\n")
    for line in lines:

        if "Total reads processed:" in line:
            value = int(line.split()[3].replace(",", ""))
            sample.stats_dfs.s2["reads_raw"] = value

        if "Reads with adapters:" in line:
            value = int(line.split()[3].replace(",", ""))
            sample.stats_dfs.s2["trim_adapter_bp_read1"] = value

        if "Quality-trimmed" in line:
            value = int(line.split()[1].replace(",", ""))
            sample.stats_dfs.s2["trim_quality_bp_read1"] = value

        if "Reads that were too short" in line:
            value = int(line.split()[5].replace(",", ""))
            sample.stats_dfs.s2["reads_filtered_by_minlen"] = value

        if "Reads with too many N" in line:
            value = int(line.split()[5].replace(",", ""))
            sample.stats_dfs.s2["reads_filtered_by_Ns"] = value

        if "Reads written (passing filters):" in line:
            value = int(line.split()[4].replace(",", ""))
            sample.stats_dfs.s2["reads_passed_filter"] = value

    ## save to stats summary
    if sample.stats_dfs.s2.reads_passed_filter:
        sample.stats.state = 2
        sample.stats.reads_passed_filter = sample.stats_dfs.s2.reads_passed_filter
        sample.files.edits = [
            (OPJ(data.dirs.edits, sample.name+".trimmed_R1_.fastq.gz"), 0)]
        ## write the long form output to the log file.
        LOGGER.info(res1)

    else:
        print("{}No reads passed filtering in Sample: {}".format(data._spacer, sample.name))