def run2(data, samples, force, ipyclient):
    """
    Filter for samples that are already finished with this step, allow others
    to run, pass them to parallel client function to filter with cutadapt.
    """

    ## create output directories
    data.dirs.edits = os.path.join(os.path.realpath(
                                   data.paramsdict["project_dir"]),
                                   data.name+"_edits")
    if not os.path.exists(data.dirs.edits):
        os.makedirs(data.dirs.edits)

    ## get samples
    subsamples = choose_samples(samples, force)

    ## only allow extra adapters in filters==3,
    ## and add poly repeats if not in list of adapters
    if int(data.paramsdict["filter_adapters"]) == 3:
        if not data._hackersonly["p3_adapters_extra"]:
            for poly in ["A"*8, "T"*8, "C"*8, "G"*8]:
                data._hackersonly["p3_adapters_extra"].append(poly)
        if not data._hackersonly["p5_adapters_extra"]:
            for poly in ["A"*8, "T"*8, "C"*8, "G"*8]:
                data._hackersonly["p5_adapters_extra"].append(poly)
    else:
        data._hackersonly["p5_adapters_extra"] = []
        data._hackersonly["p3_adapters_extra"] = []

    ## concat is not parallelized (since it's disk limited, generally)
    subsamples = concat_reads(data, subsamples, ipyclient)

    ## cutadapt is parallelized by ncores/2 because cutadapt spawns threads
    lbview = ipyclient.load_balanced_view(targets=ipyclient.ids[::2])
    run_cutadapt(data, subsamples, lbview)

    ## cleanup is ...
    assembly_cleanup(data)

def run2 ( data , samples , force , ipyclient ) : data . dirs . edits = os . path . join ( os . path . realpath ( data . paramsdict [ "project_dir" ] ) , data . name + "_edits" ) if not os . path . exists ( data . dirs . edits ) : os . makedirs ( data . dirs . edits ) subsamples = choose_samples ( samples , force ) if int ( data . paramsdict [ "filter_adapters" ] ) == 3 : if not data . _hackersonly [ "p3_adapters_extra" ] : for poly in [ "A" * 8 , "T" * 8 , "C" * 8 , "G" * 8 ] : data . _hackersonly [ "p3_adapters_extra" ] . append ( poly ) if not data . _hackersonly [ "p5_adapters_extra" ] : for poly in [ "A" * 8 , "T" * 8 , "C" * 8 , "G" * 8 ] : data . _hackersonly [ "p5_adapters_extra" ] . append ( poly ) else : data . _hackersonly [ "p5_adapters_extra" ] = [ ] data . _hackersonly [ "p3_adapters_extra" ] = [ ] subsamples = concat_reads ( data , subsamples , ipyclient ) lbview = ipyclient . load_balanced_view ( targets = ipyclient . ids [ : : 2 ] ) run_cutadapt ( data , subsamples , lbview ) assembly_cleanup ( data )

Filter for samples that are already finished with this step allow others to run pass them to parallel client function to filter with cutadapt .


def concat_reads(data, subsamples, ipyclient):
    """ concatenate if multiple input files for a single samples """

    ## concatenate reads if they come from merged assemblies.
    if any([len(i.files.fastqs) > 1 for i in subsamples]):
        ## run on single engine for now
        start = time.time()
        printstr = " concatenating inputs  | {} | s2 |"
        finished = 0
        catjobs = {}
        for sample in subsamples:
            if len(sample.files.fastqs) > 1:
                catjobs[sample.name] = ipyclient[0].apply(\
                                       concat_multiple_inputs, *(data, sample))
            else:
                sample.files.concat = sample.files.fastqs

        ## wait for all to finish
        while 1:
            finished = sum([i.ready() for i in catjobs.values()])
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(len(catjobs), finished, printstr.format(elapsed), spacer=data._spacer)
            time.sleep(0.1)
            if finished == len(catjobs):
                print("")
                break

        ## collect results, which are concat file handles.
        for async in catjobs:
            if catjobs[async].successful():
                data.samples[async].files.concat = catjobs[async].result()
            else:
                error = catjobs[async].result()#exception()
                LOGGER.error("error in step2 concat %s", error)
                raise IPyradWarningExit("error in step2 concat: {}".format(error))
    else:
        for sample in subsamples:
            ## just copy fastqs handles to concat attribute
            sample.files.concat = sample.files.fastqs

    return subsamples

def
concat_reads
(
data
,
subsamples
,
ipyclient
)
:
if
any
(
[
len
(
i
.
files
.
fastqs
)
>
1
for
i
in
subsamples
]
)
:
start
=
time
.
time
(
)
printstr
=
" concatenating inputs  | {} | s2 |"
finished
=
0
catjobs
=
{
}
for
sample
in
subsamples
:
if
len
(
sample
.
files
.
fastqs
)
>
1
:
catjobs
[
sample
.
name
]
=
ipyclient
[
0
]
.
apply
(
concat_multiple_inputs
,
*
(
data
,
sample
)
)
else
:
sample
.
files
.
concat
=
sample
.
files
.
fastqs
while
1
:
finished
=
sum
(
[
i
.
ready
(
)
for
i
in
catjobs
.
values
(
)
]
)
elapsed
=
datetime
.
timedelta
(
seconds
=
int
(
time
.
time
(
)
start
)
)
progressbar
(
len
(
catjobs
)
,
finished
,
printstr
.
format
(
elapsed
)
,
spacer
=
data
.
_spacer
)
time
.
sleep
(
0.1
)
if
finished
==
len
(
catjobs
)
:
print
(
""
)
break
for
async
in
catjobs
:
if
catjobs
[
async
]
.
successful
(
)
:
data
.
samples
[
async
]
.
files
.
concat
=
catjobs
[
async
]
.
result
(
)
else
:
error
=
catjobs
[
async
]
.
result
(
)
LOGGER
.
error
(
"error in step2 concat %s"
,
error
)
raise
IPyradWarningExit
(
"error in step2 concat: {}"
.
format
(
error
)
)
else
:
for
sample
in
subsamples
:
sample
.
files
.
concat
=
sample
.
files
.
fastqs
return
subsamples


def run_cutadapt(data, subsamples, lbview):
    """
    sends fastq files to cutadapt
    """
    ## choose cutadapt function based on datatype
    start = time.time()
    printstr = " processing reads      | {} | s2 |"
    finished = 0
    rawedits = {}

    ## sort subsamples so that the biggest files get submitted first
    subsamples.sort(key=lambda x: x.stats.reads_raw, reverse=True)
    LOGGER.info([i.stats.reads_raw for i in subsamples])

    ## send samples to cutadapt filtering
    if "pair" in data.paramsdict["datatype"]:
        for sample in subsamples:
            rawedits[sample.name] = lbview.apply(cutadaptit_pairs, *(data, sample))
    else:
        for sample in subsamples:
            rawedits[sample.name] = lbview.apply(cutadaptit_single, *(data, sample))

    ## wait for all to finish
    while 1:
        finished = sum([i.ready() for i in rawedits.values()])
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(rawedits), finished, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if finished == len(rawedits):
            print("")
            break

    ## collect results, report failures, and store stats. async = sample.name
    for async in rawedits:
        if rawedits[async].successful():
            res = rawedits[async].result()

            ## if single cleanup is easy
            if "pair" not in data.paramsdict["datatype"]:
                parse_single_results(data, data.samples[async], res)
            else:
                parse_pair_results(data, data.samples[async], res)
        else:
            print("  found an error in step2; see ipyrad_log.txt")
            LOGGER.error("error in run_cutadapt(): %s", rawedits[async].exception())


def concat_multiple_inputs(data, sample):
    """
    If multiple fastq files were appended into the list of fastqs for samples
    then we merge them here before proceeding.
    """

    ## if more than one tuple in fastq list
    if len(sample.files.fastqs) > 1:
        ## create a cat command to append them all (doesn't matter if they
        ## are gzipped, cat still works). Grab index 0 of tuples for R1s.
        cmd1 = ["cat"] + [i[0] for i in sample.files.fastqs]

        isgzip = ".gz"
        if not sample.files.fastqs[0][0].endswith(".gz"):
            isgzip = ""

        ## write to new concat handle
        conc1 = os.path.join(data.dirs.edits, sample.name+"_R1_concat.fq{}".format(isgzip))
        with open(conc1, 'w') as cout1:
            proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=cout1, close_fds=True)
            res1 = proc1.communicate()[0]
        if proc1.returncode:
            raise IPyradWarningExit("error in: {}, {}".format(cmd1, res1))

        ## Only set conc2 if R2 actually exists
        conc2 = 0
        if "pair" in data.paramsdict["datatype"]:
            cmd2 = ["cat"] + [i[1] for i in sample.files.fastqs]
            conc2 = os.path.join(data.dirs.edits, sample.name+"_R2_concat.fq{}".format(isgzip))
            with open(conc2, 'w') as cout2:
                proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=cout2, close_fds=True)
                res2 = proc2.communicate()[0]
            if proc2.returncode:
                raise IPyradWarningExit("Error concatenating fastq files. Make sure all "\
                    + "these files exist: {}\nError message: {}".format(cmd2, proc2.returncode))

        ## store new file handles
        sample.files.concat = [(conc1, conc2)]
    return sample.files.concat

If multiple fastq files were appended into the list of fastqs for samples then we merge them here before proceeding .


def make( data, samples ):
    """ Convert vcf from step6 to .loci format to facilitate downstream format conversion """
    invcffile   =  os.path.join( data.dirs.consens, data.name+".vcf" )
    outlocifile  =  os.path.join( data.dirs.outfiles, data.name+".loci" )

    importvcf( invcffile, outlocifile )

def make ( data , samples ) : invcffile = os . path . join ( data . dirs . consens , data . name + ".vcf" ) outlocifile = os . path . join ( data . dirs . outfiles , data . name + ".loci" ) importvcf ( invcffile , outlocifile )

Convert vcf from step6 to . loci format to facilitate downstream format conversion


def importvcf( vcffile, locifile ):
    """ Function for importing a vcf file into loci format. Arguments
        are the input vcffile and the loci file to write out. """

    try:
        ## Get names of all individuals in the vcf
        with open( invcffile, 'r' ) as invcf:
            for line in invcf:
                if line.split()[0] == "#CHROM":
                    ## This is maybe a little clever. The names in the vcf are everything after
                    ## the "FORMAT" column, so find that index, then slice everything after it.
                    names_col = line.split().index( "FORMAT" ) + 1
                    names = line.split()[ names_col:]
                    LOGGER.debug( "Got names - %s", names )
                    break

            print( "wat" )
        ## Get the column to start reading at
    except Exception:
        print( "wat" )

def
importvcf
(
vcffile
,
locifile
)
:
try
:
with
open
(
invcffile
,
'r'
)
as
invcf
:
for
line
in
invcf
:
if
line
.
split
(
)
[
0
]
==
"#CHROM"
:
names_col
=
line
.
split
(
)
.
index
(
"FORMAT"
)
+
1
names
=
line
.
split
(
)
[
names_col
:
]
LOGGER
.
debug
(
"Got names - %s"
,
names
)
break
print
(
"wat"
)
except
Exception
:
print
(
"wat"
)

Function for importing a vcf file into loci format . Arguments are the input vcffile and the loci file to write out .


def get_targets(ipyclient):
    """
    A function to find 2 engines per hostname on the ipyclient.
    We'll assume that the CPUs are hyperthreaded, which is why
    we grab two. If they are not then no foul. Two multi-threaded
    jobs will be run on each of the 2 engines per host.
    """
    ## fill hosts with async[gethostname]
    hosts = []
    for eid in ipyclient.ids:
        engine = ipyclient[eid]
        if not engine.outstanding:
            hosts.append(engine.apply(socket.gethostname))

    ## capture results of asyncs
    hosts = [i.get() for i in hosts]
    hostset = set(hosts)
    hostzip = zip(hosts, ipyclient.ids)
    hostdict = {host: [i[1] for i in hostzip if i[0] == host] for host in hostset}
    targets = list(itertools.chain(*[hostdict[i][:2] for i in hostdict]))

    ## return first two engines from each host
    return targets

def get_targets ( ipyclient ) : hosts = [ ] for eid in ipyclient . ids : engine = ipyclient [ eid ] if not engine . outstanding : hosts . append ( engine . apply ( socket . gethostname ) ) hosts = [ i . get ( ) for i in hosts ] hostset = set ( hosts ) hostzip = zip ( hosts , ipyclient . ids ) hostdict = { host : [ i [ 1 ] for i in hostzip if i [ 0 ] == host ] for host in hostset } targets = list ( itertools . chain ( * [ hostdict [ i ] [ : 2 ] for i in hostdict ] ) ) return targets

A function to find 2 engines per hostname on the ipyclient . We ll assume that the CPUs are hyperthreaded which is why we grab two . If they are not then no foul . Two multi - threaded jobs will be run on each of the 2 engines per host .


def compute_tree_stats(self, ipyclient):
    """
    compute stats for stats file and NHX tree features
    """

    ## get name indices
    names = self.samples

    ## get majority rule consensus tree of weighted Q bootstrap trees
    if self.params.nboots:

        ## Tree object
        fulltre = ete3.Tree(self.trees.tree, format=0)
        fulltre.unroot()

        ## only grab as many boots as the last option said was max
        with open(self.trees.boots, 'r') as inboots:
            bb = [ete3.Tree(i.strip(), format=0) for i in inboots.readlines()]
            wboots = [fulltre] + bb[-self.params.nboots:]

        ## infer consensus tree and write to file
        wctre, wcounts = consensus_tree(wboots, names=names)
        self.trees.cons = os.path.join(self.dirs, self.name + ".cons")
        with open(self.trees.cons, 'w') as ocons:
            ocons.write(wctre.write(format=0))
    else:
        wctre = ete3.Tree(self.trees.tree, format=0)
        wctre.unroot()

    ## build stats file and write trees
    self.trees.nhx = os.path.join(self.dirs, self.name + ".nhx")
    with open(self.files.stats, 'w') as ostats:

        ## print Tetrad info
        #ostats.write(STATS_STRING.format(**self.stats))

        ## print bootstrap splits
        if self.params.nboots:
            ostats.write("## splits observed in {} trees\n".format(len(wboots)))
            for i, j in enumerate(self.samples):
                ostats.write("{:<3} {}\n".format(i, j))
            ostats.write("\n")
            for split, freq in wcounts:
                if split.count('1') > 1:
                    ostats.write("{}   {:.2f}\n".format(split, round(freq, 2)))
            ostats.write("\n")

    ## parallelized this function because it can be slogging
    lbview = ipyclient.load_balanced_view()

    ## store results in dicts
    qtots = {}
    qsamp = {}
    tots = sum(1 for i in wctre.iter_leaves())
    totn = set(wctre.get_leaf_names())

    ## iterate over node traversal.
    for node in wctre.traverse():
        ## this is slow, needs to look at every sampled quartet
        ## so we send it be processed on an engine
        qtots[node] = lbview.apply(_get_total, *(tots, node))
        qsamp[node] = lbview.apply(_get_sampled, *(self, totn, node))

    ## wait for jobs to finish
    ipyclient.wait()

    ## put results into tree
    for node in wctre.traverse():
        ## this is fast, just calcs n_choose_k
        total = qtots[node].result()
        sampled = qsamp[node].result()
        ## store the results to the tree
        node.add_feature("quartets_total", total)
        node.add_feature("quartets_sampled", sampled)
    features = ["quartets_total", "quartets_sampled"]

    ## return as NHX format with extra info
    with open(self.trees.nhx, 'w') as outtre:
        outtre.write(wctre.write(format=0, features=features))

def
compute_tree_stats
(
self
,
ipyclient
)
:
names
=
self
.
samples
if
self
.
params
.
nboots
:
fulltre
=
ete3
.
Tree
(
self
.
trees
.
tree
,
format
=
0
)
fulltre
.
unroot
(
)
with
open
(
self
.
trees
.
boots
,
'r'
)
as
inboots
:
bb
=
[
ete3
.
Tree
(
i
.
strip
(
)
,
format
=
0
)
for
i
in
inboots
.
readlines
(
)
]
wboots
=
[
fulltre
]
+
bb
[
self
.
params
.
nboots
:
]
wctre
,
wcounts
=
consensus_tree
(
wboots
,
names
=
names
)
self
.
trees
.
cons
=
os
.
path
.
join
(
self
.
dirs
,
self
.
name
+
".cons"
)
with
open
(
self
.
trees
.
cons
,
'w'
)
as
ocons
:
ocons
.
write
(
wctre
.
write
(
format
=
0
)
)
else
:
wctre
=
ete3
.
Tree
(
self
.
trees
.
tree
,
format
=
0
)
wctre
.
unroot
(
)
self
.
trees
.
nhx
=
os
.
path
.
join
(
self
.
dirs
,
self
.
name
+
".nhx"
)
with
open
(
self
.
files
.
stats
,
'w'
)
as
ostats
:
if
self
.
params
.
nboots
:
ostats
.
write
(
"## splits observed in {} trees\n"
.
format
(
len
(
wboots
)
)
)
for
i
,
j
in
enumerate
(
self
.
samples
)
:
ostats
.
write
(
"{:<3} {}\n"
.
format
(
i
,
j
)
)
ostats
.
write
(
"\n"
)
for
split
,
freq
in
wcounts
:
if
split
.
count
(
'1'
)
>
1
:
ostats
.
write
(
"{}   {:.2f}\n"
.
format
(
split
,
round
(
freq
,
2
)
)
)
ostats
.
write
(
"\n"
)
lbview
=
ipyclient
.
load_balanced_view
(
)
qtots
=
{
}
qsamp
=
{
}
tots
=
sum
(
1
for
i
in
wctre
.
iter_leaves
(
)
)
totn
=
set
(
wctre
.
get_leaf_names
(
)
)
for
node
in
wctre
.
traverse
(
)
:
qtots
[
node
]
=
lbview
.
apply
(
_get_total
,
*
(
tots
,
node
)
)
qsamp
[
node
]
=
lbview
.
apply
(
_get_sampled
,
*
(
self
,
totn
,
node
)
)
ipyclient
.
wait
(
)
for
node
in
wctre
.
traverse
(
)
:
total
=
qtots
[
node
]
.
result
(
)
sampled
=
qsamp
[
node
]
.
result
(
)
node
.
add_feature
(
"quartets_total"
,
total
)
node
.
add_feature
(
"quartets_sampled"
,
sampled
)
features
=
[
"quartets_total"
,
"quartets_sampled"
]
with
open
(
self
.
trees
.
nhx
,
'w'
)
as
outtre
:
outtre
.
write
(
wctre
.
write
(
format
=
0
,
features
=
features
)
)


def random_product(iter1, iter2):
    """ random sampler for equal_splits func"""
    pool1 = tuple(iter1)
    pool2 = tuple(iter2)
    ind1 = random.sample(pool1, 2)
    ind2 = random.sample(pool2, 2)
    return tuple(ind1+ind2)

def random_product ( iter1 , iter2 ) : pool1 = tuple ( iter1 ) pool2 = tuple ( iter2 ) ind1 = random . sample ( pool1 , 2 ) ind2 = random . sample ( pool2 , 2 ) return tuple ( ind1 + ind2 )


def n_choose_k(n, k):
    """ get the number of quartets as n-choose-k. This is used
    in equal splits to decide whether a split should be exhaustively sampled
    or randomly sampled. Edges near tips can be exhaustive while highly nested
    edges probably have too many quartets
    """
    return int(reduce(MUL, (Fraction(n-i, i+1) for i in range(k)), 1))

def n_choose_k ( n , k ) : return int ( reduce ( MUL , ( Fraction ( n - i , i + 1 ) for i in range ( k ) ) , 1 ) )

get the number of quartets as n - choose - k . This is used in equal splits to decide whether a split should be exhaustively sampled or randomly sampled . Edges near tips can be exhaustive while highly nested edges probably have too many quartets


def count_snps(mat):
    """
    get dstats from the count array and return as a float tuple
    """

    ## get [aabb, baba, abba, aaab]
    snps = np.zeros(4, dtype=np.uint32)

    ## get concordant (aabb) pis sites
    snps[0] = np.uint32(\
           mat[0, 5] + mat[0, 10] + mat[0, 15] + \
           mat[5, 0] + mat[5, 10] + mat[5, 15] + \
           mat[10, 0] + mat[10, 5] + mat[10, 15] + \
           mat[15, 0] + mat[15, 5] + mat[15, 10])

    ## get discordant (baba) sites
    for i in range(16):
        if i % 5:
            snps[1] += mat[i, i]

    ## get discordant (abba) sites
    snps[2] = mat[1, 4] + mat[2, 8] + mat[3, 12] +\
              mat[4, 1] + mat[6, 9] + mat[7, 13] +\
              mat[8, 2] + mat[9, 6] + mat[11, 14] +\
              mat[12, 3] + mat[13, 7] + mat[14, 11]

    ## get autapomorphy sites
    snps[3] = (mat.sum() - np.diag(mat).sum()) - snps[2]

    return snps


def chunk_to_matrices(narr, mapcol, nmask):
    """
    numba compiled code to get matrix fast.
    arr is a 4 x N seq matrix converted to np.int8
    I convert the numbers for ATGC into their respective index for the MAT
    matrix, and leave all others as high numbers, i.e., -==45, N==78.
    """

    ## get seq alignment and create an empty array for filling
    mats = np.zeros((3, 16, 16), dtype=np.uint32)

    ## replace ints with small ints that index their place in the
    ## 16x16. This no longer checks for big ints to exclude, so resolve=True
    ## is now the default, TODO.
    last_loc = -1
    for idx in xrange(mapcol.shape[0]):
        if not nmask[idx]:
            if not mapcol[idx] == last_loc:
                i = narr[:, idx]
                mats[0, (4*i[0])+i[1], (4*i[2])+i[3]] += 1
                last_loc = mapcol[idx]

    ## fill the alternates
    x = np.uint8(0)
    for y in np.array([0, 4, 8, 12], dtype=np.uint8):
        for z in np.array([0, 4, 8, 12], dtype=np.uint8):
            mats[1, y:y+np.uint8(4), z:z+np.uint8(4)] = mats[0, x].reshape(4, 4)
            mats[2, y:y+np.uint8(4), z:z+np.uint8(4)] = mats[0, x].reshape(4, 4).T
            x += np.uint8(1)

    return mats

def chunk_to_matrices ( narr , mapcol , nmask ) : mats = np . zeros ( ( 3 , 16 , 16 ) , dtype = np . uint32 ) last_loc = - 1 for idx in xrange ( mapcol . shape [ 0 ] ) : if not nmask [ idx ] : if not mapcol [ idx ] == last_loc : i = narr [ : , idx ] mats [ 0 , ( 4 * i [ 0 ] ) + i [ 1 ] , ( 4 * i [ 2 ] ) + i [ 3 ] ] += 1 last_loc = mapcol [ idx ] x = np . uint8 ( 0 ) for y in np . array ( [ 0 , 4 , 8 , 12 ] , dtype = np . uint8 ) : for z in np . array ( [ 0 , 4 , 8 , 12 ] , dtype = np . uint8 ) : mats [ 1 , y : y + np . uint8 ( 4 ) , z : z + np . uint8 ( 4 ) ] = mats [ 0 , x ] . reshape ( 4 , 4 ) mats [ 2 , y : y + np . uint8 ( 4 ) , z : z + np . uint8 ( 4 ) ] = mats [ 0 , x ] . reshape ( 4 , 4 ) . T x += np . uint8 ( 1 ) return mats

numba compiled code to get matrix fast . arr is a 4 x N seq matrix converted to np . int8 I convert the numbers for ATGC into their respective index for the MAT matrix and leave all others as high numbers i . e . - == 45 N == 78 .


def calculate(seqnon, mapcol, nmask, tests):
    """ groups together several numba compiled funcs """

    ## create empty matrices
    #LOGGER.info("tests[0] %s", tests[0])
    #LOGGER.info('seqnon[[tests[0]]] %s', seqnon[[tests[0]]])
    mats = chunk_to_matrices(seqnon, mapcol, nmask)

    ## empty svdscores for each arrangement of seqchunk
    svds = np.zeros((3, 16), dtype=np.float64)
    qscores = np.zeros(3, dtype=np.float64)
    ranks = np.zeros(3, dtype=np.float64)

    for test in range(3):
        ## get svd scores
        svds[test] = np.linalg.svd(mats[test].astype(np.float64))[1]
        ranks[test] = np.linalg.matrix_rank(mats[test].astype(np.float64))

    ## get minrank, or 11
    minrank = int(min(11, ranks.min()))
    for test in range(3):
        qscores[test] = np.sqrt(np.sum(svds[test, minrank:]**2))

    ## sort to find the best qorder
    best = np.where(qscores == qscores.min())[0]
    #best = qscores[qscores == qscores.min()][0]
    bidx = tests[best][0]
    qsnps = count_snps(mats[best][0])

    return bidx, qsnps

def calculate ( seqnon , mapcol , nmask , tests ) : mats = chunk_to_matrices ( seqnon , mapcol , nmask ) svds = np . zeros ( ( 3 , 16 ) , dtype = np . float64 ) qscores = np . zeros ( 3 , dtype = np . float64 ) ranks = np . zeros ( 3 , dtype = np . float64 ) for test in range ( 3 ) : svds [ test ] = np . linalg . svd ( mats [ test ] . astype ( np . float64 ) ) [ 1 ] ranks [ test ] = np . linalg . matrix_rank ( mats [ test ] . astype ( np . float64 ) ) minrank = int ( min ( 11 , ranks . min ( ) ) ) for test in range ( 3 ) : qscores [ test ] = np . sqrt ( np . sum ( svds [ test , minrank : ] ** 2 ) ) best = np . where ( qscores == qscores . min ( ) ) [ 0 ] bidx = tests [ best ] [ 0 ] qsnps = count_snps ( mats [ best ] [ 0 ] ) return bidx , qsnps


def nworker(data, smpchunk, tests):
    """ The workhorse function. Not numba. """

    ## tell engines to limit threads
    #numba.config.NUMBA_DEFAULT_NUM_THREADS = 1

    ## open the seqarray view, the modified array is in bootsarr
    with h5py.File(data.database.input, 'r') as io5:
        seqview = io5["bootsarr"][:]
        maparr = io5["bootsmap"][:]

    ## create an N-mask array of all seq cols (this isn't really too slow)
    nall_mask = seqview[:] == 78

    ## tried numba compiling everythign below here, but was not faster
    ## than making nmask w/ axis arg in numpy

    ## get the input arrays ready
    rquartets = np.zeros((smpchunk.shape[0], 4), dtype=np.uint16)
    rweights = None
    #rweights = np.ones(smpchunk.shape[0], dtype=np.float64)
    rdstats = np.zeros((smpchunk.shape[0], 4), dtype=np.uint32)

    #times = []
    ## fill arrays with results using numba funcs
    for idx in xrange(smpchunk.shape[0]):
        ## get seqchunk for 4 samples (4, ncols)
        sidx = smpchunk[idx]
        seqchunk = seqview[sidx]

        ## get N-containing columns in 4-array, and invariant sites.
        nmask = np.any(nall_mask[sidx], axis=0)
        nmask += np.all(seqchunk == seqchunk[0], axis=0)  ## <- do we need this?

        ## get matrices if there are any shared SNPs
        ## returns best-tree index, qscores, and qstats
        #bidx, qscores, qstats = calculate(seqchunk, maparr[:, 0], nmask, tests)
        bidx, qstats = calculate(seqchunk, maparr[:, 0], nmask, tests)

        ## get weights from the three scores sorted.
        ## Only save to file if the quartet has information
        rdstats[idx] = qstats
        rquartets[idx] = smpchunk[idx][bidx]

    return rquartets, rweights, rdstats

def nworker ( data , smpchunk , tests ) : with h5py . File ( data . database . input , 'r' ) as io5 : seqview = io5 [ "bootsarr" ] [ : ] maparr = io5 [ "bootsmap" ] [ : ] nall_mask = seqview [ : ] == 78 rquartets = np . zeros ( ( smpchunk . shape [ 0 ] , 4 ) , dtype = np . uint16 ) rweights = None rdstats = np . zeros ( ( smpchunk . shape [ 0 ] , 4 ) , dtype = np . uint32 ) for idx in xrange ( smpchunk . shape [ 0 ] ) : sidx = smpchunk [ idx ] seqchunk = seqview [ sidx ] nmask = np . any ( nall_mask [ sidx ] , axis = 0 ) nmask += np . all ( seqchunk == seqchunk [ 0 ] , axis = 0 ) bidx , qstats = calculate ( seqchunk , maparr [ : , 0 ] , nmask , tests ) rdstats [ idx ] = qstats rquartets [ idx ] = smpchunk [ idx ] [ bidx ] return rquartets , rweights , rdstats


def shuffle_cols(seqarr, newarr, cols):
    """ used in bootstrap resampling without a map file """
    for idx in xrange(cols.shape[0]):
        newarr[:, idx] = seqarr[:, cols[idx]]
    return newarr

def shuffle_cols ( seqarr , newarr , cols ) : for idx in xrange ( cols . shape [ 0 ] ) : newarr [ : , idx ] = seqarr [ : , cols [ idx ] ] return newarr


def resolve_ambigs(tmpseq):
    """ returns a seq array with 'RSKYWM' randomly replaced with resolved bases"""
    ## iterate over the bases 'RSKWYM': [82, 83, 75, 87, 89, 77]
    for ambig in np.uint8([82, 83, 75, 87, 89, 77]):
        ## get all site in this ambig
        idx, idy = np.where(tmpseq == ambig)
        ## get the two resolutions of the ambig
        res1, res2 = AMBIGS[ambig.view("S1")]
        ## randomly sample half those sites
        halfmask = np.random.choice([True, False], idx.shape[0])
        ## replace ambig bases with their resolutions
        for i in xrange(halfmask.shape[0]):
            if halfmask[i]:
                tmpseq[idx[i], idy[i]] = np.array(res1).view(np.uint8)
            else:
                tmpseq[idx[i], idy[i]] = np.array(res2).view(np.uint8)
    return tmpseq

def resolve_ambigs ( tmpseq ) : for ambig in np . uint8 ( [ 82 , 83 , 75 , 87 , 89 , 77 ] ) : idx , idy = np . where ( tmpseq == ambig ) res1 , res2 = AMBIGS [ ambig . view ( "S1" ) ] halfmask = np . random . choice ( [ True , False ] , idx . shape [ 0 ] ) for i in xrange ( halfmask . shape [ 0 ] ) : if halfmask [ i ] : tmpseq [ idx [ i ] , idy [ i ] ] = np . array ( res1 ) . view ( np . uint8 ) else : tmpseq [ idx [ i ] , idy [ i ] ] = np . array ( res2 ) . view ( np . uint8 ) return tmpseq

returns a seq array with RSKYWM randomly replaced with resolved bases


def get_spans(maparr, spans):
    """ get span distance for each locus in original seqarray """
    ## start at 0, finds change at 1-index of map file
    bidx = 1
    spans = np.zeros((maparr[-1, 0], 2), np.uint64)
    ## read through marr and record when locus id changes
    for idx in xrange(1, maparr.shape[0]):
        cur = maparr[idx, 0]
        if cur != bidx:
            idy = idx + 1
            spans[cur-2, 1] = idx
            spans[cur-1, 0] = idx
            bidx = cur
    spans[-1, 1] = maparr[-1, -1]
    return spans

def get_spans ( maparr , spans ) : bidx = 1 spans = np . zeros ( ( maparr [ - 1 , 0 ] , 2 ) , np . uint64 ) for idx in xrange ( 1 , maparr . shape [ 0 ] ) : cur = maparr [ idx , 0 ] if cur != bidx : idy = idx + 1 spans [ cur - 2 , 1 ] = idx spans [ cur - 1 , 0 ] = idx bidx = cur spans [ - 1 , 1 ] = maparr [ - 1 , - 1 ] return spans


def get_shape(spans, loci):
    """ get shape of new bootstrap resampled locus array """
    width = 0
    for idx in xrange(loci.shape[0]):
        width += spans[loci[idx], 1] - spans[loci[idx], 0]
    return width

def get_shape ( spans , loci ) : width = 0 for idx in xrange ( loci . shape [ 0 ] ) : width += spans [ loci [ idx ] , 1 ] - spans [ loci [ idx ] , 0 ] return width


def fill_boot(seqarr, newboot, newmap, spans, loci):
    """ fills the new bootstrap resampled array """
    ## column index
    cidx = 0

    ## resample each locus
    for i in xrange(loci.shape[0]):

        ## grab a random locus's columns
        x1 = spans[loci[i]][0]
        x2 = spans[loci[i]][1]
        cols = seqarr[:, x1:x2]

        ## randomize columns within colsq
        cord = np.random.choice(cols.shape[1], cols.shape[1], replace=False)
        rcols = cols[:, cord]

        ## fill bootarr with n columns from seqarr
        ## the required length was already measured
        newboot[:, cidx:cidx+cols.shape[1]] = rcols

        ## fill bootmap with new map info
        newmap[cidx: cidx+cols.shape[1], 0] = i+1

        ## advance column index
        cidx += cols.shape[1]

    ## return the concatenated cols
    return newboot, newmap

def fill_boot ( seqarr , newboot , newmap , spans , loci ) : cidx = 0 for i in xrange ( loci . shape [ 0 ] ) : x1 = spans [ loci [ i ] ] [ 0 ] x2 = spans [ loci [ i ] ] [ 1 ] cols = seqarr [ : , x1 : x2 ] cord = np . random . choice ( cols . shape [ 1 ] , cols . shape [ 1 ] , replace = False ) rcols = cols [ : , cord ] newboot [ : , cidx : cidx + cols . shape [ 1 ] ] = rcols newmap [ cidx : cidx + cols . shape [ 1 ] , 0 ] = i + 1 cidx += cols . shape [ 1 ] return newboot , newmap


def _byteify(data, ignore_dicts=False):
    """
    converts unicode to utf-8 when reading in json files
    """
    if isinstance(data, unicode):
        return data.encode("utf-8")

    if isinstance(data, list):
        return [_byteify(item, ignore_dicts=True) for item in data]

    if isinstance(data, dict) and not ignore_dicts:
        return {
            _byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True)
            for key, value in data.iteritems()
        }
    return data

def _byteify ( data , ignore_dicts = False ) : if isinstance ( data , unicode ) : return data . encode ( "utf-8" ) if isinstance ( data , list ) : return [ _byteify ( item , ignore_dicts = True ) for item in data ] if isinstance ( data , dict ) and not ignore_dicts : return { _byteify ( key , ignore_dicts = True ) : _byteify ( value , ignore_dicts = True ) for key , value in data . iteritems ( ) } return data

converts unicode to utf - 8 when reading in json files


def _parse_names(self):
        """ parse sample names from the sequence file"""
        self.samples = []
        with iter(open(self.files.data, 'r')) as infile:
            infile.next().strip().split()
            while 1:
                try:
                    self.samples.append(infile.next().split()[0])
                except StopIteration:
                    break

def _parse_names ( self ) : self . samples = [ ] with iter ( open ( self . files . data , 'r' ) ) as infile : infile . next ( ) . strip ( ) . split ( ) while 1 : try : self . samples . append ( infile . next ( ) . split ( ) [ 0 ] ) except StopIteration : break


def _run_qmc(self, boot):
        """ runs quartet max-cut on a quartets file """

        ## convert to txt file for wQMC
        self._tmp = os.path.join(self.dirs, ".tmpwtre")
        cmd = [ip.bins.qmc, "qrtt="+self.files.qdump, "otre="+self._tmp]

        ## run them
        proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
        res = proc.communicate()
        if proc.returncode:
            #LOGGER.error("Error in QMC: \n({}).".format(res))
            LOGGER.error(res)
            raise IPyradWarningExit(res[1])

        ## read in the tmp files since qmc does not pipe
        with open(self._tmp) as intree:
            ## convert int names back to str names renamer returns a newick str
            #tmp = toytree.tree(intree.read().strip())
            tmp = ete3.Tree(intree.read().strip())
            tmpwtre = self._renamer(tmp)#.tree)

        ## save the tree
        if boot:
            self.trees.boots = os.path.join(self.dirs, self.name+".boots")
            with open(self.trees.boots, 'a') as outboot:
                outboot.write(tmpwtre+"\n")
        else:
            self.trees.tree = os.path.join(self.dirs, self.name+".tree")
            with open(self.trees.tree, 'w') as outtree:
                outtree.write(tmpwtre)

        ## save JSON file checkpoint
        self._save()

def _run_qmc ( self , boot ) : self . _tmp = os . path . join ( self . dirs , ".tmpwtre" ) cmd = [ ip . bins . qmc , "qrtt=" + self . files . qdump , "otre=" + self . _tmp ] proc = subprocess . Popen ( cmd , stderr = subprocess . STDOUT , stdout = subprocess . PIPE ) res = proc . communicate ( ) if proc . returncode : LOGGER . error ( res ) raise IPyradWarningExit ( res [ 1 ] ) with open ( self . _tmp ) as intree : tmp = ete3 . Tree ( intree . read ( ) . strip ( ) ) tmpwtre = self . _renamer ( tmp ) if boot : self . trees . boots = os . path . join ( self . dirs , self . name + ".boots" ) with open ( self . trees . boots , 'a' ) as outboot : outboot . write ( tmpwtre + "\n" ) else : self . trees . tree = os . path . join ( self . dirs , self . name + ".tree" ) with open ( self . trees . tree , 'w' ) as outtree : outtree . write ( tmpwtre ) self . _save ( )

runs quartet max - cut on a quartets file


def _dump_qmc(self):
        """
        Makes a reduced array that excludes quartets with no information and
        prints the quartets and weights to a file formatted for wQMC
        """
        ## open the h5 database
        io5 = h5py.File(self.database.output, 'r')

        ## create an output file for writing
        self.files.qdump = os.path.join(self.dirs, self.name+".quartets.txt")
        LOGGER.info("qdump file %s", self.files.qdump)
        outfile = open(self.files.qdump, 'w')

        ## todo: should pull quarts order in randomly? or doesn't matter?
        for idx in xrange(0, self.params.nquartets, self._chunksize):
            ## get mask of zero weight quartets
            #mask = io5["weights"][idx:idx+self.chunksize] != 0
            #weight = io5["weights"][idx:idx+self.chunksize][mask]
            #LOGGER.info("exluded = %s, mask shape %s",
            #            self._chunksize - mask.shape[0], mask.shape)
            #LOGGER.info('q shape %s', io5["quartets"][idx:idx+self._chunksize].shape)
            masked_quartets = io5["quartets"][idx:idx+self._chunksize, :]#[mask, :]
            quarts = [list(j) for j in masked_quartets]

            ## format and print
            #chunk = ["{},{}|{},{}:{}".format(*i+[j]) for i, j \
            #                                        in zip(quarts, weight)]
            chunk = ["{},{}|{},{}".format(*i) for i in quarts]
            outfile.write("\n".join(chunk)+"\n")

        ## close output file and h5 database
        outfile.close()
        io5.close()

def
_dump_qmc
(
self
)
:
io5
=
h5py
.
File
(
self
.
database
.
output
,
'r'
)
self
.
files
.
qdump
=
os
.
path
.
join
(
self
.
dirs
,
self
.
name
+
".quartets.txt"
)
LOGGER
.
info
(
"qdump file %s"
,
self
.
files
.
qdump
)
outfile
=
open
(
self
.
files
.
qdump
,
'w'
)
for
idx
in
xrange
(
0
,
self
.
params
.
nquartets
,
self
.
_chunksize
)
:
masked_quartets
=
io5
[
"quartets"
]
[
idx
:
idx
+
self
.
_chunksize
,
:
]
quarts
=
[
list
(
j
)
for
j
in
masked_quartets
]
chunk
=
[
"{},{}|{},{}"
.
format
(
*
i
)
for
i
in
quarts
]
outfile
.
write
(
"\n"
.
join
(
chunk
)
+
"\n"
)
outfile
.
close
(
)
io5
.
close
(
)


def _renamer(self, tre):
        """ renames newick from numbers to sample names"""
        ## get the tre with numbered tree tip labels
        names = tre.get_leaves()

        ## replace numbered names with snames
        for name in names:
            name.name = self.samples[int(name.name)]

        ## return with only topology and leaf labels
        return tre.write(format=9)

def _renamer ( self , tre ) : names = tre . get_leaves ( ) for name in names : name . name = self . samples [ int ( name . name ) ] return tre . write ( format = 9 )


def _finalize_stats(self, ipyclient):
        """ write final tree files """

        ## print stats file location:
        #print(STATSOUT.format(opr(self.files.stats)))

        ## print finished tree information ---------------------
        print(FINALTREES.format(opr(self.trees.tree)))

        ## print bootstrap information --------------------------
        if self.params.nboots:
            ## get consensus, map values to tree edges, record stats file
            self._compute_tree_stats(ipyclient)
            ## print bootstrap info
            print(BOOTTREES.format(opr(self.trees.cons), opr(self.trees.boots)))

        ## print the ASCII tree only if its small
        if len(self.samples) < 20:
            if self.params.nboots:
                wctre = ete3.Tree(self.trees.cons, format=0)
                wctre.ladderize()
                print(wctre.get_ascii(show_internal=True,
                                      attributes=["dist", "name"]))
                print("")
            else:
                qtre = ete3.Tree(self.trees.tree, format=0)
                qtre.ladderize()
                #qtre = toytree.tree(self.trees.tree, format=0)
                #qtre.tree.unroot()
                print(qtre.get_ascii())
                print("")

        ## print PDF filename & tips -----------------------------
        docslink = "https://toytree.readthedocs.io/"
        citelink = "https://ipyrad.readthedocs.io/tetrad.html"
        print(LINKS.format(docslink, citelink))


def _save(self):
        """ save a JSON file representation of Tetrad Class for checkpoint"""

        ## save each attribute as dict
        fulldict = copy.deepcopy(self.__dict__)
        for i, j in fulldict.items():
            if isinstance(j, Params):
                fulldict[i] = j.__dict__
        fulldumps = json.dumps(fulldict,
                               sort_keys=False,
                               indent=4,
                               separators=(",", ":"),
                               )

        ## save to file, make dir if it wasn't made earlier
        assemblypath = os.path.join(self.dirs, self.name+".tet.json")
        if not os.path.exists(self.dirs):
            os.mkdir(self.dirs)

        ## protect save from interruption
        done = 0
        while not done:
            try:
                with open(assemblypath, 'w') as jout:
                    jout.write(fulldumps)
                done = 1
            except (KeyboardInterrupt, SystemExit):
                print('.')
                continue

def _save ( self ) : fulldict = copy . deepcopy ( self . __dict__ ) for i , j in fulldict . items ( ) : if isinstance ( j , Params ) : fulldict [ i ] = j . __dict__ fulldumps = json . dumps ( fulldict , sort_keys = False , indent = 4 , separators = ( "," , ":" ) , ) assemblypath = os . path . join ( self . dirs , self . name + ".tet.json" ) if not os . path . exists ( self . dirs ) : os . mkdir ( self . dirs ) done = 0 while not done : try : with open ( assemblypath , 'w' ) as jout : jout . write ( fulldumps ) done = 1 except ( KeyboardInterrupt , SystemExit ) : print ( '.' ) continue


def _insert_to_array(self, start, results):
        """ inputs results from workers into hdf4 array """
        qrts, wgts, qsts = results
        #qrts, wgts = results
        #print(qrts)

        with h5py.File(self.database.output, 'r+') as out:
            chunk = self._chunksize
            out['quartets'][start:start+chunk] = qrts
            ##out['weights'][start:start+chunk] = wgts

            ## entered as 0-indexed !
            if self.checkpoint.boots:
                key = "qboots/b{}".format(self.checkpoint.boots-1)
                out[key][start:start+chunk] = qsts
            else:
                out["qstats"][start:start+chunk] = qsts

def _insert_to_array ( self , start , results ) : qrts , wgts , qsts = results with h5py . File ( self . database . output , 'r+' ) as out : chunk = self . _chunksize out [ 'quartets' ] [ start : start + chunk ] = qrts if self . checkpoint . boots : key = "qboots/b{}" . format ( self . checkpoint . boots - 1 ) out [ key ] [ start : start + chunk ] = qsts else : out [ "qstats" ] [ start : start + chunk ] = qsts


def select_samples(dbsamples, samples, pidx=None):
    """
    Get the row index of samples that are included. If samples are in the
    'excluded' they were already filtered out of 'samples' during _get_samples.
    """
    ## get index from dbsamples
    samples = [i.name for i in samples]
    if pidx:
        sidx = [list(dbsamples[pidx]).index(i) for i in samples]
    else:
        sidx = [list(dbsamples).index(i) for i in samples]
    sidx.sort()
    return sidx

def select_samples ( dbsamples , samples , pidx = None ) : samples = [ i . name for i in samples ] if pidx : sidx = [ list ( dbsamples [ pidx ] ) . index ( i ) for i in samples ] else : sidx = [ list ( dbsamples ) . index ( i ) for i in samples ] sidx . sort ( ) return sidx

Get the row index of samples that are included . If samples are in the excluded they were already filtered out of samples during _get_samples .


def padnames(names):
    """ pads names for loci output """

    ## get longest name
    longname_len = max(len(i) for i in names)
    ## Padding distance between name and seq.
    padding = 5
    ## add pad to names
    pnames = [name + " " * (longname_len - len(name)+ padding) \
              for name in names]
    snppad = "//" + " " * (longname_len - 2 + padding)
    return np.array(pnames), snppad

def
padnames
(
names
)
:
longname_len
=
max
(
len
(
i
)
for
i
in
names
)
padding
=
5
pnames
=
[
name
+
" "
*
(
longname_len
len
(
name
)
+
padding
)
for
name
in
names
]
snppad
=
"//"
+
" "
*
(
longname_len
2
+
padding
)
return
np
.
array
(
pnames
)
,
snppad


def locichunk(args):
    """
    Function from make_loci to apply to chunks. smask is sample mask.
    """
    ## parse args
    data, optim, pnames, snppad, smask, start, samplecov, locuscov, upper = args

    ## this slice
    hslice = [start, start+optim]

    ## get filter db info
    co5 = h5py.File(data.database, 'r')
    afilt = co5["filters"][hslice[0]:hslice[1], ]
    aedge = co5["edges"][hslice[0]:hslice[1], ]
    asnps = co5["snps"][hslice[0]:hslice[1], ]

    ## get seqs db
    io5 = h5py.File(data.clust_database, 'r')
    if upper:
        aseqs = np.char.upper(io5["seqs"][hslice[0]:hslice[1], ])
    else:
        aseqs = io5["seqs"][hslice[0]:hslice[1], ]

    ## which loci passed all filters
    keep = np.where(np.sum(afilt, axis=1) == 0)[0]
    store = []

    ## write loci that passed after trimming edges, then write snp string
    for iloc in keep:
        edg = aedge[iloc]
        #LOGGER.info("!!!!!! iloc edg %s, %s", iloc, edg)
        args = [iloc, pnames, snppad, edg, aseqs, asnps, smask, samplecov, locuscov, start]
        if edg[4]:
            outstr, samplecov, locuscov = enter_pairs(*args)
            store.append(outstr)
        else:
            outstr, samplecov, locuscov = enter_singles(*args)
            store.append(outstr)

    ## write to file and clear store
    tmpo = os.path.join(data.dirs.outfiles, data.name+".loci.{}".format(start))
    with open(tmpo, 'w') as tmpout:
        tmpout.write("\n".join(store) + "\n")

    ## close handles
    io5.close()
    co5.close()

    ## return sample counter
    return samplecov, locuscov, start

def locichunk ( args ) : data , optim , pnames , snppad , smask , start , samplecov , locuscov , upper = args hslice = [ start , start + optim ] co5 = h5py . File ( data . database , 'r' ) afilt = co5 [ "filters" ] [ hslice [ 0 ] : hslice [ 1 ] , ] aedge = co5 [ "edges" ] [ hslice [ 0 ] : hslice [ 1 ] , ] asnps = co5 [ "snps" ] [ hslice [ 0 ] : hslice [ 1 ] , ] io5 = h5py . File ( data . clust_database , 'r' ) if upper : aseqs = np . char . upper ( io5 [ "seqs" ] [ hslice [ 0 ] : hslice [ 1 ] , ] ) else : aseqs = io5 [ "seqs" ] [ hslice [ 0 ] : hslice [ 1 ] , ] keep = np . where ( np . sum ( afilt , axis = 1 ) == 0 ) [ 0 ] store = [ ] for iloc in keep : edg = aedge [ iloc ] args = [ iloc , pnames , snppad , edg , aseqs , asnps , smask , samplecov , locuscov , start ] if edg [ 4 ] : outstr , samplecov , locuscov = enter_pairs ( * args ) store . append ( outstr ) else : outstr , samplecov , locuscov = enter_singles ( * args ) store . append ( outstr ) tmpo = os . path . join ( data . dirs . outfiles , data . name + ".loci.{}" . format ( start ) ) with open ( tmpo , 'w' ) as tmpout : tmpout . write ( "\n" . join ( store ) + "\n" ) io5 . close ( ) co5 . close ( ) return samplecov , locuscov , start

Function from make_loci to apply to chunks . smask is sample mask .


def enter_pairs(iloc, pnames, snppad, edg, aseqs, asnps, smask, samplecov, locuscov, start):
    """ enters funcs for pairs """

    ## snps was created using only the selected samples.
    LOGGER.info("edges in enter_pairs %s", edg)
    seq1 = aseqs[iloc, :, edg[0]:edg[1]+1]
    snp1 = asnps[iloc, edg[0]:edg[1]+1, ]

    ## the 2nd read edges are +5 for the spacer
    seq2 = aseqs[iloc, :, edg[2]:edg[3]+1]
    snp2 = asnps[iloc, edg[2]:edg[3]+1, ]

    ## remove rows with all Ns, seq has only selected samples
    nalln = np.all(seq1 == "N", axis=1)

    ## make mask of removed rows and excluded samples. Use the inverse
    ## of this to save the coverage for samples
    nsidx = nalln + smask
    LOGGER.info("nsidx %s, nalln %s, smask %s", nsidx, nalln, smask)
    samplecov = samplecov + np.invert(nsidx).astype(np.int32)
    LOGGER.info("samplecov %s", samplecov)
    idx = np.sum(np.invert(nsidx).astype(np.int32))
    LOGGER.info("idx %s", idx)
    locuscov[idx] += 1

    ## select the remaining names in order
    seq1 = seq1[~nsidx, ]
    seq2 = seq2[~nsidx, ]
    names = pnames[~nsidx]

    ## save string for printing, excluding names not in samples
    outstr = "\n".join(\
        [name + s1.tostring()+"nnnn"+s2.tostring() for name, s1, s2 in \
         zip(names, seq1, seq2)])

    #LOGGER.info("s1 %s", s1.tostring())
    #LOGGER.info("s2 %s", s2.tostring())

    ## get snp string and add to store
    snpstring1 = ["-" if snp1[i, 0] else \
                 "*" if snp1[i, 1] else \
                 " " for i in range(len(snp1))]
    snpstring2 = ["-" if snp2[i, 0] else \
                 "*" if snp2[i, 1] else \
                 " " for i in range(len(snp2))]

    #npis = str(snpstring1+snpstring2).count("*")
    #nvars = str(snpstring1+snpstring2).count("-") + npis
    outstr += "\n" + snppad + "".join(snpstring1)+\
              "    "+"".join(snpstring2)+"|{}|".format(iloc+start)
              #"|LOCID={},DBID={},NVAR={},NPIS={}|"\
              #.format(1+iloc+start, iloc, nvars, npis)

    return outstr, samplecov, locuscov

def
enter_pairs
(
iloc
,
pnames
,
snppad
,
edg
,
aseqs
,
asnps
,
smask
,
samplecov
,
locuscov
,
start
)
:
LOGGER
.
info
(
"edges in enter_pairs %s"
,
edg
)
seq1
=
aseqs
[
iloc
,
:
,
edg
[
0
]
:
edg
[
1
]
+
1
]
snp1
=
asnps
[
iloc
,
edg
[
0
]
:
edg
[
1
]
+
1
,
]
seq2
=
aseqs
[
iloc
,
:
,
edg
[
2
]
:
edg
[
3
]
+
1
]
snp2
=
asnps
[
iloc
,
edg
[
2
]
:
edg
[
3
]
+
1
,
]
nalln
=
np
.
all
(
seq1
==
"N"
,
axis
=
1
)
nsidx
=
nalln
+
smask
LOGGER
.
info
(
"nsidx %s, nalln %s, smask %s"
,
nsidx
,
nalln
,
smask
)
samplecov
=
samplecov
+
np
.
invert
(
nsidx
)
.
astype
(
np
.
int32
)
LOGGER
.
info
(
"samplecov %s"
,
samplecov
)
idx
=
np
.
sum
(
np
.
invert
(
nsidx
)
.
astype
(
np
.
int32
)
)
LOGGER
.
info
(
"idx %s"
,
idx
)
locuscov
[
idx
]
+=
1
seq1
=
seq1
[
~
nsidx
,
]
seq2
=
seq2
[
~
nsidx
,
]
names
=
pnames
[
~
nsidx
]
outstr
=
"\n"
.
join
(
[
name
+
s1
.
tostring
(
)
+
"nnnn"
+
s2
.
tostring
(
)
for
name
,
s1
,
s2
in
zip
(
names
,
seq1
,
seq2
)
]
)
snpstring1
=
[
"-"
if
snp1
[
i
,
0
]
else
"*"
if
snp1
[
i
,
1
]
else
" "
for
i
in
range
(
len
(
snp1
)
)
]
snpstring2
=
[
"-"
if
snp2
[
i
,
0
]
else
"*"
if
snp2
[
i
,
1
]
else
" "
for
i
in
range
(
len
(
snp2
)
)
]
outstr
+=
"\n"
+
snppad
+
""
.
join
(
snpstring1
)
+
"    "
+
""
.
join
(
snpstring2
)
+
"|{}|"
.
format
(
iloc
+
start
)
return
outstr
,
samplecov
,
locuscov


def enter_singles(iloc, pnames, snppad, edg, aseqs, asnps, smask, samplecov, locuscov, start):
    """ enter funcs for SE or merged data """

    ## grab all seqs between edges
    seq = aseqs[iloc, :, edg[0]:edg[1]+1]
    ## snps was created using only the selected samples, and is edge masked.
    ## The mask is for counting snps quickly, but trimming is still needed here
    ## to make the snps line up with the seqs in the snp string.
    snp = asnps[iloc, edg[0]:edg[1]+1, ]

    ## remove rows with all Ns, seq has only selected samples
    nalln = np.all(seq == "N", axis=1)

    ## make mask of removed rows and excluded samples. Use the inverse
    ## of this to save the coverage for samples
    nsidx = nalln + smask
    samplecov = samplecov + np.invert(nsidx).astype(np.int32)
    idx = np.sum(np.invert(nsidx).astype(np.int32))
    locuscov[idx] += 1

    ## select the remaining names in order
    seq = seq[~nsidx, ]
    names = pnames[~nsidx]

    ## save string for printing, excluding names not in samples
    outstr = "\n".join(\
        [name + s.tostring() for name, s in zip(names, seq)])

    ## get snp string and add to store
    snpstring = ["-" if snp[i, 0] else \
                 "*" if snp[i, 1] else \
                 " " for i in range(len(snp))]
    outstr += "\n" + snppad + "".join(snpstring) + "|{}|".format(iloc+start)
    #LOGGER.info("outstr %s", outstr)
    return outstr, samplecov, locuscov

def
enter_singles
(
iloc
,
pnames
,
snppad
,
edg
,
aseqs
,
asnps
,
smask
,
samplecov
,
locuscov
,
start
)
:
seq
=
aseqs
[
iloc
,
:
,
edg
[
0
]
:
edg
[
1
]
+
1
]
snp
=
asnps
[
iloc
,
edg
[
0
]
:
edg
[
1
]
+
1
,
]
nalln
=
np
.
all
(
seq
==
"N"
,
axis
=
1
)
nsidx
=
nalln
+
smask
samplecov
=
samplecov
+
np
.
invert
(
nsidx
)
.
astype
(
np
.
int32
)
idx
=
np
.
sum
(
np
.
invert
(
nsidx
)
.
astype
(
np
.
int32
)
)
locuscov
[
idx
]
+=
1
seq
=
seq
[
~
nsidx
,
]
names
=
pnames
[
~
nsidx
]
outstr
=
"\n"
.
join
(
[
name
+
s
.
tostring
(
)
for
name
,
s
in
zip
(
names
,
seq
)
]
)
snpstring
=
[
"-"
if
snp
[
i
,
0
]
else
"*"
if
snp
[
i
,
1
]
else
" "
for
i
in
range
(
len
(
snp
)
)
]
outstr
+=
"\n"
+
snppad
+
""
.
join
(
snpstring
)
+
"|{}|"
.
format
(
iloc
+
start
)
return
outstr
,
samplecov
,
locuscov


def init_arrays(data):
    """
    Create database file for storing final filtered snps data as hdf5 array.
    Copies splits and duplicates info from clust_database to database.
    """

    ## get stats from step6 h5 and create new h5
    co5 = h5py.File(data.clust_database, 'r')
    io5 = h5py.File(data.database, 'w')

    ## get maxlen and chunk len
    maxlen = data._hackersonly["max_fragment_length"] + 20
    chunks = co5["seqs"].attrs["chunksize"][0]
    nloci = co5["seqs"].shape[0]

    ## make array for snp string, 2 cols, - and *
    snps = io5.create_dataset("snps", (nloci, maxlen, 2),
                              dtype=np.bool,
                              chunks=(chunks, maxlen, 2),
                              compression='gzip')
    snps.attrs["chunksize"] = chunks
    snps.attrs["names"] = ["-", "*"]

    ## array for filters that will be applied in step7
    filters = io5.create_dataset("filters", (nloci, 6), dtype=np.bool)
    filters.attrs["filters"] = ["duplicates", "max_indels",
                                "max_snps", "max_shared_hets",
                                "min_samps", "max_alleles"]

    ## array for edgetrimming
    edges = io5.create_dataset("edges", (nloci, 5),
                               dtype=np.uint16,
                               chunks=(chunks, 5),
                               compression="gzip")
    edges.attrs["chunksize"] = chunks
    edges.attrs["names"] = ["R1_L", "R1_R", "R2_L", "R2_R", "sep"]

    ## xfer data from clustdb to finaldb
    edges[:, 4] = co5["splits"][:]
    filters[:, 0] = co5["duplicates"][:]

    ## close h5s
    io5.close()
    co5.close()

def init_arrays ( data ) : co5 = h5py . File ( data . clust_database , 'r' ) io5 = h5py . File ( data . database , 'w' ) maxlen = data . _hackersonly [ "max_fragment_length" ] + 20 chunks = co5 [ "seqs" ] . attrs [ "chunksize" ] [ 0 ] nloci = co5 [ "seqs" ] . shape [ 0 ] snps = io5 . create_dataset ( "snps" , ( nloci , maxlen , 2 ) , dtype = np . bool , chunks = ( chunks , maxlen , 2 ) , compression = 'gzip' ) snps . attrs [ "chunksize" ] = chunks snps . attrs [ "names" ] = [ "-" , "*" ] filters = io5 . create_dataset ( "filters" , ( nloci , 6 ) , dtype = np . bool ) filters . attrs [ "filters" ] = [ "duplicates" , "max_indels" , "max_snps" , "max_shared_hets" , "min_samps" , "max_alleles" ] edges = io5 . create_dataset ( "edges" , ( nloci , 5 ) , dtype = np . uint16 , chunks = ( chunks , 5 ) , compression = "gzip" ) edges . attrs [ "chunksize" ] = chunks edges . attrs [ "names" ] = [ "R1_L" , "R1_R" , "R2_L" , "R2_R" , "sep" ] edges [ : , 4 ] = co5 [ "splits" ] [ : ] filters [ : , 0 ] = co5 [ "duplicates" ] [ : ] io5 . close ( ) co5 . close ( )

Create database file for storing final filtered snps data as hdf5 array . Copies splits and duplicates info from clust_database to database .


def snpcount_numba(superints, snpsarr):
    """
    Used to count the number of unique bases in a site for snpstring.
    """
    ## iterate over all loci
    for iloc in xrange(superints.shape[0]):
        for site in xrange(superints.shape[2]):

            ## make new array
            catg = np.zeros(4, dtype=np.int16)

            ## a list for only catgs
            ncol = superints[iloc, :, site]
            for idx in range(ncol.shape[0]):
                if ncol[idx] == 67: #C
                    catg[0] += 1
                elif ncol[idx] == 65: #A
                    catg[1] += 1
                elif ncol[idx] == 84: #T
                    catg[2] += 1
                elif ncol[idx] == 71: #G
                    catg[3] += 1
                elif ncol[idx] == 82: #R
                    catg[1] += 1        #A
                    catg[3] += 1        #G
                elif ncol[idx] == 75: #K
                    catg[2] += 1        #T
                    catg[3] += 1        #G
                elif ncol[idx] == 83: #S
                    catg[0] += 1        #C
                    catg[3] += 1        #G
                elif ncol[idx] == 89: #Y
                    catg[0] += 1        #C
                    catg[2] += 1        #T
                elif ncol[idx] == 87: #W
                    catg[1] += 1        #A
                    catg[2] += 1        #T
                elif ncol[idx] == 77: #M
                    catg[0] += 1        #C
                    catg[1] += 1        #A

            ## get second most common site
            catg.sort()
            ## if invariant e.g., [0, 0, 0, 9], then nothing (" ")
            if not catg[2]:
                pass
            else:
                if catg[2] > 1:
                    snpsarr[iloc, site, 1] = True
                else:
                    snpsarr[iloc, site, 0] = True
    return snpsarr

def snpcount_numba ( superints , snpsarr ) : for iloc in xrange ( superints . shape [ 0 ] ) : for site in xrange ( superints . shape [ 2 ] ) : catg = np . zeros ( 4 , dtype = np . int16 ) ncol = superints [ iloc , : , site ] for idx in range ( ncol . shape [ 0 ] ) : if ncol [ idx ] == 67 : catg [ 0 ] += 1 elif ncol [ idx ] == 65 : catg [ 1 ] += 1 elif ncol [ idx ] == 84 : catg [ 2 ] += 1 elif ncol [ idx ] == 71 : catg [ 3 ] += 1 elif ncol [ idx ] == 82 : catg [ 1 ] += 1 catg [ 3 ] += 1 elif ncol [ idx ] == 75 : catg [ 2 ] += 1 catg [ 3 ] += 1 elif ncol [ idx ] == 83 : catg [ 0 ] += 1 catg [ 3 ] += 1 elif ncol [ idx ] == 89 : catg [ 0 ] += 1 catg [ 2 ] += 1 elif ncol [ idx ] == 87 : catg [ 1 ] += 1 catg [ 2 ] += 1 elif ncol [ idx ] == 77 : catg [ 0 ] += 1 catg [ 1 ] += 1 catg . sort ( ) if not catg [ 2 ] : pass else : if catg [ 2 ] > 1 : snpsarr [ iloc , site , 1 ] = True else : snpsarr [ iloc , site , 0 ] = True return snpsarr

Used to count the number of unique bases in a site for snpstring .


def maxind_numba(block):
    """ filter for indels """
    ## remove terminal edges
    inds = 0
    for row in xrange(block.shape[0]):
        where = np.where(block[row] != 45)[0]
        if len(where) == 0:
            obs = 100
        else:
            left = np.min(where)
            right = np.max(where)
            obs = np.sum(block[row, left:right] == 45)
        if obs > inds:
            inds = obs
    return inds

def maxind_numba ( block ) : inds = 0 for row in xrange ( block . shape [ 0 ] ) : where = np . where ( block [ row ] != 45 ) [ 0 ] if len ( where ) == 0 : obs = 100 else : left = np . min ( where ) right = np . max ( where ) obs = np . sum ( block [ row , left : right ] == 45 ) if obs > inds : inds = obs return inds


def write_snps_map(data):
    """ write a map file with linkage information for SNPs file"""

    ## grab map data from tmparr
    start = time.time()
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    with h5py.File(tmparrs, 'r') as io5:
        maparr = io5["maparr"][:]

        ## get last data
        end = np.where(np.all(maparr[:] == 0, axis=1))[0]
        if np.any(end):
            end = end.min()
        else:
            end = maparr.shape[0]

        ## write to map file (this is too slow...)
        outchunk = []
        with open(data.outfiles.snpsmap, 'w') as out:
            for idx in xrange(end):
                ## build to list
                line = maparr[idx, :]
                #print(line)
                outchunk.append(\
                    "{}\trad{}_snp{}\t{}\t{}\n"\
                    .format(line[0], line[1], line[2], 0, line[3]))
                ## clear list
                if not idx % 10000:
                    out.write("".join(outchunk))
                    outchunk = []
            ## write remaining
            out.write("".join(outchunk))
    LOGGER.debug("finished writing snps_map in: %s", time.time() - start)

def
write_snps_map
(
data
)
:
start
=
time
.
time
(
)
tmparrs
=
os
.
path
.
join
(
data
.
dirs
.
outfiles
,
"tmp-{}.h5"
.
format
(
data
.
name
)
)
with
h5py
.
File
(
tmparrs
,
'r'
)
as
io5
:
maparr
=
io5
[
"maparr"
]
[
:
]
end
=
np
.
where
(
np
.
all
(
maparr
[
:
]
==
0
,
axis
=
1
)
)
[
0
]
if
np
.
any
(
end
)
:
end
=
end
.
min
(
)
else
:
end
=
maparr
.
shape
[
0
]
outchunk
=
[
]
with
open
(
data
.
outfiles
.
snpsmap
,
'w'
)
as
out
:
for
idx
in
xrange
(
end
)
:
line
=
maparr
[
idx
,
:
]
outchunk
.
append
(
"{}\trad{}_snp{}\t{}\t{}\n"
.
format
(
line
[
0
]
,
line
[
1
]
,
line
[
2
]
,
0
,
line
[
3
]
)
)
if
not
idx
%
10000
:
out
.
write
(
""
.
join
(
outchunk
)
)
outchunk
=
[
]
out
.
write
(
""
.
join
(
outchunk
)
)
LOGGER
.
debug
(
"finished writing snps_map in: %s"
,
time
.
time
(
)
start
)


def write_usnps(data, sidx, pnames):
    """ write the bisnp string """

    ## grab bis data from tmparr
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    with h5py.File(tmparrs, 'r') as io5:
        bisarr = io5["bisarr"]

        ## trim to size b/c it was made longer than actual
        end = np.where(np.all(bisarr[:] == "", axis=0))[0]
        if np.any(end):
            end = end.min()
        else:
            end = bisarr.shape[1]

        ## write to usnps file
        with open(data.outfiles.usnpsphy, 'w') as out:
            out.write("{} {}\n".format(bisarr.shape[0], end))
            for idx, name in enumerate(pnames):
                out.write("{}{}\n".format(name, "".join(bisarr[idx, :end])))

def
write_usnps
(
data
,
sidx
,
pnames
)
:
tmparrs
=
os
.
path
.
join
(
data
.
dirs
.
outfiles
,
"tmp-{}.h5"
.
format
(
data
.
name
)
)
with
h5py
.
File
(
tmparrs
,
'r'
)
as
io5
:
bisarr
=
io5
[
"bisarr"
]
end
=
np
.
where
(
np
.
all
(
bisarr
[
:
]
==
""
,
axis
=
0
)
)
[
0
]
if
np
.
any
(
end
)
:
end
=
end
.
min
(
)
else
:
end
=
bisarr
.
shape
[
1
]
with
open
(
data
.
outfiles
.
usnpsphy
,
'w'
)
as
out
:
out
.
write
(
"{} {}\n"
.
format
(
bisarr
.
shape
[
0
]
,
end
)
)
for
idx
,
name
in
enumerate
(
pnames
)
:
out
.
write
(
"{}{}\n"
.
format
(
name
,
""
.
join
(
bisarr
[
idx
,
:
end
]
)
)
)


def write_str(data, sidx, pnames):
    """ Write STRUCTURE format for all SNPs and unlinked SNPs """

    ## grab snp and bis data from tmparr
    start = time.time()
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    with h5py.File(tmparrs, 'r') as io5:
        snparr = io5["snparr"]
        bisarr = io5["bisarr"]

        ## trim to size b/c it was made longer than actual
        bend = np.where(np.all(bisarr[:] == "", axis=0))[0]
        if np.any(bend):
            bend = bend.min()
        else:
            bend = bisarr.shape[1]

        send = np.where(np.all(snparr[:] == "", axis=0))[0]
        if np.any(send):
            send = send.min()
        else:
            send = snparr.shape[1]

        ## write to str and ustr
        out1 = open(data.outfiles.str, 'w')
        out2 = open(data.outfiles.ustr, 'w')
        numdict = {'A': '0', 'T': '1', 'G': '2', 'C': '3', 'N': '-9', '-': '-9'}
        if data.paramsdict["max_alleles_consens"] > 1:
            for idx, name in enumerate(pnames):
                out1.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][0]] for i in snparr[idx, :send]])))
                out1.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][1]] for i in snparr[idx, :send]])))
                out2.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][0]] for i in bisarr[idx, :bend]])))
                out2.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][1]] for i in bisarr[idx, :bend]])))
        else:
            ## haploid output
            for idx, name in enumerate(pnames):
                out1.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][0]] for i in snparr[idx, :send]])))
                out2.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][0]] for i in bisarr[idx, :bend]])))
        out1.close()
        out2.close()
    LOGGER.debug("finished writing str in: %s", time.time() - start)

def
write_str
(
data
,
sidx
,
pnames
)
:
start
=
time
.
time
(
)
tmparrs
=
os
.
path
.
join
(
data
.
dirs
.
outfiles
,
"tmp-{}.h5"
.
format
(
data
.
name
)
)
with
h5py
.
File
(
tmparrs
,
'r'
)
as
io5
:
snparr
=
io5
[
"snparr"
]
bisarr
=
io5
[
"bisarr"
]
bend
=
np
.
where
(
np
.
all
(
bisarr
[
:
]
==
""
,
axis
=
0
)
)
[
0
]
if
np
.
any
(
bend
)
:
bend
=
bend
.
min
(
)
else
:
bend
=
bisarr
.
shape
[
1
]
send
=
np
.
where
(
np
.
all
(
snparr
[
:
]
==
""
,
axis
=
0
)
)
[
0
]
if
np
.
any
(
send
)
:
send
=
send
.
min
(
)
else
:
send
=
snparr
.
shape
[
1
]
out1
=
open
(
data
.
outfiles
.
str
,
'w'
)
out2
=
open
(
data
.
outfiles
.
ustr
,
'w'
)
numdict
=
{
'A'
:
'0'
,
'T'
:
'1'
,
'G'
:
'2'
,
'C'
:
'3'
,
'N'
:
'-9'
,
'-'
:
'-9'
}
if
data
.
paramsdict
[
"max_alleles_consens"
]
>
1
:
for
idx
,
name
in
enumerate
(
pnames
)
:
out1
.
write
(
"{}\t\t\t\t\t{}\n"
.
format
(
name
,
"\t"
.
join
(
[
numdict
[
DUCT
[
i
]
[
0
]
]
for
i
in
snparr
[
idx
,
:
send
]
]
)
)
)
out1
.
write
(
"{}\t\t\t\t\t{}\n"
.
format
(
name
,
"\t"
.
join
(
[
numdict
[
DUCT
[
i
]
[
1
]
]
for
i
in
snparr
[
idx
,
:
send
]
]
)
)
)
out2
.
write
(
"{}\t\t\t\t\t{}\n"
.
format
(
name
,
"\t"
.
join
(
[
numdict
[
DUCT
[
i
]
[
0
]
]
for
i
in
bisarr
[
idx
,
:
bend
]
]
)
)
)
out2
.
write
(
"{}\t\t\t\t\t{}\n"
.
format
(
name
,
"\t"
.
join
(
[
numdict
[
DUCT
[
i
]
[
1
]
]
for
i
in
bisarr
[
idx
,
:
bend
]
]
)
)
)
else
:
for
idx
,
name
in
enumerate
(
pnames
)
:
out1
.
write
(
"{}\t\t\t\t\t{}\n"
.
format
(
name
,
"\t"
.
join
(
[
numdict
[
DUCT
[
i
]
[
0
]
]
for
i
in
snparr
[
idx
,
:
send
]
]
)
)
)
out2
.
write
(
"{}\t\t\t\t\t{}\n"
.
format
(
name
,
"\t"
.
join
(
[
numdict
[
DUCT
[
i
]
[
0
]
]
for
i
in
bisarr
[
idx
,
:
bend
]
]
)
)
)
out1
.
close
(
)
out2
.
close
(
)
LOGGER
.
debug
(
"finished writing str in: %s"
,
time
.
time
(
)
start
)


def concat_vcf(data, names, full):
    """
    Sorts, concatenates, and gzips VCF chunks. Also cleans up chunks.
    """
    ## open handle and write headers
    if not full:
        writer = open(data.outfiles.vcf, 'w')
    else:
        writer = gzip.open(data.outfiles.VCF, 'w')
    vcfheader(data, names, writer)
    writer.close()

    ## get vcf chunks
    vcfchunks = glob.glob(data.outfiles.vcf+".*")
    vcfchunks.sort(key=lambda x: int(x.rsplit(".")[-1]))

    ## concatenate
    if not full:
        writer = open(data.outfiles.vcf, 'a')
    else:
        writer = gzip.open(data.outfiles.VCF, 'a')

    ## what order do users want? The order in the original ref file?
    ## Sorted by the size of chroms? that is the order in faidx.
    ## If reference mapping then it's nice to sort the vcf data by
    ## CHROM and POS. This is doing a very naive sort right now, so the
    ## CHROM will be ordered, but not the pos within each chrom.
    if data.paramsdict["assembly_method"] in ["reference", "denovo+reference"]:
        ## Some unix sorting magic to get POS sorted within CHROM
        ## First you sort by POS (-k 2,2), then you do a `stable` sort
        ## by CHROM. You end up with POS ordered and grouped correctly by CHROM
        ## but relatively unordered CHROMs (locus105 will be before locus11).
        cmd = ["cat"] + vcfchunks + [" | sort -k 2,2 -n | sort -k 1,1 -s"]
        cmd = " ".join(cmd)
        proc = sps.Popen(cmd, shell=True, stderr=sps.STDOUT, stdout=writer, close_fds=True)
    else:
        proc = sps.Popen(["cat"] + vcfchunks, stderr=sps.STDOUT, stdout=writer, close_fds=True)

    err = proc.communicate()[0]
    if proc.returncode:
        raise IPyradWarningExit("err in concat_vcf: %s", err)
    writer.close()

    for chunk in vcfchunks:
        os.remove(chunk)

def
concat_vcf
(
data
,
names
,
full
)
:
if
not
full
:
writer
=
open
(
data
.
outfiles
.
vcf
,
'w'
)
else
:
writer
=
gzip
.
open
(
data
.
outfiles
.
VCF
,
'w'
)
vcfheader
(
data
,
names
,
writer
)
writer
.
close
(
)
vcfchunks
=
glob
.
glob
(
data
.
outfiles
.
vcf
+
".*"
)
vcfchunks
.
sort
(
key
=
lambda
x
:
int
(
x
.
rsplit
(
"."
)
[
1
]
)
)
if
not
full
:
writer
=
open
(
data
.
outfiles
.
vcf
,
'a'
)
else
:
writer
=
gzip
.
open
(
data
.
outfiles
.
VCF
,
'a'
)
if
data
.
paramsdict
[
"assembly_method"
]
in
[
"reference"
,
"denovo+reference"
]
:
cmd
=
[
"cat"
]
+
vcfchunks
+
[
" | sort -k 2,2 -n | sort -k 1,1 -s"
]
cmd
=
" "
.
join
(
cmd
)
proc
=
sps
.
Popen
(
cmd
,
shell
=
True
,
stderr
=
sps
.
STDOUT
,
stdout
=
writer
,
close_fds
=
True
)
else
:
proc
=
sps
.
Popen
(
[
"cat"
]
+
vcfchunks
,
stderr
=
sps
.
STDOUT
,
stdout
=
writer
,
close_fds
=
True
)
err
=
proc
.
communicate
(
)
[
0
]
if
proc
.
returncode
:
raise
IPyradWarningExit
(
"err in concat_vcf: %s"
,
err
)
writer
.
close
(
)
for
chunk
in
vcfchunks
:
os
.
remove
(
chunk
)

Sorts concatenates and gzips VCF chunks . Also cleans up chunks .


def reftrick(iseq, consdict):
    """ Returns the most common base at each site in order. """

    altrefs = np.zeros((iseq.shape[1], 4), dtype=np.uint8)
    altrefs[:, 1] = 46

    for col in xrange(iseq.shape[1]):
        ## expand colums with ambigs and remove N-
        fcounts = np.zeros(111, dtype=np.int64)
        counts = np.bincount(iseq[:, col])#, minlength=90)
        fcounts[:counts.shape[0]] = counts
        ## set N and - to zero, wish numba supported minlen arg
        fcounts[78] = 0
        fcounts[45] = 0
        ## add ambig counts to true bases
        for aidx in xrange(consdict.shape[0]):
            nbases = fcounts[consdict[aidx, 0]]
            for _ in xrange(nbases):
                fcounts[consdict[aidx, 1]] += 1
                fcounts[consdict[aidx, 2]] += 1
            fcounts[consdict[aidx, 0]] = 0

        ## now get counts from the modified counts arr
        who = np.argmax(fcounts)
        altrefs[col, 0] = who
        fcounts[who] = 0

        ## if an alt allele fill over the "." placeholder
        who = np.argmax(fcounts)
        if who:
            altrefs[col, 1] = who
            fcounts[who] = 0

            ## if 3rd or 4th alleles observed then add to arr
            who = np.argmax(fcounts)
            altrefs[col, 2] = who
            fcounts[who] = 0

            ## if 3rd or 4th alleles observed then add to arr
            who = np.argmax(fcounts)
            altrefs[col, 3] = who

    return altrefs

def reftrick ( iseq , consdict ) : altrefs = np . zeros ( ( iseq . shape [ 1 ] , 4 ) , dtype = np . uint8 ) altrefs [ : , 1 ] = 46 for col in xrange ( iseq . shape [ 1 ] ) : fcounts = np . zeros ( 111 , dtype = np . int64 ) counts = np . bincount ( iseq [ : , col ] ) fcounts [ : counts . shape [ 0 ] ] = counts fcounts [ 78 ] = 0 fcounts [ 45 ] = 0 for aidx in xrange ( consdict . shape [ 0 ] ) : nbases = fcounts [ consdict [ aidx , 0 ] ] for _ in xrange ( nbases ) : fcounts [ consdict [ aidx , 1 ] ] += 1 fcounts [ consdict [ aidx , 2 ] ] += 1 fcounts [ consdict [ aidx , 0 ] ] = 0 who = np . argmax ( fcounts ) altrefs [ col , 0 ] = who fcounts [ who ] = 0 who = np . argmax ( fcounts ) if who : altrefs [ col , 1 ] = who fcounts [ who ] = 0 who = np . argmax ( fcounts ) altrefs [ col , 2 ] = who fcounts [ who ] = 0 who = np . argmax ( fcounts ) altrefs [ col , 3 ] = who return altrefs

Returns the most common base at each site in order .


def _collapse_outgroup(tree, taxdicts):
    """ collapse outgroup in ete Tree for easier viewing """
    ## check that all tests have the same outgroup
    outg = taxdicts[0]["p4"]
    if not all([i["p4"] == outg for i in taxdicts]):
        raise Exception("no good")

    ## prune tree, keep only one sample from outgroup
    tre = ete.Tree(tree.write(format=1)) #tree.copy(method="deepcopy")
    alltax = [i for i in tre.get_leaf_names() if i not in outg]
    alltax += [outg[0]]
    tre.prune(alltax)
    tre.search_nodes(name=outg[0])[0].name = "outgroup"
    tre.ladderize()

    ## remove other ougroups from taxdicts
    taxd = copy.deepcopy(taxdicts)
    newtaxdicts = []
    for test in taxd:
        #test["p4"] = [outg[0]]
        test["p4"] = ["outgroup"]
        newtaxdicts.append(test)

    return tre, newtaxdicts

def
_collapse_outgroup
(
tree
,
taxdicts
)
:
outg
=
taxdicts
[
0
]
[
"p4"
]
if
not
all
(
[
i
[
"p4"
]
==
outg
for
i
in
taxdicts
]
)
:
raise
Exception
(
"no good"
)
tre
=
ete
.
Tree
(
tree
.
write
(
format
=
1
)
)
alltax
=
[
i
for
i
in
tre
.
get_leaf_names
(
)
if
i
not
in
outg
]
alltax
+=
[
outg
[
0
]
]
tre
.
prune
(
alltax
)
tre
.
search_nodes
(
name
=
outg
[
0
]
)
[
0
]
.
name
=
"outgroup"
tre
.
ladderize
(
)
taxd
=
copy
.
deepcopy
(
taxdicts
)
newtaxdicts
=
[
]
for
test
in
taxd
:
test
[
"p4"
]
=
[
"outgroup"
]
newtaxdicts
.
append
(
test
)
return
tre
,
newtaxdicts


def draw(
        self,
        show_tip_labels=True,
        show_node_support=False,
        use_edge_lengths=False,
        orient="right",
        print_args=False,
        *args,
        **kwargs):
        """
        plot the tree using toyplot.graph.

        Parameters:
        -----------
            show_tip_labels: bool
                Show tip names from tree.
            use_edge_lengths: bool
                Use edge lengths from newick tree.
            show_node_support: bool
                Show support values at nodes using a set of default
                options.

            ...
        """
        ## re-decompose tree for new orient and edges args
        self._decompose_tree(orient=orient, use_edge_lengths=use_edge_lengths)

        ## update kwargs with entered args and all other kwargs
        dwargs = {}
        dwargs["show_tip_labels"] = show_tip_labels
        dwargs["show_node_support"] = show_node_support
        dwargs.update(kwargs)

        ## pass to panel plotter
        canvas, axes, panel = tree_panel_plot(self, print_args, **dwargs)
        return canvas, axes, panel

def draw ( self , show_tip_labels = True , show_node_support = False , use_edge_lengths = False , orient = "right" , print_args = False , * args , ** kwargs ) : self . _decompose_tree ( orient = orient , use_edge_lengths = use_edge_lengths ) dwargs = { } dwargs [ "show_tip_labels" ] = show_tip_labels dwargs [ "show_node_support" ] = show_node_support dwargs . update ( kwargs ) canvas , axes , panel = tree_panel_plot ( self , print_args , ** dwargs ) return canvas , axes , panel


def get_quick_depths(data, sample):
    """ iterate over clustS files to get data """

    ## use existing sample cluster path if it exists, since this
    ## func can be used in step 4 and that can occur after merging
    ## assemblies after step3, and if we then referenced by data.dirs.clusts
    ## the path would be broken.
    ##
    ## If branching at step 3 to test different clust thresholds, the
    ## branched samples will retain the samples.files.clusters of the
    ## parent (which have the clust_threshold value of the parent), so
    ## it will look like nothing has changed. If we call this func
    ## from step 3 then it indicates we are in a branch and should
    ## reset the sample.files.clusters handle to point to the correct
    ## data.dirs.clusts directory. See issue #229.
    ## Easier to just always trust that samples.files.clusters is right,
    ## no matter what step?
    #if sample.files.clusters and not sample.stats.state == 3:
    #    pass
    #else:
    #    ## set cluster file handles
    sample.files.clusters = os.path.join(
         data.dirs.clusts, sample.name+".clustS.gz")

    ## get new clustered loci
    fclust = data.samples[sample.name].files.clusters
    clusters = gzip.open(fclust, 'r')
    pairdealer = itertools.izip(*[iter(clusters)]*2)

    ## storage
    depths = []
    maxlen = []

    ## start with cluster 0
    tdepth = 0
    tlen = 0

    ## iterate until empty
    while 1:
        ## grab next
        try:
            name, seq = pairdealer.next()
        except StopIteration:
            break

        ## if not the end of a cluster
        #print name.strip(), seq.strip()
        if name.strip() == seq.strip():
            depths.append(tdepth)
            maxlen.append(tlen)
            tlen = 0
            tdepth = 0

        else:
            tdepth += int(name.split(";")[-2][5:])
            tlen = len(seq)

    ## return
    clusters.close()
    return np.array(maxlen), np.array(depths)

def get_quick_depths ( data , sample ) : sample . files . clusters = os . path . join ( data . dirs . clusts , sample . name + ".clustS.gz" ) fclust = data . samples [ sample . name ] . files . clusters clusters = gzip . open ( fclust , 'r' ) pairdealer = itertools . izip ( * [ iter ( clusters ) ] * 2 ) depths = [ ] maxlen = [ ] tdepth = 0 tlen = 0 while 1 : try : name , seq = pairdealer . next ( ) except StopIteration : break if name . strip ( ) == seq . strip ( ) : depths . append ( tdepth ) maxlen . append ( tlen ) tlen = 0 tdepth = 0 else : tdepth += int ( name . split ( ";" ) [ - 2 ] [ 5 : ] ) tlen = len ( seq ) clusters . close ( ) return np . array ( maxlen ) , np . array ( depths )


def align_and_parse(handle, max_internal_indels=5, is_gbs=False):
    """ much faster implementation for aligning chunks """

    ## data are already chunked, read in the whole thing. bail if no data.
    try:
        with open(handle, 'rb') as infile:
            clusts = infile.read().split("//\n//\n")
            ## remove any empty spots
            clusts = [i for i in clusts if i]
            ## Skip entirely empty chunks
            if not clusts:
                raise IPyradError
    except (IOError, IPyradError):
        LOGGER.debug("skipping empty chunk - {}".format(handle))
        return 0

    ## count discarded clusters for printing to stats later
    highindels = 0

    ## iterate over clusters sending each to muscle, splits and aligns pairs
    try:
        aligned = persistent_popen_align3(clusts, 200, is_gbs)
    except Exception as inst:
        LOGGER.debug("Error in handle - {} - {}".format(handle, inst))
        #raise IPyradWarningExit("error hrere {}".format(inst))
        aligned = []

    ## store good alignments to be written to file
    refined = []

    ## filter and trim alignments
    for clust in aligned:

        ## check for too many internal indels
        filtered = aligned_indel_filter(clust, max_internal_indels)

        ## reverse complement matches. No longer implemented.
        #filtered = overshoot_filter(clust)

        ## finally, add to outstack if alignment is good
        if not filtered:
            refined.append(clust)#"\n".join(stack))
        else:
            highindels += 1

    ## write to file after
    if refined:
        outhandle = handle.rsplit(".", 1)[0]+".aligned"
        with open(outhandle, 'wb') as outfile:
            outfile.write("\n//\n//\n".join(refined)+"\n")

    ## remove the old tmp file
    log_level = logging.getLevelName(LOGGER.getEffectiveLevel())
    if not log_level == "DEBUG":
        os.remove(handle)
    return highindels

def
align_and_parse
(
handle
,
max_internal_indels
=
5
,
is_gbs
=
False
)
:
try
:
with
open
(
handle
,
'rb'
)
as
infile
:
clusts
=
infile
.
read
(
)
.
split
(
"//\n//\n"
)
clusts
=
[
i
for
i
in
clusts
if
i
]
if
not
clusts
:
raise
IPyradError
except
(
IOError
,
IPyradError
)
:
LOGGER
.
debug
(
"skipping empty chunk - {}"
.
format
(
handle
)
)
return
0
highindels
=
0
try
:
aligned
=
persistent_popen_align3
(
clusts
,
200
,
is_gbs
)
except
Exception
as
inst
:
LOGGER
.
debug
(
"Error in handle - {} - {}"
.
format
(
handle
,
inst
)
)
aligned
=
[
]
refined
=
[
]
for
clust
in
aligned
:
filtered
=
aligned_indel_filter
(
clust
,
max_internal_indels
)
if
not
filtered
:
refined
.
append
(
clust
)
else
:
highindels
+=
1
if
refined
:
outhandle
=
handle
.
rsplit
(
"."
,
1
)
[
0
]
+
".aligned"
with
open
(
outhandle
,
'wb'
)
as
outfile
:
outfile
.
write
(
"\n//\n//\n"
.
join
(
refined
)
+
"\n"
)
log_level
=
logging
.
getLevelName
(
LOGGER
.
getEffectiveLevel
(
)
)
if
not
log_level
==
"DEBUG"
:
os
.
remove
(
handle
)
return
highindels


def aligned_indel_filter(clust, max_internal_indels):
    """ checks for too many internal indels in muscle aligned clusters """

    ## make into list
    lclust = clust.split()

    ## paired or not
    try:
        seq1 = [i.split("nnnn")[0] for i in lclust[1::2]]
        seq2 = [i.split("nnnn")[1] for i in lclust[1::2]]
        intindels1 = [i.rstrip("-").lstrip("-").count("-") for i in seq1]
        intindels2 = [i.rstrip("-").lstrip("-").count("-") for i in seq2]
        intindels = intindels1 + intindels2
        if max(intindels) > max_internal_indels:
            return 1

    except IndexError:
        seq1 = lclust[1::2]
        intindels = [i.rstrip("-").lstrip("-").count("-") for i in seq1]
        if max(intindels) > max_internal_indels:
            return 1

    return 0

def aligned_indel_filter ( clust , max_internal_indels ) : lclust = clust . split ( ) try : seq1 = [ i . split ( "nnnn" ) [ 0 ] for i in lclust [ 1 : : 2 ] ] seq2 = [ i . split ( "nnnn" ) [ 1 ] for i in lclust [ 1 : : 2 ] ] intindels1 = [ i . rstrip ( "-" ) . lstrip ( "-" ) . count ( "-" ) for i in seq1 ] intindels2 = [ i . rstrip ( "-" ) . lstrip ( "-" ) . count ( "-" ) for i in seq2 ] intindels = intindels1 + intindels2 if max ( intindels ) > max_internal_indels : return 1 except IndexError : seq1 = lclust [ 1 : : 2 ] intindels = [ i . rstrip ( "-" ) . lstrip ( "-" ) . count ( "-" ) for i in seq1 ] if max ( intindels ) > max_internal_indels : return 1 return 0


def setup_dirs(data):
    """ sets up directories for step3 data """
    ## make output folder for clusters
    pdir = os.path.realpath(data.paramsdict["project_dir"])
    data.dirs.clusts = os.path.join(pdir, "{}_clust_{}"\
                       .format(data.name, data.paramsdict["clust_threshold"]))
    if not os.path.exists(data.dirs.clusts):
        os.mkdir(data.dirs.clusts)

    ## make a tmpdir for align files
    data.tmpdir = os.path.abspath(os.path.expanduser(
        os.path.join(pdir, data.name+'-tmpalign')))
    if not os.path.exists(data.tmpdir):
        os.mkdir(data.tmpdir)

    ## If ref mapping, init samples and make the refmapping output directory.
    if not data.paramsdict["assembly_method"] == "denovo":
        ## make output directory for read mapping process
        data.dirs.refmapping = os.path.join(pdir, "{}_refmapping".format(data.name))
        if not os.path.exists(data.dirs.refmapping):
            os.mkdir(data.dirs.refmapping)

def setup_dirs ( data ) : pdir = os . path . realpath ( data . paramsdict [ "project_dir" ] ) data . dirs . clusts = os . path . join ( pdir , "{}_clust_{}" . format ( data . name , data . paramsdict [ "clust_threshold" ] ) ) if not os . path . exists ( data . dirs . clusts ) : os . mkdir ( data . dirs . clusts ) data . tmpdir = os . path . abspath ( os . path . expanduser ( os . path . join ( pdir , data . name + '-tmpalign' ) ) ) if not os . path . exists ( data . tmpdir ) : os . mkdir ( data . tmpdir ) if not data . paramsdict [ "assembly_method" ] == "denovo" : data . dirs . refmapping = os . path . join ( pdir , "{}_refmapping" . format ( data . name ) ) if not os . path . exists ( data . dirs . refmapping ) : os . mkdir ( data . dirs . refmapping )


def build_dag(data, samples):
    """
    build a directed acyclic graph describing jobs to be run in order.
    """

    ## Create DAGs for the assembly method being used, store jobs in nodes
    snames = [i.name for i in samples]
    dag = nx.DiGraph()

    ## get list of pre-align jobs from globals based on assembly method
    joborder = JOBORDER[data.paramsdict["assembly_method"]]

    ## WHICH JOBS TO RUN: iterate over the sample names
    for sname in snames:
        ## append pre-align job for each sample to nodes list
        for func in joborder:
            dag.add_node("{}-{}-{}".format(func, 0, sname))

        ## append align func jobs, each will have max 10
        for chunk in xrange(10):
            dag.add_node("{}-{}-{}".format("muscle_align", chunk, sname))

        ## append final reconcat jobs
        dag.add_node("{}-{}-{}".format("reconcat", 0, sname))

    ## ORDER OF JOBS: add edges/dependency between jobs: (first-this, then-that)
    for sname in snames:
        for sname2 in snames:
            ## enforce that clust/map cannot start until derep is done for ALL
            ## samples. This is b/c...
            dag.add_edge("{}-{}-{}".format(joborder[0], 0, sname2),
                         "{}-{}-{}".format(joborder[1], 0, sname))

        ## add remaining pre-align jobs
        for idx in xrange(2, len(joborder)):
            dag.add_edge("{}-{}-{}".format(joborder[idx-1], 0, sname),
                         "{}-{}-{}".format(joborder[idx], 0, sname))

        ## Add 10 align jobs, none of which can start until all chunker jobs
        ## are finished. Similarly, reconcat jobs cannot start until all align
        ## jobs are finished.
        for sname2 in snames:
            for chunk in range(10):
                dag.add_edge("{}-{}-{}".format("muscle_chunker", 0, sname2),
                             "{}-{}-{}".format("muscle_align", chunk, sname))
                ## add that the final reconcat job can't start until after
                ## each chunk of its own sample has finished aligning.
                dag.add_edge("{}-{}-{}".format("muscle_align", chunk, sname),
                             "{}-{}-{}".format("reconcat", 0, sname))
    ## return the dag
    return dag, joborder

def build_dag ( data , samples ) : snames = [ i . name for i in samples ] dag = nx . DiGraph ( ) joborder = JOBORDER [ data . paramsdict [ "assembly_method" ] ] for sname in snames : for func in joborder : dag . add_node ( "{}-{}-{}" . format ( func , 0 , sname ) ) for chunk in xrange ( 10 ) : dag . add_node ( "{}-{}-{}" . format ( "muscle_align" , chunk , sname ) ) dag . add_node ( "{}-{}-{}" . format ( "reconcat" , 0 , sname ) ) for sname in snames : for sname2 in snames : dag . add_edge ( "{}-{}-{}" . format ( joborder [ 0 ] , 0 , sname2 ) , "{}-{}-{}" . format ( joborder [ 1 ] , 0 , sname ) ) for idx in xrange ( 2 , len ( joborder ) ) : dag . add_edge ( "{}-{}-{}" . format ( joborder [ idx - 1 ] , 0 , sname ) , "{}-{}-{}" . format ( joborder [ idx ] , 0 , sname ) ) for sname2 in snames : for chunk in range ( 10 ) : dag . add_edge ( "{}-{}-{}" . format ( "muscle_chunker" , 0 , sname2 ) , "{}-{}-{}" . format ( "muscle_align" , chunk , sname ) ) dag . add_edge ( "{}-{}-{}" . format ( "muscle_align" , chunk , sname ) , "{}-{}-{}" . format ( "reconcat" , 0 , sname ) ) return dag , joborder

build a directed acyclic graph describing jobs to be run in order .


def _plot_dag(dag, results, snames):
    """
    makes plot to help visualize the DAG setup. For developers only.
    """
    try:
        import matplotlib.pyplot as plt
        from matplotlib.dates import date2num
        from matplotlib.cm import gist_rainbow

        ## first figure is dag layout
        plt.figure("dag_layout", figsize=(10, 10))
        nx.draw(dag,
                pos=nx.spring_layout(dag),
                node_color='pink',
                with_labels=True)
        plt.savefig("./dag_layout.png", bbox_inches='tight', dpi=200)

        ## second figure is times for steps
        pos = {}
        colors = {}

        for node in dag:
            #jobkey = "{}-{}".format(node, sample)
            mtd = results[node].metadata
            start = date2num(mtd.started)
            #runtime = date2num(md.completed)# - start
            ## sample id to separate samples on x-axis
            _, _, sname = node.split("-", 2)
            sid = snames.index(sname)
            ## 1e6 to separate on y-axis
            pos[node] = (start+sid, start*1e6)
            colors[node] = mtd.engine_id

        ## x just spaces out samples;
        ## y is start time of each job with edge leading to next job
        ## color is the engine that ran the job
        ## all jobs were submitted as 3 second wait times
        plt.figure("dag_starttimes", figsize=(10, 16))
        nx.draw(dag, pos,
                node_list=colors.keys(),
                node_color=colors.values(),
                cmap=gist_rainbow,
                with_labels=True)
        plt.savefig("./dag_starttimes.png", bbox_inches='tight', dpi=200)

    except Exception as inst:
        LOGGER.warning(inst)

def _plot_dag ( dag , results , snames ) : try : import matplotlib . pyplot as plt from matplotlib . dates import date2num from matplotlib . cm import gist_rainbow plt . figure ( "dag_layout" , figsize = ( 10 , 10 ) ) nx . draw ( dag , pos = nx . spring_layout ( dag ) , node_color = 'pink' , with_labels = True ) plt . savefig ( "./dag_layout.png" , bbox_inches = 'tight' , dpi = 200 ) pos = { } colors = { } for node in dag : mtd = results [ node ] . metadata start = date2num ( mtd . started ) _ , _ , sname = node . split ( "-" , 2 ) sid = snames . index ( sname ) pos [ node ] = ( start + sid , start * 1e6 ) colors [ node ] = mtd . engine_id plt . figure ( "dag_starttimes" , figsize = ( 10 , 16 ) ) nx . draw ( dag , pos , node_list = colors . keys ( ) , node_color = colors . values ( ) , cmap = gist_rainbow , with_labels = True ) plt . savefig ( "./dag_starttimes.png" , bbox_inches = 'tight' , dpi = 200 ) except Exception as inst : LOGGER . warning ( inst )

makes plot to help visualize the DAG setup . For developers only .


def trackjobs(func, results, spacer):
    """
    Blocks and prints progress for just the func being requested from a list
    of submitted engine jobs. Returns whether any of the jobs failed.

    func = str
    results = dict of asyncs
    """

    ## TODO: try to insert a better way to break on KBD here.
    LOGGER.info("inside trackjobs of %s", func)

    ## get just the jobs from results that are relevant to this func
    asyncs = [(i, results[i]) for i in results if i.split("-", 2)[0] == func]

    ## progress bar
    start = time.time()
    while 1:
        ## how many of this func have finished so far
        ready = [i[1].ready() for i in asyncs]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        printstr = " {}    | {} | s3 |".format(PRINTSTR[func], elapsed)
        progressbar(len(ready), sum(ready), printstr, spacer=spacer)
        time.sleep(0.1)
        if len(ready) == sum(ready):
            print("")
            break

    sfails = []
    errmsgs = []
    for job in asyncs:
        if not job[1].successful():
            sfails.append(job[0])
            errmsgs.append(job[1].result())

    return func, sfails, errmsgs

def
trackjobs
(
func
,
results
,
spacer
)
:
LOGGER
.
info
(
"inside trackjobs of %s"
,
func
)
asyncs
=
[
(
i
,
results
[
i
]
)
for
i
in
results
if
i
.
split
(
"-"
,
2
)
[
0
]
==
func
]
start
=
time
.
time
(
)
while
1
:
ready
=
[
i
[
1
]
.
ready
(
)
for
i
in
asyncs
]
elapsed
=
datetime
.
timedelta
(
seconds
=
int
(
time
.
time
(
)
start
)
)
printstr
=
" {}    | {} | s3 |"
.
format
(
PRINTSTR
[
func
]
,
elapsed
)
progressbar
(
len
(
ready
)
,
sum
(
ready
)
,
printstr
,
spacer
=
spacer
)
time
.
sleep
(
0.1
)
if
len
(
ready
)
==
sum
(
ready
)
:
print
(
""
)
break
sfails
=
[
]
errmsgs
=
[
]
for
job
in
asyncs
:
if
not
job
[
1
]
.
successful
(
)
:
sfails
.
append
(
job
[
0
]
)
errmsgs
.
append
(
job
[
1
]
.
result
(
)
)
return
func
,
sfails
,
errmsgs

Blocks and prints progress for just the func being requested from a list of submitted engine jobs . Returns whether any of the jobs failed .


def concat_multiple_edits(data, sample):
    """
    if multiple fastq files were appended into the list of fastqs for samples
    then we merge them here before proceeding.
    """

    ## if more than one tuple in fastq list
    if len(sample.files.edits) > 1:
        ## create a cat command to append them all (doesn't matter if they
        ## are gzipped, cat still works). Grab index 0 of tuples for R1s.
        cmd1 = ["cat"] + [i[0] for i in sample.files.edits]

        ## write to new concat handle
        conc1 = os.path.join(data.dirs.edits, sample.name+"_R1_concatedit.fq.gz")
        with open(conc1, 'w') as cout1:
            proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=cout1, close_fds=True)
            res1 = proc1.communicate()[0]
        if proc1.returncode:
            raise IPyradWarningExit("error in: %s, %s", cmd1, res1)

        ## Only set conc2 if R2 actually exists
        conc2 = 0
        if os.path.exists(str(sample.files.edits[0][1])):
            cmd2 = ["cat"] + [i[1] for i in sample.files.edits]
            conc2 = os.path.join(data.dirs.edits, sample.name+"_R2_concatedit.fq.gz")
            with gzip.open(conc2, 'w') as cout2:
                proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=cout2, close_fds=True)
                res2 = proc2.communicate()[0]
            if proc2.returncode:
                raise IPyradWarningExit("error in: %s, %s", cmd2, res2)

        ## store new file handles
        sample.files.edits = [(conc1, conc2)]
    return sample.files.edits

def
concat_multiple_edits
(
data
,
sample
)
:
if
len
(
sample
.
files
.
edits
)
>
1
:
cmd1
=
[
"cat"
]
+
[
i
[
0
]
for
i
in
sample
.
files
.
edits
]
conc1
=
os
.
path
.
join
(
data
.
dirs
.
edits
,
sample
.
name
+
"_R1_concatedit.fq.gz"
)
with
open
(
conc1
,
'w'
)
as
cout1
:
proc1
=
sps
.
Popen
(
cmd1
,
stderr
=
sps
.
STDOUT
,
stdout
=
cout1
,
close_fds
=
True
)
res1
=
proc1
.
communicate
(
)
[
0
]
if
proc1
.
returncode
:
raise
IPyradWarningExit
(
"error in: %s, %s"
,
cmd1
,
res1
)
conc2
=
0
if
os
.
path
.
exists
(
str
(
sample
.
files
.
edits
[
0
]
[
1
]
)
)
:
cmd2
=
[
"cat"
]
+
[
i
[
1
]
for
i
in
sample
.
files
.
edits
]
conc2
=
os
.
path
.
join
(
data
.
dirs
.
edits
,
sample
.
name
+
"_R2_concatedit.fq.gz"
)
with
gzip
.
open
(
conc2
,
'w'
)
as
cout2
:
proc2
=
sps
.
Popen
(
cmd2
,
stderr
=
sps
.
STDOUT
,
stdout
=
cout2
,
close_fds
=
True
)
res2
=
proc2
.
communicate
(
)
[
0
]
if
proc2
.
returncode
:
raise
IPyradWarningExit
(
"error in: %s, %s"
,
cmd2
,
res2
)
sample
.
files
.
edits
=
[
(
conc1
,
conc2
)
]
return
sample
.
files
.
edits

if multiple fastq files were appended into the list of fastqs for samples then we merge them here before proceeding .


def cluster(data, sample, nthreads, force):
    """
    Calls vsearch for clustering. cov varies by data type, values were chosen
    based on experience, but could be edited by users
    """

    ## get the dereplicated reads
    if "reference" in data.paramsdict["assembly_method"]:
        derephandle = os.path.join(data.dirs.edits, sample.name+"-refmap_derep.fastq")
        ## In the event all reads for all samples map successfully then clustering
        ## the unmapped reads makes no sense, so just bail out.
        if not os.stat(derephandle).st_size:
            ## In this case you do have to create empty, dummy vsearch output
            ## files so building_clusters will not fail.
            uhandle = os.path.join(data.dirs.clusts, sample.name+".utemp")
            usort = os.path.join(data.dirs.clusts, sample.name+".utemp.sort")
            hhandle = os.path.join(data.dirs.clusts, sample.name+".htemp")
            for f in [uhandle, usort, hhandle]:
                open(f, 'a').close()
            return
    else:
        derephandle = os.path.join(data.dirs.edits, sample.name+"_derep.fastq")

    ## create handles for the outfiles
    uhandle = os.path.join(data.dirs.clusts, sample.name+".utemp")
    temphandle = os.path.join(data.dirs.clusts, sample.name+".htemp")

    ## If derep file doesn't exist then bail out
    if not os.path.isfile(derephandle):
        LOGGER.warn("Bad derephandle - {}".format(derephandle))
        raise IPyradError("Input file for clustering doesn't exist - {}"\
                        .format(derephandle))

    ## testing one sample fail
    #if sample.name == "1C_0":
    #    x

    ## datatype specific optimization
    ## minsl: the percentage of the seed that must be matched
    ##    smaller values for RAD/ddRAD where we might want to combine, say 50bp
    ##    reads and 100bp reads in the same analysis.
    ## query_cov: the percentage of the query sequence that must match seed
    ##    smaller values are needed for gbs where only the tips might overlap
    ##    larger values for pairgbs where they should overlap near completely
    ##    small minsl and high query cov allows trimmed reads to match to untrim
    ##    seed for rad/ddrad/pairddrad.
    strand = "plus"
    cov = 0.75
    minsl = 0.5
    if data.paramsdict["datatype"] in ["gbs", "2brad"]:
        strand = "both"
        cov = 0.5
        minsl = 0.5
    elif data.paramsdict["datatype"] == 'pairgbs':
        strand = "both"
        cov = 0.75
        minsl = 0.75

    ## If this value is not null (which is the default) then override query cov
    if data._hackersonly["query_cov"]:
        cov = str(data._hackersonly["query_cov"])
        assert float(cov) <= 1, "query_cov must be <= 1.0"

    ## get call string
    cmd = [ipyrad.bins.vsearch,
           "-cluster_smallmem", derephandle,
           "-strand", strand,
           "-query_cov", str(cov),
           "-id", str(data.paramsdict["clust_threshold"]),
           "-minsl", str(minsl),
           "-userout", uhandle,
           "-userfields", "query+target+id+gaps+qstrand+qcov",
           "-maxaccepts", "1",
           "-maxrejects", "0",
           "-threads", str(nthreads),
           "-notmatched", temphandle,
           "-fasta_width", "0",
           "-fastq_qmax", "100",
           "-fulldp",
           "-usersort"]

    ## not sure what the benefit of this option is exactly, needs testing,
    ## might improve indel detection on left side, but we don't want to enforce
    ## aligning on left side if not necessarily, since quality trimmed reads
    ## might lose bases on left side in step2 and no longer align.
    #if data.paramsdict["datatype"] in ["rad", "ddrad", "pairddrad"]:
    #    cmd += ["-leftjust"]

    ## run vsearch
    LOGGER.debug("%s", cmd)
    proc = sps.Popen(cmd, stderr=sps.STDOUT, stdout=sps.PIPE, close_fds=True)

    ## This is long running so we wrap it to make sure we can kill it
    try:
        res = proc.communicate()[0]
    except KeyboardInterrupt:
        proc.kill()
        raise KeyboardInterrupt

    ## check for errors
    if proc.returncode:
        LOGGER.error("error %s: %s", cmd, res)
        raise IPyradWarningExit("cmd {}: {}".format(cmd, res))

Calls vsearch for clustering . cov varies by data type values were chosen based on experience but could be edited by users