def muscle_chunker(data, sample):
    """
    Splits the muscle alignment into chunks. Each chunk is run on a separate
    computing core. Because the largest clusters are at the beginning of the
    clusters file, assigning equal clusters to each file would put all of the
    large cluster, that take longer to align, near the top. So instead we
    randomly distribute the clusters among the files. If assembly method is
    reference then this step is just a placeholder and nothing happens.
    """
    ## log our location for debugging
    LOGGER.info("inside muscle_chunker")

    ## only chunk up denovo data, refdata has its own chunking method which
    ## makes equal size chunks, instead of uneven chunks like in denovo
    if data.paramsdict["assembly_method"] != "reference":
        ## get the number of clusters
        clustfile = os.path.join(data.dirs.clusts, sample.name+".clust.gz")
        with iter(gzip.open(clustfile, 'rb')) as clustio:
            nloci = sum(1 for i in clustio if "//" in i) // 2
            #tclust = clustio.read().count("//")//2
            optim = (nloci//20) + (nloci%20)
            LOGGER.info("optim for align chunks: %s", optim)

        ## write optim clusters to each tmp file
        clustio = gzip.open(clustfile, 'rb')
        inclusts = iter(clustio.read().strip().split("//\n//\n"))

        ## splitting loci so first file is smaller and last file is bigger
        inc = optim // 10
        for idx in range(10):
            ## how big is this chunk?
            this = optim + (idx * inc)
            left = nloci-this
            if idx == 9:
                ## grab everything left
                grabchunk = list(itertools.islice(inclusts, int(1e9)))
            else:
                ## grab next chunks-worth of data
                grabchunk = list(itertools.islice(inclusts, this))
                nloci = left

            ## write the chunk to file
            tmpfile = os.path.join(data.tmpdir, sample.name+"_chunk_{}.ali".format(idx))
            with open(tmpfile, 'wb') as out:
                out.write("//\n//\n".join(grabchunk))

        ## write the chunk to file
        #grabchunk = list(itertools.islice(inclusts, left))
        #if grabchunk:
        #    tmpfile = os.path.join(data.tmpdir, sample.name+"_chunk_9.ali")
        #    with open(tmpfile, 'a') as out:
        #        out.write("\n//\n//\n".join(grabchunk))
        clustio.close()

def
muscle_chunker
(
data
,
sample
)
:
LOGGER
.
info
(
"inside muscle_chunker"
)
if
data
.
paramsdict
[
"assembly_method"
]
!=
"reference"
:
clustfile
=
os
.
path
.
join
(
data
.
dirs
.
clusts
,
sample
.
name
+
".clust.gz"
)
with
iter
(
gzip
.
open
(
clustfile
,
'rb'
)
)
as
clustio
:
nloci
=
sum
(
1
for
i
in
clustio
if
"//"
in
i
)
//
2
optim
=
(
nloci
//
20
)
+
(
nloci
%
20
)
LOGGER
.
info
(
"optim for align chunks: %s"
,
optim
)
clustio
=
gzip
.
open
(
clustfile
,
'rb'
)
inclusts
=
iter
(
clustio
.
read
(
)
.
strip
(
)
.
split
(
"//\n//\n"
)
)
inc
=
optim
//
10
for
idx
in
range
(
10
)
:
this
=
optim
+
(
idx
*
inc
)
left
=
nloci
this
if
idx
==
9
:
grabchunk
=
list
(
itertools
.
islice
(
inclusts
,
int
(
1e9
)
)
)
else
:
grabchunk
=
list
(
itertools
.
islice
(
inclusts
,
this
)
)
nloci
=
left
tmpfile
=
os
.
path
.
join
(
data
.
tmpdir
,
sample
.
name
+
"_chunk_{}.ali"
.
format
(
idx
)
)
with
open
(
tmpfile
,
'wb'
)
as
out
:
out
.
write
(
"//\n//\n"
.
join
(
grabchunk
)
)
clustio
.
close
(
)

Splits the muscle alignment into chunks . Each chunk is run on a separate computing core . Because the largest clusters are at the beginning of the clusters file assigning equal clusters to each file would put all of the large cluster that take longer to align near the top . So instead we randomly distribute the clusters among the files . If assembly method is reference then this step is just a placeholder and nothing happens .


def derep_concat_split(data, sample, nthreads, force):
    """
    Running on remote Engine. Refmaps, then merges, then dereplicates,
    then denovo clusters reads.
    """

    ## report location for debugging
    LOGGER.info("INSIDE derep %s", sample.name)

    ## MERGED ASSEMBIES ONLY:
    ## concatenate edits files within Samples. Returns a new sample.files.edits
    ## with the concat file. No change if not merged Assembly.
    mergefile = os.path.join(data.dirs.edits, sample.name+"_merged_.fastq")
    if not force:
        if not os.path.exists(mergefile):
            sample.files.edits = concat_multiple_edits(data, sample)
        else:
            LOGGER.info("skipped concat_multiple_edits: {} exists"\
                        .format(mergefile))
    else:
        sample.files.edits = concat_multiple_edits(data, sample)

    ## PAIRED DATA ONLY:
    ## Denovo: merge or concat fastq pairs [sample.files.pairs]
    ## Reference: only concat fastq pairs  []
    ## Denovo + Reference: ...
    if 'pair' in data.paramsdict['datatype']:
        ## the output file handle for merged reads

        ## modify behavior of merging vs concating if reference
        if "reference" in data.paramsdict["assembly_method"]:
            nmerged = merge_pairs(data, sample.files.edits, mergefile, 0, 0)
        else:
            nmerged = merge_pairs(data, sample.files.edits, mergefile, 1, 1)

        ## store results
        sample.files.edits = [(mergefile, )]
        sample.stats.reads_merged = nmerged

    ## 3rad uses random adapters to identify pcr duplicates. We will
    ## remove pcr dupes here. Basically append the radom adapter to
    ## each sequence, do a regular old vsearch derep, then trim
    ## off the adapter, and push it down the pipeline. This will
    ## remove all identical seqs with identical random i5 adapters.
    if "3rad" in data.paramsdict["datatype"]:
        declone_3rad(data, sample)
        derep_and_sort(data,
                os.path.join(data.dirs.edits, sample.name+"_declone.fastq"),
                os.path.join(data.dirs.edits, sample.name+"_derep.fastq"),
                nthreads)
    else:
        ## convert fastq to fasta, then derep and sort reads by their size.
        ## we pass in only one file b/c paired should be merged by now.
        derep_and_sort(data,
                sample.files.edits[0][0],
                os.path.join(data.dirs.edits, sample.name+"_derep.fastq"),
                nthreads)

Running on remote Engine . Refmaps then merges then dereplicates then denovo clusters reads .


def branch_assembly(args, parsedict):
    """
    Load the passed in assembly and create a branch. Copy it
    to a new assembly, and also write out the appropriate params.txt
    """

    ## Get the current assembly
    data = getassembly(args, parsedict)

    ## get arguments to branch command
    bargs = args.branch

    ## get new name, trim off .txt if it was accidentally added
    newname = bargs[0]
    if newname.endswith(".txt"):
        newname = newname[:-4]

    ## look for subsamples
    if len(bargs) > 1:
        ## Branching and subsampling at step 6 is a bad idea, it messes up
        ## indexing into the hdf5 cluster file. Warn against this.
        if any([x.stats.state == 6 for x in data.samples.values()]):
            pass
            ## TODODODODODO
            #print("wat")

        ## are we removing or keeping listed samples?
        subsamples = bargs[1:]

        ## drop the matching samples
        if bargs[1] == "-":
            ## check drop names
            fails = [i for i in subsamples[1:] if i not in data.samples.keys()]
            if any(fails):
                raise IPyradWarningExit("\
                    \n  Failed: unrecognized names requested, check spelling:\n  {}"\
                    .format("\n  ".join([i for i in fails])))
            print("  dropping {} samples".format(len(subsamples)-1))
            subsamples = list(set(data.samples.keys()) - set(subsamples))

        ## If the arg after the new param name is a file that exists
        if os.path.exists(bargs[1]):
            new_data = data.branch(newname, infile=bargs[1])
        else:
            new_data = data.branch(newname, subsamples)

    ## keeping all samples
    else:
        new_data = data.branch(newname, None)

    print("  creating a new branch called '{}' with {} Samples".\
             format(new_data.name, len(new_data.samples)))

    print("  writing new params file to {}"\
            .format("params-"+new_data.name+".txt\n"))
    new_data.write_params("params-"+new_data.name+".txt", force=args.force)

Load the passed in assembly and create a branch . Copy it to a new assembly and also write out the appropriate params . txt


def getassembly(args, parsedict):
    """
    loads assembly or creates a new one and set its params from
    parsedict. Does not launch ipcluster.
    """

    ## Creating an assembly with a full path in the name will "work"
    ## but it is potentially dangerous, so here we have assembly_name
    ## and assembly_file, name is used for creating new in cwd, file is
    ## used for loading existing.
    ##
    ## Be nice if the user includes the extension.
    #project_dir = ip.core.assembly._expander(parsedict['1'])
    #assembly_name = parsedict['0']
    project_dir = ip.core.assembly._expander(parsedict['project_dir'])
    assembly_name = parsedict['assembly_name']
    assembly_file = os.path.join(project_dir, assembly_name)

    ## Assembly creation will handle error checking  on
    ## the format of the assembly_name

    ## make sure the working directory exists.
    if not os.path.exists(project_dir):
        os.mkdir(project_dir)

    try:
        ## If 1 and force then go ahead and create a new assembly
        if ('1' in args.steps) and args.force:
            data = ip.Assembly(assembly_name, cli=True)
        else:
            data = ip.load_json(assembly_file, cli=True)
            data._cli = True

    except IPyradWarningExit as _:
        ## if no assembly is found then go ahead and make one
        if '1' not in args.steps:
            raise IPyradWarningExit(\
                "  Error: You must first run step 1 on the assembly: {}"\
                .format(assembly_file))
        else:
            ## create a new assembly object
            data = ip.Assembly(assembly_name, cli=True)

    ## for entering some params...
    for param in parsedict:

        ## trap assignment of assembly_name since it is immutable.
        if param == "assembly_name":
            ## Raise error if user tried to change assembly name
            if parsedict[param] != data.name:
                data.set_params(param, parsedict[param])
        else:
            ## all other params should be handled by set_params
            try:
                data.set_params(param, parsedict[param])
            except IndexError as _:
                print("  Malformed params file: {}".format(args.params))
                print("  Bad parameter {} - {}".format(param, parsedict[param]))
                sys.exit(-1)
    return data

loads assembly or creates a new one and set its params from parsedict . Does not launch ipcluster .


def get_binom(base1, base2, estE, estH):
    """
    return probability of base call
    """

    prior_homo = (1. - estH) / 2.
    prior_hete = estH

    ## calculate probs
    bsum = base1 + base2
    hetprob = scipy.misc.comb(bsum, base1)/(2. **(bsum))
    homoa = scipy.stats.binom.pmf(base2, bsum, estE)
    homob = scipy.stats.binom.pmf(base1, bsum, estE)

    ## calculate probs
    hetprob *= prior_hete
    homoa *= prior_homo
    homob *= prior_homo

    ## final
    probabilities = [homoa, homob, hetprob]
    bestprob = max(probabilities)/float(sum(probabilities))

    ## return
    if hetprob > homoa:
        return True, bestprob
    else:
        return False, bestprob

def get_binom ( base1 , base2 , estE , estH ) : prior_homo = ( 1. - estH ) / 2. prior_hete = estH bsum = base1 + base2 hetprob = scipy . misc . comb ( bsum , base1 ) / ( 2. ** ( bsum ) ) homoa = scipy . stats . binom . pmf ( base2 , bsum , estE ) homob = scipy . stats . binom . pmf ( base1 , bsum , estE ) hetprob *= prior_hete homoa *= prior_homo homob *= prior_homo probabilities = [ homoa , homob , hetprob ] bestprob = max ( probabilities ) / float ( sum ( probabilities ) ) if hetprob > homoa : return True , bestprob else : return False , bestprob


def basecaller(arrayed, mindepth_majrule, mindepth_statistical, estH, estE):
    """
    call all sites in a locus array.
    """

    ## an array to fill with consensus site calls
    cons = np.zeros(arrayed.shape[1], dtype=np.uint8)
    cons.fill(78)
    arr = arrayed.view(np.uint8)

    ## iterate over columns
    for col in xrange(arr.shape[1]):
        ## the site of focus
        carr = arr[:, col]

        ## make mask of N and - sites
        mask = carr == 45
        mask += carr == 78
        marr = carr[~mask]

        ## skip if only empties (e.g., N-)
        if not marr.shape[0]:
            cons[col] = 78

        ## skip if not variable
        elif np.all(marr == marr[0]):
            cons[col] = marr[0]

        ## estimate variable site call
        else:
            ## get allele freqs (first-most, second, third = p, q, r)
            counts = np.bincount(marr)

            pbase = np.argmax(counts)
            nump = counts[pbase]
            counts[pbase] = 0

            qbase = np.argmax(counts)
            numq = counts[qbase]
            counts[qbase] = 0

            rbase = np.argmax(counts)
            numr = counts[rbase]

            ## based on biallelic depth
            bidepth = nump + numq
            if bidepth < mindepth_majrule:
                cons[col] = 78

            else:
                ## if depth is too high, reduce to sampled int
                if bidepth > 500:
                    base1 = int(500 * (nump / float(bidepth)))
                    base2 = int(500 * (numq / float(bidepth)))
                else:
                    base1 = nump
                    base2 = numq

                ## make statistical base call
                if bidepth >= mindepth_statistical:
                    ishet, prob = get_binom(base1, base2, estE, estH)
                    #LOGGER.info("ishet, prob, b1, b2: %s %s %s %s", ishet, prob, base1, base2)
                    if prob < 0.95:
                        cons[col] = 78
                    else:
                        if ishet:
                            cons[col] = TRANS[(pbase, qbase)]
                        else:
                            cons[col] = pbase

                ## make majrule base call
                else: #if bidepth >= mindepth_majrule:
                    if nump == numq:
                        cons[col] = TRANS[(pbase, qbase)]
                    else:
                        cons[col] = pbase
    return cons.view("S1")

def basecaller ( arrayed , mindepth_majrule , mindepth_statistical , estH , estE ) : cons = np . zeros ( arrayed . shape [ 1 ] , dtype = np . uint8 ) cons . fill ( 78 ) arr = arrayed . view ( np . uint8 ) for col in xrange ( arr . shape [ 1 ] ) : carr = arr [ : , col ] mask = carr == 45 mask += carr == 78 marr = carr [ ~ mask ] if not marr . shape [ 0 ] : cons [ col ] = 78 elif np . all ( marr == marr [ 0 ] ) : cons [ col ] = marr [ 0 ] else : counts = np . bincount ( marr ) pbase = np . argmax ( counts ) nump = counts [ pbase ] counts [ pbase ] = 0 qbase = np . argmax ( counts ) numq = counts [ qbase ] counts [ qbase ] = 0 rbase = np . argmax ( counts ) numr = counts [ rbase ] bidepth = nump + numq if bidepth < mindepth_majrule : cons [ col ] = 78 else : if bidepth > 500 : base1 = int ( 500 * ( nump / float ( bidepth ) ) ) base2 = int ( 500 * ( numq / float ( bidepth ) ) ) else : base1 = nump base2 = numq if bidepth >= mindepth_statistical : ishet , prob = get_binom ( base1 , base2 , estE , estH ) if prob < 0.95 : cons [ col ] = 78 else : if ishet : cons [ col ] = TRANS [ ( pbase , qbase ) ] else : cons [ col ] = pbase else : if nump == numq : cons [ col ] = TRANS [ ( pbase , qbase ) ] else : cons [ col ] = pbase return cons . view ( "S1" )


def nfilter1(data, reps):
    """ applies read depths filter """
    if sum(reps) >= data.paramsdict["mindepth_majrule"] and \
        sum(reps) <= data.paramsdict["maxdepth"]:
        return 1
    else:
        return 0

def nfilter1 ( data , reps ) : if sum ( reps ) >= data . paramsdict [ "mindepth_majrule" ] and sum ( reps ) <= data . paramsdict [ "maxdepth" ] : return 1 else : return 0


def storealleles(consens, hidx, alleles):
    """ store phased allele data for diploids """
    ## find the first hetero site and choose the priority base
    ## example, if W: then priority base in A and not T. PRIORITY=(order: CATG)
    bigbase = PRIORITY[consens[hidx[0]]]

    ## find which allele has priority based on bigbase
    bigallele = [i for i in alleles if i[0] == bigbase][0]

    ## uplow other bases relative to this one and the priority list
    ## e.g., if there are two hetero sites (WY) and the two alleles are
    ## AT and TC, then since bigbase of (W) is A second hetero site should
    ## be stored as y, since the ordering is swapped in this case; the priority
    ## base (C versus T) is C, but C goes with the minor base at h site 1.
    #consens = list(consens)
    for hsite, pbase in zip(hidx[1:], bigallele[1:]):
        if PRIORITY[consens[hsite]] != pbase:
            consens[hsite] = consens[hsite].lower()

    ## return consens
    return consens

def storealleles ( consens , hidx , alleles ) : bigbase = PRIORITY [ consens [ hidx [ 0 ] ] ] bigallele = [ i for i in alleles if i [ 0 ] == bigbase ] [ 0 ] for hsite , pbase in zip ( hidx [ 1 : ] , bigallele [ 1 : ] ) : if PRIORITY [ consens [ hsite ] ] != pbase : consens [ hsite ] = consens [ hsite ] . lower ( ) return consens


def chunk_clusters(data, sample):
    """ split job into bits and pass to the client """

    ## counter for split job submission
    num = 0

    ## set optim size for chunks in N clusters. The first few chunks take longer
    ## because they contain larger clusters, so we create 4X as many chunks as
    ## processors so that they are split more evenly.
    optim = int((sample.stats.clusters_total // data.cpus) + \
                (sample.stats.clusters_total % data.cpus))

    ## break up the file into smaller tmp files for each engine
    ## chunking by cluster is a bit trickier than chunking by N lines
    chunkslist = []

    ## open to clusters
    with gzip.open(sample.files.clusters, 'rb') as clusters:
        ## create iterator to sample 2 lines at a time
        pairdealer = itertools.izip(*[iter(clusters)]*2)

        ## Use iterator to sample til end of cluster
        done = 0
        while not done:
            ## grab optim clusters and write to file.
            done, chunk = clustdealer(pairdealer, optim)
            chunkhandle = os.path.join(data.dirs.clusts,
                                    "tmp_"+str(sample.name)+"."+str(num*optim))
            if chunk:
                chunkslist.append((optim, chunkhandle))
                with open(chunkhandle, 'wb') as outchunk:
                    outchunk.write("//\n//\n".join(chunk)+"//\n//\n")
                num += 1

    return chunkslist

def chunk_clusters ( data , sample ) : num = 0 optim = int ( ( sample . stats . clusters_total // data . cpus ) + ( sample . stats . clusters_total % data . cpus ) ) chunkslist = [ ] with gzip . open ( sample . files . clusters , 'rb' ) as clusters : pairdealer = itertools . izip ( * [ iter ( clusters ) ] * 2 ) done = 0 while not done : done , chunk = clustdealer ( pairdealer , optim ) chunkhandle = os . path . join ( data . dirs . clusts , "tmp_" + str ( sample . name ) + "." + str ( num * optim ) ) if chunk : chunkslist . append ( ( optim , chunkhandle ) ) with open ( chunkhandle , 'wb' ) as outchunk : outchunk . write ( "//\n//\n" . join ( chunk ) + "//\n//\n" ) num += 1 return chunkslist


def run(data, samples, force, ipyclient):
    """ checks if the sample should be run and passes the args """
    ## prepare dirs
    data.dirs.consens = os.path.join(data.dirs.project, data.name+"_consens")
    if not os.path.exists(data.dirs.consens):
        os.mkdir(data.dirs.consens)

    ## zap any tmp files that might be leftover
    tmpcons = glob.glob(os.path.join(data.dirs.consens, "*_tmpcons.*"))
    tmpcats = glob.glob(os.path.join(data.dirs.consens, "*_tmpcats.*"))
    for tmpfile in tmpcons+tmpcats:
        os.remove(tmpfile)

    ## filter through samples for those ready
    samples = get_subsamples(data, samples, force)

    ## set up parallel client: how many cores?
    lbview = ipyclient.load_balanced_view()
    data.cpus = data._ipcluster["cores"]
    if not data.cpus:
        data.cpus = len(ipyclient.ids)

    ## wrap everything to ensure destruction of temp files
    inst = ""
    try:
        ## calculate depths, if they changed.
        samples = calculate_depths(data, samples, lbview)

        ## chunk clusters into bits for parallel processing
        lasyncs = make_chunks(data, samples, lbview)

        ## process chunks and cleanup
        process_chunks(data, samples, lasyncs, lbview)

    except KeyboardInterrupt as inst:
        raise inst

    finally:
        ## if process failed at any point delete tmp files
        tmpcons = glob.glob(os.path.join(data.dirs.clusts, "tmp_*.[0-9]*"))
        tmpcons += glob.glob(os.path.join(data.dirs.consens, "*_tmpcons.*"))
        tmpcons += glob.glob(os.path.join(data.dirs.consens, "*_tmpcats.*"))
        for tmpchunk in tmpcons:
            os.remove(tmpchunk)

        ## Finished step 5. Set step 6 checkpoint to 0 to force
        ## re-running from scratch.
        data._checkpoint = 0

def run ( data , samples , force , ipyclient ) : data . dirs . consens = os . path . join ( data . dirs . project , data . name + "_consens" ) if not os . path . exists ( data . dirs . consens ) : os . mkdir ( data . dirs . consens ) tmpcons = glob . glob ( os . path . join ( data . dirs . consens , "*_tmpcons.*" ) ) tmpcats = glob . glob ( os . path . join ( data . dirs . consens , "*_tmpcats.*" ) ) for tmpfile in tmpcons + tmpcats : os . remove ( tmpfile ) samples = get_subsamples ( data , samples , force ) lbview = ipyclient . load_balanced_view ( ) data . cpus = data . _ipcluster [ "cores" ] if not data . cpus : data . cpus = len ( ipyclient . ids ) inst = "" try : samples = calculate_depths ( data , samples , lbview ) lasyncs = make_chunks ( data , samples , lbview ) process_chunks ( data , samples , lasyncs , lbview ) except KeyboardInterrupt as inst : raise inst finally : tmpcons = glob . glob ( os . path . join ( data . dirs . clusts , "tmp_*.[0-9]*" ) ) tmpcons += glob . glob ( os . path . join ( data . dirs . consens , "*_tmpcons.*" ) ) tmpcons += glob . glob ( os . path . join ( data . dirs . consens , "*_tmpcats.*" ) ) for tmpchunk in tmpcons : os . remove ( tmpchunk ) data . _checkpoint = 0


def calculate_depths(data, samples, lbview):
    """
    check whether mindepth has changed, and thus whether clusters_hidepth
    needs to be recalculated, and get new maxlen for new highdepth clusts.
    if mindepth not changed then nothing changes.
    """

    ## send jobs to be processed on engines
    start = time.time()
    printstr = " calculating depths    | {} | s5 |"
    recaljobs = {}
    maxlens = []
    for sample in samples:
        recaljobs[sample.name] = lbview.apply(recal_hidepth, *(data, sample))

    ## block until finished
    while 1:
        ready = [i.ready() for i in recaljobs.values()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(ready), sum(ready), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if len(ready) == sum(ready):
            print("")
            break

    ## check for failures and collect results
    modsamples = []
    for sample in samples:
        if not recaljobs[sample.name].successful():
            LOGGER.error("  sample %s failed: %s", sample.name, recaljobs[sample.name].exception())
        else:
            modsample, _, maxlen, _, _ = recaljobs[sample.name].result()
            modsamples.append(modsample)
            maxlens.append(maxlen)

    ## reset global maxlen if something changed
    data._hackersonly["max_fragment_length"] = int(max(maxlens)) + 4

    return samples

def
calculate_depths
(
data
,
samples
,
lbview
)
:
start
=
time
.
time
(
)
printstr
=
" calculating depths    | {} | s5 |"
recaljobs
=
{
}
maxlens
=
[
]
for
sample
in
samples
:
recaljobs
[
sample
.
name
]
=
lbview
.
apply
(
recal_hidepth
,
*
(
data
,
sample
)
)
while
1
:
ready
=
[
i
.
ready
(
)
for
i
in
recaljobs
.
values
(
)
]
elapsed
=
datetime
.
timedelta
(
seconds
=
int
(
time
.
time
(
)
start
)
)
progressbar
(
len
(
ready
)
,
sum
(
ready
)
,
printstr
.
format
(
elapsed
)
,
spacer
=
data
.
_spacer
)
time
.
sleep
(
0.1
)
if
len
(
ready
)
==
sum
(
ready
)
:
print
(
""
)
break
modsamples
=
[
]
for
sample
in
samples
:
if
not
recaljobs
[
sample
.
name
]
.
successful
(
)
:
LOGGER
.
error
(
"  sample %s failed: %s"
,
sample
.
name
,
recaljobs
[
sample
.
name
]
.
exception
(
)
)
else
:
modsample
,
_
,
maxlen
,
_
,
_
=
recaljobs
[
sample
.
name
]
.
result
(
)
modsamples
.
append
(
modsample
)
maxlens
.
append
(
maxlen
)
data
.
_hackersonly
[
"max_fragment_length"
]
=
int
(
max
(
maxlens
)
)
+
4
return
samples

check whether mindepth has changed and thus whether clusters_hidepth needs to be recalculated and get new maxlen for new highdepth clusts . if mindepth not changed then nothing changes .


def make_chunks(data, samples, lbview):
    """
    calls chunk_clusters and tracks progress.
    """
    ## first progress bar
    start = time.time()
    printstr = " chunking clusters     | {} | s5 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(10, 0, printstr.format(elapsed), spacer=data._spacer)

    ## send off samples to be chunked
    lasyncs = {}
    for sample in samples:
        lasyncs[sample.name] = lbview.apply(chunk_clusters, *(data, sample))

    ## block until finished
    while 1:
        ready = [i.ready() for i in lasyncs.values()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(ready), sum(ready), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if len(ready) == sum(ready):
            print("")
            break

    ## check for failures
    for sample in samples:
        if not lasyncs[sample.name].successful():
            LOGGER.error("  sample %s failed: %s", sample.name,
                        lasyncs[sample.name].exception())

    return lasyncs

def
make_chunks
(
data
,
samples
,
lbview
)
:
start
=
time
.
time
(
)
printstr
=
" chunking clusters     | {} | s5 |"
elapsed
=
datetime
.
timedelta
(
seconds
=
int
(
time
.
time
(
)
start
)
)
progressbar
(
10
,
0
,
printstr
.
format
(
elapsed
)
,
spacer
=
data
.
_spacer
)
lasyncs
=
{
}
for
sample
in
samples
:
lasyncs
[
sample
.
name
]
=
lbview
.
apply
(
chunk_clusters
,
*
(
data
,
sample
)
)
while
1
:
ready
=
[
i
.
ready
(
)
for
i
in
lasyncs
.
values
(
)
]
elapsed
=
datetime
.
timedelta
(
seconds
=
int
(
time
.
time
(
)
start
)
)
progressbar
(
len
(
ready
)
,
sum
(
ready
)
,
printstr
.
format
(
elapsed
)
,
spacer
=
data
.
_spacer
)
time
.
sleep
(
0.1
)
if
len
(
ready
)
==
sum
(
ready
)
:
print
(
""
)
break
for
sample
in
samples
:
if
not
lasyncs
[
sample
.
name
]
.
successful
(
)
:
LOGGER
.
error
(
"  sample %s failed: %s"
,
sample
.
name
,
lasyncs
[
sample
.
name
]
.
exception
(
)
)
return
lasyncs

calls chunk_clusters and tracks progress .


def make(data, samples):
    """ reads in .loci and builds alleles from case characters """

    #read in loci file
    outfile = open(os.path.join(data.dirs.outfiles, data.name+".alleles"), 'w')
    lines = open(os.path.join(data.dirs.outfiles, data.name+".loci"), 'r')

    ## Get the longest sample name for pretty printing
    longname = max(len(x) for x in data.samples.keys())

    ## Padding between name and sequence in output file. This should be the
    ## same as write_outfiles.write_tmp_loci.name_padding
    name_padding = 5
    writing = []
    loc = 0
    for line in lines:
        if ">" in line:
            name, seq = line.split(" ")[0], line.split(" ")[-1]
            allele1, allele2 = splitalleles(seq.strip())

            ## Format the output string. the "-2" below accounts for the additional
            ## 2 characters added to the sample name that don't get added to the
            ## snpsites line, so you gotta bump this line back 2 to make it
            ## line up right.
            writing.append(name+"_0"+" "*(longname-len(name)-2+name_padding)+allele1)
            writing.append(name+"_1"+" "*(longname-len(name)-2+name_padding)+allele2)
        else:
            writing.append(line.strip())
        loc += 1

        ## print every 10K loci "
        if not loc % 10000:
            outfile.write("\n".join(writing)+"\n")
            writing = []

    outfile.write("\n".join(writing))
    outfile.close()

def
make
(
data
,
samples
)
:
outfile
=
open
(
os
.
path
.
join
(
data
.
dirs
.
outfiles
,
data
.
name
+
".alleles"
)
,
'w'
)
lines
=
open
(
os
.
path
.
join
(
data
.
dirs
.
outfiles
,
data
.
name
+
".loci"
)
,
'r'
)
longname
=
max
(
len
(
x
)
for
x
in
data
.
samples
.
keys
(
)
)
name_padding
=
5
writing
=
[
]
loc
=
0
for
line
in
lines
:
if
">"
in
line
:
name
,
seq
=
line
.
split
(
" "
)
[
0
]
,
line
.
split
(
" "
)
[
1
]
allele1
,
allele2
=
splitalleles
(
seq
.
strip
(
)
)
writing
.
append
(
name
+
"_0"
+
" "
*
(
longname
len
(
name
)
2
+
name_padding
)
+
allele1
)
writing
.
append
(
name
+
"_1"
+
" "
*
(
longname
len
(
name
)
2
+
name_padding
)
+
allele2
)
else
:
writing
.
append
(
line
.
strip
(
)
)
loc
+=
1
if
not
loc
%
10000
:
outfile
.
write
(
"\n"
.
join
(
writing
)
+
"\n"
)
writing
=
[
]
outfile
.
write
(
"\n"
.
join
(
writing
)
)
outfile
.
close
(
)

reads in . loci and builds alleles from case characters


def cluster_info(ipyclient, spacer=""):
    """ reports host and engine info for an ipyclient """
    ## get engine data, skips busy engines.
    hosts = []
    for eid in ipyclient.ids:
        engine = ipyclient[eid]
        if not engine.outstanding:
            hosts.append(engine.apply(_socket.gethostname))

    ## report it
    hosts = [i.get() for i in hosts]
    result = []
    for hostname in set(hosts):
        result.append("{}host compute node: [{} cores] on {}"\
            .format(spacer, hosts.count(hostname), hostname))
    print "\n".join(result)

def
cluster_info
(
ipyclient
,
spacer
=
""
)
:
hosts
=
[
]
for
eid
in
ipyclient
.
ids
:
engine
=
ipyclient
[
eid
]
if
not
engine
.
outstanding
:
hosts
.
append
(
engine
.
apply
(
_socket
.
gethostname
)
)
hosts
=
[
i
.
get
(
)
for
i
in
hosts
]
result
=
[
]
for
hostname
in
set
(
hosts
)
:
result
.
append
(
"{}host compute node: [{} cores] on {}"
.
format
(
spacer
,
hosts
.
count
(
hostname
)
,
hostname
)
)
print
"\n"
.
join
(
result
)


def _set_debug_dict(__loglevel__):
    """ set the debug dict """

    _lconfig.dictConfig({
    'version': 1,
    'disable_existing_loggers': False,

    'formatters': {
        'standard': {
            'format': "%(asctime)s \t"\
                     +"pid=%(process)d \t"\
                     +"[%(filename)s]\t"\
                     +"%(levelname)s \t"\
                     +"%(message)s"
        },
    },
    'handlers': {
        __name__: {
            'level':__loglevel__,
            'class':'logging.FileHandler',
            'filename':__debugfile__,
            'formatter':"standard",
            'mode':'a+'
        }
    },
    'loggers':{
        __name__: {
            'handlers': [__name__],
            'level': __loglevel__,
            'propogate': True
        }
    }
    })

def
_set_debug_dict
(
__loglevel__
)
:
_lconfig
.
dictConfig
(
{
'version'
:
1
,
'disable_existing_loggers'
:
False
,
'formatters'
:
{
'standard'
:
{
'format'
:
"%(asctime)s \t"
+
"pid=%(process)d \t"
+
"[%(filename)s]\t"
+
"%(levelname)s \t"
+
"%(message)s"
}
,
}
,
'handlers'
:
{
__name__
:
{
'level'
:
__loglevel__
,
'class'
:
'logging.FileHandler'
,
'filename'
:
__debugfile__
,
'formatter'
:
"standard"
,
'mode'
:
'a+'
}
}
,
'loggers'
:
{
__name__
:
{
'handlers'
:
[
__name__
]
,
'level'
:
__loglevel__
,
'propogate'
:
True
}
}
}
)


def _debug_off():
    """ turns off debugging by removing hidden tmp file """
    if _os.path.exists(__debugflag__):
        _os.remove(__debugflag__)
    __loglevel__ = "ERROR"
    _LOGGER.info("debugging turned off")
    _set_debug_dict(__loglevel__)

def
_debug_off
(
)
:
if
_os
.
path
.
exists
(
__debugflag__
)
:
_os
.
remove
(
__debugflag__
)
__loglevel__
=
"ERROR"
_LOGGER
.
info
(
"debugging turned off"
)
_set_debug_dict
(
__loglevel__
)


def _cmd_exists(cmd):
    """ check if dependency program is there """
    return _subprocess.call("type " + cmd,
                           shell=True,
                           stdout=_subprocess.PIPE,
                           stderr=_subprocess.PIPE) == 0

def
_cmd_exists
(
cmd
)
:
return
_subprocess
.
call
(
"type "
+
cmd
,
shell
=
True
,
stdout
=
_subprocess
.
PIPE
,
stderr
=
_subprocess
.
PIPE
)
==
0


def _getbins():
    """ gets the right version of vsearch, muscle, and smalt
    depending on linux vs osx """

    # Return error if system is 32-bit arch.
    # This is straight from the python docs:
    # https://docs.python.org/2/library/platform.html#cross-platform
    if not _sys.maxsize > 2**32:
        _sys.exit("ipyrad requires 64bit architecture")

    ## get platform mac or linux
    _platform = _sys.platform

    ## get current location
    if 'VIRTUAL_ENV' in _os.environ:
        ipyrad_path = _os.environ['VIRTUAL_ENV']
    else:
        path = _os.path.abspath(_os.path.dirname(__file__))
        ipyrad_path = _os.path.dirname(path)

    ## find bin directory
    ipyrad_path = _os.path.dirname(path)
    bin_path = _os.path.join(ipyrad_path, "bin")

    ## get the correct binaries
    if 'linux' in _platform:
        vsearch = _os.path.join(
                       _os.path.abspath(bin_path),
                       "vsearch-linux-x86_64")
        muscle = _os.path.join(
                       _os.path.abspath(bin_path),
                       "muscle-linux-x86_64")
        smalt = _os.path.join(
                       _os.path.abspath(bin_path),
                       "smalt-linux-x86_64")
        bwa = _os.path.join(
                       _os.path.abspath(bin_path),
                       "bwa-linux-x86_64")
        samtools = _os.path.join(
                       _os.path.abspath(bin_path),
                       "samtools-linux-x86_64")
        bedtools = _os.path.join(
                       _os.path.abspath(bin_path),
                       "bedtools-linux-x86_64")
        qmc = _os.path.join(
                       _os.path.abspath(bin_path),
                       "QMC-linux-x86_64")
    else:
        vsearch = _os.path.join(
                       _os.path.abspath(bin_path),
                       "vsearch-osx-x86_64")
        muscle = _os.path.join(
                       _os.path.abspath(bin_path),
                       "muscle-osx-x86_64")
        smalt = _os.path.join(
                       _os.path.abspath(bin_path),
                       "smalt-osx-x86_64")
        bwa = _os.path.join(
                       _os.path.abspath(bin_path),
                       "bwa-osx-x86_64")
        samtools = _os.path.join(
                       _os.path.abspath(bin_path),
                       "samtools-osx-x86_64")
        bedtools = _os.path.join(
                       _os.path.abspath(bin_path),
                       "bedtools-osx-x86_64")
        ## only one compiled version available, works for all?
        qmc = _os.path.join(
                       _os.path.abspath(bin_path),
                       "QMC-osx-x86_64")

    # Test for existence of binaries
    assert _cmd_exists(muscle), "muscle not found here: "+muscle
    assert _cmd_exists(vsearch), "vsearch not found here: "+vsearch
    assert _cmd_exists(smalt), "smalt not found here: "+smalt
    assert _cmd_exists(bwa), "bwa not found here: "+bwa
    assert _cmd_exists(samtools), "samtools not found here: "+samtools
    assert _cmd_exists(bedtools), "bedtools not found here: "+bedtools
    #assert _cmd_exists(qmc), "wQMC not found here: "+qmc
    return vsearch, muscle, smalt, bwa, samtools, bedtools, qmc

def
_getbins
(
)
:
if
not
_sys
.
maxsize
>
2
**
32
:
_sys
.
exit
(
"ipyrad requires 64bit architecture"
)
_platform
=
_sys
.
platform
if
'VIRTUAL_ENV'
in
_os
.
environ
:
ipyrad_path
=
_os
.
environ
[
'VIRTUAL_ENV'
]
else
:
path
=
_os
.
path
.
abspath
(
_os
.
path
.
dirname
(
__file__
)
)
ipyrad_path
=
_os
.
path
.
dirname
(
path
)
ipyrad_path
=
_os
.
path
.
dirname
(
path
)
bin_path
=
_os
.
path
.
join
(
ipyrad_path
,
"bin"
)
if
'linux'
in
_platform
:
vsearch
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"vsearch-linux-x86_64"
)
muscle
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"muscle-linux-x86_64"
)
smalt
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"smalt-linux-x86_64"
)
bwa
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"bwa-linux-x86_64"
)
samtools
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"samtools-linux-x86_64"
)
bedtools
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"bedtools-linux-x86_64"
)
qmc
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"QMC-linux-x86_64"
)
else
:
vsearch
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"vsearch-osx-x86_64"
)
muscle
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"muscle-osx-x86_64"
)
smalt
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"smalt-osx-x86_64"
)
bwa
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"bwa-osx-x86_64"
)
samtools
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"samtools-osx-x86_64"
)
bedtools
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"bedtools-osx-x86_64"
)
qmc
=
_os
.
path
.
join
(
_os
.
path
.
abspath
(
bin_path
)
,
"QMC-osx-x86_64"
)
assert
_cmd_exists
(
muscle
)
,
"muscle not found here: "
+
muscle
assert
_cmd_exists
(
vsearch
)
,
"vsearch not found here: "
+
vsearch
assert
_cmd_exists
(
smalt
)
,
"smalt not found here: "
+
smalt
assert
_cmd_exists
(
bwa
)
,
"bwa not found here: "
+
bwa
assert
_cmd_exists
(
samtools
)
,
"samtools not found here: "
+
samtools
assert
_cmd_exists
(
bedtools
)
,
"bedtools not found here: "
+
bedtools
return
vsearch
,
muscle
,
smalt
,
bwa
,
samtools
,
bedtools
,
qmc

gets the right version of vsearch muscle and smalt depending on linux vs osx


def nworker(data, chunk):
    """
    Worker to distribute work to jit funcs. Wraps everything on an
    engine to run single-threaded to maximize efficiency for
    multi-processing.
    """

    ## set the thread limit on the remote engine
    oldlimit = set_mkl_thread_limit(1)

    ## open seqarray view, the modified arr is in bootstarr
    with h5py.File(data.database.input, 'r') as io5:
        seqview = io5["bootsarr"][:]
        maparr = io5["bootsmap"][:, 0]
        smps = io5["quartets"][chunk:chunk+data._chunksize]

        ## create an N-mask array of all seq cols
        nall_mask = seqview[:] == 78

    ## init arrays to fill with results
    rquartets = np.zeros((smps.shape[0], 4), dtype=np.uint16)
    rinvariants = np.zeros((smps.shape[0], 16, 16), dtype=np.uint16)

    ## fill arrays with results as we compute them. This iterates
    ## over all of the quartet sets in this sample chunk. It would
    ## be nice to have this all numbified.
    for idx in xrange(smps.shape[0]):
        sidx = smps[idx]
        seqs = seqview[sidx]

        ## these axis calls cannot be numbafied, but I can't
        ## find a faster way that is JIT compiled, and I've
        ## really, really, really tried. Tried again now that
        ## numba supports axis args for np.sum. Still can't
        ## get speed improvements by numbifying this loop.
        nmask = np.any(nall_mask[sidx], axis=0)
        nmask += np.all(seqs == seqs[0], axis=0)

        ## here are the jitted funcs
        bidx, invar = calculate(seqs, maparr, nmask, TESTS)

        ## store results
        rquartets[idx] = smps[idx][bidx]
        rinvariants[idx] = invar

    ## reset thread limit
    set_mkl_thread_limit(oldlimit)

    ## return results...
    return rquartets, rinvariants

def nworker ( data , chunk ) : oldlimit = set_mkl_thread_limit ( 1 ) with h5py . File ( data . database . input , 'r' ) as io5 : seqview = io5 [ "bootsarr" ] [ : ] maparr = io5 [ "bootsmap" ] [ : , 0 ] smps = io5 [ "quartets" ] [ chunk : chunk + data . _chunksize ] nall_mask = seqview [ : ] == 78 rquartets = np . zeros ( ( smps . shape [ 0 ] , 4 ) , dtype = np . uint16 ) rinvariants = np . zeros ( ( smps . shape [ 0 ] , 16 , 16 ) , dtype = np . uint16 ) for idx in xrange ( smps . shape [ 0 ] ) : sidx = smps [ idx ] seqs = seqview [ sidx ] nmask = np . any ( nall_mask [ sidx ] , axis = 0 ) nmask += np . all ( seqs == seqs [ 0 ] , axis = 0 ) bidx , invar = calculate ( seqs , maparr , nmask , TESTS ) rquartets [ idx ] = smps [ idx ] [ bidx ] rinvariants [ idx ] = invar set_mkl_thread_limit ( oldlimit ) return rquartets , rinvariants

Worker to distribute work to jit funcs . Wraps everything on an engine to run single - threaded to maximize efficiency for multi - processing .


def store_all(self):
    """
    Populate array with all possible quartets. This allows us to
    sample from the total, and also to continue from a checkpoint
    """

    with h5py.File(self.database.input, 'a') as io5:
        fillsets = io5["quartets"]

        ## generator for all quartet sets
        qiter = itertools.combinations(xrange(len(self.samples)), 4)
        i = 0
        while i < self.params.nquartets:
            ## sample a chunk of the next ordered N set of quartets
            dat = np.array(list(itertools.islice(qiter, self._chunksize)))
            end = min(self.params.nquartets, dat.shape[0]+i)
            fillsets[i:end] = dat[:end-i]
            i += self._chunksize

            ## send progress update to stdout on engine
            print(min(i, self.params.nquartets))

def store_all ( self ) : with h5py . File ( self . database . input , 'a' ) as io5 : fillsets = io5 [ "quartets" ] qiter = itertools . combinations ( xrange ( len ( self . samples ) ) , 4 ) i = 0 while i < self . params . nquartets : dat = np . array ( list ( itertools . islice ( qiter , self . _chunksize ) ) ) end = min ( self . params . nquartets , dat . shape [ 0 ] + i ) fillsets [ i : end ] = dat [ : end - i ] i += self . _chunksize print ( min ( i , self . params . nquartets ) )

Populate array with all possible quartets . This allows us to sample from the total and also to continue from a checkpoint


def store_random(self):
    """
    Populate array with random quartets sampled from a generator.
    Holding all sets in memory might take a lot, but holding a very
    large list of random numbers for which ones to sample will fit
    into memory for most reasonable sized sets. So we'll load a
    list of random numbers in the range of the length of total
    sets that can be generated, then only keep sets from the set
    generator if they are in the int list. I did several tests to
    check that random pairs are as likely as 0 & 1 to come up together
    in a random quartet set.
    """

    with h5py.File(self.database.input, 'a') as io5:
        fillsets = io5["quartets"]

        ## set generators
        qiter = itertools.combinations(xrange(len(self.samples)), 4)
        rand = np.arange(0, n_choose_k(len(self.samples), 4))
        np.random.shuffle(rand)
        rslice = rand[:self.params.nquartets]
        rss = np.sort(rslice)
        riter = iter(rss)
        del rand, rslice

        ## print progress update 1 to the engine stdout
        print(self._chunksize)

        ## set to store
        rando = riter.next()
        tmpr = np.zeros((self.params.nquartets, 4), dtype=np.uint16)
        tidx = 0
        while 1:
            try:
                for i, j in enumerate(qiter):
                    if i == rando:
                        tmpr[tidx] = j
                        tidx += 1
                        rando = riter.next()

                    ## print progress bar update to engine stdout
                    if not i % self._chunksize:
                        print(min(i, self.params.nquartets))

            except StopIteration:
                break
        ## store into database
        fillsets[:] = tmpr
        del tmpr

def store_random ( self ) : with h5py . File ( self . database . input , 'a' ) as io5 : fillsets = io5 [ "quartets" ] qiter = itertools . combinations ( xrange ( len ( self . samples ) ) , 4 ) rand = np . arange ( 0 , n_choose_k ( len ( self . samples ) , 4 ) ) np . random . shuffle ( rand ) rslice = rand [ : self . params . nquartets ] rss = np . sort ( rslice ) riter = iter ( rss ) del rand , rslice print ( self . _chunksize ) rando = riter . next ( ) tmpr = np . zeros ( ( self . params . nquartets , 4 ) , dtype = np . uint16 ) tidx = 0 while 1 : try : for i , j in enumerate ( qiter ) : if i == rando : tmpr [ tidx ] = j tidx += 1 rando = riter . next ( ) if not i % self . _chunksize : print ( min ( i , self . params . nquartets ) ) except StopIteration : break fillsets [ : ] = tmpr del tmpr

Populate array with random quartets sampled from a generator . Holding all sets in memory might take a lot but holding a very large list of random numbers for which ones to sample will fit into memory for most reasonable sized sets . So we ll load a list of random numbers in the range of the length of total sets that can be generated then only keep sets from the set generator if they are in the int list . I did several tests to check that random pairs are as likely as 0 & 1 to come up together in a random quartet set .


def random_combination(nsets, n, k):
    """
    Returns nsets unique random quartet sets sampled from
    n-choose-k without replacement combinations.
    """
    sets = set()
    while len(sets) < nsets:
        newset = tuple(sorted(np.random.choice(n, k, replace=False)))
        sets.add(newset)
    return tuple(sets)

def random_combination ( nsets , n , k ) : sets = set ( ) while len ( sets ) < nsets : newset = tuple ( sorted ( np . random . choice ( n , k , replace = False ) ) ) sets . add ( newset ) return tuple ( sets )

Returns nsets unique random quartet sets sampled from n - choose - k without replacement combinations .


def random_product(iter1, iter2):
    """
    Random sampler for equal_splits functions
    """
    iter4 = np.concatenate([
        np.random.choice(iter1, 2, replace=False),
        np.random.choice(iter2, 2, replace=False)
        ])
    return iter4

def random_product ( iter1 , iter2 ) : iter4 = np . concatenate ( [ np . random . choice ( iter1 , 2 , replace = False ) , np . random . choice ( iter2 , 2 , replace = False ) ] ) return iter4


def resolve_ambigs(tmpseq):
    """
    Randomly resolve ambiguous bases. This is applied to each boot
    replicate so that over reps the random resolutions don't matter.
    Sites are randomly resolved, so best for unlinked SNPs since
    otherwise linked SNPs are losing their linkage information...
    though it's not like we're using it anyways.
    """

    ## the order of rows in GETCONS
    for aidx in xrange(6):
        #np.uint([82, 75, 83, 89, 87, 77]):
        ambig, res1, res2 = GETCONS[aidx]

        ## get true wherever tmpseq is ambig
        idx, idy = np.where(tmpseq == ambig)
        halfmask = np.random.choice(np.array([True, False]), idx.shape[0])

        for col in xrange(idx.shape[0]):
            if halfmask[col]:
                tmpseq[idx[col], idy[col]] = res1
            else:
                tmpseq[idx[col], idy[col]] = res2
    return tmpseq

def resolve_ambigs ( tmpseq ) : for aidx in xrange ( 6 ) : ambig , res1 , res2 = GETCONS [ aidx ] idx , idy = np . where ( tmpseq == ambig ) halfmask = np . random . choice ( np . array ( [ True , False ] ) , idx . shape [ 0 ] ) for col in xrange ( idx . shape [ 0 ] ) : if halfmask [ col ] : tmpseq [ idx [ col ] , idy [ col ] ] = res1 else : tmpseq [ idx [ col ] , idy [ col ] ] = res2 return tmpseq

Randomly resolve ambiguous bases . This is applied to each boot replicate so that over reps the random resolutions don t matter . Sites are randomly resolved so best for unlinked SNPs since otherwise linked SNPs are losing their linkage information ... though it s not like we re using it anyways .


def set_mkl_thread_limit(cores):
    """
    set mkl thread limit and return old value so we can reset
    when finished.
    """
    if "linux" in sys.platform:
        mkl_rt = ctypes.CDLL('libmkl_rt.so')
    else:
        mkl_rt = ctypes.CDLL('libmkl_rt.dylib')
    oldlimit = mkl_rt.mkl_get_max_threads()
    mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(cores)))
    return oldlimit

def set_mkl_thread_limit ( cores ) : if "linux" in sys . platform : mkl_rt = ctypes . CDLL ( 'libmkl_rt.so' ) else : mkl_rt = ctypes . CDLL ( 'libmkl_rt.dylib' ) oldlimit = mkl_rt . mkl_get_max_threads ( ) mkl_rt . mkl_set_num_threads ( ctypes . byref ( ctypes . c_int ( cores ) ) ) return oldlimit

set mkl thread limit and return old value so we can reset when finished .


def get_total(tots, node):
    """ get total number of quartets possible for a split"""
    if (node.is_leaf() or node.is_root()):
        return 0
    else:
        ## Get counts on down edges.
        ## How to treat polytomies here?
        if len(node.children) > 2:
            down_r = node.children[0]
            down_l = node.children[1]
            for child in node.children[2:]:
                down_l += child
        else:
            down_r, down_l = node.children
        lendr = sum(1 for i in down_r.iter_leaves())
        lendl = sum(1 for i in down_l.iter_leaves())

        ## get count on up edge sister
        up_r = node.get_sisters()[0]
        lenur = sum(1 for i in up_r.iter_leaves())

        ## everyone else
        lenul = tots - (lendr + lendl + lenur)

        ## return product
        return lendr * lendl * lenur * lenul

def get_total ( tots , node ) : if ( node . is_leaf ( ) or node . is_root ( ) ) : return 0 else : if len ( node . children ) > 2 : down_r = node . children [ 0 ] down_l = node . children [ 1 ] for child in node . children [ 2 : ] : down_l += child else : down_r , down_l = node . children lendr = sum ( 1 for i in down_r . iter_leaves ( ) ) lendl = sum ( 1 for i in down_l . iter_leaves ( ) ) up_r = node . get_sisters ( ) [ 0 ] lenur = sum ( 1 for i in up_r . iter_leaves ( ) ) lenul = tots - ( lendr + lendl + lenur ) return lendr * lendl * lenur * lenul


def get_sampled(data, totn, node):
    """ get total number of quartets sampled for a split"""
    ## convert tip names to ints
    names = sorted(totn)
    cdict = {name: idx for idx, name in enumerate(names)}

    ## skip some nodes
    if (node.is_leaf() or node.is_root()):
        return 0
    else:
        ## get counts on down edges
        if len(node.children) > 2:
            down_r = node.children[0]
            down_l = node.children[1]
            for child in node.children[2:]:
                down_l += child
        else:
            down_r, down_l = node.children

        lendr = set(cdict[i] for i in down_r.get_leaf_names())
        lendl = set(cdict[i] for i in down_l.get_leaf_names())

        ## get count on up edge sister
        up_r = node.get_sisters()[0]
        lenur = set(cdict[i] for i in up_r.get_leaf_names())

        ## everyone else
        lenul = set(cdict[i] for i in totn) - set.union(lendr, lendl, lenur)

    idx = 0
    sampled = 0
    with h5py.File(data.database.output, 'r') as io5:
        end = io5["quartets"].shape[0]
        while 1:
            ## break condition
            if idx >= end:
                break

            ## counts matches
            qrts = io5["quartets"][idx:idx+data._chunksize]
            for qrt in qrts:
                sqrt = set(qrt)
                if all([sqrt.intersection(i) for i in [lendr, lendl, lenur, lenul]]):
                    sampled += 1

            ## increase span
            idx += data._chunksize
    return sampled

def get_sampled ( data , totn , node ) : names = sorted ( totn ) cdict = { name : idx for idx , name in enumerate ( names ) } if ( node . is_leaf ( ) or node . is_root ( ) ) : return 0 else : if len ( node . children ) > 2 : down_r = node . children [ 0 ] down_l = node . children [ 1 ] for child in node . children [ 2 : ] : down_l += child else : down_r , down_l = node . children lendr = set ( cdict [ i ] for i in down_r . get_leaf_names ( ) ) lendl = set ( cdict [ i ] for i in down_l . get_leaf_names ( ) ) up_r = node . get_sisters ( ) [ 0 ] lenur = set ( cdict [ i ] for i in up_r . get_leaf_names ( ) ) lenul = set ( cdict [ i ] for i in totn ) - set . union ( lendr , lendl , lenur ) idx = 0 sampled = 0 with h5py . File ( data . database . output , 'r' ) as io5 : end = io5 [ "quartets" ] . shape [ 0 ] while 1 : if idx >= end : break qrts = io5 [ "quartets" ] [ idx : idx + data . _chunksize ] for qrt in qrts : sqrt = set ( qrt ) if all ( [ sqrt . intersection ( i ) for i in [ lendr , lendl , lenur , lenul ] ] ) : sampled += 1 idx += data . _chunksize return sampled


def _run_qmc(self, boot):
        """
        Runs quartet max-cut QMC on the quartets qdump file.
        """

        ## build command
        self._tmp = os.path.join(self.dirs, ".tmptre")
        cmd = [ip.bins.qmc, "qrtt="+self.files.qdump, "otre="+self._tmp]

        ## run it
        proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
        res = proc.communicate()
        if proc.returncode:
            raise IPyradWarningExit(res[1])

        ## parse tmp file written by qmc into a tree and rename it
        with open(self._tmp, 'r') as intree:
            tre = ete3.Tree(intree.read().strip())
            names = tre.get_leaves()
            for name in names:
                name.name = self.samples[int(name.name)]
            tmptre = tre.write(format=9)

        ## save the tree to file
        if boot:
            self.trees.boots = os.path.join(self.dirs, self.name+".boots")
            with open(self.trees.boots, 'a') as outboot:
                outboot.write(tmptre+"\n")
        else:
            self.trees.tree = os.path.join(self.dirs, self.name+".tree")
            with open(self.trees.tree, 'w') as outtree:
                outtree.write(tmptre)

        ## save the file
        self._save()

def _run_qmc ( self , boot ) : self . _tmp = os . path . join ( self . dirs , ".tmptre" ) cmd = [ ip . bins . qmc , "qrtt=" + self . files . qdump , "otre=" + self . _tmp ] proc = subprocess . Popen ( cmd , stderr = subprocess . STDOUT , stdout = subprocess . PIPE ) res = proc . communicate ( ) if proc . returncode : raise IPyradWarningExit ( res [ 1 ] ) with open ( self . _tmp , 'r' ) as intree : tre = ete3 . Tree ( intree . read ( ) . strip ( ) ) names = tre . get_leaves ( ) for name in names : name . name = self . samples [ int ( name . name ) ] tmptre = tre . write ( format = 9 ) if boot : self . trees . boots = os . path . join ( self . dirs , self . name + ".boots" ) with open ( self . trees . boots , 'a' ) as outboot : outboot . write ( tmptre + "\n" ) else : self . trees . tree = os . path . join ( self . dirs , self . name + ".tree" ) with open ( self . trees . tree , 'w' ) as outtree : outtree . write ( tmptre ) self . _save ( )

Runs quartet max - cut QMC on the quartets qdump file .


def _insert_to_array(self, chunk, results):
        """
        Enters results arrays into the HDF5 database.
        """

        ## two result arrs
        chunksize = self._chunksize
        qrts, invs = results

        ## enter into db
        with h5py.File(self.database.output, 'r+') as io5:
            io5['quartets'][chunk:chunk+chunksize] = qrts

            ## entered as 0-indexed !
            if self.params.save_invariants:
                if self.checkpoint.boots:
                    key = "invariants/boot{}".format(self.checkpoint.boots)
                    io5[key][chunk:chunk+chunksize] = invs
                else:
                    io5["invariants/boot0"][chunk:chunk+chunksize] = invs

def _insert_to_array ( self , chunk , results ) : chunksize = self . _chunksize qrts , invs = results with h5py . File ( self . database . output , 'r+' ) as io5 : io5 [ 'quartets' ] [ chunk : chunk + chunksize ] = qrts if self . params . save_invariants : if self . checkpoint . boots : key = "invariants/boot{}" . format ( self . checkpoint . boots ) io5 [ key ] [ chunk : chunk + chunksize ] = invs else : io5 [ "invariants/boot0" ] [ chunk : chunk + chunksize ] = invs

Enters results arrays into the HDF5 database .


def get_client(cluster_id, profile, engines, timeout, cores, quiet, spacer, **kwargs):
    """
    Creates a client to view ipcluster engines for a given profile and
    returns it with at least one engine spun up and ready to go. If no
    engines are found after nwait amount of time then an error is raised.
    If engines==MPI it waits a bit longer to find engines. If the number
    of engines is set then it waits even longer to try to find that number
    of engines.
    """

    ## save stds for later, we're gonna hide them to prevent external printing
    save_stdout = sys.stdout
    save_stderr = sys.stderr
    sys.stdout = cStringIO.StringIO()
    sys.stderr = cStringIO.StringIO()

    ## get cluster_info print string
    connection_string = "{}establishing parallel connection:".format(spacer)

    ## wrapped search for ipcluster
    try:
        ## are we looking for a running ipcluster instance?
        if profile not in [None, "default"]:
            args = {'profile': profile, "timeout": timeout}
        else:
            clusterargs = [cluster_id, profile, timeout]
            argnames = ["cluster_id", "profile", "timeout"]
            args = {key:value for key, value in zip(argnames, clusterargs)}

        ## get connection within timeout window of wait time and hide messages
        ipyclient = ipp.Client(**args)
        sys.stdout = save_stdout
        sys.stderr = save_stderr

        ## check that all engines have connected
        if (engines == "MPI") or ("ipyrad-cli-" in cluster_id):
            if not quiet:
                print(connection_string)

        for _ in range(6000):
            initid = len(ipyclient)
            time.sleep(0.01)
            ## If MPI then wait for all engines to start so we can report
            ## how many cores are on each host. If Local then only wait for
            ## one engine to be ready and then just go.
            if (engines == "MPI") or ("ipyrad-cli-" in cluster_id):
                ## wait for cores to be connected
                if cores:
                    time.sleep(0.1)
                    if initid == cores:
                        break
                if initid:
                    time.sleep(3)
                    if len(ipyclient) == initid:
                        break
            else:
                if cores:
                    if initid == cores:
                        break
                else:
                    if initid:
                        break

    except KeyboardInterrupt as inst:
        ## ensure stdout is reset even if Exception was raised
        sys.stdout = save_stdout
        sys.stderr = save_stderr
        raise inst

    ## This is raised if ipcluster is not running ------------
    except IOError as inst:
        ## ensure stdout is reset even if Exception was raised
        sys.stdout = save_stdout
        sys.stderr = save_stderr
        if "ipyrad-cli-" in cluster_id:
            raise IPyradWarningExit(NO_IPCLUSTER_CLI)
        else:
            raise IPyradWarningExit(NO_IPCLUSTER_API)

    except (ipp.TimeoutError, ipp.NoEnginesRegistered) as inst:
        ## raised by ipp if no connection file is found for 'nwait' seconds
        sys.stdout = save_stdout
        sys.stderr = save_stderr
        raise inst

    except Exception as inst:
        ## if any other exceptions were missed...
        sys.stdout = save_stdout
        sys.stderr = save_stderr
        raise inst

    finally:
        ## ensure that no matter what we reset the stds
        sys.stdout = save_stdout
        sys.stderr = save_stderr

    return ipyclient

def
get_client
(
cluster_id
,
profile
,
engines
,
timeout
,
cores
,
quiet
,
spacer
,
**
kwargs
)
:
save_stdout
=
sys
.
stdout
save_stderr
=
sys
.
stderr
sys
.
stdout
=
cStringIO
.
StringIO
(
)
sys
.
stderr
=
cStringIO
.
StringIO
(
)
connection_string
=
"{}establishing parallel connection:"
.
format
(
spacer
)
try
:
if
profile
not
in
[
,
"default"
]
:
args
=
{
'profile'
:
profile
,
"timeout"
:
timeout
}
else
:
clusterargs
=
[
cluster_id
,
profile
,
timeout
]
argnames
=
[
"cluster_id"
,
"profile"
,
"timeout"
]
args
=
{
key
:
value
for
key
,
value
in
zip
(
argnames
,
clusterargs
)
}
ipyclient
=
ipp
.
Client
(
**
args
)
sys
.
stdout
=
save_stdout
sys
.
stderr
=
save_stderr
if
(
engines
==
"MPI"
)
or
(
"ipyrad-cli-"
in
cluster_id
)
:
if
not
quiet
:
print
(
connection_string
)
for
_
in
range
(
6000
)
:
initid
=
len
(
ipyclient
)
time
.
sleep
(
0.01
)
if
(
engines
==
"MPI"
)
or
(
"ipyrad-cli-"
in
cluster_id
)
:
if
cores
:
time
.
sleep
(
0.1
)
if
initid
==
cores
:
break
if
initid
:
time
.
sleep
(
3
)
if
len
(
ipyclient
)
==
initid
:
break
else
:
if
cores
:
if
initid
==
cores
:
break
else
:
if
initid
:
break
except
KeyboardInterrupt
as
inst
:
sys
.
stdout
=
save_stdout
sys
.
stderr
=
save_stderr
raise
inst
except
IOError
as
inst
:
sys
.
stdout
=
save_stdout
sys
.
stderr
=
save_stderr
if
"ipyrad-cli-"
in
cluster_id
:
raise
IPyradWarningExit
(
NO_IPCLUSTER_CLI
)
else
:
raise
IPyradWarningExit
(
NO_IPCLUSTER_API
)
except
(
ipp
.
TimeoutError
,
ipp
.
NoEnginesRegistered
)
as
inst
:
sys
.
stdout
=
save_stdout
sys
.
stderr
=
save_stderr
raise
inst
except
Exception
as
inst
:
sys
.
stdout
=
save_stdout
sys
.
stderr
=
save_stderr
raise
inst
finally
:
sys
.
stdout
=
save_stdout
sys
.
stderr
=
save_stderr
return
ipyclient

Creates a client to view ipcluster engines for a given profile and returns it with at least one engine spun up and ready to go . If no engines are found after nwait amount of time then an error is raised . If engines == MPI it waits a bit longer to find engines . If the number of engines is set then it waits even longer to try to find that number of engines .


def memoize(func):
    """ Memoization decorator for a function taking one or more arguments. """
    class Memodict(dict):
        """ just a dict"""
        def __getitem__(self, *key):
            return dict.__getitem__(self, key)

        def __missing__(self, key):
            """ this makes it faster """
            ret = self[key] = func(*key)
            return ret

    return Memodict().__getitem__

def memoize ( func ) : class Memodict ( dict ) : def __getitem__ ( self , * key ) : return dict . __getitem__ ( self , key ) def __missing__ ( self , key ) : ret = self [ key ] = func ( * key ) return ret return Memodict ( ) . __getitem__

Memoization decorator for a function taking one or more arguments .


def ambigcutters(seq):
    """
    Returns both resolutions of a cut site that has an ambiguous base in
    it, else the single cut site
    """
    resos = []
    if any([i in list("RKSYWM") for i in seq]):
        for base in list("RKSYWM"):
            if base in seq:
                resos.append(seq.replace(base, AMBIGS[base][0]))
                resos.append(seq.replace(base, AMBIGS[base][1]))
        return resos
    else:
        return [seq, ""]

def ambigcutters ( seq ) : resos = [ ] if any ( [ i in list ( "RKSYWM" ) for i in seq ] ) : for base in list ( "RKSYWM" ) : if base in seq : resos . append ( seq . replace ( base , AMBIGS [ base ] [ 0 ] ) ) resos . append ( seq . replace ( base , AMBIGS [ base ] [ 1 ] ) ) return resos else : return [ seq , "" ]

Returns both resolutions of a cut site that has an ambiguous base in it else the single cut site


def splitalleles(consensus):
    """ takes diploid consensus alleles with phase data stored as a mixture
    of upper and lower case characters and splits it into 2 alleles """

    ## store two alleles, allele1 will start with bigbase
    allele1 = list(consensus)
    allele2 = list(consensus)
    hidx = [i for (i, j) in enumerate(consensus) if j in "RKSWYMrkswym"]

    ## do remaining h sites
    for idx in hidx:
        hsite = consensus[idx]
        if hsite.isupper():
            allele1[idx] = PRIORITY[hsite]
            allele2[idx] = MINOR[hsite]
        else:
            allele1[idx] = MINOR[hsite.upper()]
            allele2[idx] = PRIORITY[hsite.upper()]

    ## convert back to strings
    allele1 = "".join(allele1)
    allele2 = "".join(allele2)

    return allele1, allele2

def splitalleles ( consensus ) : allele1 = list ( consensus ) allele2 = list ( consensus ) hidx = [ i for ( i , j ) in enumerate ( consensus ) if j in "RKSWYMrkswym" ] for idx in hidx : hsite = consensus [ idx ] if hsite . isupper ( ) : allele1 [ idx ] = PRIORITY [ hsite ] allele2 [ idx ] = MINOR [ hsite ] else : allele1 [ idx ] = MINOR [ hsite . upper ( ) ] allele2 [ idx ] = PRIORITY [ hsite . upper ( ) ] allele1 = "" . join ( allele1 ) allele2 = "" . join ( allele2 ) return allele1 , allele2


def comp(seq):
    """ returns a seq with complement. Preserves little n's for splitters."""
    ## makes base to its small complement then makes upper
    return seq.replace("A", 't')\
              .replace('T', 'a')\
              .replace('C', 'g')\
              .replace('G', 'c')\
              .replace('n', 'Z')\
              .upper()\
              .replace("Z", "n")

def comp ( seq ) : return seq . replace ( "A" , 't' ) . replace ( 'T' , 'a' ) . replace ( 'C' , 'g' ) . replace ( 'G' , 'c' ) . replace ( 'n' , 'Z' ) . upper ( ) . replace ( "Z" , "n" )

returns a seq with complement . Preserves little n s for splitters .


def fullcomp(seq):
    """ returns complement of sequence including ambiguity characters,
    and saves lower case info for multiple hetero sequences"""
    ## this is surely not the most efficient...
    seq = seq.replace("A", 'u')\
             .replace('T', 'v')\
             .replace('C', 'p')\
             .replace('G', 'z')\
             .replace('u', 'T')\
             .replace('v', 'A')\
             .replace('p', 'G')\
             .replace('z', 'C')

    ## No complement for S & W b/c complements are S & W, respectively
    seq = seq.replace('R', 'u')\
             .replace('K', 'v')\
             .replace('Y', 'b')\
             .replace('M', 'o')\
             .replace('u', 'Y')\
             .replace('v', 'M')\
             .replace('b', 'R')\
             .replace('o', 'K')

    seq = seq.replace('r', 'u')\
             .replace('k', 'v')\
             .replace('y', 'b')\
             .replace('m', 'o')\
             .replace('u', 'y')\
             .replace('v', 'm')\
             .replace('b', 'r')\
             .replace('o', 'k')
    return seq

def fullcomp ( seq ) : seq = seq . replace ( "A" , 'u' ) . replace ( 'T' , 'v' ) . replace ( 'C' , 'p' ) . replace ( 'G' , 'z' ) . replace ( 'u' , 'T' ) . replace ( 'v' , 'A' ) . replace ( 'p' , 'G' ) . replace ( 'z' , 'C' ) seq = seq . replace ( 'R' , 'u' ) . replace ( 'K' , 'v' ) . replace ( 'Y' , 'b' ) . replace ( 'M' , 'o' ) . replace ( 'u' , 'Y' ) . replace ( 'v' , 'M' ) . replace ( 'b' , 'R' ) . replace ( 'o' , 'K' ) seq = seq . replace ( 'r' , 'u' ) . replace ( 'k' , 'v' ) . replace ( 'y' , 'b' ) . replace ( 'm' , 'o' ) . replace ( 'u' , 'y' ) . replace ( 'v' , 'm' ) . replace ( 'b' , 'r' ) . replace ( 'o' , 'k' ) return seq

returns complement of sequence including ambiguity characters and saves lower case info for multiple hetero sequences


def fastq_touchup_for_vsearch_merge(read, outfile, reverse=False):
    """ option to change orientation of reads and sets Qscore to B """

    counts = 0
    with open(outfile, 'w') as out:
        ## read in paired end read files 4 lines at a time
        if read.endswith(".gz"):
            fr1 = gzip.open(read, 'rb')
        else:
            fr1 = open(read, 'rb')
        quarts = itertools.izip(*[iter(fr1)]*4)

        ## a list to store until writing
        writing = []

        while 1:
            try:
                lines = quarts.next()
            except StopIteration:
                break
            if reverse:
                seq = lines[1].strip()[::-1]
            else:
                seq = lines[1].strip()
            writing.append("".join([
                lines[0],
                seq+"\n",
                lines[2],
                "B"*len(seq)
            ]))

            ## write to disk
            counts += 1
            if not counts % 1000:
                out.write("\n".join(writing)+"\n")
                writing = []
        if writing:
            out.write("\n".join(writing))

    out.close()
    fr1.close()

def fastq_touchup_for_vsearch_merge ( read , outfile , reverse = False ) : counts = 0 with open ( outfile , 'w' ) as out : if read . endswith ( ".gz" ) : fr1 = gzip . open ( read , 'rb' ) else : fr1 = open ( read , 'rb' ) quarts = itertools . izip ( * [ iter ( fr1 ) ] * 4 ) writing = [ ] while 1 : try : lines = quarts . next ( ) except StopIteration : break if reverse : seq = lines [ 1 ] . strip ( ) [ : : - 1 ] else : seq = lines [ 1 ] . strip ( ) writing . append ( "" . join ( [ lines [ 0 ] , seq + "\n" , lines [ 2 ] , "B" * len ( seq ) ] ) ) counts += 1 if not counts % 1000 : out . write ( "\n" . join ( writing ) + "\n" ) writing = [ ] if writing : out . write ( "\n" . join ( writing ) ) out . close ( ) fr1 . close ( )


def revcomp(sequence):
    "returns reverse complement of a string"
    sequence = sequence[::-1].strip()\
                             .replace("A", "t")\
                             .replace("T", "a")\
                             .replace("C", "g")\
                             .replace("G", "c").upper()
    return sequence


def clustdealer(pairdealer, optim):
    """ return optim clusters given iterators, and whether it got all or not"""
    ccnt = 0
    chunk = []
    while ccnt < optim:
        ## try refreshing taker, else quit
        try:
            taker = itertools.takewhile(lambda x: x[0] != "//\n", pairdealer)
            oneclust = ["".join(taker.next())]
        except StopIteration:
            #LOGGER.debug('last chunk %s', chunk)
            return 1, chunk

        ## load one cluster
        while 1:
            try:
                oneclust.append("".join(taker.next()))
            except StopIteration:
                break
        chunk.append("".join(oneclust))
        ccnt += 1
    return 0, chunk

def clustdealer ( pairdealer , optim ) : ccnt = 0 chunk = [ ] while ccnt < optim : try : taker = itertools . takewhile ( lambda x : x [ 0 ] != "//\n" , pairdealer ) oneclust = [ "" . join ( taker . next ( ) ) ] except StopIteration : return 1 , chunk while 1 : try : oneclust . append ( "" . join ( taker . next ( ) ) ) except StopIteration : break chunk . append ( "" . join ( oneclust ) ) ccnt += 1 return 0 , chunk

return optim clusters given iterators and whether it got all or not


def progressbar(njobs, finished, msg="", spacer="  "):
    """ prints a progress bar """
    if njobs:
        progress = 100*(finished / float(njobs))
    else:
        progress = 100

    hashes = '#'*int(progress/5.)
    nohash = ' '*int(20-len(hashes))
    if not ipyrad.__interactive__:
        msg = msg.rsplit("|", 2)[0]

    args = [spacer, hashes+nohash, int(progress), msg]
    print("\r{}[{}] {:>3}% {} ".format(*args), end="")
    sys.stdout.flush()

def
progressbar
(
njobs
,
finished
,
msg
=
""
,
spacer
=
"  "
)
:
if
njobs
:
progress
=
100
*
(
finished
/
float
(
njobs
)
)
else
:
progress
=
100
hashes
=
'#'
*
int
(
progress
/
5.
)
nohash
=
' '
*
int
(
20
len
(
hashes
)
)
if
not
ipyrad
.
__interactive__
:
msg
=
msg
.
rsplit
(
"|"
,
2
)
[
0
]
args
=
[
spacer
,
hashes
+
nohash
,
int
(
progress
)
,
msg
]
print
(
"\r{}[{}] {:>3}% {} "
.
format
(
*
args
)
,
end
=
""
)
sys
.
stdout
.
flush
(
)


def get_threaded_view(ipyclient, split=True):
    """ gets optimum threaded view of ids given the host setup """
    ## engine ids
    ## e.g., [0, 1, 2, 3, 4, 5, 6, 7, 8]
    eids = ipyclient.ids

    ## get host names
    ## e.g., ['a', 'a', 'b', 'b', 'a', 'c', 'c', 'c', 'c']
    dview = ipyclient.direct_view()
    hosts = dview.apply_sync(socket.gethostname)

    ## group ids into a dict by their hostnames
    ## e.g., {a: [0, 1, 4], b: [2, 3], c: [5, 6, 7, 8]}
    hostdict = defaultdict(list)
    for host, eid in zip(hosts, eids):
        hostdict[host].append(eid)

    ## Now split threads on the same host into separate proc if there are many
    hostdictkeys = hostdict.keys()
    for key in hostdictkeys:
        gids = hostdict[key]
        maxt = len(gids)
        if len(gids) >= 4:
            maxt = 2
        ## if 4 nodes and 4 ppn, put one sample per host
        if (len(gids) == 4) and (len(hosts) >= 4):
            maxt = 4
        if len(gids) >= 6:
            maxt = 3
        if len(gids) >= 8:
            maxt = 4
        if len(gids) >= 16:
            maxt = 4
        ## split ids into groups of maxt
        threaded = [gids[i:i+maxt] for i in xrange(0, len(gids), maxt)]
        lth = len(threaded)
        ## if anything was split (lth>1) update hostdict with new proc
        if lth > 1:
            hostdict.pop(key)
            for hostid in range(lth):
                hostdict[str(key)+"_"+str(hostid)] = threaded[hostid]

    ## make sure split numbering is correct
    #threaded = hostdict.values()
    #assert len(ipyclient.ids) <= len(list(itertools.chain(*threaded)))
    LOGGER.info("threaded_view: %s", dict(hostdict))
    return hostdict

def
get_threaded_view
(
ipyclient
,
split
=
True
)
:
eids
=
ipyclient
.
ids
dview
=
ipyclient
.
direct_view
(
)
hosts
=
dview
.
apply_sync
(
socket
.
gethostname
)
hostdict
=
defaultdict
(
list
)
for
host
,
eid
in
zip
(
hosts
,
eids
)
:
hostdict
[
host
]
.
append
(
eid
)
hostdictkeys
=
hostdict
.
keys
(
)
for
key
in
hostdictkeys
:
gids
=
hostdict
[
key
]
maxt
=
len
(
gids
)
if
len
(
gids
)
>=
4
:
maxt
=
2
if
(
len
(
gids
)
==
4
)
and
(
len
(
hosts
)
>=
4
)
:
maxt
=
4
if
len
(
gids
)
>=
6
:
maxt
=
3
if
len
(
gids
)
>=
8
:
maxt
=
4
if
len
(
gids
)
>=
16
:
maxt
=
4
threaded
=
[
gids
[
i
:
i
+
maxt
]
for
i
in
xrange
(
0
,
len
(
gids
)
,
maxt
)
]
lth
=
len
(
threaded
)
if
lth
>
1
:
hostdict
.
pop
(
key
)
for
hostid
in
range
(
lth
)
:
hostdict
[
str
(
key
)
+
"_"
+
str
(
hostid
)
]
=
threaded
[
hostid
]
LOGGER
.
info
(
"threaded_view: %s"
,
dict
(
hostdict
)
)
return
hostdict


def detect_cpus():
    """
    Detects the number of CPUs on a system. This is better than asking
    ipyparallel since ipp has to wait for Engines to spin up.
    """
    # Linux, Unix and MacOS:
    if hasattr(os, "sysconf"):
        if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
            # Linux & Unix:
            ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
            if isinstance(ncpus, int) and ncpus > 0:
                return ncpus
        else: # OSX:
            return int(os.popen2("sysctl -n hw.ncpu")[1].read())
    # Windows:
    if os.environ.has_key("NUMBER_OF_PROCESSORS"):
        ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
        if ncpus > 0:
            return ncpus
    return 1

def
detect_cpus
(
)
:
if
hasattr
(
os
,
"sysconf"
)
:
if
os
.
sysconf_names
.
has_key
(
"SC_NPROCESSORS_ONLN"
)
:
ncpus
=
os
.
sysconf
(
"SC_NPROCESSORS_ONLN"
)
if
isinstance
(
ncpus
,
int
)
and
ncpus
>
0
:
return
ncpus
else
:
return
int
(
os
.
popen2
(
"sysctl -n hw.ncpu"
)
[
1
]
.
read
(
)
)
if
os
.
environ
.
has_key
(
"NUMBER_OF_PROCESSORS"
)
:
ncpus
=
int
(
os
.
environ
[
"NUMBER_OF_PROCESSORS"
]
)
if
ncpus
>
0
:
return
ncpus
return
1

Detects the number of CPUs on a system . This is better than asking ipyparallel since ipp has to wait for Engines to spin up .


def _call_structure(mname, ename, sname, name, workdir, seed, ntaxa, nsites, kpop, rep):
    """ make the subprocess call to structure """
    ## create call string
    outname = os.path.join(workdir, "{}-K-{}-rep-{}".format(name, kpop, rep))

    cmd = ["structure",
           "-m", mname,
           "-e", ename,
           "-K", str(kpop),
           "-D", str(seed),
           "-N", str(ntaxa),
           "-L", str(nsites),
           "-i", sname,
           "-o", outname]

    ## call the shell function
    proc = subprocess.Popen(cmd,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    comm = proc.communicate()

    ## cleanup
    oldfiles = [mname, ename, sname]
    for oldfile in oldfiles:
        if os.path.exists(oldfile):
            os.remove(oldfile)
    return comm

def _call_structure ( mname , ename , sname , name , workdir , seed , ntaxa , nsites , kpop , rep ) : outname = os . path . join ( workdir , "{}-K-{}-rep-{}" . format ( name , kpop , rep ) ) cmd = [ "structure" , "-m" , mname , "-e" , ename , "-K" , str ( kpop ) , "-D" , str ( seed ) , "-N" , str ( ntaxa ) , "-L" , str ( nsites ) , "-i" , sname , "-o" , outname ] proc = subprocess . Popen ( cmd , stdout = subprocess . PIPE , stderr = subprocess . STDOUT ) comm = proc . communicate ( ) oldfiles = [ mname , ename , sname ] for oldfile in oldfiles : if os . path . exists ( oldfile ) : os . remove ( oldfile ) return comm


def _get_clumpp_table(self, kpop, max_var_multiple, quiet):
    """ private function to clumpp results"""

    ## concat results for k=x
    reps, excluded = _concat_reps(self, kpop, max_var_multiple, quiet)
    if reps:
        ninds = reps[0].inds
        nreps = len(reps)
    else:
        ninds = nreps = 0
    if not reps:
        return "no result files found"

    clumphandle = os.path.join(self.workdir, "tmp.clumppparams.txt")
    self.clumppparams.kpop = kpop
    self.clumppparams.c = ninds
    self.clumppparams.r = nreps
    with open(clumphandle, 'w') as tmp_c:
        tmp_c.write(self.clumppparams._asfile())

    ## create CLUMPP args string
    outfile = os.path.join(self.workdir,
                "{}-K-{}.outfile".format(self.name, kpop))
    indfile = os.path.join(self.workdir,
                "{}-K-{}.indfile".format(self.name, kpop))
    miscfile = os.path.join(self.workdir,
                "{}-K-{}.miscfile".format(self.name, kpop))
    cmd = ["CLUMPP", clumphandle,
           "-i", indfile,
           "-o", outfile,
           "-j", miscfile,
           "-r", str(nreps),
           "-c", str(ninds),
           "-k", str(kpop)]

    ## call clumpp
    proc = subprocess.Popen(cmd,
                            stderr=subprocess.STDOUT,
                            stdout=subprocess.PIPE)
    _ = proc.communicate()

    ## cleanup
    for rfile in [indfile, miscfile]:
        if os.path.exists(rfile):
            os.remove(rfile)

    ## parse clumpp results file
    ofile = os.path.join(self.workdir, "{}-K-{}.outfile".format(self.name, kpop))
    if os.path.exists(ofile):
        csvtable = pd.read_csv(ofile, delim_whitespace=True, header=None)
        table = csvtable.loc[:, 5:]

        ## apply names to cols and rows
        table.columns = range(table.shape[1])
        table.index = self.labels
        if not quiet:
            sys.stderr.write(
                "[K{}] {}/{} results permuted across replicates (max_var={}).\n"\
                .format(kpop, nreps, nreps+excluded, max_var_multiple))
        return table

    else:
        sys.stderr.write("No files ready for {}-K-{} in {}\n"\
                         .format(self.name, kpop, self.workdir))
        return


def _get_evanno_table(self, kpops, max_var_multiple, quiet):
    """
    Calculates Evanno method K value scores for a series
    of permuted clumpp results.
    """

    ## iterate across k-vals
    kpops = sorted(kpops)
    replnliks = []

    for kpop in kpops:
        ## concat results for k=x
        reps, excluded = _concat_reps(self, kpop, max_var_multiple, quiet)

        ## report if some results were excluded
        if excluded:
            if not quiet:
                sys.stderr.write(
                "[K{}] {} reps excluded (not converged) see 'max_var_multiple'.\n"\
                .format(kpop, excluded))

        if reps:
            ninds = reps[0].inds
            nreps = len(reps)
        else:
            ninds = nreps = 0
        if not reps:
            print "no result files found"

        ## all we really need is the lnlik
        replnliks.append([i.est_lnlik for i in reps])

    ## compare lnlik and var of results
    if len(replnliks) > 1:
        lnmean = [np.mean(i) for i in replnliks]
        lnstds = [np.std(i, ddof=1) for i in replnliks]
    else:
        lnmean = replnliks
        lnstds = np.nan

    tab = pd.DataFrame(
        index=kpops,
        data={
            "Nreps": [len(i) for i in replnliks],
            "lnPK": [0] * len(kpops),
            "lnPPK": [0] * len(kpops),
            "deltaK": [0] * len(kpops),
            "estLnProbMean": lnmean,
            "estLnProbStdev": lnstds,
        }
        )

    ## calculate Evanno's
    for kpop in kpops[1:]:
        tab.loc[kpop, "lnPK"] = tab.loc[kpop, "estLnProbMean"] \
                              - tab.loc[kpop-1, "estLnProbMean"]

    for kpop in kpops[1:-1]:
        tab.loc[kpop, "lnPPK"] = abs(tab.loc[kpop+1, "lnPK"]
                                     - tab.loc[kpop, "lnPK"])
        tab.loc[kpop, "deltaK"] = (abs(
                                    tab.loc[kpop+1, "estLnProbMean"] - \
                                    2.0 * tab.loc[kpop, "estLnProbMean"] + \
                                    tab.loc[kpop-1, "estLnProbMean"]) / \
                                   tab.loc[kpop, "estLnProbStdev"])

    ## return table
    return tab

Calculates Evanno method K value scores for a series of permuted clumpp results .


def result_files(self):
        """ returns a list of files that have finished structure """
        reps = OPJ(self.workdir, self.name+"-K-*-rep-*_f")
        repfiles = glob.glob(reps)
        return repfiles

def result_files ( self ) : reps = OPJ ( self . workdir , self . name + "-K-*-rep-*_f" ) repfiles = glob . glob ( reps ) return repfiles


def get_evanno_table(self, kvalues, max_var_multiple=0, quiet=False):
        """
        Calculates the Evanno table from results files for tests with
        K-values in the input list kvalues. The values lnPK, lnPPK,
        and deltaK are calculated. The max_var_multiplier arg can be used
        to exclude results files based on variance of the likelihood as a
        proxy for convergence.

        Parameters:
        -----------
        kvalues : list
            The list of K-values for which structure was run for this object.
            e.g., kvalues = [3, 4, 5]

        max_var_multiple: int
            A multiplier value to use as a filter for convergence of runs.
            Default=0=no filtering. As an example, if 10 replicates
            were run then the variance of the run with the minimum variance is
            used as a benchmark. If other runs have a variance that is N times
            greater then that run will be excluded. Remember, if replicate runs
            sampled different distributions of SNPs then it is not unexpected that
            they will have very different variances. However, you may still want
            to exclude runs with very high variance since they likely have
            not converged.

        quiet: bool
            Suppresses printed messages about convergence.

        Returns:
        --------
        table : pandas.DataFrame
            A data frame with LPK, LNPPK, and delta K. The latter is typically
            used to find the best fitting value of K. But be wary of over
            interpreting a single best K value.
        """
        ## do not allow bad vals
        if max_var_multiple:
            if max_var_multiple < 1:
                raise ValueError('max_variance_multiplier must be >1')

        table = _get_evanno_table(self, kvalues, max_var_multiple, quiet)
        return table

def
get_evanno_table
(
self
,
kvalues
,
max_var_multiple
=
0
,
quiet
=
False
)
:
if
max_var_multiple
:
if
max_var_multiple
<
1
:
raise
ValueError
(
'max_variance_multiplier must be >1'
)
table
=
_get_evanno_table
(
self
,
kvalues
,
max_var_multiple
,
quiet
)
return
table

Calculates the Evanno table from results files for tests with K - values in the input list kvalues . The values lnPK lnPPK and deltaK are calculated . The max_var_multiplier arg can be used to exclude results files based on variance of the likelihood as a proxy for convergence .


def parse(self, psearch, dsearch):
        """ parse an _f structure output file """
        stable = ""
        with open(self.repfile) as orep:
            dat = orep.readlines()
            for line in dat:
                ## stat lines
                if "Estimated Ln Prob of Data" in line:
                    self.est_lnlik = float(line.split()[-1])
                if "Mean value of ln likelihood" in line:
                    self.mean_lnlik = float(line.split()[-1])
                if "Variance of ln likelihood" in line:
                    self.var_lnlik = float(line.split()[-1])
                if "Mean value of alpha" in line:
                    self.alpha = float(line.split()[-1])

                ## matrix lines
                nonline = psearch.search(line)
                popline = dsearch.search(line)

                #if ")   :  " in line:
                if nonline:
                    ## check if sample is supervised...
                    abc = line.strip().split()
                    outstr = "{}{}{}".format(
                        " ".join([abc[0], abc[0], abc[2],
                                  abc[0].split('.')[0]]),
                        " :  ",
                        " ".join(abc[4:])
                    )
                    self.inds += 1
                    stable += outstr+"\n"

                elif popline:
                    ## check if sample is supervised...
                    abc = line.strip().split()
                    prop = ["0.000"] * self.kpop
                    pidx = int(abc[3]) - 1
                    prop[pidx] = "1.000"
                    outstr = "{}{}{}".format(
                        " ".join([abc[0], abc[0], abc[2],
                                  abc[0].split('.')[0]]),
                        " :  ",
                        " ".join(prop)
                    )
                    self.inds += 1
                    stable += outstr+"\n"

            stable += "\n"
        return stable

def
parse
(
self
,
psearch
,
dsearch
)
:
stable
=
""
with
open
(
self
.
repfile
)
as
orep
:
dat
=
orep
.
readlines
(
)
for
line
in
dat
:
if
"Estimated Ln Prob of Data"
in
line
:
self
.
est_lnlik
=
float
(
line
.
split
(
)
[
1
]
)
if
"Mean value of ln likelihood"
in
line
:
self
.
mean_lnlik
=
float
(
line
.
split
(
)
[
1
]
)
if
"Variance of ln likelihood"
in
line
:
self
.
var_lnlik
=
float
(
line
.
split
(
)
[
1
]
)
if
"Mean value of alpha"
in
line
:
self
.
alpha
=
float
(
line
.
split
(
)
[
1
]
)
nonline
=
psearch
.
search
(
line
)
popline
=
dsearch
.
search
(
line
)
if
nonline
:
abc
=
line
.
strip
(
)
.
split
(
)
outstr
=
"{}{}{}"
.
format
(
" "
.
join
(
[
abc
[
0
]
,
abc
[
0
]
,
abc
[
2
]
,
abc
[
0
]
.
split
(
'.'
)
[
0
]
]
)
,
" :  "
,
" "
.
join
(
abc
[
4
:
]
)
)
self
.
inds
+=
1
stable
+=
outstr
+
"\n"
elif
popline
:
abc
=
line
.
strip
(
)
.
split
(
)
prop
=
[
"0.000"
]
*
self
.
kpop
pidx
=
int
(
abc
[
3
]
)
1
prop
[
pidx
]
=
"1.000"
outstr
=
"{}{}{}"
.
format
(
" "
.
join
(
[
abc
[
0
]
,
abc
[
0
]
,
abc
[
2
]
,
abc
[
0
]
.
split
(
'.'
)
[
0
]
]
)
,
" :  "
,
" "
.
join
(
prop
)
)
self
.
inds
+=
1
stable
+=
outstr
+
"\n"
stable
+=
"\n"
return
stable


def _call_raxml(command_list):
    """ call the command as sps """
    proc = subprocess.Popen(
        command_list,
        stderr=subprocess.STDOUT,
        stdout=subprocess.PIPE
        )
    comm = proc.communicate()
    return comm

def _call_raxml ( command_list ) : proc = subprocess . Popen ( command_list , stderr = subprocess . STDOUT , stdout = subprocess . PIPE ) comm = proc . communicate ( ) return comm


def run(self,
        ipyclient=None,
        quiet=False,
        force=False,
        block=False,
        ):
        """
        Submits raxml job to run. If no ipyclient object is provided then
        the function will block until the raxml run is finished. If an ipyclient
        is provided then the job is sent to a remote engine and an asynchronous
        result object is returned which can be queried or awaited until it finishes.

        Parameters
        -----------
        ipyclient:
            Not yet supported...
        quiet:
            suppress print statements
        force:
            overwrite existing results files with this job name.
        block:
            will block progress in notebook until job finishes, even if job
            is running on a remote ipyclient.
        """

        ## stop before trying in raxml
        if force:
            for key, oldfile in self.trees:
                if os.path.exists(oldfile):
                    os.remove(oldfile)
        if os.path.exists(self.trees.info):
            print("Error: set a new name for this job or use Force flag.\nFile exists: {}"\
                  .format(self.trees.info))
            return

        ## TODO: add a progress bar tracker here. It could even read it from
        ## the info file that is being written.
        ## submit it
        if not ipyclient:
            proc = _call_raxml(self._command_list)
            self.stdout = proc[0]
            self.stderr = proc[1]
        else:
            ## find all hosts and submit job to the host with most available engines
            lbview = ipyclient.load_balanced_view()
            self.async = lbview.apply(_call_raxml, self._command_list)

        ## initiate random seed
        if not quiet:
            if not ipyclient:
                ## look for errors
                if "Overall execution time" not in self.stdout:
                    print("Error in raxml run\n" + self.stdout)
                else:
                    print("job {} finished successfully".format(self.params.n))
            else:
                print("job {} submitted to cluster".format(self.params.n))

Submits raxml job to run . If no ipyclient object is provided then the function will block until the raxml run is finished . If an ipyclient is provided then the job is sent to a remote engine and an asynchronous result object is returned which can be queried or awaited until it finishes .


def _get_binary(self):
        """ find binaries available"""

        ## check for binary
        backup_binaries = ["raxmlHPC-PTHREADS", "raxmlHPC-PTHREADS-SSE3"]

        ## check user binary first, then backups
        for binary in [self.params.binary] + backup_binaries:
            proc = subprocess.Popen(["which", self.params.binary],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT).communicate()
            ## update the binary
            if proc:
                self.params.binary = binary

        ## if none then raise error
        if not proc[0]:
            raise Exception(BINARY_ERROR.format(self.params.binary))

def _get_binary ( self ) : backup_binaries = [ "raxmlHPC-PTHREADS" , "raxmlHPC-PTHREADS-SSE3" ] for binary in [ self . params . binary ] + backup_binaries : proc = subprocess . Popen ( [ "which" , self . params . binary ] , stdout = subprocess . PIPE , stderr = subprocess . STDOUT ) . communicate ( ) if proc : self . params . binary = binary if not proc [ 0 ] : raise Exception ( BINARY_ERROR . format ( self . params . binary ) )


def dstat(inarr, taxdict, mindict=1, nboots=1000, name=0):
    """ private function to perform a single D-stat test"""

    #if isinstance(inarr, str):
    #    with open(inarr, 'r') as infile:
    #        inarr = infile.read().strip().split("|\n")

    # ## get data as an array from loci file
    # ## if loci-list then parse arr from loci
    if isinstance(inarr, list):
        arr, _ = _loci_to_arr(inarr, taxdict, mindict)

    # ## if it's an array already then go ahead
    # elif isinstance(inarr, np.ndarray):
    #     arr = inarr
    # ## if it's a simulation object get freqs from array
    # elif isinstance(inarr, Sim):
    #     arr = _msp_to_arr(inarr, taxdict)

    #elif isinstance(inarr, types.GeneratorType):
    #    arr = _msp_to_arr(inarr, taxdict)
    #elif isinstance(inarr, list):
    #    arr = _msp_to_arr(inarr, taxdict)
    ## get data from Sim object, do not digest the ms generator
    #else:
    #    raise Exception("Must enter either a 'locifile' or 'arr'")

    ## run tests
    #if len(taxdict) == 4:
    if arr.shape[1] == 4:

        ## get results
        res, boots = _get_signif_4(arr, nboots)

        ## make res into a nice DataFrame
        res = pd.DataFrame(res,
            columns=[name],
            index=["Dstat", "bootmean", "bootstd", "Z", "ABBA", "BABA", "nloci"])

    else:
        ## get results
        res, boots = _get_signif_5(arr, nboots)
         ## make int a DataFrame
        res = pd.DataFrame(res,
            index=["p3", "p4", "shared"],
            columns=["Dstat", "bootmean", "bootstd", "Z", "ABxxA", "BAxxA", "nloci"]
            )

    return res.T, boots

def dstat ( inarr , taxdict , mindict = 1 , nboots = 1000 , name = 0 ) : if isinstance ( inarr , list ) : arr , _ = _loci_to_arr ( inarr , taxdict , mindict ) if arr . shape [ 1 ] == 4 : res , boots = _get_signif_4 ( arr , nboots ) res = pd . DataFrame ( res , columns = [ name ] , index = [ "Dstat" , "bootmean" , "bootstd" , "Z" , "ABBA" , "BABA" , "nloci" ] ) else : res , boots = _get_signif_5 ( arr , nboots ) res = pd . DataFrame ( res , index = [ "p3" , "p4" , "shared" ] , columns = [ "Dstat" , "bootmean" , "bootstd" , "Z" , "ABxxA" , "BAxxA" , "nloci" ] ) return res . T , boots

private function to perform a single D - stat test


def _get_boots(arr, nboots):
    """
    return array of bootstrap D-stats
    """
    ## hold results (nboots, [dstat, ])
    boots = np.zeros((nboots,))

    ## iterate to fill boots
    for bidx in xrange(nboots):
        ## sample with replacement
        lidx = np.random.randint(0, arr.shape[0], arr.shape[0])
        tarr = arr[lidx]
        _, _, dst = _prop_dstat(tarr)
        boots[bidx] = dst

    ## return bootarr
    return boots

def _get_boots ( arr , nboots ) : boots = np . zeros ( ( nboots , ) ) for bidx in xrange ( nboots ) : lidx = np . random . randint ( 0 , arr . shape [ 0 ] , arr . shape [ 0 ] ) tarr = arr [ lidx ] _ , _ , dst = _prop_dstat ( tarr ) boots [ bidx ] = dst return boots


def taxon_table(self):
        """
        Returns the .tests list of taxa as a pandas dataframe.
        By auto-generating this table from tests it means that
        the table itself cannot be modified unless it is returned
        and saved.
        """
        if self.tests:
            keys = sorted(self.tests[0].keys())
            if isinstance(self.tests, list):
                ld = [[(key, i[key]) for key in keys] for i in self.tests]
                dd = [dict(i) for i in ld]
                df = pd.DataFrame(dd)
                return df
            else:
                return pd.DataFrame(pd.Series(self.tests)).T
        else:
            return None

def taxon_table ( self ) : if self . tests : keys = sorted ( self . tests [ 0 ] . keys ( ) ) if isinstance ( self . tests , list ) : ld = [ [ ( key , i [ key ] ) for key in keys ] for i in self . tests ] dd = [ dict ( i ) for i in ld ] df = pd . DataFrame ( dd ) return df else : return pd . DataFrame ( pd . Series ( self . tests ) ) . T else : return None

Returns the . tests list of taxa as a pandas dataframe . By auto - generating this table from tests it means that the table itself cannot be modified unless it is returned and saved .


def nexmake(mdict, nlocus, dirs, mcmc_burnin, mcmc_ngen, mcmc_sample_freq):
    """
    function that takes a dictionary mapping names to
    sequences, and a locus number, and writes it as a NEXUS
    file with a mrbayes analysis block.
    """
    ## create matrix as a string
    max_name_len = max([len(i) for i in mdict])
    namestring = "{:<" + str(max_name_len+1) + "} {}\n"
    matrix = ""
    for i in mdict.items():
        matrix += namestring.format(i[0], i[1])

    ## write nexus block
    handle = os.path.join(dirs, "{}.nex".format(nlocus))
    with open(handle, 'w') as outnex:
        outnex.write(NEXBLOCK.format(**{
            "ntax": len(mdict),
            "nchar": len(mdict.values()[0]),
            "matrix": matrix,
            "ngen": mcmc_ngen,
            "sfreq": mcmc_sample_freq,
            "burnin": mcmc_burnin,
            }))

def
nexmake
(
mdict
,
nlocus
,
dirs
,
mcmc_burnin
,
mcmc_ngen
,
mcmc_sample_freq
)
:
max_name_len
=
max
(
[
len
(
i
)
for
i
in
mdict
]
)
namestring
=
"{:<"
+
str
(
max_name_len
+
1
)
+
"} {}\n"
matrix
=
""
for
i
in
mdict
.
items
(
)
:
matrix
+=
namestring
.
format
(
i
[
0
]
,
i
[
1
]
)
handle
=
os
.
path
.
join
(
dirs
,
"{}.nex"
.
format
(
nlocus
)
)
with
open
(
handle
,
'w'
)
as
outnex
:
outnex
.
write
(
NEXBLOCK
.
format
(
**
{
"ntax"
:
len
(
mdict
)
,
"nchar"
:
len
(
mdict
.
values
(
)
[
0
]
)
,
"matrix"
:
matrix
,
"ngen"
:
mcmc_ngen
,
"sfreq"
:
mcmc_sample_freq
,
"burnin"
:
mcmc_burnin
,
}
)
)

function that takes a dictionary mapping names to sequences and a locus number and writes it as a NEXUS file with a mrbayes analysis block .


def call_fastq_dump_on_SRRs(self, srr, outname, paired):
    """
    calls fastq-dump on SRRs, relabels fastqs by their accession
    names, and writes them to the workdir. Saves temp sra files
    in the designated tmp folder and immediately removes them.
    """

    ## build command for fastq-dumping
    fd_cmd = [
        "fastq-dump", srr,
        "--accession", outname,
        "--outdir", self.workdir,
        "--gzip",
        ]
    if paired:
        fd_cmd += ["--split-files"]

    ## call fq dump command
    proc = sps.Popen(fd_cmd, stderr=sps.STDOUT, stdout=sps.PIPE)
    o, e = proc.communicate()

    ## delete the stupid temp sra file from the place
    ## that it is very hard-coded to be written to, and
    ## LEFT IN, for some crazy reason.
    srafile = os.path.join(self.workdir, "sra", srr+".sra")
    if os.path.exists(srafile):
        os.remove(srafile)

def call_fastq_dump_on_SRRs ( self , srr , outname , paired ) : fd_cmd = [ "fastq-dump" , srr , "--accession" , outname , "--outdir" , self . workdir , "--gzip" , ] if paired : fd_cmd += [ "--split-files" ] proc = sps . Popen ( fd_cmd , stderr = sps . STDOUT , stdout = sps . PIPE ) o , e = proc . communicate ( ) srafile = os . path . join ( self . workdir , "sra" , srr + ".sra" ) if os . path . exists ( srafile ) : os . remove ( srafile )

calls fastq - dump on SRRs relabels fastqs by their accession names and writes them to the workdir . Saves temp sra files in the designated tmp folder and immediately removes them .