async def get_access_token(consumer_key, consumer_secret,
                           oauth_token, oauth_token_secret,
                           oauth_verifier, **kwargs):
    """
        get the access token of the user

    Parameters
    ----------
    consumer_key : str
        Your consumer key
    consumer_secret : str
        Your consumer secret
    oauth_token : str
        OAuth token from :func:`get_oauth_token`
    oauth_token_secret : str
        OAuth token secret from :func:`get_oauth_token`
    oauth_verifier : str
        OAuth verifier from :func:`get_oauth_verifier`

    Returns
    -------
    dict
        Access tokens
    """

    client = BasePeonyClient(consumer_key=consumer_key,
                             consumer_secret=consumer_secret,
                             access_token=oauth_token,
                             access_token_secret=oauth_token_secret,
                             api_version="",
                             suffix="")

    response = await client.api.oauth.access_token.get(
        _suffix="",
        oauth_verifier=oauth_verifier
    )

    return parse_token(response)

async def get_access_token ( consumer_key , consumer_secret , oauth_token , oauth_token_secret , oauth_verifier , ** kwargs ) : client = BasePeonyClient ( consumer_key = consumer_key , consumer_secret = consumer_secret , access_token = oauth_token , access_token_secret = oauth_token_secret , api_version = "" , suffix = "" ) response = await client . api . oauth . access_token . get ( _suffix = "" , oauth_verifier = oauth_verifier ) return parse_token ( response )


def parse_token(response):
    """
    parse the responses containing the tokens

    Parameters
    ----------
    response : str
        The response containing the tokens

    Returns
    -------
    dict
        The parsed tokens
    """
    items = response.split("&")
    items = [item.split("=") for item in items]

    return {key: value for key, value in items}

def parse_token ( response ) : items = response . split ( "&" ) items = [ item . split ( "=" ) for item in items ] return { key : value for key , value in items }


def predict(self, sequences):
        """
        Return netChop predictions for each position in each sequence.

        Parameters
        -----------
        sequences : list of string
            Amino acid sequences to predict cleavage for

        Returns
        -----------
        list of list of float

        The i'th list corresponds to the i'th sequence. Each list gives
        the cleavage probability for each position in the sequence.
        """
        with tempfile.NamedTemporaryFile(suffix=".fsa", mode="w") as input_fd:
            for (i, sequence) in enumerate(sequences):
                input_fd.write("> %d\n" % i)
                input_fd.write(sequence)
                input_fd.write("\n")
            input_fd.flush()
            try:
                output = subprocess.check_output(["netChop", input_fd.name])
            except subprocess.CalledProcessError as e:
                logging.error("Error calling netChop: %s:\n%s" % (e, e.output))
                raise

        parsed = self.parse_netchop(output)
        assert len(parsed) == len(sequences), \
            "Expected %d results but got %d" % (
                len(sequences), len(parsed))
        assert [len(x) for x in parsed] == [len(x) for x in sequences]
        return parsed

def
predict
(
self
,
sequences
)
:
with
tempfile
.
NamedTemporaryFile
(
suffix
=
".fsa"
,
mode
=
"w"
)
as
input_fd
:
for
(
i
,
sequence
)
in
enumerate
(
sequences
)
:
input_fd
.
write
(
"> %d\n"
%
i
)
input_fd
.
write
(
sequence
)
input_fd
.
write
(
"\n"
)
input_fd
.
flush
(
)
try
:
output
=
subprocess
.
check_output
(
[
"netChop"
,
input_fd
.
name
]
)
except
subprocess
.
CalledProcessError
as
e
:
logging
.
error
(
"Error calling netChop: %s:\n%s"
%
(
e
,
e
.
output
)
)
raise
parsed
=
self
.
parse_netchop
(
output
)
assert
len
(
parsed
)
==
len
(
sequences
)
,
"Expected %d results but got %d"
%
(
len
(
sequences
)
,
len
(
parsed
)
)
assert
[
len
(
x
)
for
x
in
parsed
]
==
[
len
(
x
)
for
x
in
sequences
]
return
parsed

Return netChop predictions for each position in each sequence .


def parse_netchop(netchop_output):
        """
        Parse netChop stdout.
        """
        line_iterator = iter(netchop_output.decode().split("\n"))
        scores = []
        for line in line_iterator:
            if "pos" in line and 'AA' in line and 'score' in line:
                scores.append([])
                if "----" not in next(line_iterator):
                    raise ValueError("Dashes expected")
                line = next(line_iterator)
                while '-------' not in line:
                    score = float(line.split()[3])
                    scores[-1].append(score)
                    line = next(line_iterator)
        return scores

def
parse_netchop
(
netchop_output
)
:
line_iterator
=
iter
(
netchop_output
.
decode
(
)
.
split
(
"\n"
)
)
scores
=
[
]
for
line
in
line_iterator
:
if
"pos"
in
line
and
'AA'
in
line
and
'score'
in
line
:
scores
.
append
(
[
]
)
if
"----"
not
in
next
(
line_iterator
)
:
raise
ValueError
(
"Dashes expected"
)
line
=
next
(
line_iterator
)
while
'-------'
not
in
line
:
score
=
float
(
line
.
split
(
)
[
3
]
)
scores
[
1
]
.
append
(
score
)
line
=
next
(
line_iterator
)
return
scores


def to_dataframe(
            self,
            columns=BindingPrediction.fields + ("length",)):
        """
        Converts collection of BindingPrediction objects to DataFrame
        """
        return pd.DataFrame.from_records(
            [tuple([getattr(x, name) for name in columns]) for x in self],
            columns=columns)

def to_dataframe ( self , columns = BindingPrediction . fields + ( "length" , ) ) : return pd . DataFrame . from_records ( [ tuple ( [ getattr ( x , name ) for name in columns ] ) for x in self ] , columns = columns )


def NetMHC(alleles,
           default_peptide_lengths=[9],
           program_name="netMHC"):
    """
    This function wraps NetMHC3 and NetMHC4 to automatically detect which class
    to use. Currently based on running the '-h' command and looking for
    discriminating substrings between the versions.
    """
    # run NetMHC's help command and parse discriminating substrings out of
    # the resulting str output
    with open(os.devnull, 'w') as devnull:
        help_output = check_output([program_name, "-h"], stderr=devnull)
    help_output_str = help_output.decode("ascii", "ignore")

    substring_to_netmhc_class = {
        "-listMHC": NetMHC4,
        "--Alleles": NetMHC3,
    }

    successes = []

    for substring, netmhc_class in substring_to_netmhc_class.items():
        if substring in help_output_str:
            successes.append(netmhc_class)

    if len(successes) > 1:
        raise SystemError("Command %s is valid for multiple NetMHC versions. "
                          "This is likely an mhctools bug." % program_name)
    if len(successes) == 0:
        raise SystemError("Command %s is not a valid way of calling any NetMHC software."
                          % program_name)

    netmhc_class = successes[0]
    return netmhc_class(
        alleles=alleles,
        default_peptide_lengths=default_peptide_lengths,
        program_name=program_name)

This function wraps NetMHC3 and NetMHC4 to automatically detect which class to use . Currently based on running the - h command and looking for discriminating substrings between the versions .


def predict_peptides(self, peptides):
        """
        Predict MHC affinity for peptides.
        """

        # importing locally to avoid slowing down CLI applications which
        # don't use MHCflurry
        from mhcflurry.encodable_sequences import EncodableSequences

        binding_predictions = []
        encodable_sequences = EncodableSequences.create(peptides)
        for allele in self.alleles:
            predictions_df = self.predictor.predict_to_dataframe(
                encodable_sequences, allele=allele)
            for (_, row) in predictions_df.iterrows():
                binding_prediction = BindingPrediction(
                    allele=allele,
                    peptide=row.peptide,
                    affinity=row.prediction,
                    percentile_rank=(
                        row.prediction_percentile
                        if 'prediction_percentile' in row else nan),
                    prediction_method_name="mhcflurry"
                )
                binding_predictions.append(binding_prediction)
        return BindingPredictionCollection(binding_predictions)

def predict_peptides ( self , peptides ) : from mhcflurry . encodable_sequences import EncodableSequences binding_predictions = [ ] encodable_sequences = EncodableSequences . create ( peptides ) for allele in self . alleles : predictions_df = self . predictor . predict_to_dataframe ( encodable_sequences , allele = allele ) for ( _ , row ) in predictions_df . iterrows ( ) : binding_prediction = BindingPrediction ( allele = allele , peptide = row . peptide , affinity = row . prediction , percentile_rank = ( row . prediction_percentile if 'prediction_percentile' in row else nan ) , prediction_method_name = "mhcflurry" ) binding_predictions . append ( binding_prediction ) return BindingPredictionCollection ( binding_predictions )


def seq_to_str(obj, sep=","):
    """
    Given a sequence convert it to a comma separated string.
    If, however, the argument is a single object, return its string
    representation.
    """
    if isinstance(obj, string_classes):
        return obj
    elif isinstance(obj, (list, tuple)):
        return sep.join([str(x) for x in obj])
    else:
        return str(obj)

def seq_to_str ( obj , sep = "," ) : if isinstance ( obj , string_classes ) : return obj elif isinstance ( obj , ( list , tuple ) ) : return sep . join ( [ str ( x ) for x in obj ] ) else : return str ( obj )

Given a sequence convert it to a comma separated string . If however the argument is a single object return its string representation .


def create_input_peptides_files(
        peptides,
        max_peptides_per_file=None,
        group_by_length=False):
    """
    Creates one or more files containing one peptide per line,
    returns names of files.
    """
    if group_by_length:
        peptide_lengths = {len(p) for p in peptides}
        peptide_groups = {l: [] for l in peptide_lengths}
        for p in peptides:
            peptide_groups[len(p)].append(p)
    else:
        peptide_groups = {"": peptides}

    file_names = []
    for key, group in peptide_groups.items():
        n_peptides = len(group)
        if not max_peptides_per_file:
            max_peptides_per_file = n_peptides
        input_file = None
        for i, p in enumerate(group):
            if i % max_peptides_per_file == 0:
                if input_file is not None:
                    file_names.append(input_file.name)
                    input_file.close()
                input_file = make_writable_tempfile(
                    prefix_number=i // max_peptides_per_file,
                    prefix_name=key,
                    suffix=".txt")
            input_file.write("%s\n" % p)
        if input_file is not None:
            file_names.append(input_file.name)
            input_file.close()
    return file_names

def create_input_peptides_files ( peptides , max_peptides_per_file = None , group_by_length = False ) : if group_by_length : peptide_lengths = { len ( p ) for p in peptides } peptide_groups = { l : [ ] for l in peptide_lengths } for p in peptides : peptide_groups [ len ( p ) ] . append ( p ) else : peptide_groups = { "" : peptides } file_names = [ ] for key , group in peptide_groups . items ( ) : n_peptides = len ( group ) if not max_peptides_per_file : max_peptides_per_file = n_peptides input_file = None for i , p in enumerate ( group ) : if i % max_peptides_per_file == 0 : if input_file is not None : file_names . append ( input_file . name ) input_file . close ( ) input_file = make_writable_tempfile ( prefix_number = i // max_peptides_per_file , prefix_name = key , suffix = ".txt" ) input_file . write ( "%s\n" % p ) if input_file is not None : file_names . append ( input_file . name ) input_file . close ( ) return file_names

Creates one or more files containing one peptide per line returns names of files .


def _check_peptide_lengths(self, peptide_lengths=None):
        """
        If peptide lengths not specified, then try using the default
        lengths associated with this predictor object. If those aren't
        a valid non-empty sequence of integers, then raise an exception.
        Otherwise return the peptide lengths.
        """
        if not peptide_lengths:
            peptide_lengths = self.default_peptide_lengths

        if not peptide_lengths:
            raise ValueError(
                ("Must either provide 'peptide_lengths' argument "
                "or set 'default_peptide_lengths"))
        if isinstance(peptide_lengths, int):
            peptide_lengths = [peptide_lengths]
        require_iterable_of(peptide_lengths, int)
        for peptide_length in peptide_lengths:
            if (self.min_peptide_length is not None and
                    peptide_length < self.min_peptide_length):
                raise ValueError(
                    "Invalid peptide length %d, shorter than min %d" % (
                        peptide_length,
                        self.min_peptide_length))
            elif (self.max_peptide_length is not None and
                    peptide_length > self.max_peptide_length):
                raise ValueError(
                    "Invalid peptide length %d, longer than max %d" % (
                        peptide_length,
                        self.max_peptide_length))
        return peptide_lengths

If peptide lengths not specified then try using the default lengths associated with this predictor object . If those aren t a valid non - empty sequence of integers then raise an exception . Otherwise return the peptide lengths .


def _check_peptide_inputs(self, peptides):
        """
        Check peptide sequences to make sure they are valid for this predictor.
        """
        require_iterable_of(peptides, string_types)
        check_X = not self.allow_X_in_peptides
        check_lower = not self.allow_lowercase_in_peptides
        check_min_length = self.min_peptide_length is not None
        min_length = self.min_peptide_length
        check_max_length = self.max_peptide_length is not None
        max_length = self.max_peptide_length
        for p in peptides:
            if not p.isalpha():
                raise ValueError("Invalid characters in peptide '%s'" % p)
            elif check_X and "X" in p:
                raise ValueError("Invalid character 'X' in peptide '%s'" % p)
            elif check_lower and not p.isupper():
                raise ValueError("Invalid lowercase letters in peptide '%s'" % p)
            elif check_min_length and len(p) < min_length:
                raise ValueError(
                    "Peptide '%s' too short (%d chars), must be at least %d" % (
                        p, len(p), min_length))
            elif check_max_length and len(p) > max_length:
                raise ValueError(
                    "Peptide '%s' too long (%d chars), must be at least %d" % (
                        p, len(p), max_length))

Check peptide sequences to make sure they are valid for this predictor .


def predict_subsequences(
            self,
            sequence_dict,
            peptide_lengths=None):
        """
        Given a dictionary mapping sequence names to amino acid strings,
        and an optional list of peptide lengths, returns a
        BindingPredictionCollection.
        """
        if isinstance(sequence_dict, string_types):
            sequence_dict = {"seq": sequence_dict}
        elif isinstance(sequence_dict, (list, tuple)):
            sequence_dict = {seq: seq for seq in sequence_dict}

        peptide_lengths = self._check_peptide_lengths(peptide_lengths)

        # convert long protein sequences to set of peptides and
        # associated sequence name / offsets that each peptide may have come
        # from
        peptide_set = set([])
        peptide_to_name_offset_pairs = defaultdict(list)

        for name, sequence in sequence_dict.items():
            for peptide_length in peptide_lengths:
                for i in range(len(sequence) - peptide_length + 1):
                    peptide = sequence[i:i + peptide_length]
                    peptide_set.add(peptide)
                    peptide_to_name_offset_pairs[peptide].append((name, i))
        peptide_list = sorted(peptide_set)

        binding_predictions = self.predict_peptides(peptide_list)

        # create BindingPrediction objects with sequence name and offset
        results = []
        for binding_prediction in binding_predictions:
            for name, offset in peptide_to_name_offset_pairs[
                    binding_prediction.peptide]:
                results.append(binding_prediction.clone_with_updates(
                    source_sequence_name=name,
                    offset=offset))
        self._check_results(
            results,
            peptides=peptide_set,
            alleles=self.alleles)
        return BindingPredictionCollection(results)

def predict_subsequences ( self , sequence_dict , peptide_lengths = None ) : if isinstance ( sequence_dict , string_types ) : sequence_dict = { "seq" : sequence_dict } elif isinstance ( sequence_dict , ( list , tuple ) ) : sequence_dict = { seq : seq for seq in sequence_dict } peptide_lengths = self . _check_peptide_lengths ( peptide_lengths ) peptide_set = set ( [ ] ) peptide_to_name_offset_pairs = defaultdict ( list ) for name , sequence in sequence_dict . items ( ) : for peptide_length in peptide_lengths : for i in range ( len ( sequence ) - peptide_length + 1 ) : peptide = sequence [ i : i + peptide_length ] peptide_set . add ( peptide ) peptide_to_name_offset_pairs [ peptide ] . append ( ( name , i ) ) peptide_list = sorted ( peptide_set ) binding_predictions = self . predict_peptides ( peptide_list ) results = [ ] for binding_prediction in binding_predictions : for name , offset in peptide_to_name_offset_pairs [ binding_prediction . peptide ] : results . append ( binding_prediction . clone_with_updates ( source_sequence_name = name , offset = offset ) ) self . _check_results ( results , peptides = peptide_set , alleles = self . alleles ) return BindingPredictionCollection ( results )

Given a dictionary mapping sequence names to amino acid strings and an optional list of peptide lengths returns a BindingPredictionCollection .


def _check_hla_alleles(
            alleles,
            valid_alleles=None):
        """
        Given a list of HLA alleles and an optional list of valid
        HLA alleles, return a set of alleles that we will pass into
        the MHC binding predictor.
        """
        require_iterable_of(alleles, string_types, "HLA alleles")

        # Don't run the MHC predictor twice for homozygous alleles,
        # only run it for unique alleles
        alleles = {
            normalize_allele_name(allele.strip().upper())
            for allele in alleles
        }
        if valid_alleles:
            # For some reason netMHCpan drops the '*' in names, so
            # 'HLA-A*03:01' becomes 'HLA-A03:01'
            missing_alleles = [
                allele
                for allele in alleles
                if allele not in valid_alleles
            ]
            if len(missing_alleles) > 0:
                raise UnsupportedAllele(
                    "Unsupported HLA alleles: %s" % missing_alleles)

        return list(alleles)

def
_check_hla_alleles
(
alleles
,
valid_alleles
=
)
:
require_iterable_of
(
alleles
,
string_types
,
"HLA alleles"
)
alleles
=
{
normalize_allele_name
(
allele
.
strip
(
)
.
upper
(
)
)
for
allele
in
alleles
}
if
valid_alleles
:
missing_alleles
=
[
allele
for
allele
in
alleles
if
allele
not
in
valid_alleles
]
if
len
(
missing_alleles
)
>
0
:
raise
UnsupportedAllele
(
"Unsupported HLA alleles: %s"
%
missing_alleles
)
return
list
(
alleles
)

Given a list of HLA alleles and an optional list of valid HLA alleles return a set of alleles that we will pass into the MHC binding predictor .


async def _connect(self):
        """
            Connect to the stream

        Returns
        -------
        asyncio.coroutine
            The streaming response
        """
        logger.debug("connecting to the stream")
        await self.client.setup
        if self.session is None:
            self.session = self.client._session
        kwargs = await self.client.headers.prepare_request(**self.kwargs)
        request = self.client.error_handler(self.session.request)

        return await request(timeout=0, **kwargs)

async
def
_connect
(
self
)
:
logger
.
debug
(
"connecting to the stream"
)
await
self
.
client
.
setup
if
self
.
session
is
:
self
.
session
=
self
.
client
.
_session
kwargs
=
await
self
.
client
.
headers
.
prepare_request
(
**
self
.
kwargs
)
request
=
self
.
client
.
error_handler
(
self
.
session
.
request
)
return
await
request
(
timeout
=
0
,
**
kwargs
)


async def connect(self):
        """
            Create the connection

        Returns
        -------
        self

        Raises
        ------
        exception.PeonyException
            On a response status in 4xx that are not status 420 or 429
            Also on statuses in 1xx or 3xx since this should not be the status
            received here
        """
        with async_timeout.timeout(self.timeout):
            self.response = await self._connect()

        if self.response.status in range(200, 300):
            self._error_timeout = 0
            self.state = NORMAL
        elif self.response.status == 500:
            self.state = DISCONNECTION
        elif self.response.status in range(501, 600):
            self.state = RECONNECTION
        elif self.response.status in (420, 429):
            self.state = ENHANCE_YOUR_CALM
        else:
            logger.debug("raising error during stream connection")
            raise await exceptions.throw(self.response,
                                         loads=self.client._loads,
                                         url=self.kwargs['url'])

        logger.debug("stream state: %d" % self.state)


def with_prefix(self, prefix, strict=False):
        """
            decorator to handle commands with prefixes

        Parameters
        ----------
        prefix : str
            the prefix of the command
        strict : bool, optional
            If set to True the command must be at the beginning
            of the message. Defaults to False.

        Returns
        -------
        function
            a decorator that returns an :class:`EventHandler` instance
        """

        def decorated(func):
            return EventHandler(func=func, event=self.event,
                                prefix=prefix, strict=strict)

        return decorated

def with_prefix ( self , prefix , strict = False ) : def decorated ( func ) : return EventHandler ( func = func , event = self . event , prefix = prefix , strict = strict ) return decorated


async def set_tz(self):
        """
            set the environment timezone to the timezone
            set in your twitter settings
        """
        settings = await self.api.account.settings.get()

        tz = settings.time_zone.tzinfo_name

        os.environ['TZ'] = tz
        time.tzset()

async def set_tz ( self ) : settings = await self . api . account . settings . get ( ) tz = settings . time_zone . tzinfo_name os . environ [ 'TZ' ] = tz time . tzset ( )


def run_command(args, **kwargs):
    """
    Given a list whose first element is a command name, followed by arguments,
    execute it and show timing info.
    """
    assert len(args) > 0
    start_time = time.time()
    process = AsyncProcess(args, **kwargs)
    process.wait()
    elapsed_time = time.time() - start_time
    logger.info("%s took %0.4f seconds", args[0], elapsed_time)

def
run_command
(
args
,
**
kwargs
)
:
assert
len
(
args
)
>
0
start_time
=
time
.
time
(
)
process
=
AsyncProcess
(
args
,
**
kwargs
)
process
.
wait
(
)
elapsed_time
=
time
.
time
(
)
start_time
logger
.
info
(
"%s took %0.4f seconds"
,
args
[
0
]
,
elapsed_time
)

Given a list whose first element is a command name followed by arguments execute it and show timing info .


def run_multiple_commands_redirect_stdout(
        multiple_args_dict,
        print_commands=True,
        process_limit=-1,
        polling_freq=0.5,
        **kwargs):
    """
    Run multiple shell commands in parallel, write each of their
    stdout output to files associated with each command.

    Parameters
    ----------
    multiple_args_dict : dict
        A dictionary whose keys are files and values are args list.
        Run each args list as a subprocess and write stdout to the
        corresponding file.

    print_commands : bool
        Print shell commands before running them.

    process_limit : int
        Limit the number of concurrent processes to this number. 0
        if there is no limit, -1 to use max number of processors

    polling_freq : int
        Number of seconds between checking for done processes, if
        we have a process limit
    """
    assert len(multiple_args_dict) > 0
    assert all(len(args) > 0 for args in multiple_args_dict.values())
    assert all(hasattr(f, 'name') for f in multiple_args_dict.keys())
    if process_limit < 0:
        logger.debug("Using %d processes" % cpu_count())
        process_limit = cpu_count()

    start_time = time.time()
    processes = Queue(maxsize=process_limit)

    def add_to_queue(process):
        process.start()
        if print_commands:
            handler = logging.FileHandler(process.redirect_stdout_file.name)
            handler.setLevel(logging.DEBUG)
            logger.addHandler(handler)
            logger.debug(" ".join(process.args))
            logger.removeHandler(handler)
        processes.put(process)

    for f, args in multiple_args_dict.items():
        p = AsyncProcess(
                args,
                redirect_stdout_file=f,
                **kwargs)
        if not processes.full():
            add_to_queue(p)
        else:
            while processes.full():
                # Are there any done processes?
                to_remove = []
                for possibly_done in processes.queue:
                    if possibly_done.poll() is not None:
                        possibly_done.wait()
                        to_remove.append(possibly_done)
                # Remove them from the queue and stop checking
                if to_remove:
                    for process_to_remove in to_remove:
                        processes.queue.remove(process_to_remove)
                    break
                # Check again in a second if there weren't
                time.sleep(polling_freq)
            add_to_queue(p)

    # Wait for all the rest of the processes
    while not processes.empty():
        processes.get().wait()

    elapsed_time = time.time() - start_time
    logger.info(
        "Ran %d commands in %0.4f seconds",
        len(multiple_args_dict),
        elapsed_time)

def
run_multiple_commands_redirect_stdout
(
multiple_args_dict
,
print_commands
=
True
,
process_limit
=
1
,
polling_freq
=
0.5
,
**
kwargs
)
:
assert
len
(
multiple_args_dict
)
>
0
assert
all
(
len
(
args
)
>
0
for
args
in
multiple_args_dict
.
values
(
)
)
assert
all
(
hasattr
(
f
,
'name'
)
for
f
in
multiple_args_dict
.
keys
(
)
)
if
process_limit
<
0
:
logger
.
debug
(
"Using %d processes"
%
cpu_count
(
)
)
process_limit
=
cpu_count
(
)
start_time
=
time
.
time
(
)
processes
=
Queue
(
maxsize
=
process_limit
)
def
add_to_queue
(
process
)
:
process
.
start
(
)
if
print_commands
:
handler
=
logging
.
FileHandler
(
process
.
redirect_stdout_file
.
name
)
handler
.
setLevel
(
logging
.
DEBUG
)
logger
.
addHandler
(
handler
)
logger
.
debug
(
" "
.
join
(
process
.
args
)
)
logger
.
removeHandler
(
handler
)
processes
.
put
(
process
)
for
f
,
args
in
multiple_args_dict
.
items
(
)
:
p
=
AsyncProcess
(
args
,
redirect_stdout_file
=
f
,
**
kwargs
)
if
not
processes
.
full
(
)
:
add_to_queue
(
p
)
else
:
while
processes
.
full
(
)
:
to_remove
=
[
]
for
possibly_done
in
processes
.
queue
:
if
possibly_done
.
poll
(
)
is
not
:
possibly_done
.
wait
(
)
to_remove
.
append
(
possibly_done
)
if
to_remove
:
for
process_to_remove
in
to_remove
:
processes
.
queue
.
remove
(
process_to_remove
)
break
time
.
sleep
(
polling_freq
)
add_to_queue
(
p
)
while
not
processes
.
empty
(
)
:
processes
.
get
(
)
.
wait
(
)
elapsed_time
=
time
.
time
(
)
start_time
logger
.
info
(
"Ran %d commands in %0.4f seconds"
,
len
(
multiple_args_dict
)
,
elapsed_time
)

Run multiple shell commands in parallel write each of their stdout output to files associated with each command .


def loads(json_data, encoding="utf-8", **kwargs):
    """
        Custom loads function with an object_hook and automatic decoding

    Parameters
    ----------
    json_data : str
        The JSON data to decode
    *args
        Positional arguments, passed to :func:`json.loads`
    encoding : :obj:`str`, optional
        The encoding of the bytestring
    **kwargs
        Keyword arguments passed to :func:`json.loads`

    Returns
    -------
    :obj:`dict` or :obj:`list`
        Decoded json data
    """
    if isinstance(json_data, bytes):
        json_data = json_data.decode(encoding)

    return json.loads(json_data, object_hook=JSONData, **kwargs)

def loads ( json_data , encoding = "utf-8" , ** kwargs ) : if isinstance ( json_data , bytes ) : json_data = json_data . decode ( encoding ) return json . loads ( json_data , object_hook = JSONData , ** kwargs )


async def read(response, loads=loads, encoding=None):
    """
        read the data of the response

    Parameters
    ----------
    response : aiohttp.ClientResponse
        response
    loads : callable
        json loads function
    encoding : :obj:`str`, optional
        character encoding of the response, if set to None
        aiohttp should guess the right encoding

    Returns
    -------
    :obj:`bytes`, :obj:`str`, :obj:`dict` or :obj:`list`
        the data returned depends on the response
    """
    ctype = response.headers.get('Content-Type', "").lower()

    try:
        if "application/json" in ctype:
            logger.info("decoding data as json")
            return await response.json(encoding=encoding, loads=loads)

        if "text" in ctype:
            logger.info("decoding data as text")
            return await response.text(encoding=encoding)

    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
        data = await response.read()
        raise exceptions.PeonyDecodeError(response=response,
                                          data=data,
                                          exception=exc)

    return await response.read()

async
def
read
(
response
,
loads
=
loads
,
encoding
=
)
:
ctype
=
response
.
headers
.
get
(
'Content-Type'
,
""
)
.
lower
(
)
try
:
if
"application/json"
in
ctype
:
logger
.
info
(
"decoding data as json"
)
return
await
response
.
json
(
encoding
=
encoding
,
loads
=
loads
)
if
"text"
in
ctype
:
logger
.
info
(
"decoding data as text"
)
return
await
response
.
text
(
encoding
=
encoding
)
except
(
UnicodeDecodeError
,
json
.
JSONDecodeError
)
as
exc
:
data
=
await
response
.
read
(
)
raise
exceptions
.
PeonyDecodeError
(
response
=
response
,
data
=
data
,
exception
=
exc
)
return
await
response
.
read
(
)


def doc(func):
    """
        Find the message shown when someone calls the help command

    Parameters
    ----------
    func : function
        the function

    Returns
    -------
    str
        The help message for this command
    """
    stripped_chars = " \t"

    if hasattr(func, '__doc__'):
        docstring = func.__doc__.lstrip(" \n\t")
        if "\n" in docstring:
            i = docstring.index("\n")
            return docstring[:i].rstrip(stripped_chars)
        elif docstring:
            return docstring.rstrip(stripped_chars)

    return ""

def
doc
(
func
)
:
stripped_chars
=
" \t"
if
hasattr
(
func
,
'__doc__'
)
:
docstring
=
func
.
__doc__
.
lstrip
(
" \n\t"
)
if
"\n"
in
docstring
:
i
=
docstring
.
index
(
"\n"
)
return
docstring
[
:
i
]
.
rstrip
(
stripped_chars
)
elif
docstring
:
return
docstring
.
rstrip
(
stripped_chars
)
return
""


def permission_check(data, command_permissions,
                     command=None, permissions=None):
    """
        Check the permissions of the user requesting a command

    Parameters
    ----------
    data : dict
        message data
    command_permissions : dict
        permissions of the command, contains all the roles as key and users
        with these permissions as values
    command : function
        the command that is run
    permissions : tuple or list
        a list of permissions for the command

    Returns
    -------
    bool
        True if the user has the right permissions, False otherwise
    """
    if permissions:
        pass
    elif command:
        if hasattr(command, 'permissions'):
            permissions = command.permissions
        else:
            return True  # true if no permission is required
    else:
        msg = "{name} must be called with command or permissions argument"
        raise RuntimeError(msg.format(name="_permission_check"))

    return any(data['sender']['id'] in command_permissions[permission]
               for permission in permissions
               if permission in command_permissions)


def main(args_list=None):
    """
    Script to make pMHC binding predictions from amino acid sequences.

    Usage example:
        mhctools
            --sequence SFFPIQQQQQAAALLLI \
            --sequence SILQQQAQAQQAQAASSSC \
            --extract-subsequences \
            --mhc-predictor netmhc \
            --mhc-alleles HLA-A0201 H2-Db \
            --mhc-predictor netmhc \
            --output-csv epitope.csv
    """
    args = parse_args(args_list)
    binding_predictions = run_predictor(args)
    df = binding_predictions.to_dataframe()
    logger.info('\n%s', df)
    if args.output_csv:
        df.to_csv(args.output_csv, index=False)
        print("Wrote: %s" % args.output_csv)

def
main
(
args_list
=
)
:
args
=
parse_args
(
args_list
)
binding_predictions
=
run_predictor
(
args
)
df
=
binding_predictions
.
to_dataframe
(
)
logger
.
info
(
'\n%s'
,
df
)
if
args
.
output_csv
:
df
.
to_csv
(
args
.
output_csv
,
index
=
False
)
print
(
"Wrote: %s"
%
args
.
output_csv
)

Script to make pMHC binding predictions from amino acid sequences .


def _prepare_drb_allele_name(self, parsed_beta_allele):
        """
        Assume that we're dealing with a human DRB allele
        which NetMHCIIpan treats differently because there is
        little population diversity in the DR-alpha gene
        """
        if "DRB" not in parsed_beta_allele.gene:
            raise ValueError("Unexpected allele %s" % parsed_beta_allele)
        return "%s_%s%s" % (
            parsed_beta_allele.gene,
            parsed_beta_allele.allele_family,
            parsed_beta_allele.allele_code)

def
_prepare_drb_allele_name
(
self
,
parsed_beta_allele
)
:
if
"DRB"
not
in
parsed_beta_allele
.
gene
:
raise
ValueError
(
"Unexpected allele %s"
%
parsed_beta_allele
)
return
"%s_%s%s"
%
(
parsed_beta_allele
.
gene
,
parsed_beta_allele
.
allele_family
,
parsed_beta_allele
.
allele_code
)

Assume that we re dealing with a human DRB allele which NetMHCIIpan treats differently because there is little population diversity in the DR - alpha gene


def get_error(data):
    """ return the error if there is a corresponding exception """
    if isinstance(data, dict):
        if 'errors' in data:
            error = data['errors'][0]
        else:
            error = data.get('error', None)

        if isinstance(error, dict):
            if error.get('code') in errors:
                return error

def get_error ( data ) : if isinstance ( data , dict ) : if 'errors' in data : error = data [ 'errors' ] [ 0 ] else : error = data . get ( 'error' , None ) if isinstance ( error , dict ) : if error . get ( 'code' ) in errors : return error


async def throw(response, loads=None, encoding=None, **kwargs):
    """ Get the response data if possible and raise an exception """
    if loads is None:
        loads = data_processing.loads

    data = await data_processing.read(response, loads=loads,
                                      encoding=encoding)

    error = get_error(data)
    if error is not None:
        exception = errors[error['code']]
        raise exception(response=response, error=error, data=data, **kwargs)

    if response.status in statuses:
        exception = statuses[response.status]
        raise exception(response=response, data=data, **kwargs)

    # raise PeonyException if no specific exception was found
    raise PeonyException(response=response, data=data, **kwargs)

async def throw ( response , loads = None , encoding = None , ** kwargs ) : if loads is None : loads = data_processing . loads data = await data_processing . read ( response , loads = loads , encoding = encoding ) error = get_error ( data ) if error is not None : exception = errors [ error [ 'code' ] ] raise exception ( response = response , error = error , data = data , ** kwargs ) if response . status in statuses : exception = statuses [ response . status ] raise exception ( response = response , data = data , ** kwargs ) raise PeonyException ( response = response , data = data , ** kwargs )


def code(self, code):
        """ Decorator to associate a code to an exception """
        def decorator(exception):
            self[code] = exception
            return exception

        return decorator

def code ( self , code ) : def decorator ( exception ) : self [ code ] = exception return exception return decorator


async def prepare_request(self, method, url,
                              headers=None,
                              skip_params=False,
                              proxy=None,
                              **kwargs):
        """
        prepare all the arguments for the request

        Parameters
        ----------
        method : str
            HTTP method used by the request
        url : str
            The url to request
        headers : dict, optional
            Additionnal headers
        proxy : str
            proxy of the request
        skip_params : bool
            Don't use the parameters to sign the request

        Returns
        -------
        dict
            Parameters of the request correctly formatted
        """

        if method.lower() == "post":
            key = 'data'
        else:
            key = 'params'

        if key in kwargs and not skip_params:
            request_params = {key: kwargs.pop(key)}
        else:
            request_params = {}

        request_params.update(dict(method=method.upper(), url=url))

        coro = self.sign(**request_params, skip_params=skip_params,
                         headers=headers)
        request_params['headers'] = await utils.execute(coro)
        request_params['proxy'] = proxy

        kwargs.update(request_params)

        return kwargs

async def prepare_request ( self , method , url , headers = None , skip_params = False , proxy = None , ** kwargs ) : if method . lower ( ) == "post" : key = 'data' else : key = 'params' if key in kwargs and not skip_params : request_params = { key : kwargs . pop ( key ) } else : request_params = { } request_params . update ( dict ( method = method . upper ( ) , url = url ) ) coro = self . sign ( ** request_params , skip_params = skip_params , headers = headers ) request_params [ 'headers' ] = await utils . execute ( coro ) request_params [ 'proxy' ] = proxy kwargs . update ( request_params ) return kwargs


def _user_headers(self, headers=None):
        """ Make sure the user doesn't override the Authorization header """
        h = self.copy()

        if headers is not None:
            keys = set(headers.keys())
            if h.get('Authorization', False):
                keys -= {'Authorization'}

            for key in keys:
                h[key] = headers[key]

        return h

def _user_headers ( self , headers = None ) : h = self . copy ( ) if headers is not None : keys = set ( headers . keys ( ) ) if h . get ( 'Authorization' , False ) : keys -= { 'Authorization' } for key in keys : h [ key ] = headers [ key ] return h

Make sure the user doesn t override the Authorization header


def process_keys(func):
    """
    Raise error for keys that are not strings
    and add the prefix if it is missing
    """

    @wraps(func)
    def decorated(self, k, *args):
        if not isinstance(k, str):
            msg = "%s: key must be a string" % self.__class__.__name__
            raise ValueError(msg)

        if not k.startswith(self.prefix):
            k = self.prefix + k

        return func(self, k, *args)

    return decorated

def
process_keys
(
func
)
:
@
wraps
(
func
)
def
decorated
(
self
,
k
,
*
args
)
:
if
not
isinstance
(
k
,
str
)
:
msg
=
"%s: key must be a string"
%
self
.
__class__
.
__name__
raise
ValueError
(
msg
)
if
not
k
.
startswith
(
self
.
prefix
)
:
k
=
self
.
prefix
+
k
return
func
(
self
,
k
,
*
args
)
return
decorated


def _get(self, text):
        """
            Analyze the text to get the right function

        Parameters
        ----------
        text : str
            The text that could call a function
        """
        if self.strict:
            match = self.prog.match(text)
            if match:
                cmd = match.group()
                if cmd in self:
                    return cmd
        else:
            words = self.prog.findall(text)
            for word in words:
                if word in self:
                    return word

def _get ( self , text ) : if self . strict : match = self . prog . match ( text ) if match : cmd = match . group ( ) if cmd in self : return cmd else : words = self . prog . findall ( text ) for word in words : if word in self : return word


async def run(self, *args, data):
        """ run the function you want """
        cmd = self._get(data.text)

        try:
            if cmd is not None:
                command = self[cmd](*args, data=data)
                return await peony.utils.execute(command)

        except:
            fmt = "Error occurred while running function {cmd}:"
            peony.utils.log_error(fmt.format(cmd=cmd))


def simplified_edges(self):
        """
        A generator for getting all of the edges without consuming extra
        memory.
        """
        for group, edgelist in self.edges.items():
            for u, v, d in edgelist:
                yield (u, v)

def simplified_edges ( self ) : for group , edgelist in self . edges . items ( ) : for u , v , d in edgelist : yield ( u , v )

A generator for getting all of the edges without consuming extra memory .


def has_edge_within_group(self, group):
        """
        Checks whether there are within-group edges or not.
        """
        assert group in self.nodes.keys(),\
            "{0} not one of the group of nodes".format(group)
        nodelist = self.nodes[group]
        for n1, n2 in self.simplified_edges():
            if n1 in nodelist and n2 in nodelist:
                return True

def
has_edge_within_group
(
self
,
group
)
:
assert
group
in
self
.
nodes
.
keys
(
)
,
"{0} not one of the group of nodes"
.
format
(
group
)
nodelist
=
self
.
nodes
[
group
]
for
n1
,
n2
in
self
.
simplified_edges
(
)
:
if
n1
in
nodelist
and
n2
in
nodelist
:
return
True

Checks whether there are within - group edges or not .


def plot_axis(self, rs, theta):
        """
        Renders the axis.
        """
        xs, ys = get_cartesian(rs, theta)
        self.ax.plot(xs, ys, 'black', alpha=0.3)

def plot_axis ( self , rs , theta ) : xs , ys = get_cartesian ( rs , theta ) self . ax . plot ( xs , ys , 'black' , alpha = 0.3 )


def plot_nodes(self, nodelist, theta, group):
        """
        Plots nodes to screen.
        """
        for i, node in enumerate(nodelist):
            r = self.internal_radius + i * self.scale
            x, y = get_cartesian(r, theta)
            circle = plt.Circle(xy=(x, y), radius=self.dot_radius,
                                color=self.node_colormap[group], linewidth=0)
            self.ax.add_patch(circle)

def plot_nodes ( self , nodelist , theta , group ) : for i , node in enumerate ( nodelist ) : r = self . internal_radius + i * self . scale x , y = get_cartesian ( r , theta ) circle = plt . Circle ( xy = ( x , y ) , radius = self . dot_radius , color = self . node_colormap [ group ] , linewidth = 0 ) self . ax . add_patch ( circle )


def group_theta(self, group):
        """
        Computes the theta along which a group's nodes are aligned.
        """
        for i, g in enumerate(self.nodes.keys()):
            if g == group:
                break

        return i * self.major_angle

def group_theta ( self , group ) : for i , g in enumerate ( self . nodes . keys ( ) ) : if g == group : break return i * self . major_angle

Computes the theta along which a group s nodes are aligned .


def find_node_group_membership(self, node):
        """
        Identifies the group for which a node belongs to.
        """
        for group, nodelist in self.nodes.items():
            if node in nodelist:
                return group

def find_node_group_membership ( self , node ) : for group , nodelist in self . nodes . items ( ) : if node in nodelist : return group

Identifies the group for which a node belongs to .


def get_idx(self, node):
        """
        Finds the index of the node in the sorted list.
        """
        group = self.find_node_group_membership(node)
        return self.nodes[group].index(node)

def get_idx ( self , node ) : group = self . find_node_group_membership ( node ) return self . nodes [ group ] . index ( node )

Finds the index of the node in the sorted list .


def node_radius(self, node):
        """
        Computes the radial position of the node.
        """
        return self.get_idx(node) * self.scale + self.internal_radius

def node_radius ( self , node ) : return self . get_idx ( node ) * self . scale + self . internal_radius

Computes the radial position of the node .


def node_theta(self, node):
        """
        Convenience function to find the node's theta angle.
        """
        group = self.find_node_group_membership(node)
        return self.group_theta(group)

def node_theta ( self , node ) : group = self . find_node_group_membership ( node ) return self . group_theta ( group )

Convenience function to find the node s theta angle .


def add_edges(self):
        """
        Draws all of the edges in the graph.
        """
        for group, edgelist in self.edges.items():
            for (u, v, d) in edgelist:
                self.draw_edge(u, v, d, group)

def add_edges ( self ) : for group , edgelist in self . edges . items ( ) : for ( u , v , d ) in edgelist : self . draw_edge ( u , v , d , group )


def draw(self):
        """
        The master function that is called that draws everything.
        """
        self.ax.set_xlim(-self.plot_radius(), self.plot_radius())
        self.ax.set_ylim(-self.plot_radius(), self.plot_radius())

        self.add_axes_and_nodes()
        self.add_edges()

        self.ax.axis('off')

def draw ( self ) : self . ax . set_xlim ( - self . plot_radius ( ) , self . plot_radius ( ) ) self . ax . set_ylim ( - self . plot_radius ( ) , self . plot_radius ( ) ) self . add_axes_and_nodes ( ) self . add_edges ( ) self . ax . axis ( 'off' )

The master function that is called that draws everything .


def adjust_angles(self, start_node, start_angle, end_node, end_angle):
        """
        This function adjusts the start and end angles to correct for
        duplicated axes.
        """
        start_group = self.find_node_group_membership(start_node)
        end_group = self.find_node_group_membership(end_node)

        if start_group == 0 and end_group == len(self.nodes.keys())-1:
            if self.has_edge_within_group(start_group):
                start_angle = correct_negative_angle(start_angle -
                                                     self.minor_angle)
            if self.has_edge_within_group(end_group):
                end_angle = correct_negative_angle(end_angle +
                                                   self.minor_angle)

        elif start_group == len(self.nodes.keys())-1 and end_group == 0:
            if self.has_edge_within_group(start_group):
                start_angle = correct_negative_angle(start_angle +
                                                     self.minor_angle)
            if self.has_edge_within_group(end_group):
                end_angle = correct_negative_angle(end_angle -
                                                   self.minor_angle)

        elif start_group < end_group:
            if self.has_edge_within_group(end_group):
                end_angle = correct_negative_angle(end_angle -
                                                   self.minor_angle)
            if self.has_edge_within_group(start_group):
                start_angle = correct_negative_angle(start_angle +
                                                     self.minor_angle)

        elif end_group < start_group:
            if self.has_edge_within_group(start_group):
                start_angle = correct_negative_angle(start_angle -
                                                     self.minor_angle)
            if self.has_edge_within_group(end_group):
                end_angle = correct_negative_angle(end_angle +
                                                   self.minor_angle)

        return start_angle, end_angle

def adjust_angles ( self , start_node , start_angle , end_node , end_angle ) : start_group = self . find_node_group_membership ( start_node ) end_group = self . find_node_group_membership ( end_node ) if start_group == 0 and end_group == len ( self . nodes . keys ( ) ) - 1 : if self . has_edge_within_group ( start_group ) : start_angle = correct_negative_angle ( start_angle - self . minor_angle ) if self . has_edge_within_group ( end_group ) : end_angle = correct_negative_angle ( end_angle + self . minor_angle ) elif start_group == len ( self . nodes . keys ( ) ) - 1 and end_group == 0 : if self . has_edge_within_group ( start_group ) : start_angle = correct_negative_angle ( start_angle + self . minor_angle ) if self . has_edge_within_group ( end_group ) : end_angle = correct_negative_angle ( end_angle - self . minor_angle ) elif start_group < end_group : if self . has_edge_within_group ( end_group ) : end_angle = correct_negative_angle ( end_angle - self . minor_angle ) if self . has_edge_within_group ( start_group ) : start_angle = correct_negative_angle ( start_angle + self . minor_angle ) elif end_group < start_group : if self . has_edge_within_group ( start_group ) : start_angle = correct_negative_angle ( start_angle - self . minor_angle ) if self . has_edge_within_group ( end_group ) : end_angle = correct_negative_angle ( end_angle + self . minor_angle ) return start_angle , end_angle

This function adjusts the start and end angles to correct for duplicated axes .


def mods_genre(self):
		"""
		Guesses an appropriate MODS XML genre type.
		"""

		type2genre = {
				'conference': 'conference publication',
				'book chapter': 'bibliography',
				'unpublished': 'article'
			}
		tp = str(self.type).lower()
		return type2genre.get(tp, tp)

def
mods_genre
(
self
)
:
type2genre
=
{
'conference'
:
'conference publication'
,
'book chapter'
:
'bibliography'
,
'unpublished'
:
'article'
}
tp
=
str
(
self
.
type
)
.
lower
(
)
return
type2genre
.
get
(
tp
,
tp
)

Guesses an appropriate MODS XML genre type .


def get_publications(context, template='publications/publications.html'):
	"""
	Get all publications.
	"""

	types = Type.objects.filter(hidden=False)
	publications = Publication.objects.select_related()
	publications = publications.filter(external=False, type__in=types)
	publications = publications.order_by('-year', '-month', '-id')

	if not publications:
		return ''

	# load custom links and files
	populate(publications)

	return render_template(template, context['request'], {'publications': publications})

def get_publications ( context , template = 'publications/publications.html' ) : types = Type . objects . filter ( hidden = False ) publications = Publication . objects . select_related ( ) publications = publications . filter ( external = False , type__in = types ) publications = publications . order_by ( '-year' , '-month' , '-id' ) if not publications : return '' populate ( publications ) return render_template ( template , context [ 'request' ] , { 'publications' : publications } )


def get_publication(context, id):
	"""
	Get a single publication.
	"""

	pbl = Publication.objects.filter(pk=int(id))

	if len(pbl) < 1:
		return ''

	pbl[0].links = pbl[0].customlink_set.all()
	pbl[0].files = pbl[0].customfile_set.all()

	return render_template(
		'publications/publication.html', context['request'], {'publication': pbl[0]})

def get_publication ( context , id ) : pbl = Publication . objects . filter ( pk = int ( id ) ) if len ( pbl ) < 1 : return '' pbl [ 0 ] . links = pbl [ 0 ] . customlink_set . all ( ) pbl [ 0 ] . files = pbl [ 0 ] . customfile_set . all ( ) return render_template ( 'publications/publication.html' , context [ 'request' ] , { 'publication' : pbl [ 0 ] } )


def get_publication_list(context, list, template='publications/publications.html'):
	"""
	Get a publication list.
	"""

	list = List.objects.filter(list__iexact=list)

	if not list:
		return ''

	list = list[0]
	publications = list.publication_set.all()
	publications = publications.order_by('-year', '-month', '-id')

	if not publications:
		return ''

	# load custom links and files
	populate(publications)

	return render_template(
		template, context['request'], {'list': list, 'publications': publications})

def get_publication_list ( context , list , template = 'publications/publications.html' ) : list = List . objects . filter ( list__iexact = list ) if not list : return '' list = list [ 0 ] publications = list . publication_set . all ( ) publications = publications . order_by ( '-year' , '-month' , '-id' ) if not publications : return '' populate ( publications ) return render_template ( template , context [ 'request' ] , { 'list' : list , 'publications' : publications } )


def tex_parse(string):
	"""
	Renders some basic TeX math to HTML.
	"""

	string = string.replace('{', '').replace('}', '')
	def tex_replace(match):
		return \
			sub(r'\^(\w)', r'\1',
			sub(r'\^\{(.*?)\}', r'\1',
			sub(r'\_(\w)', r'\1',
			sub(r'\_\{(.*?)\}', r'\1',
			sub(r'\\(' + GREEK_LETTERS + ')', r'&\1;', match.group(1))))))
	return mark_safe(sub(r'\$([^\$]*)\$', tex_replace, escape(string)))

def tex_parse ( string ) : string = string . replace ( '{' , '' ) . replace ( '}' , '' ) def tex_replace ( match ) : return sub ( r'\^(\w)' , r'\1' , sub ( r'\^\{(.*?)\}' , r'\1' , sub ( r'\_(\w)' , r'\1' , sub ( r'\_\{(.*?)\}' , r'\1' , sub ( r'\\(' + GREEK_LETTERS + ')' , r'&\1;' , match . group ( 1 ) ) ) ) ) ) return mark_safe ( sub ( r'\$([^\$]*)\$' , tex_replace , escape ( string ) ) )


def parse(string):
	"""
	Takes a string in BibTex format and returns a list of BibTex entries, where
	each entry is a dictionary containing the entries' key-value pairs.

	@type  string: string
	@param string: bibliography in BibTex format

	@rtype: list
	@return: a list of dictionaries representing a bibliography
	"""

	# bibliography
	bib = []

	# make sure we are dealing with unicode strings
	if not isinstance(string, six.text_type):
		string = string.decode('utf-8')

	# replace special characters
	for key, value in special_chars:
		string = string.replace(key, value)
	string = re.sub(r'\\[cuHvs]{?([a-zA-Z])}?', r'\1', string)

	# split into BibTex entries
	entries = re.findall(
		r'(?u)@(\w+)[ \t]?{[ \t]*([^,\s]*)[ \t]*,?\s*((?:[^=,\s]+\s*\=\s*(?:"[^"]*"|{(?:[^{}]*|{[^{}]*})*}|[^,}]*),?\s*?)+)\s*}',
		string)

	for entry in entries:
		# parse entry
		pairs = re.findall(r'(?u)([^=,\s]+)\s*\=\s*("[^"]*"|{(?:[^{}]*|{[^{}]*})*}|[^,]*)', entry[2])

		# add to bibliography
		bib.append({'type': entry[0].lower(), 'key': entry[1]})

		for key, value in pairs:
			# post-process key and value
			key = key.lower()
			if value and value[0] == '"' and value[-1] == '"':
				value = value[1:-1]
			if value and value[0] == '{' and value[-1] == '}':
				value = value[1:-1]
			if key not in ['booktitle', 'title']:
				value = value.replace('}', '').replace('{', '')
			else:
				if value.startswith('{') and value.endswith('}'):
					value = value[1:]
					value = value[:-1]
			value = value.strip()
			value = re.sub(r'\s+', ' ', value)

			# store pair in bibliography
			bib[-1][key] = value

	return bib

Takes a string in BibTex format and returns a list of BibTex entries where each entry is a dictionary containing the entries key - value pairs .


def swap(self, qs):
        """
        Swap the positions of this object with a reference object.
        """
        try:
            replacement = qs[0]
        except IndexError:
            # already first/last
            return
        if not self._valid_ordering_reference(replacement):
            raise ValueError(
                "%r can only be swapped with instances of %r which %s equals %r." % (
                    self, self.__class__, self.order_with_respect_to,
                    self._get_order_with_respect_to()
                )
            )
        self.order, replacement.order = replacement.order, self.order
        self.save()
        replacement.save()

Swap the positions of this object with a reference object .


def up(self):
        """
        Move this object up one position.
        """
        self.swap(self.get_ordering_queryset().filter(order__lt=self.order).order_by('-order'))

def up ( self ) : self . swap ( self . get_ordering_queryset ( ) . filter ( order__lt = self . order ) . order_by ( '-order' ) )


def down(self):
        """
        Move this object down one position.
        """
        self.swap(self.get_ordering_queryset().filter(order__gt=self.order))

def down ( self ) : self . swap ( self . get_ordering_queryset ( ) . filter ( order__gt = self . order ) )


def to(self, order):
        """
        Move object to a certain position, updating all affected objects to move accordingly up or down.
        """
        if order is None or self.order == order:
            # object is already at desired position
            return
        qs = self.get_ordering_queryset()
        if self.order > order:
            qs.filter(order__lt=self.order, order__gte=order).update(order=F('order') + 1)
        else:
            qs.filter(order__gt=self.order, order__lte=order).update(order=F('order') - 1)
        self.order = order
        self.save()

def to ( self , order ) : if order is None or self . order == order : return qs = self . get_ordering_queryset ( ) if self . order > order : qs . filter ( order__lt = self . order , order__gte = order ) . update ( order = F ( 'order' ) + 1 ) else : qs . filter ( order__gt = self . order , order__lte = order ) . update ( order = F ( 'order' ) - 1 ) self . order = order self . save ( )

Move object to a certain position updating all affected objects to move accordingly up or down .


def above(self, ref):
        """
        Move this object above the referenced object.
        """
        if not self._valid_ordering_reference(ref):
            raise ValueError(
                "%r can only be moved above instances of %r which %s equals %r." % (
                    self, self.__class__, self.order_with_respect_to,
                    self._get_order_with_respect_to()
                )
            )
        if self.order == ref.order:
            return
        if self.order > ref.order:
            o = ref.order
        else:
            o = self.get_ordering_queryset().filter(order__lt=ref.order).aggregate(Max('order')).get('order__max') or 0
        self.to(o)

Move this object above the referenced object .


def below(self, ref):
        """
        Move this object below the referenced object.
        """
        if not self._valid_ordering_reference(ref):
            raise ValueError(
                "%r can only be moved below instances of %r which %s equals %r." % (
                    self, self.__class__, self.order_with_respect_to,
                    self._get_order_with_respect_to()
                )
            )
        if self.order == ref.order:
            return
        if self.order > ref.order:
            o = self.get_ordering_queryset().filter(order__gt=ref.order).aggregate(Min('order')).get('order__min') or 0
        else:
            o = ref.order
        self.to(o)

Move this object below the referenced object .


def top(self):
        """
        Move this object to the top of the ordered stack.
        """
        o = self.get_ordering_queryset().aggregate(Min('order')).get('order__min')
        self.to(o)

def top ( self ) : o = self . get_ordering_queryset ( ) . aggregate ( Min ( 'order' ) ) . get ( 'order__min' ) self . to ( o )

Move this object to the top of the ordered stack .


def bottom(self):
        """
        Move this object to the bottom of the ordered stack.
        """
        o = self.get_ordering_queryset().aggregate(Max('order')).get('order__max')
        self.to(o)

def bottom ( self ) : o = self . get_ordering_queryset ( ) . aggregate ( Max ( 'order' ) ) . get ( 'order__max' ) self . to ( o )

Move this object to the bottom of the ordered stack .


def populate(publications):
	"""
	Load custom links and files from database and attach to publications.
	"""

	customlinks = CustomLink.objects.filter(publication__in=publications)
	customfiles = CustomFile.objects.filter(publication__in=publications)

	publications_ = {}
	for publication in publications:
		publication.links = []
		publication.files = []
		publications_[publication.id] = publication

	for link in customlinks:
		publications_[link.publication_id].links.append(link)
	for file in customfiles:
		publications_[file.publication_id].files.append(file)

def populate ( publications ) : customlinks = CustomLink . objects . filter ( publication__in = publications ) customfiles = CustomFile . objects . filter ( publication__in = publications ) publications_ = { } for publication in publications : publication . links = [ ] publication . files = [ ] publications_ [ publication . id ] = publication for link in customlinks : publications_ [ link . publication_id ] . links . append ( link ) for file in customfiles : publications_ [ file . publication_id ] . files . append ( file )

Load custom links and files from database and attach to publications .


def worker(self):
    """
    Calculates the quartet weights for the test at a random
    subsampled chunk of loci.
    """

    ## subsample loci
    fullseqs = self.sample_loci()

    ## find all iterations of samples for this quartet
    liters = itertools.product(*self.imap.values())

    ## run tree inference for each iteration of sampledict
    hashval = uuid.uuid4().hex
    weights = []
    for ridx, lidx in enumerate(liters):

        ## get subalignment for this iteration and make to nex
        a,b,c,d = lidx
        sub = {}
        for i in lidx:
            if self.rmap[i] == "p1":
                sub["A"] = fullseqs[i]
            elif self.rmap[i] == "p2":
                sub["B"] = fullseqs[i]
            elif self.rmap[i] == "p3":
                sub["C"] = fullseqs[i]
            else:
                sub["D"] = fullseqs[i]

        ## write as nexus file
        nex = []
        for tax in list("ABCD"):
            nex.append(">{}         {}".format(tax, sub[tax]))

        ## check for too much missing or lack of variants
        nsites, nvar = count_var(nex)

        ## only run test if there's variation present
        if nvar > self.minsnps:

            ## format as nexus file
            nexus = "{} {}\n".format(4, len(fullseqs[a])) + "\n".join(nex)

            ## infer ML tree
            treeorder = self.run_tree_inference(nexus, "{}.{}".format(hashval, ridx))

            ## add to list
            weights.append(treeorder)

    ## cleanup - remove all files with the hash val
    rfiles = glob.glob(os.path.join(tempfile.tempdir, "*{}*".format(hashval)))
    for rfile in rfiles:
        if os.path.exists(rfile):
            os.remove(rfile)

    ## return result as weights for the set topologies.
    trees = ["ABCD", "ACBD", "ADBC"]
    wdict = {i:float(weights.count(i))/len(weights) for i in trees}
    return wdict

def
worker
(
self
)
:
fullseqs
=
self
.
sample_loci
(
)
liters
=
itertools
.
product
(
*
self
.
imap
.
values
(
)
)
hashval
=
uuid
.
uuid4
(
)
.
hex
weights
=
[
]
for
ridx
,
lidx
in
enumerate
(
liters
)
:
a
,
b
,
c
,
d
=
lidx
sub
=
{
}
for
i
in
lidx
:
if
self
.
rmap
[
i
]
==
"p1"
:
sub
[
"A"
]
=
fullseqs
[
i
]
elif
self
.
rmap
[
i
]
==
"p2"
:
sub
[
"B"
]
=
fullseqs
[
i
]
elif
self
.
rmap
[
i
]
==
"p3"
:
sub
[
"C"
]
=
fullseqs
[
i
]
else
:
sub
[
"D"
]
=
fullseqs
[
i
]
nex
=
[
]
for
tax
in
list
(
"ABCD"
)
:
nex
.
append
(
">{}         {}"
.
format
(
tax
,
sub
[
tax
]
)
)
nsites
,
nvar
=
count_var
(
nex
)
if
nvar
>
self
.
minsnps
:
nexus
=
"{} {}\n"
.
format
(
4
,
len
(
fullseqs
[
a
]
)
)
+
"\n"
.
join
(
nex
)
treeorder
=
self
.
run_tree_inference
(
nexus
,
"{}.{}"
.
format
(
hashval
,
ridx
)
)
weights
.
append
(
treeorder
)
rfiles
=
glob
.
glob
(
os
.
path
.
join
(
tempfile
.
tempdir
,
"*{}*"
.
format
(
hashval
)
)
)
for
rfile
in
rfiles
:
if
os
.
path
.
exists
(
rfile
)
:
os
.
remove
(
rfile
)
trees
=
[
"ABCD"
,
"ACBD"
,
"ADBC"
]
wdict
=
{
i
:
float
(
weights
.
count
(
i
)
)
/
len
(
weights
)
for
i
in
trees
}
return
wdict

Calculates the quartet weights for the test at a random subsampled chunk of loci .


def get_order(tre):
    """
    return tree order
    """
    anode = tre.tree&">A"
    sister = anode.get_sisters()[0]
    sisters = (anode.name[1:], sister.name[1:])
    others = [i for i in list("ABCD") if i not in sisters]
    return sorted(sisters) + sorted(others)

def get_order ( tre ) : anode = tre . tree & ">A" sister = anode . get_sisters ( ) [ 0 ] sisters = ( anode . name [ 1 : ] , sister . name [ 1 : ] ) others = [ i for i in list ( "ABCD" ) if i not in sisters ] return sorted ( sisters ) + sorted ( others )


def count_var(nex):
    """
    count number of sites with cov=4, and number of variable sites.
    """
    arr = np.array([list(i.split()[-1]) for i in nex])
    miss = np.any(arr=="N", axis=0)
    nomiss = arr[:, ~miss]
    nsnps = np.invert(np.all(nomiss==nomiss[0, :], axis=0)).sum()
    return nomiss.shape[1], nsnps

def count_var ( nex ) : arr = np . array ( [ list ( i . split ( ) [ - 1 ] ) for i in nex ] ) miss = np . any ( arr == "N" , axis = 0 ) nomiss = arr [ : , ~ miss ] nsnps = np . invert ( np . all ( nomiss == nomiss [ 0 , : ] , axis = 0 ) ) . sum ( ) return nomiss . shape [ 1 ] , nsnps

count number of sites with cov = 4 and number of variable sites .


def sample_loci(self):
        """ finds loci with sufficient sampling for this test"""

        ## store idx of passing loci
        idxs = np.random.choice(self.idxs, self.ntests)

        ## open handle, make a proper generator to reduce mem
        with open(self.data) as indata:
            liter = (indata.read().strip().split("|\n"))

        ## store data as dict
        seqdata = {i:"" for i in self.samples}

        ## put chunks into a list
        for idx, loc in enumerate(liter):
            if idx in idxs:
                ## parse chunk
                lines = loc.split("\n")[:-1]
                names = [i.split()[0] for i in lines]
                seqs = [i.split()[1] for i in lines]
                dd = {i:j for i,j in zip(names, seqs)}

                ## add data to concatenated seqdict
                for name in seqdata:
                    if name in names:
                        seqdata[name] += dd[name]
                    else:
                        seqdata[name] += "N"*len(seqs[0])

        ## concatenate into a phylip file
        return seqdata

def sample_loci ( self ) : idxs = np . random . choice ( self . idxs , self . ntests ) with open ( self . data ) as indata : liter = ( indata . read ( ) . strip ( ) . split ( "|\n" ) ) seqdata = { i : "" for i in self . samples } for idx , loc in enumerate ( liter ) : if idx in idxs : lines = loc . split ( "\n" ) [ : - 1 ] names = [ i . split ( ) [ 0 ] for i in lines ] seqs = [ i . split ( ) [ 1 ] for i in lines ] dd = { i : j for i , j in zip ( names , seqs ) } for name in seqdata : if name in names : seqdata [ name ] += dd [ name ] else : seqdata [ name ] += "N" * len ( seqs [ 0 ] ) return seqdata


def run_tree_inference(self, nexus, idx):
        """
        Write nexus to tmpfile, runs phyml tree inference, and parses
        and returns the resulting tree.
        """
        ## create a tmpdir for this test
        tmpdir = tempfile.tempdir
        tmpfile = os.path.join(tempfile.NamedTemporaryFile(
            delete=False,
            prefix=str(idx),
            dir=tmpdir,
        ))

        ## write nexus to tmpfile
        tmpfile.write(nexus)
        tmpfile.flush()

        ## infer the tree
        rax = raxml(name=str(idx), data=tmpfile.name, workdir=tmpdir, N=1, T=2)
        rax.run(force=True, block=True, quiet=True)

        ## clean up
        tmpfile.close()

        ## return tree order
        order = get_order(toytree.tree(rax.trees.bestTree))
        return "".join(order)

def run_tree_inference ( self , nexus , idx ) : tmpdir = tempfile . tempdir tmpfile = os . path . join ( tempfile . NamedTemporaryFile ( delete = False , prefix = str ( idx ) , dir = tmpdir , ) ) tmpfile . write ( nexus ) tmpfile . flush ( ) rax = raxml ( name = str ( idx ) , data = tmpfile . name , workdir = tmpdir , N = 1 , T = 2 ) rax . run ( force = True , block = True , quiet = True ) tmpfile . close ( ) order = get_order ( toytree . tree ( rax . trees . bestTree ) ) return "" . join ( order )

Write nexus to tmpfile runs phyml tree inference and parses and returns the resulting tree .


def plot(self):
        """
        return a toyplot barplot of the results table.
        """
        if self.results_table == None:
            return "no results found"
        else:
            bb = self.results_table.sort_values(
                by=["ABCD", "ACBD"],
                ascending=[False, True],
                )

            ## make a barplot
            import toyplot
            c = toyplot.Canvas(width=600, height=200)
            a = c.cartesian()
            m = a.bars(bb)
            return c, a, m

def
plot
(
self
)
:
if
self
.
results_table
==
:
return
"no results found"
else
:
bb
=
self
.
results_table
.
sort_values
(
by
=
[
"ABCD"
,
"ACBD"
]
,
ascending
=
[
False
,
True
]
,
)
import
toyplot
c
=
toyplot
.
Canvas
(
width
=
600
,
height
=
200
)
a
=
c
.
cartesian
(
)
m
=
a
.
bars
(
bb
)
return
c
,
a
,
m

return a toyplot barplot of the results table .


def plot_pairwise_dist(self, labels=None, ax=None, cmap=None, cdict=None, metric="euclidean"):
        """
        Plot pairwise distances between all samples

        labels: bool or list
                by default labels aren't included. If labels == True, then labels are read in
                from the vcf file. Alternatively, labels can be passed in as a list, should
                be same length as the number of samples.
        """
        allele_counts = self.genotypes.to_n_alt()
        dist = allel.pairwise_distance(allele_counts, metric=metric)
        if not ax:
            fig = plt.figure(figsize=(5, 5))
            ax = fig.add_subplot(1, 1, 1)

        if isinstance(labels, bool):
            if labels:
                labels = list(self.samples_vcforder)
        elif isinstance(labels, type(None)):
            pass
        else:
            ## If not bool or None (default), then check to make sure the list passed in
            ## is the right length
            if not len(labels) == len(self.samples_vcforder):
                raise IPyradError(LABELS_LENGTH_ERROR.format(len(labels), len(self.samples_vcforder)))

        allel.plot.pairwise_distance(dist, labels=labels, ax=ax, colorbar=False)

def plot_pairwise_dist ( self , labels = None , ax = None , cmap = None , cdict = None , metric = "euclidean" ) : allele_counts = self . genotypes . to_n_alt ( ) dist = allel . pairwise_distance ( allele_counts , metric = metric ) if not ax : fig = plt . figure ( figsize = ( 5 , 5 ) ) ax = fig . add_subplot ( 1 , 1 , 1 ) if isinstance ( labels , bool ) : if labels : labels = list ( self . samples_vcforder ) elif isinstance ( labels , type ( None ) ) : pass else : if not len ( labels ) == len ( self . samples_vcforder ) : raise IPyradError ( LABELS_LENGTH_ERROR . format ( len ( labels ) , len ( self . samples_vcforder ) ) ) allel . plot . pairwise_distance ( dist , labels = labels , ax = ax , colorbar = False )


def copy(self):
        """ returns a copy of the pca analysis object """
        cp = copy.deepcopy(self)
        cp.genotypes = allel.GenotypeArray(self.genotypes, copy=True)
        return cp

def copy ( self ) : cp = copy . deepcopy ( self ) cp . genotypes = allel . GenotypeArray ( self . genotypes , copy = True ) return cp


def loci2migrate(name, locifile, popdict, mindict=1):
    """
    A function to build an input file for the program migrate from an ipyrad
    .loci file, and a dictionary grouping Samples into populations.

    Parameters:
    -----------
    name: (str)
       The name prefix for the migrate formatted output file.
    locifile: (str)
       The path to the .loci file produced by ipyrad.
    popdict: (dict)
       A Python dictionary grouping Samples into Populations.

    Examples:
    ---------
    You can create the population dictionary by hand, and pass in the path
    to your .loci file as a string.
       >> popdict = {'A': ['a', 'b', 'c'], 'B': ['d', 'e', 'f']}
       >> loci2migrate("outfile.migrate", "./mydata.loci", popdict)

    Or, if you load your ipyrad.Assembly object from it's JSON file, you can
    access the loci file path and population information from there directly.
       >> data = ip.load_json("mydata.json")
       >> loci2migrate("outfile.migrate", data.outfiles.loci, data.populations)

    """

    ## I/O
    outfile = open(name+".migrate", 'w')
    infile = open(locifile, 'r')

    ## minhits dictionary can be an int (all same) or a dictionary (set each)
    if isinstance(mindict, int):
        mindict = {pop: mindict for pop in popdict}
    else:
        mindict = mindict

    ## filter data to only the loci that have data for mindict setting
    keep = []
    MINS = zip(taxa.keys(), minhits)

    ## read in data to sample names
    loci  = infile.read().strip().split("|")[:-1]
    for loc in loci:
        samps = [i.split()[0].replace(">","") for i in loc.split("\n") if ">" in i]
        ## filter for coverage
        GG = []
        for group,mins in MINS:
            GG.append( sum([i in samps for i in taxa[group]]) >= int(mins) )
        if all(GG):
            keep.append(loc)

    ## print data to file
    print >>outfile, len(taxa), len(keep), "( npops nloci for data set", data.name+".loci",")"

    ## print all data for each population at a time
    done = 0
    for group in taxa:
        ## print a list of lengths of each locus
        if not done:
            loclens = [len(loc.split("\n")[1].split()[-1].replace("x","n").replace("n","")) for loc in keep]
            print >>outfile, " ".join(map(str,loclens))
            done += 1

        ## print a list of number of individuals in each locus
        indslist = []
        for loc in keep:
            samps = [i.split()[0].replace(">","") for i in loc.split("\n") if ">" in i]
            inds = sum([i in samps for i in taxa[group]])
            indslist.append(inds)
        print >>outfile, " ".join(map(str,indslist)), group

        ## print sample id, spaces, and sequence data
        #for loc in range(len(keep)):
        for loc in range(len(keep)):
            seqs = [i.split()[-1] for i in keep[loc].split("\n") if \
                    i.split()[0].replace(">","") in taxa[group]]
            for i in range(len(seqs)):
                print >>outfile, group[0:8]+"_"+str(i)+\
                      (" "*(10-len(group[0:8]+"_"+str(i))))+seqs[i].replace("x","n").replace("n","")

    outfile.close()

def
loci2migrate
(
name
,
locifile
,
popdict
,
mindict
=
1
)
:
outfile
=
open
(
name
+
".migrate"
,
'w'
)
infile
=
open
(
locifile
,
'r'
)
if
isinstance
(
mindict
,
int
)
:
mindict
=
{
pop
:
mindict
for
pop
in
popdict
}
else
:
mindict
=
mindict
keep
=
[
]
MINS
=
zip
(
taxa
.
keys
(
)
,
minhits
)
loci
=
infile
.
read
(
)
.
strip
(
)
.
split
(
"|"
)
[
:
1
]
for
loc
in
loci
:
samps
=
[
i
.
split
(
)
[
0
]
.
replace
(
">"
,
""
)
for
i
in
loc
.
split
(
"\n"
)
if
">"
in
i
]
GG
=
[
]
for
group
,
mins
in
MINS
:
GG
.
append
(
sum
(
[
i
in
samps
for
i
in
taxa
[
group
]
]
)
>=
int
(
mins
)
)
if
all
(
GG
)
:
keep
.
append
(
loc
)
print
>>
outfile
,
len
(
taxa
)
,
len
(
keep
)
,
"( npops nloci for data set"
,
data
.
name
+
".loci"
,
")"
done
=
0
for
group
in
taxa
:
if
not
done
:
loclens
=
[
len
(
loc
.
split
(
"\n"
)
[
1
]
.
split
(
)
[
1
]
.
replace
(
"x"
,
"n"
)
.
replace
(
"n"
,
""
)
)
for
loc
in
keep
]
print
>>
outfile
,
" "
.
join
(
map
(
str
,
loclens
)
)
done
+=
1
indslist
=
[
]
for
loc
in
keep
:
samps
=
[
i
.
split
(
)
[
0
]
.
replace
(
">"
,
""
)
for
i
in
loc
.
split
(
"\n"
)
if
">"
in
i
]
inds
=
sum
(
[
i
in
samps
for
i
in
taxa
[
group
]
]
)
indslist
.
append
(
inds
)
print
>>
outfile
,
" "
.
join
(
map
(
str
,
indslist
)
)
,
group
for
loc
in
range
(
len
(
keep
)
)
:
seqs
=
[
i
.
split
(
)
[
1
]
for
i
in
keep
[
loc
]
.
split
(
"\n"
)
if
i
.
split
(
)
[
0
]
.
replace
(
">"
,
""
)
in
taxa
[
group
]
]
for
i
in
range
(
len
(
seqs
)
)
:
print
>>
outfile
,
group
[
0
:
8
]
+
"_"
+
str
(
i
)
+
(
" "
*
(
10
len
(
group
[
0
:
8
]
+
"_"
+
str
(
i
)
)
)
)
+
seqs
[
i
]
.
replace
(
"x"
,
"n"
)
.
replace
(
"n"
,
""
)
outfile
.
close
(
)

A function to build an input file for the program migrate from an ipyrad . loci file and a dictionary grouping Samples into populations .


def update(assembly, idict, count):
    """ updates dictionary with the next .5M reads from the super long string
    phylip file. Makes for faster reading. """

    data = iter(open(os.path.join(assembly.dirs.outfiles,
                     assembly.name+".phy"), 'r'))

    ntax, nchar = data.next().strip().split()

    ## read in max N bp at a time
    for line in data:
        tax, seq = line.strip().split()
        idict[tax] = idict[tax][100000:]
        idict[tax] += seq[count:count+100000]
    del line

    return idict

def update ( assembly , idict , count ) : data = iter ( open ( os . path . join ( assembly . dirs . outfiles , assembly . name + ".phy" ) , 'r' ) ) ntax , nchar = data . next ( ) . strip ( ) . split ( ) for line in data : tax , seq = line . strip ( ) . split ( ) idict [ tax ] = idict [ tax ] [ 100000 : ] idict [ tax ] += seq [ count : count + 100000 ] del line return idict

updates dictionary with the next . 5M reads from the super long string phylip file . Makes for faster reading .


def make(assembly, samples):
    """ Make phylip and nexus formats. This is hackish since I'm recycling the
    code whole-hog from pyrad V3. Probably could be good to go back through
    and clean up the conversion code some time.
    """

    ## get the longest name
    longname = max([len(i) for i in assembly.samples.keys()])
    names = [i.name for i in samples]

    partitions = makephy(assembly, samples, longname)
    makenex(assembly, names, longname, partitions)

def make ( assembly , samples ) : longname = max ( [ len ( i ) for i in assembly . samples . keys ( ) ] ) names = [ i . name for i in samples ] partitions = makephy ( assembly , samples , longname ) makenex ( assembly , names , longname , partitions )

Make phylip and nexus formats . This is hackish since I m recycling the code whole - hog from pyrad V3 . Probably could be good to go back through and clean up the conversion code some time .


def sample_cleanup(data, sample):
    """
    Clean up a bunch of loose files.
    """
    umap1file = os.path.join(data.dirs.edits, sample.name+"-tmp-umap1.fastq")
    umap2file = os.path.join(data.dirs.edits, sample.name+"-tmp-umap2.fastq")
    unmapped = os.path.join(data.dirs.refmapping, sample.name+"-unmapped.bam")
    samplesam = os.path.join(data.dirs.refmapping, sample.name+".sam")
    split1 = os.path.join(data.dirs.edits, sample.name+"-split1.fastq")
    split2 = os.path.join(data.dirs.edits, sample.name+"-split2.fastq")
    refmap_derep = os.path.join(data.dirs.edits, sample.name+"-refmap_derep.fastq")
    for f in [umap1file, umap2file, unmapped, samplesam, split1, split2, refmap_derep]:
        try:
            os.remove(f)
        except:
            pass

def sample_cleanup ( data , sample ) : umap1file = os . path . join ( data . dirs . edits , sample . name + "-tmp-umap1.fastq" ) umap2file = os . path . join ( data . dirs . edits , sample . name + "-tmp-umap2.fastq" ) unmapped = os . path . join ( data . dirs . refmapping , sample . name + "-unmapped.bam" ) samplesam = os . path . join ( data . dirs . refmapping , sample . name + ".sam" ) split1 = os . path . join ( data . dirs . edits , sample . name + "-split1.fastq" ) split2 = os . path . join ( data . dirs . edits , sample . name + "-split2.fastq" ) refmap_derep = os . path . join ( data . dirs . edits , sample . name + "-refmap_derep.fastq" ) for f in [ umap1file , umap2file , unmapped , samplesam , split1 , split2 , refmap_derep ] : try : os . remove ( f ) except : pass


def index_reference_sequence(data, force=False):
    """
    Index the reference sequence, unless it already exists. Also make a mapping
    of scaffolds to index numbers for later user in steps 5-6.
    """

    ## get ref file from params
    refseq_file = data.paramsdict['reference_sequence']
    index_files = []

    ## Check for existence of index files. Default to bwa unless you specify smalt
    if "smalt" in data._hackersonly["aligner"]:
        # These are smalt index files. Only referenced here to ensure they exist
        index_files.extend([".sma", ".smi"])
    else:
        index_files.extend([".amb", ".ann", ".bwt", ".pac", ".sa"])

    ## samtools specific index
    index_files.extend([".fai"])

    ## If reference sequence already exists then bail out of this func
    if not force:
        if all([os.path.isfile(refseq_file+i) for i in index_files]):
            return
    #if data._headers:
    #    print(INDEX_MSG.format(data._hackersonly["aligner"]))

    if "smalt" in data._hackersonly["aligner"]:
        ## Create smalt index for mapping
        ## smalt index [-k <wordlen>] [-s <stepsiz>]  <index_name> <reference_file>
        cmd1 = [ipyrad.bins.smalt, "index",
                "-k", str(data._hackersonly["smalt_index_wordlen"]),
                refseq_file,
                refseq_file]
    else:
        ## bwa index <reference_file>
        cmd1 = [ipyrad.bins.bwa, "index", refseq_file]

    ## call the command
    LOGGER.info(" ".join(cmd1))
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    error1 = proc1.communicate()[0]

    ## simple samtools index for grabbing ref seqs
    cmd2 = [ipyrad.bins.samtools, "faidx", refseq_file]
    LOGGER.info(" ".join(cmd2))
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)
    error2 = proc2.communicate()[0]

    ## error handling
    if proc1.returncode:
        raise IPyradWarningExit(error1)
    if error2:
        if "please use bgzip" in error2:
            raise IPyradWarningExit(NO_ZIP_BINS.format(refseq_file))
        else:
            raise IPyradWarningExit(error2)

def
index_reference_sequence
(
data
,
force
=
False
)
:
refseq_file
=
data
.
paramsdict
[
'reference_sequence'
]
index_files
=
[
]
if
"smalt"
in
data
.
_hackersonly
[
"aligner"
]
:
index_files
.
extend
(
[
".sma"
,
".smi"
]
)
else
:
index_files
.
extend
(
[
".amb"
,
".ann"
,
".bwt"
,
".pac"
,
".sa"
]
)
index_files
.
extend
(
[
".fai"
]
)
if
not
force
:
if
all
(
[
os
.
path
.
isfile
(
refseq_file
+
i
)
for
i
in
index_files
]
)
:
return
if
"smalt"
in
data
.
_hackersonly
[
"aligner"
]
:
cmd1
=
[
ipyrad
.
bins
.
smalt
,
"index"
,
"-k"
,
str
(
data
.
_hackersonly
[
"smalt_index_wordlen"
]
)
,
refseq_file
,
refseq_file
]
else
:
cmd1
=
[
ipyrad
.
bins
.
bwa
,
"index"
,
refseq_file
]
LOGGER
.
info
(
" "
.
join
(
cmd1
)
)
proc1
=
sps
.
Popen
(
cmd1
,
stderr
=
sps
.
STDOUT
,
stdout
=
sps
.
PIPE
)
error1
=
proc1
.
communicate
(
)
[
0
]
cmd2
=
[
ipyrad
.
bins
.
samtools
,
"faidx"
,
refseq_file
]
LOGGER
.
info
(
" "
.
join
(
cmd2
)
)
proc2
=
sps
.
Popen
(
cmd2
,
stderr
=
sps
.
STDOUT
,
stdout
=
sps
.
PIPE
)
error2
=
proc2
.
communicate
(
)
[
0
]
if
proc1
.
returncode
:
raise
IPyradWarningExit
(
error1
)
if
error2
:
if
"please use bgzip"
in
error2
:
raise
IPyradWarningExit
(
NO_ZIP_BINS
.
format
(
refseq_file
)
)
else
:
raise
IPyradWarningExit
(
error2
)

Index the reference sequence unless it already exists . Also make a mapping of scaffolds to index numbers for later user in steps 5 - 6 .


def fetch_cluster_se(data, samfile, chrom, rstart, rend):
    """
    Builds a single end cluster from the refmapped data.
    """

    ## If SE then we enforce the minimum overlap distance to avoid the
    ## staircase syndrome of multiple reads overlapping just a little.
    overlap_buffer = data._hackersonly["min_SE_refmap_overlap"]

    ## the *_buff variables here are because we have to play patty
    ## cake here with the rstart/rend vals because we want pysam to
    ## enforce the buffer for SE, but we want the reference sequence
    ## start and end positions to print correctly for downstream.
    rstart_buff = rstart + overlap_buffer
    rend_buff = rend - overlap_buffer

    ## Reads that map to only very short segements of the reference
    ## sequence will return buffer end values that are before the
    ## start values causing pysam to complain. Very short mappings.
    if rstart_buff > rend_buff:
        tmp = rstart_buff
        rstart_buff = rend_buff
        rend_buff = tmp
    ## Buffering can't make start and end equal or pysam returns nothing.
    if rstart_buff == rend_buff:
        rend_buff += 1

    ## store pairs
    rdict = {}
    clust = []
    iterreg = []

    iterreg = samfile.fetch(chrom, rstart_buff, rend_buff)

    ## use dict to match up read pairs
    for read in iterreg:
        if read.qname not in rdict:
            rdict[read.qname] = read

    ## sort dict keys so highest derep is first ('seed')
    sfunc = lambda x: int(x.split(";size=")[1].split(";")[0])
    rkeys = sorted(rdict.keys(), key=sfunc, reverse=True)

    ## get blocks from the seed for filtering, bail out if seed is not paired
    try:
        read1 = rdict[rkeys[0]]
    except ValueError:
        LOGGER.error("Found bad cluster, skipping - key:{} rdict:{}".format(rkeys[0], rdict))
        return ""

    ## the starting blocks for the seed
    poss = read1.get_reference_positions(full_length=True)
    seed_r1start = min(poss)
    seed_r1end = max(poss)

    ## store the seed -------------------------------------------
    if read1.is_reverse:
        seq = revcomp(read1.seq)
    else:
        seq = read1.seq

    ## store, could write orient but just + for now.
    size = sfunc(rkeys[0])
    clust.append(">{}:{}:{};size={};*\n{}"\
        .format(chrom, seed_r1start, seed_r1end, size, seq))

    ## If there's only one hit in this region then rkeys will only have
    ## one element and the call to `rkeys[1:]` will raise. Test for this.
    if len(rkeys) > 1:
        ## store the hits to the seed -------------------------------
        for key in rkeys[1:]:
            skip = False
            try:
                read1 = rdict[key]
            except ValueError:
                ## enter values that will make this read get skipped
                read1 = rdict[key][0]
                skip = True

            ## orient reads only if not skipping
            if not skip:
                poss = read1.get_reference_positions(full_length=True)
                minpos = min(poss)
                maxpos = max(poss)
                ## store the seq
                if read1.is_reverse:
                    seq = revcomp(read1.seq)
                else:
                    seq = read1.seq
                ## store, could write orient but just + for now.
                size = sfunc(key)
                clust.append(">{}:{}:{};size={};+\n{}"\
                    .format(chrom, minpos, maxpos, size, seq))
            else:
                ## seq is excluded, though, we could save it and return
                ## it as a separate cluster that will be aligned separately.
                pass

    return clust

Builds a single end cluster from the refmapped data .


def ref_build_and_muscle_chunk(data, sample):
    """
    1. Run bedtools to get all overlapping regions
    2. Parse out reads from regions using pysam and dump into chunk files.
       We measure it out to create 10 chunk files per sample.
    3. If we really wanted to speed this up, though it is pretty fast already,
       we could parallelize it since we can easily break the regions into
       a list of chunks.
    """

    ## get regions using bedtools
    regions = bedtools_merge(data, sample).strip().split("\n")
    nregions = len(regions)
    chunksize = (nregions / 10) + (nregions % 10)

    LOGGER.debug("nregions {} chunksize {}".format(nregions, chunksize))
    ## create an output file to write clusters to
    idx = 0
    tmpfile = os.path.join(data.tmpdir, sample.name+"_chunk_{}.ali")
    ## remove old files if they exist to avoid append errors
    for i in range(11):
        if os.path.exists(tmpfile.format(i)):
            os.remove(tmpfile.format(i))
    fopen = open

    ## If reference+denovo we drop the reads back into clust.gz
    ## and let the muscle_chunker do it's thing back in cluster_within
    if data.paramsdict["assembly_method"] == "denovo+reference":
        tmpfile = os.path.join(data.dirs.clusts, sample.name+".clust.gz")
        fopen = gzip.open

    ## build clusters for aligning with muscle from the sorted bam file
    samfile = pysam.AlignmentFile(sample.files.mapped_reads, 'rb')
    #"./tortas_refmapping/PZ70-mapped-sorted.bam", "rb")

    ## fill clusts list and dump periodically
    clusts = []
    nclusts = 0
    for region in regions:
        chrom, pos1, pos2 = region.split()
        try:
            ## fetches pairs quickly but then goes slow to merge them.
            if "pair" in data.paramsdict["datatype"]:
                clust = fetch_cluster_pairs(data, samfile, chrom, int(pos1), int(pos2))

            ## fetch but no need to merge
            else:
                clust = fetch_cluster_se(data, samfile, chrom, int(pos1), int(pos2))
        except IndexError as inst:
            LOGGER.error("Bad region chrom:start-end {}:{}-{}".format(chrom, pos1, pos2))
            continue
        if clust:
            clusts.append("\n".join(clust))
            nclusts += 1

            if nclusts == chunksize:
                ## write to file
                tmphandle = tmpfile.format(idx)
                with fopen(tmphandle, 'a') as tmp:
                    #LOGGER.debug("Writing tmpfile - {}".format(tmpfile.format(idx)))
                    #if data.paramsdict["assembly_method"] == "denovo+reference":
                    #    ## This is dumb, but for this method you need to prepend the
                    #    ## separator to maintain proper formatting of clust.gz
                    tmp.write("\n//\n//\n".join(clusts)+"\n//\n//\n")
                idx += 1
                nclusts = 0
                clusts = []
    if clusts:
        ## write remaining to file
        with fopen(tmpfile.format(idx), 'a') as tmp:
            #tmp.write("\n//\n//\n" + ("\n//\n//\n".join(clusts)))
            tmp.write("\n//\n//\n".join(clusts)+"\n//\n//\n")
        clusts = []

    if not data.paramsdict["assembly_method"] == "denovo+reference":
        chunkfiles = glob.glob(os.path.join(data.tmpdir, sample.name+"_chunk_*.ali"))
        LOGGER.info("created chunks %s", chunkfiles)

    ## cleanup
    samfile.close()

def
ref_build_and_muscle_chunk
(
data
,
sample
)
:
regions
=
bedtools_merge
(
data
,
sample
)
.
strip
(
)
.
split
(
"\n"
)
nregions
=
len
(
regions
)
chunksize
=
(
nregions
/
10
)
+
(
nregions
%
10
)
LOGGER
.
debug
(
"nregions {} chunksize {}"
.
format
(
nregions
,
chunksize
)
)
idx
=
0
tmpfile
=
os
.
path
.
join
(
data
.
tmpdir
,
sample
.
name
+
"_chunk_{}.ali"
)
for
i
in
range
(
11
)
:
if
os
.
path
.
exists
(
tmpfile
.
format
(
i
)
)
:
os
.
remove
(
tmpfile
.
format
(
i
)
)
fopen
=
open
if
data
.
paramsdict
[
"assembly_method"
]
==
"denovo+reference"
:
tmpfile
=
os
.
path
.
join
(
data
.
dirs
.
clusts
,
sample
.
name
+
".clust.gz"
)
fopen
=
gzip
.
open
samfile
=
pysam
.
AlignmentFile
(
sample
.
files
.
mapped_reads
,
'rb'
)
clusts
=
[
]
nclusts
=
0
for
region
in
regions
:
chrom
,
pos1
,
pos2
=
region
.
split
(
)
try
:
if
"pair"
in
data
.
paramsdict
[
"datatype"
]
:
clust
=
fetch_cluster_pairs
(
data
,
samfile
,
chrom
,
int
(
pos1
)
,
int
(
pos2
)
)
else
:
clust
=
fetch_cluster_se
(
data
,
samfile
,
chrom
,
int
(
pos1
)
,
int
(
pos2
)
)
except
IndexError
as
inst
:
LOGGER
.
error
(
"Bad region chrom:start-end {}:{}-{}"
.
format
(
chrom
,
pos1
,
pos2
)
)
continue
if
clust
:
clusts
.
append
(
"\n"
.
join
(
clust
)
)
nclusts
+=
1
if
nclusts
==
chunksize
:
tmphandle
=
tmpfile
.
format
(
idx
)
with
fopen
(
tmphandle
,
'a'
)
as
tmp
:
tmp
.
write
(
"\n//\n//\n"
.
join
(
clusts
)
+
"\n//\n//\n"
)
idx
+=
1
nclusts
=
0
clusts
=
[
]
if
clusts
:
with
fopen
(
tmpfile
.
format
(
idx
)
,
'a'
)
as
tmp
:
tmp
.
write
(
"\n//\n//\n"
.
join
(
clusts
)
+
"\n//\n//\n"
)
clusts
=
[
]
if
not
data
.
paramsdict
[
"assembly_method"
]
==
"denovo+reference"
:
chunkfiles
=
glob
.
glob
(
os
.
path
.
join
(
data
.
tmpdir
,
sample
.
name
+
"_chunk_*.ali"
)
)
LOGGER
.
info
(
"created chunks %s"
,
chunkfiles
)
samfile
.
close
(
)

1 . Run bedtools to get all overlapping regions 2 . Parse out reads from regions using pysam and dump into chunk files . We measure it out to create 10 chunk files per sample . 3 . If we really wanted to speed this up though it is pretty fast already we could parallelize it since we can easily break the regions into a list of chunks .


def ref_muscle_chunker(data, sample):
    """
    Run bedtools to get all overlapping regions. Pass this list into the func
    'get_overlapping_reads' which will write fastq chunks to the clust.gz file.

    1) Run bedtools merge to get a list of all contiguous blocks of bases
    in the reference seqeunce where one or more of our reads overlap.
    The output will look like this:
            1       45230754        45230783
            1       74956568        74956596
            ...
            1       116202035       116202060
    """
    LOGGER.info('entering ref_muscle_chunker')

    ## Get regions, which will be a giant list of 5-tuples, of which we're
    ## only really interested in the first three: (chrom, start, end) position.
    regions = bedtools_merge(data, sample)

    if len(regions) > 0:
        ## this calls bam_region_to_fasta a billion times
        get_overlapping_reads(data, sample, regions)
    else:
        msg = "No reads mapped to reference sequence - {}".format(sample.name)
        LOGGER.warn(msg)

Run bedtools to get all overlapping regions . Pass this list into the func get_overlapping_reads which will write fastq chunks to the clust . gz file .