def export_variants(adapter, collaborator, document_id=None, case_id=None):
    """Export causative variants for a collaborator

    Args:
        adapter(MongoAdapter)
        collaborator(str)
        document_id(str): Search for a specific variant
        case_id(str): Search causative variants for a case

    Yields:
        variant_obj(scout.Models.Variant): Variants marked as causative ordered by position.
    """

    # Store the variants in a list for sorting
    variants = []
    if document_id:
        yield adapter.variant(document_id)
        return

    variant_ids = adapter.get_causatives(
        institute_id=collaborator,
        case_id=case_id
        )
    ##TODO add check so that same variant is not included more than once
    for document_id in variant_ids:

        variant_obj = adapter.variant(document_id)
        chrom = variant_obj['chromosome']
        # Convert chromosome to integer for sorting
        chrom_int = CHROMOSOME_INTEGERS.get(chrom)
        if not chrom_int:
            LOG.info("Unknown chromosome %s", chrom)
            continue

        # Add chromosome and position to prepare for sorting
        variants.append((chrom_int, variant_obj['position'], variant_obj))

    # Sort varants based on position
    variants.sort(key=lambda x: (x[0], x[1]))

    for variant in variants:
        variant_obj = variant[2]
        yield variant_obj

def
export_variants
(
adapter
,
collaborator
,
document_id
=
,
case_id
=
)
:
variants
=
[
]
if
document_id
:
yield
adapter
.
variant
(
document_id
)
return
variant_ids
=
adapter
.
get_causatives
(
institute_id
=
collaborator
,
case_id
=
case_id
)
for
document_id
in
variant_ids
:
variant_obj
=
adapter
.
variant
(
document_id
)
chrom
=
variant_obj
[
'chromosome'
]
chrom_int
=
CHROMOSOME_INTEGERS
.
get
(
chrom
)
if
not
chrom_int
:
LOG
.
info
(
"Unknown chromosome %s"
,
chrom
)
continue
variants
.
append
(
(
chrom_int
,
variant_obj
[
'position'
]
,
variant_obj
)
)
variants
.
sort
(
key
=
lambda
x
:
(
x
[
0
]
,
x
[
1
]
)
)
for
variant
in
variants
:
variant_obj
=
variant
[
2
]
yield
variant_obj


def export_verified_variants(aggregate_variants, unique_callers):
    """Create the lines for an excel file with verified variants for
        an institute

        Args:
            aggregate_variants(list): a list of variants with aggregates case data
            unique_callers(set): a unique list of available callers

        Returns:
            document_lines(list): list of lines to include in the document
    """
    document_lines = []
    for variant in aggregate_variants:
        # get genotype and allele depth for each sample
        samples = []
        for sample in variant['samples']:
            line = [] # line elements corespond to contants.variants_export.VERIFIED_VARIANTS_HEADER
            line.append(variant['institute'])
            line.append(variant['_id']) # variant database ID
            line.append(variant['category'])
            line.append(variant['variant_type'])
            line.append(variant['display_name'][:30]) # variant display name
            # Build local link to variant:
            case_name = variant['case_obj']['display_name']  # case display name
            local_link = '/'.join([ '', variant['institute'], case_name, variant['_id'] ])
            line.append(local_link)
            line.append(variant.get('validation'))
            line.append(case_name)
            case_individual = next(ind for ind in variant['case_obj']['individuals'] if ind['individual_id'] == sample['sample_id'])
            if case_individual['phenotype'] == 2:
                line.append(' '.join([sample.get('display_name'),'(A)'])) # label sample as affected
            else:
                line.append(sample.get('display_name'))
            line.append(''.join(['chr',variant['chromosome'],':',str(variant['position'])])) # position
            line.append('>'.join([variant.get('reference')[:10],variant.get('alternative')[:10]])) # change
            genes = []
            prot_effect = []
            funct_anno = []
            for gene in variant.get('genes'): # this will be a unique long field in the document
                genes.append(gene.get('hgnc_symbol',''))
                funct_anno.append(gene.get('functional_annotation'))
                for transcript in gene.get('transcripts'):
                    if transcript.get('is_canonical') and transcript.get('protein_sequence_name'):
                        prot_effect.append(urllib.parse.unquote(transcript.get('protein_sequence_name')))
            line.append(','.join(prot_effect))
            line.append(','.join(funct_anno))
            line.append(','.join(genes))
            line.append(variant.get('rank_score'))
            line.append(variant.get('cadd_score'))
            line.append(sample.get('genotype_call'))
            line.append(sample['allele_depths'][0])
            line.append(sample['allele_depths'][1])
            line.append(sample['genotype_quality'])

            # Set callers values. One cell per caller, leave blank if not applicable
            for caller in unique_callers:
                if variant.get(caller):
                    line.append(variant.get(caller))
                else:
                    line.append('-')
            document_lines.append(line)
    return document_lines

def
export_verified_variants
(
aggregate_variants
,
unique_callers
)
:
document_lines
=
[
]
for
variant
in
aggregate_variants
:
samples
=
[
]
for
sample
in
variant
[
'samples'
]
:
line
=
[
]
line
.
append
(
variant
[
'institute'
]
)
line
.
append
(
variant
[
'_id'
]
)
line
.
append
(
variant
[
'category'
]
)
line
.
append
(
variant
[
'variant_type'
]
)
line
.
append
(
variant
[
'display_name'
]
[
:
30
]
)
case_name
=
variant
[
'case_obj'
]
[
'display_name'
]
local_link
=
'/'
.
join
(
[
''
,
variant
[
'institute'
]
,
case_name
,
variant
[
'_id'
]
]
)
line
.
append
(
local_link
)
line
.
append
(
variant
.
get
(
'validation'
)
)
line
.
append
(
case_name
)
case_individual
=
next
(
ind
for
ind
in
variant
[
'case_obj'
]
[
'individuals'
]
if
ind
[
'individual_id'
]
==
sample
[
'sample_id'
]
)
if
case_individual
[
'phenotype'
]
==
2
:
line
.
append
(
' '
.
join
(
[
sample
.
get
(
'display_name'
)
,
'(A)'
]
)
)
else
:
line
.
append
(
sample
.
get
(
'display_name'
)
)
line
.
append
(
''
.
join
(
[
'chr'
,
variant
[
'chromosome'
]
,
':'
,
str
(
variant
[
'position'
]
)
]
)
)
line
.
append
(
'>'
.
join
(
[
variant
.
get
(
'reference'
)
[
:
10
]
,
variant
.
get
(
'alternative'
)
[
:
10
]
]
)
)
genes
=
[
]
prot_effect
=
[
]
funct_anno
=
[
]
for
gene
in
variant
.
get
(
'genes'
)
:
genes
.
append
(
gene
.
get
(
'hgnc_symbol'
,
''
)
)
funct_anno
.
append
(
gene
.
get
(
'functional_annotation'
)
)
for
transcript
in
gene
.
get
(
'transcripts'
)
:
if
transcript
.
get
(
'is_canonical'
)
and
transcript
.
get
(
'protein_sequence_name'
)
:
prot_effect
.
append
(
urllib
.
parse
.
unquote
(
transcript
.
get
(
'protein_sequence_name'
)
)
)
line
.
append
(
','
.
join
(
prot_effect
)
)
line
.
append
(
','
.
join
(
funct_anno
)
)
line
.
append
(
','
.
join
(
genes
)
)
line
.
append
(
variant
.
get
(
'rank_score'
)
)
line
.
append
(
variant
.
get
(
'cadd_score'
)
)
line
.
append
(
sample
.
get
(
'genotype_call'
)
)
line
.
append
(
sample
[
'allele_depths'
]
[
0
]
)
line
.
append
(
sample
[
'allele_depths'
]
[
1
]
)
line
.
append
(
sample
[
'genotype_quality'
]
)
for
caller
in
unique_callers
:
if
variant
.
get
(
caller
)
:
line
.
append
(
variant
.
get
(
caller
)
)
else
:
line
.
append
(
'-'
)
document_lines
.
append
(
line
)
return
document_lines


def export_mt_variants(variants, sample_id):
    """Export mitochondrial variants for a case to create a MT excel report

    Args:
        variants(list): all MT variants for a case, sorted by position
        sample_id(str) : the id of a sample within the case

    Returns:
        document_lines(list): list of lines to include in the document
    """
    document_lines = []
    for variant in variants:
        line = []
        position = variant.get('position')
        change = '>'.join([variant.get('reference'),variant.get('alternative')])
        line.append(position)
        line.append(change)
        line.append(str(position)+change)
        genes = []
        prot_effect = []
        for gene in variant.get('genes'):
            genes.append(gene.get('hgnc_symbol',''))
            for transcript in gene.get('transcripts'):
                if transcript.get('is_canonical') and transcript.get('protein_sequence_name'):
                    prot_effect.append(urllib.parse.unquote(transcript.get('protein_sequence_name')))
        line.append(','.join(prot_effect))
        line.append(','.join(genes))
        ref_ad = ''
        alt_ad = ''
        for sample in variant['samples']:
            if sample.get('sample_id') == sample_id:
                ref_ad = sample['allele_depths'][0]
                alt_ad = sample['allele_depths'][1]
        line.append(ref_ad)
        line.append(alt_ad)
        document_lines.append(line)
    return document_lines

def export_mt_variants ( variants , sample_id ) : document_lines = [ ] for variant in variants : line = [ ] position = variant . get ( 'position' ) change = '>' . join ( [ variant . get ( 'reference' ) , variant . get ( 'alternative' ) ] ) line . append ( position ) line . append ( change ) line . append ( str ( position ) + change ) genes = [ ] prot_effect = [ ] for gene in variant . get ( 'genes' ) : genes . append ( gene . get ( 'hgnc_symbol' , '' ) ) for transcript in gene . get ( 'transcripts' ) : if transcript . get ( 'is_canonical' ) and transcript . get ( 'protein_sequence_name' ) : prot_effect . append ( urllib . parse . unquote ( transcript . get ( 'protein_sequence_name' ) ) ) line . append ( ',' . join ( prot_effect ) ) line . append ( ',' . join ( genes ) ) ref_ad = '' alt_ad = '' for sample in variant [ 'samples' ] : if sample . get ( 'sample_id' ) == sample_id : ref_ad = sample [ 'allele_depths' ] [ 0 ] alt_ad = sample [ 'allele_depths' ] [ 1 ] line . append ( ref_ad ) line . append ( alt_ad ) document_lines . append ( line ) return document_lines


def user(context, user_id, update_role, add_institute, remove_admin, remove_institute):
    """
    Update a user in the database
    """
    adapter = context.obj['adapter']

    user_obj = adapter.user(user_id)

    if not user_obj:
        LOG.warning("User %s could not be found", user_id)
        context.abort()

    existing_roles = set(user_obj.get('roles',[]))
    if update_role:
        if not update_role in user_obj['roles']:
            existing_roles = set(user_obj['roles'])
            existing_roles.add(update_role)
            LOG.info("Adding role %s to user", update_role)
        else:
            LOG.warning("User already have role %s", update_role)

    if remove_admin:
        try:
            existing_roles.remove('admin')
            LOG.info("Removing admin rights from user %s", user_id)
        except KeyError as err:
            LOG.info("User %s does not have admin rights", user_id)

    user_obj['roles'] = list(existing_roles)

    existing_institutes = set(user_obj.get('institutes',[]))
    for institute_id in add_institute:
        institute_obj = adapter.institute(institute_id)
        if not institute_obj:
            LOG.warning("Institute %s could not be found", institute_id)
        else:
            existing_institutes.add(institute_id)
            LOG.info("Adding institute %s to user", institute_id)

    for institute_id in remove_institute:
        try:
            existing_institutes.remove(institute_id)
            LOG.info("Removing institute %s from user", institute_id)
        except KeyError as err:
            LOG.info("User does not have access to institute %s", institute_id)

    user_obj['institutes'] = list(existing_institutes)

    updated_user = adapter.update_user(user_obj)


def str_variants(institute_id, case_name):
    """Display a list of STR variants."""
    page = int(request.args.get('page', 1))
    variant_type = request.args.get('variant_type', 'clinical')

    form = StrFiltersForm(request.args)

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)

    query = form.data
    query['variant_type'] = variant_type

    variants_query = store.variants(case_obj['_id'], category='str',
        query=query)
    data = controllers.str_variants(store, institute_obj, case_obj,
        variants_query, page)
    return dict(institute=institute_obj, case=case_obj,
        variant_type = variant_type, form=form, page=page, **data)

def str_variants ( institute_id , case_name ) : page = int ( request . args . get ( 'page' , 1 ) ) variant_type = request . args . get ( 'variant_type' , 'clinical' ) form = StrFiltersForm ( request . args ) institute_obj , case_obj = institute_and_case ( store , institute_id , case_name ) query = form . data query [ 'variant_type' ] = variant_type variants_query = store . variants ( case_obj [ '_id' ] , category = 'str' , query = query ) data = controllers . str_variants ( store , institute_obj , case_obj , variants_query , page ) return dict ( institute = institute_obj , case = case_obj , variant_type = variant_type , form = form , page = page , ** data )


def sv_variant(institute_id, case_name, variant_id):
    """Display a specific structural variant."""
    data = controllers.sv_variant(store, institute_id, case_name, variant_id)
    return data

def sv_variant ( institute_id , case_name , variant_id ) : data = controllers . sv_variant ( store , institute_id , case_name , variant_id ) return data


def str_variant(institute_id, case_name, variant_id):
    """Display a specific STR variant."""
    data = controllers.str_variant(store, institute_id, case_name, variant_id)
    return data

def str_variant ( institute_id , case_name , variant_id ) : data = controllers . str_variant ( store , institute_id , case_name , variant_id ) return data


def verify(institute_id, case_name, variant_id, variant_category, order):
    """Start procedure to validate variant using other techniques."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj = store.variant(variant_id)
    user_obj = store.user(current_user.email)

    comment = request.form.get('verification_comment')

    try:
        controllers.variant_verification(store=store, mail=mail, institute_obj=institute_obj, case_obj=case_obj, user_obj=user_obj, comment=comment,
                           variant_obj=variant_obj, sender=current_app.config['MAIL_USERNAME'], variant_url=request.referrer, order=order, url_builder=url_for)
    except controllers.MissingVerificationRecipientError:
        flash('No verification recipients added to institute.', 'danger')

    return redirect(request.referrer)

Start procedure to validate variant using other techniques .


def clinvar(institute_id, case_name, variant_id):
    """Build a clinVar submission form for a variant."""
    data = controllers.clinvar_export(store, institute_id, case_name, variant_id)
    if request.method == 'GET':
        return data
    else: #POST
        form_dict = request.form.to_dict()
        submission_objects = set_submission_objects(form_dict) # A tuple of submission objects (variants and casedata objects)

        # Add submission data to an open clinvar submission object,
        # or create a new if no open submission is found in database
        open_submission = store.get_open_clinvar_submission(current_user.email, institute_id)
        updated_submission = store.add_to_submission(open_submission['_id'], submission_objects)

        # Redirect to clinvar submissions handling page, and pass it the updated_submission_object
        return redirect(url_for('cases.clinvar_submissions', institute_id=institute_id))

def clinvar ( institute_id , case_name , variant_id ) : data = controllers . clinvar_export ( store , institute_id , case_name , variant_id ) if request . method == 'GET' : return data else : form_dict = request . form . to_dict ( ) submission_objects = set_submission_objects ( form_dict ) open_submission = store . get_open_clinvar_submission ( current_user . email , institute_id ) updated_submission = store . add_to_submission ( open_submission [ '_id' ] , submission_objects ) return redirect ( url_for ( 'cases.clinvar_submissions' , institute_id = institute_id ) )

Build a clinVar submission form for a variant .


def cancer_variants(institute_id, case_name):
    """Show cancer variants overview."""
    data = controllers.cancer_variants(store, request.args, institute_id, case_name)
    return data

def cancer_variants ( institute_id , case_name ) : data = controllers . cancer_variants ( store , request . args , institute_id , case_name ) return data


def variant_acmg(institute_id, case_name, variant_id):
    """ACMG classification form."""
    if request.method == 'GET':
        data = controllers.variant_acmg(store, institute_id, case_name, variant_id)
        return data
    else:
        criteria = []
        criteria_terms = request.form.getlist('criteria')
        for term in criteria_terms:
            criteria.append(dict(
                term=term,
                comment=request.form.get("comment-{}".format(term)),
                links=[request.form.get("link-{}".format(term))],
            ))
        acmg = controllers.variant_acmg_post(store, institute_id, case_name, variant_id,
                                             current_user.email, criteria)
        flash("classified as: {}".format(acmg), 'info')
        return redirect(url_for('.variant', institute_id=institute_id, case_name=case_name,
                                variant_id=variant_id))

def
variant_acmg
(
institute_id
,
case_name
,
variant_id
)
:
if
request
.
method
==
'GET'
:
data
=
controllers
.
variant_acmg
(
store
,
institute_id
,
case_name
,
variant_id
)
return
data
else
:
criteria
=
[
]
criteria_terms
=
request
.
form
.
getlist
(
'criteria'
)
for
term
in
criteria_terms
:
criteria
.
append
(
dict
(
term
=
term
,
comment
=
request
.
form
.
get
(
"comment-{}"
.
format
(
term
)
)
,
links
=
[
request
.
form
.
get
(
"link-{}"
.
format
(
term
)
)
]
,
)
)
acmg
=
controllers
.
variant_acmg_post
(
store
,
institute_id
,
case_name
,
variant_id
,
current_user
.
email
,
criteria
)
flash
(
"classified as: {}"
.
format
(
acmg
)
,
'info'
)
return
redirect
(
url_for
(
'.variant'
,
institute_id
=
institute_id
,
case_name
=
case_name
,
variant_id
=
variant_id
)
)


def evaluation(evaluation_id):
    """Show or delete an ACMG evaluation."""
    evaluation_obj = store.get_evaluation(evaluation_id)
    controllers.evaluation(store, evaluation_obj)
    if request.method == 'POST':
        link = url_for('.variant', institute_id=evaluation_obj['institute']['_id'],
                       case_name=evaluation_obj['case']['display_name'],
                       variant_id=evaluation_obj['variant_specific'])
        store.delete_evaluation(evaluation_obj)
        return redirect(link)
    return dict(evaluation=evaluation_obj, institute=evaluation_obj['institute'],
                case=evaluation_obj['case'], variant=evaluation_obj['variant'],
                CRITERIA=ACMG_CRITERIA)

def evaluation ( evaluation_id ) : evaluation_obj = store . get_evaluation ( evaluation_id ) controllers . evaluation ( store , evaluation_obj ) if request . method == 'POST' : link = url_for ( '.variant' , institute_id = evaluation_obj [ 'institute' ] [ '_id' ] , case_name = evaluation_obj [ 'case' ] [ 'display_name' ] , variant_id = evaluation_obj [ 'variant_specific' ] ) store . delete_evaluation ( evaluation_obj ) return redirect ( link ) return dict ( evaluation = evaluation_obj , institute = evaluation_obj [ 'institute' ] , case = evaluation_obj [ 'case' ] , variant = evaluation_obj [ 'variant' ] , CRITERIA = ACMG_CRITERIA )


def acmg():
    """Calculate an ACMG classification from submitted criteria."""
    criteria = request.args.getlist('criterion')
    classification = get_acmg(criteria)
    return jsonify(dict(classification=classification))

def acmg ( ) : criteria = request . args . getlist ( 'criterion' ) classification = get_acmg ( criteria ) return jsonify ( dict ( classification = classification ) )

Calculate an ACMG classification from submitted criteria .


def upload_panel(institute_id, case_name):
    """Parse gene panel file and fill in HGNC symbols for filter."""
    file = form.symbol_file.data

    if file.filename == '':
        flash('No selected file', 'warning')
        return redirect(request.referrer)

    try:
        stream = io.StringIO(file.stream.read().decode('utf-8'), newline=None)
    except UnicodeDecodeError as error:
        flash("Only text files are supported!", 'warning')
        return redirect(request.referrer)

    category = request.args.get('category')

    if(category == 'sv'):
        form = SvFiltersForm(request.args)
    else:
        form = FiltersForm(request.args)

    hgnc_symbols = set(form.hgnc_symbols.data)
    new_hgnc_symbols = controllers.upload_panel(store, institute_id, case_name, stream)
    hgnc_symbols.update(new_hgnc_symbols)
    form.hgnc_symbols.data = ','.join(hgnc_symbols)
    # reset gene panels
    form.gene_panels.data = ''
    # HTTP redirect code 307 asks that the browser preserves the method of request (POST).
    if(category == 'sv'):
        return redirect(url_for('.sv_variants', institute_id=institute_id, case_name=case_name,
                            **form.data), code=307)
    else:
        return redirect(url_for('.variants', institute_id=institute_id, case_name=case_name,
                            **form.data), code=307)

def
upload_panel
(
institute_id
,
case_name
)
:
file
=
form
.
symbol_file
.
data
if
file
.
filename
==
''
:
flash
(
'No selected file'
,
'warning'
)
return
redirect
(
request
.
referrer
)
try
:
stream
=
io
.
StringIO
(
file
.
stream
.
read
(
)
.
decode
(
'utf-8'
)
,
newline
=
)
except
UnicodeDecodeError
as
error
:
flash
(
"Only text files are supported!"
,
'warning'
)
return
redirect
(
request
.
referrer
)
category
=
request
.
args
.
get
(
'category'
)
if
(
category
==
'sv'
)
:
form
=
SvFiltersForm
(
request
.
args
)
else
:
form
=
FiltersForm
(
request
.
args
)
hgnc_symbols
=
set
(
form
.
hgnc_symbols
.
data
)
new_hgnc_symbols
=
controllers
.
upload_panel
(
store
,
institute_id
,
case_name
,
stream
)
hgnc_symbols
.
update
(
new_hgnc_symbols
)
form
.
hgnc_symbols
.
data
=
','
.
join
(
hgnc_symbols
)
form
.
gene_panels
.
data
=
''
if
(
category
==
'sv'
)
:
return
redirect
(
url_for
(
'.sv_variants'
,
institute_id
=
institute_id
,
case_name
=
case_name
,
**
form
.
data
)
,
code
=
307
)
else
:
return
redirect
(
url_for
(
'.variants'
,
institute_id
=
institute_id
,
case_name
=
case_name
,
**
form
.
data
)
,
code
=
307
)

Parse gene panel file and fill in HGNC symbols for filter .


def download_verified():
    """Download all verified variants for user's cases"""
    user_obj = store.user(current_user.email)
    user_institutes = user_obj.get('institutes')
    temp_excel_dir = os.path.join(variants_bp.static_folder, 'verified_folder')
    os.makedirs(temp_excel_dir, exist_ok=True)

    written_files = controllers.verified_excel_file(store, user_institutes, temp_excel_dir)
    if written_files:
        today = datetime.datetime.now().strftime('%Y-%m-%d')
        # zip the files on the fly and serve the archive to the user
        data = io.BytesIO()
        with zipfile.ZipFile(data, mode='w') as z:
            for f_name in pathlib.Path(temp_excel_dir).iterdir():
                zipfile.ZipFile
                z.write(f_name, os.path.basename(f_name))
        data.seek(0)

        # remove temp folder with excel files in it
        shutil.rmtree(temp_excel_dir)

        return send_file(
            data,
            mimetype='application/zip',
            as_attachment=True,
            attachment_filename='_'.join(['scout', 'verified_variants', today])+'.zip'
        )
    else:
        flash("No verified variants could be exported for user's institutes", 'warning')
        return redirect(request.referrer)

Download all verified variants for user s cases


def genes_by_alias(hgnc_genes):
    """Return a dictionary with hgnc symbols as keys

    Value of the dictionaries are information about the hgnc ids for a symbol.
    If the symbol is primary for a gene then 'true_id' will exist.
    A list of hgnc ids that the symbol points to is in ids.

    Args:
        hgnc_genes(dict): a dictionary with hgnc_id as key and gene info as value

    Returns:
        alias_genes(dict):
            {
                'hgnc_symbol':{
                    'true_id': int,
                    'ids': list(int)
                    }
            }
    """
    alias_genes = {}
    for hgnc_id in hgnc_genes:
        gene = hgnc_genes[hgnc_id]
        # This is the primary symbol:
        hgnc_symbol = gene['hgnc_symbol']

        for alias in gene['previous_symbols']:
            true_id = None
            if alias == hgnc_symbol:
                true_id = hgnc_id
            if alias in alias_genes:
                alias_genes[alias.upper()]['ids'].add(hgnc_id)
                if true_id:
                    alias_genes[alias.upper()]['true_id'] = hgnc_id
            else:
                alias_genes[alias.upper()] = {
                    'true': true_id,
                    'ids': set([hgnc_id])
                }

    return alias_genes

def genes_by_alias ( hgnc_genes ) : alias_genes = { } for hgnc_id in hgnc_genes : gene = hgnc_genes [ hgnc_id ] hgnc_symbol = gene [ 'hgnc_symbol' ] for alias in gene [ 'previous_symbols' ] : true_id = None if alias == hgnc_symbol : true_id = hgnc_id if alias in alias_genes : alias_genes [ alias . upper ( ) ] [ 'ids' ] . add ( hgnc_id ) if true_id : alias_genes [ alias . upper ( ) ] [ 'true_id' ] = hgnc_id else : alias_genes [ alias . upper ( ) ] = { 'true' : true_id , 'ids' : set ( [ hgnc_id ] ) } return alias_genes


def add_incomplete_penetrance(genes, alias_genes, hpo_lines):
    """Add information of incomplete penetrance"""
    LOG.info("Add incomplete penetrance info")
    for hgnc_symbol in get_incomplete_penetrance_genes(hpo_lines):
        for hgnc_id in get_correct_ids(hgnc_symbol, alias_genes):
            genes[hgnc_id]['incomplete_penetrance'] = True

def
add_incomplete_penetrance
(
genes
,
alias_genes
,
hpo_lines
)
:
LOG
.
info
(
"Add incomplete penetrance info"
)
for
hgnc_symbol
in
get_incomplete_penetrance_genes
(
hpo_lines
)
:
for
hgnc_id
in
get_correct_ids
(
hgnc_symbol
,
alias_genes
)
:
genes
[
hgnc_id
]
[
'incomplete_penetrance'
]
=
True


def link_genes(ensembl_lines, hgnc_lines, exac_lines, mim2gene_lines,
               genemap_lines, hpo_lines):
    """Gather information from different sources and return a gene dict

    Extract information collected from a number of sources and combine them
    into a gene dict with HGNC symbols as keys.

    hgnc_id works as the primary symbol and it is from this source we gather
    as much information as possible (hgnc_complete_set.txt)

    Coordinates are gathered from ensemble and the entries are linked from hgnc
    to ensembl via ENSGID.

    From exac the gene intolerance scores are collected, genes are linked to hgnc
    via hgnc symbol. This is a unstable symbol since they often change.

        Args:
            ensembl_lines(iterable(str)): Strings with ensembl gene information
            hgnc_lines(iterable(str)): Strings with hgnc gene information
            exac_lines(iterable(str)): Strings with exac PLi score info
            mim2gene_lines(iterable(str))
            genemap_lines(iterable(str))
            hpo_lines(iterable(str)): Strings with hpo gene information

        Yields:
            gene(dict): A dictionary with gene information
    """
    genes = {}
    LOG.info("Linking genes")
    # HGNC genes are the main source, these define the gene dataset to use
    # Try to use as much information as possible from hgnc
    for hgnc_gene in parse_hgnc_genes(hgnc_lines):
        hgnc_id = hgnc_gene['hgnc_id']
        genes[hgnc_id] = hgnc_gene

    add_ensembl_info(genes, ensembl_lines)

    symbol_to_id = genes_by_alias(genes)

    add_exac_info(genes, symbol_to_id, exac_lines)

    add_omim_info(genes, symbol_to_id, genemap_lines, mim2gene_lines)

    add_incomplete_penetrance(genes, symbol_to_id, hpo_lines)

    return genes


def matchmaker_request(url, token, method, content_type=None, accept=None, data=None):
    """Send a request to MatchMaker and return its response

    Args:
        url(str): url to send request to
        token(str): MME server authorization token
        method(str): 'GET', 'POST' or 'DELETE'
        content_type(str): MME request Content-Type
        accept(str): accepted response
        data(dict): eventual data to send in request

    Returns:
        json_response(dict): server response
    """
    headers = Headers()
    headers = { 'X-Auth-Token': token}
    if content_type:
        headers['Content-Type'] = content_type
    if accept:
        headers['Accept'] = accept

    #sending data anyway so response will not be cached
    req_data = data or {'timestamp' : datetime.datetime.now().timestamp()}
    json_response = None
    try:
        LOG.info('Sending {} request to MME url {}. Data sent: {}'.format(
            method, url, req_data))
        resp = requests.request(
            method = method,
            url = url,
            headers = headers,
            data = json.dumps(req_data)
        )
        json_response = resp.json()
        LOG.info('MME server response was:{}'.format(json_response))

        if isinstance(json_response, str):
            json_response = {
                'message' : json_response,
            }
        elif isinstance(json_response, list): #asking for connected nodes
            return json_response
        json_response['status_code'] = resp.status_code
    except Exception as err:
        LOG.info('An error occurred while sending HTTP request to server ({})'.format(err))
        json_response = {
            'message' : str(err)
        }
    return json_response


def mme_nodes(mme_base_url, token):
    """Return the available MatchMaker nodes

    Args:
        mme_base_url(str): base URL of MME service
        token(str): MME server authorization token

    Returns:
        nodes(list): a list of node disctionaries
    """
    nodes = []
    if not mme_base_url or not token:
        return nodes
    url = ''.join([mme_base_url, '/nodes'])
    nodes = matchmaker_request(url=url, token=token, method='GET')
    LOG.info('Matchmaker has the following connected nodes:{}'.format(nodes))
    return nodes


def get_cytoband_coordinates(chrom, pos):
    """Get the cytoband coordinate for a position

    Args:
        chrom(str)
        pos(int)

    Returns:
        coordinate(str)
    """
    coordinate = ""

    if chrom in CYTOBANDS:
        for interval in CYTOBANDS[chrom][pos]:
            coordinate = interval.data

    return coordinate

def get_cytoband_coordinates ( chrom , pos ) : coordinate = "" if chrom in CYTOBANDS : for interval in CYTOBANDS [ chrom ] [ pos ] : coordinate = interval . data return coordinate


def get_sub_category(alt_len, ref_len, category, svtype=None):
    """Get the subcategory for a VCF variant

    The sub categories are:
        'snv', 'indel', 'del', 'ins', 'dup', 'bnd', 'inv'

    Args:
        alt_len(int)
        ref_len(int)
        category(str)
        svtype(str)

    Returns:
        subcategory(str)
    """
    subcategory = ''

    if category in ('snv', 'indel', 'cancer'):
        if ref_len == alt_len:
            subcategory = 'snv'
        else:
            subcategory = 'indel'
    elif category == 'sv':
        subcategory = svtype

    return subcategory

def get_sub_category ( alt_len , ref_len , category , svtype = None ) : subcategory = '' if category in ( 'snv' , 'indel' , 'cancer' ) : if ref_len == alt_len : subcategory = 'snv' else : subcategory = 'indel' elif category == 'sv' : subcategory = svtype return subcategory


def get_length(alt_len, ref_len, category, pos, end, svtype=None, svlen=None):
    """Return the length of a variant

    Args:
        alt_len(int)
        ref_len(int)
        category(str)
        svtype(str)
        svlen(int)
    """
    # -1 would indicate uncertain length
    length = -1
    if category in ('snv', 'indel', 'cancer'):
        if ref_len == alt_len:
            length = alt_len
        else:
            length = abs(ref_len - alt_len)

    elif category == 'sv':
        if svtype == 'bnd':
            length = int(10e10)
        else:
            if svlen:
                length = abs(int(svlen))
            # Some software does not give a length but they give END
            elif end:
                if end != pos:
                    length = end - pos
    return length

def get_length ( alt_len , ref_len , category , pos , end , svtype = None , svlen = None ) : length = - 1 if category in ( 'snv' , 'indel' , 'cancer' ) : if ref_len == alt_len : length = alt_len else : length = abs ( ref_len - alt_len ) elif category == 'sv' : if svtype == 'bnd' : length = int ( 10e10 ) else : if svlen : length = abs ( int ( svlen ) ) elif end : if end != pos : length = end - pos return length


def get_end(pos, alt, category, snvend=None, svend=None, svlen=None):
    """Return the end coordinate for a variant

    Args:
        pos(int)
        alt(str)
        category(str)
        snvend(str)
        svend(int)
        svlen(int)

    Returns:
        end(int)
    """
    # If nothing is known we set end to be same as start
    end = pos
    # If variant is snv or indel we know that cyvcf2 can handle end pos
    if category in ('snv', 'indel', 'cancer'):
        end = snvend

    # With SVs we have to be a bit more careful
    elif category == 'sv':
        # The END field from INFO usually works fine
        end = svend

        # For some cases like insertions the callers set end to same as pos
        # In those cases we can hope that there is a svlen...
        if svend == pos:
            if svlen:
                end = pos + svlen
        # If variant is 'BND' they have ':' in alt field
        # Information about other end is in the alt field
        if ':' in alt:
            match = BND_ALT_PATTERN.match(alt)
            if match:
                end = int(match.group(2))

    return end

def get_end ( pos , alt , category , snvend = None , svend = None , svlen = None ) : end = pos if category in ( 'snv' , 'indel' , 'cancer' ) : end = snvend elif category == 'sv' : end = svend if svend == pos : if svlen : end = pos + svlen if ':' in alt : match = BND_ALT_PATTERN . match ( alt ) if match : end = int ( match . group ( 2 ) ) return end


def parse_coordinates(variant, category):
    """Find out the coordinates for a variant

    Args:
        variant(cyvcf2.Variant)

    Returns:
        coordinates(dict): A dictionary on the form:
        {
            'position':<int>,
            'end':<int>,
            'end_chrom':<str>,
            'length':<int>,
            'sub_category':<str>,
            'mate_id':<str>,
            'cytoband_start':<str>,
            'cytoband_end':<str>,
        }
    """
    ref = variant.REF

    if variant.ALT:
        alt = variant.ALT[0]
    if category=="str" and not variant.ALT:
        alt = '.'

    chrom_match = CHR_PATTERN.match(variant.CHROM)
    chrom = chrom_match.group(2)

    svtype = variant.INFO.get('SVTYPE')
    if svtype:
        svtype = svtype.lower()

    mate_id = variant.INFO.get('MATEID')

    svlen = variant.INFO.get('SVLEN')

    svend = variant.INFO.get('END')
    snvend = int(variant.end)

    position = int(variant.POS)

    ref_len = len(ref)
    alt_len = len(alt)

    sub_category = get_sub_category(alt_len, ref_len, category, svtype)
    end = get_end(position, alt, category, snvend, svend)

    length = get_length(alt_len, ref_len, category, position, end, svtype, svlen)
    end_chrom = chrom

    if sub_category == 'bnd':
        if ':' in alt:
            match = BND_ALT_PATTERN.match(alt)
            # BND will often be translocations between different chromosomes
            if match:
                other_chrom = match.group(1)
                match = CHR_PATTERN.match(other_chrom)
                end_chrom = match.group(2)

    cytoband_start = get_cytoband_coordinates(chrom, position)
    cytoband_end = get_cytoband_coordinates(end_chrom, end)

    coordinates = {
        'position': position,
        'end': end,
        'length': length,
        'sub_category': sub_category,
        'mate_id': mate_id,
        'cytoband_start': cytoband_start,
        'cytoband_end': cytoband_end,
        'end_chrom': end_chrom,
    }

    return coordinates

def parse_coordinates ( variant , category ) : ref = variant . REF if variant . ALT : alt = variant . ALT [ 0 ] if category == "str" and not variant . ALT : alt = '.' chrom_match = CHR_PATTERN . match ( variant . CHROM ) chrom = chrom_match . group ( 2 ) svtype = variant . INFO . get ( 'SVTYPE' ) if svtype : svtype = svtype . lower ( ) mate_id = variant . INFO . get ( 'MATEID' ) svlen = variant . INFO . get ( 'SVLEN' ) svend = variant . INFO . get ( 'END' ) snvend = int ( variant . end ) position = int ( variant . POS ) ref_len = len ( ref ) alt_len = len ( alt ) sub_category = get_sub_category ( alt_len , ref_len , category , svtype ) end = get_end ( position , alt , category , snvend , svend ) length = get_length ( alt_len , ref_len , category , position , end , svtype , svlen ) end_chrom = chrom if sub_category == 'bnd' : if ':' in alt : match = BND_ALT_PATTERN . match ( alt ) if match : other_chrom = match . group ( 1 ) match = CHR_PATTERN . match ( other_chrom ) end_chrom = match . group ( 2 ) cytoband_start = get_cytoband_coordinates ( chrom , position ) cytoband_end = get_cytoband_coordinates ( end_chrom , end ) coordinates = { 'position' : position , 'end' : end , 'length' : length , 'sub_category' : sub_category , 'mate_id' : mate_id , 'cytoband_start' : cytoband_start , 'cytoband_end' : cytoband_end , 'end_chrom' : end_chrom , } return coordinates


def cli(infile):
    """docstring for cli"""
    lines = get_file_handle(infile)
    cytobands = parse_cytoband(lines)

    print("Check some coordinates:")

    print("checking chrom 1 pos 2")
    intervals = cytobands['1'][2]
    for interval in intervals:
        print(interval)
        print(interval.begin)
        print(interval.end)
        print(interval.data)
        # print(interval.__dict__)
    print(cytobands['1'][2])

    print("checking chrom 8 pos 101677777")
    print(cytobands['8'][101677777])

    print("checking chrom X pos 4200000 - 6000000")
    print(cytobands['X'][4200000:6000000])


def panels():
    """Show all panels for a case."""
    if request.method == 'POST':
        # update an existing panel
        csv_file = request.files['csv_file']
        content = csv_file.stream.read()
        lines = None
        try:
            if b'\n' in content:
                lines = content.decode('utf-8', 'ignore').split('\n')
            else:
                lines = content.decode('windows-1252').split('\r')
        except Exception as err:
            flash('Something went wrong while parsing the panel CSV file! ({})'.format(err), 'danger')
            return redirect(request.referrer)

        new_panel_name = request.form.get('new_panel_name')
        if new_panel_name: #create a new panel
            new_panel_id = controllers.new_panel(
                store=store,
                institute_id=request.form['institute'],
                panel_name=new_panel_name,
                display_name=request.form['display_name'],
                csv_lines=lines,
            )
            if new_panel_id is None:
                flash('Something went wrong and the panel list was not updated!','warning')
                return redirect(request.referrer)
            else:
                flash("new gene panel added, {}!".format(new_panel_name),'success')
            return redirect(url_for('panels.panel', panel_id=new_panel_id))

        else: # modify an existing panel
            update_option = request.form['modify_option']
            panel_obj= controllers.update_panel(
                                   store=store,
                                   panel_name=request.form['panel_name'],
                                   csv_lines=lines,
                                   option=update_option
             )
            if panel_obj is None:
                return abort(404, "gene panel not found: {}".format(request.form['panel_name']))
            else:
                return redirect(url_for('panels.panel', panel_id=panel_obj['_id']))

    institutes = list(user_institutes(store, current_user))
    panel_names = [name
                   for institute in institutes
                   for name in
                   store.gene_panels(institute_id=institute['_id']).distinct('panel_name')]

    panel_versions = {}
    for name in panel_names:
        panel_versions[name]=store.gene_panels(panel_id=name)

    panel_groups = []
    for institute_obj in institutes:
         institute_panels = store.latest_panels(institute_obj['_id'])
         panel_groups.append((institute_obj, institute_panels))

    return dict(panel_groups=panel_groups, panel_names=panel_names,
                panel_versions=panel_versions, institutes=institutes)


def panel_update(panel_id):
    """Update panel to a new version."""
    panel_obj = store.panel(panel_id)
    update_version = request.form.get('version', None)
    new_panel_id = store.apply_pending(panel_obj, update_version)
    return redirect(url_for('panels.panel', panel_id=new_panel_id))

def panel_update ( panel_id ) : panel_obj = store . panel ( panel_id ) update_version = request . form . get ( 'version' , None ) new_panel_id = store . apply_pending ( panel_obj , update_version ) return redirect ( url_for ( 'panels.panel' , panel_id = new_panel_id ) )


def panel_export(panel_id):
    """Export panel to PDF file"""
    panel_obj = store.panel(panel_id)
    data = controllers.panel_export(store, panel_obj)
    data['report_created_at'] = datetime.datetime.now().strftime("%Y-%m-%d")
    html_report = render_template('panels/panel_pdf_simple.html', **data)
    return render_pdf(HTML(string=html_report), download_filename=data['panel']['panel_name']+'_'+str(data['panel']['version'])+'_'+datetime.datetime.now().strftime("%Y-%m-%d")+'_scout.pdf')

def panel_export ( panel_id ) : panel_obj = store . panel ( panel_id ) data = controllers . panel_export ( store , panel_obj ) data [ 'report_created_at' ] = datetime . datetime . now ( ) . strftime ( "%Y-%m-%d" ) html_report = render_template ( 'panels/panel_pdf_simple.html' , ** data ) return render_pdf ( HTML ( string = html_report ) , download_filename = data [ 'panel' ] [ 'panel_name' ] + '_' + str ( data [ 'panel' ] [ 'version' ] ) + '_' + datetime . datetime . now ( ) . strftime ( "%Y-%m-%d" ) + '_scout.pdf' )


def gene_edit(panel_id, hgnc_id):
    """Edit additional information about a panel gene."""
    panel_obj = store.panel(panel_id)
    hgnc_gene = store.hgnc_gene(hgnc_id)
    panel_gene = controllers.existing_gene(store, panel_obj, hgnc_id)

    form = PanelGeneForm()
    transcript_choices = []
    for transcript in hgnc_gene['transcripts']:
        if transcript.get('refseq_id'):
            refseq_id = transcript.get('refseq_id')
            transcript_choices.append((refseq_id, refseq_id))
    form.disease_associated_transcripts.choices = transcript_choices

    if form.validate_on_submit():
        action = 'edit' if panel_gene else 'add'
        info_data = form.data.copy()
        if 'csrf_token' in info_data:
            del info_data['csrf_token']
        store.add_pending(panel_obj, hgnc_gene, action=action, info=info_data)
        return redirect(url_for('.panel', panel_id=panel_id))

    if panel_gene:
        for field_key in ['disease_associated_transcripts', 'reduced_penetrance',
                          'mosaicism', 'inheritance_models', 'database_entry_version', 'comment']:
            form_field = getattr(form, field_key)
            if not form_field.data:
                panel_value = panel_gene.get(field_key)
                if panel_value is not None:
                    form_field.process_data(panel_value)
    return dict(panel=panel_obj, form=form, gene=hgnc_gene, panel_gene=panel_gene)

def gene_edit ( panel_id , hgnc_id ) : panel_obj = store . panel ( panel_id ) hgnc_gene = store . hgnc_gene ( hgnc_id ) panel_gene = controllers . existing_gene ( store , panel_obj , hgnc_id ) form = PanelGeneForm ( ) transcript_choices = [ ] for transcript in hgnc_gene [ 'transcripts' ] : if transcript . get ( 'refseq_id' ) : refseq_id = transcript . get ( 'refseq_id' ) transcript_choices . append ( ( refseq_id , refseq_id ) ) form . disease_associated_transcripts . choices = transcript_choices if form . validate_on_submit ( ) : action = 'edit' if panel_gene else 'add' info_data = form . data . copy ( ) if 'csrf_token' in info_data : del info_data [ 'csrf_token' ] store . add_pending ( panel_obj , hgnc_gene , action = action , info = info_data ) return redirect ( url_for ( '.panel' , panel_id = panel_id ) ) if panel_gene : for field_key in [ 'disease_associated_transcripts' , 'reduced_penetrance' , 'mosaicism' , 'inheritance_models' , 'database_entry_version' , 'comment' ] : form_field = getattr ( form , field_key ) if not form_field . data : panel_value = panel_gene . get ( field_key ) if panel_value is not None : form_field . process_data ( panel_value ) return dict ( panel = panel_obj , form = form , gene = hgnc_gene , panel_gene = panel_gene )

Edit additional information about a panel gene .


def delivery_report(context, case_id, report_path,
                    update):
    """Add delivery report to an existing case."""

    adapter = context.obj['adapter']

    try:
        load_delivery_report(adapter=adapter, case_id=case_id,
                             report_path=report_path, update=update)
        LOG.info("saved report to case!")
    except Exception as e:
        LOG.error(e)
        context.abort()

def
delivery_report
(
context
,
case_id
,
report_path
,
update
)
:
adapter
=
context
.
obj
[
'adapter'
]
try
:
load_delivery_report
(
adapter
=
adapter
,
case_id
=
case_id
,
report_path
=
report_path
,
update
=
update
)
LOG
.
info
(
"saved report to case!"
)
except
Exception
as
e
:
LOG
.
error
(
e
)
context
.
abort
(
)

Add delivery report to an existing case .


def hpo_terms(store, query = None, limit = None):
    """Retrieves a list of HPO terms from scout database

    Args:
        store (obj): an adapter to the scout database
        query (str): the term to search in the database
        limit (str): the number of desired results

    Returns:
        hpo_phenotypes (dict): the complete list of HPO objects stored in scout

    """
    hpo_phenotypes = {}
    if limit:
        limit=int(limit)

    hpo_phenotypes['phenotypes'] = list(store.hpo_terms(text=query, limit=limit))
    return hpo_phenotypes

def hpo_terms ( store , query = None , limit = None ) : hpo_phenotypes = { } if limit : limit = int ( limit ) hpo_phenotypes [ 'phenotypes' ] = list ( store . hpo_terms ( text = query , limit = limit ) ) return hpo_phenotypes


def whitelist(context):
    """Show all objects in the whitelist collection"""
    LOG.info("Running scout view users")
    adapter = context.obj['adapter']

    ## TODO add a User interface to the adapter
    for whitelist_obj in adapter.whitelist_collection.find():
        click.echo(whitelist_obj['_id'])

def
whitelist
(
context
)
:
LOG
.
info
(
"Running scout view users"
)
adapter
=
context
.
obj
[
'adapter'
]
for
whitelist_obj
in
adapter
.
whitelist_collection
.
find
(
)
:
click
.
echo
(
whitelist_obj
[
'_id'
]
)


def build_phenotype(phenotype_id, adapter):
    """Build a small phenotype object

        Build a dictionary with phenotype_id and description

    Args:
        phenotype_id (str): The phenotype id
        adapter (scout.adapter.MongoAdapter)

    Returns:
        phenotype_obj (dict):

        dict(
            phenotype_id = str,
            feature = str, # description of phenotype
            )
    """
    phenotype_obj = {}
    phenotype = adapter.hpo_term(phenotype_id)
    if phenotype:
        phenotype_obj['phenotype_id'] = phenotype['hpo_id']
        phenotype_obj['feature'] = phenotype['description']
    return phenotype

def build_phenotype ( phenotype_id , adapter ) : phenotype_obj = { } phenotype = adapter . hpo_term ( phenotype_id ) if phenotype : phenotype_obj [ 'phenotype_id' ] = phenotype [ 'hpo_id' ] phenotype_obj [ 'feature' ] = phenotype [ 'description' ] return phenotype


def gene(store, hgnc_id):
    """Parse information about a gene."""
    res = {'builds': {'37': None, '38': None}, 'symbol': None, 'description': None, 'ensembl_id': None, 'record': None}

    for build in res['builds']:
        record = store.hgnc_gene(hgnc_id, build=build)
        if record:

            record['position'] = "{this[chromosome]}:{this[start]}-{this[end]}".format(this=record)
            res['aliases'] = record['aliases']
            res['hgnc_id'] = record['hgnc_id']
            res['description'] = record['description']
            res['builds'][build] = record
            res['symbol'] = record['hgnc_symbol']
            res['description'] = record['description']
            res['entrez_id'] = record.get('entrez_id')
            res['pli_score'] = record.get('pli_score')

            add_gene_links(record, int(build))

            res['omim_id'] = record.get('omim_id')
            res['incomplete_penetrance'] = record.get('incomplete_penetrance',False)
            res['inheritance_models'] = record.get('inheritance_models',[])
            for transcript in record['transcripts']:
                transcript['position'] = ("{this[chrom]}:{this[start]}-{this[end]}"
                                          .format(this=transcript))
                add_tx_links(transcript, build)

            for phenotype in record.get('phenotypes',[]):
                phenotype['omim_link'] = omim(phenotype.get('mim_number'))

            if not res['record']:
                res['record'] = record

    # If none of the genes where found
    if not any(res.values()):
        raise ValueError

    return res


def genes_to_json(store, query):
    """Fetch matching genes and convert to JSON."""
    gene_query = store.hgnc_genes(query, search=True)
    json_terms = [{'name': "{} | {} ({})".format(gene['hgnc_id'], gene['hgnc_symbol'],
                                                 ', '.join(gene['aliases'])),
                   'id': gene['hgnc_id']} for gene in gene_query]
    return json_terms

def
genes_to_json
(
store
,
query
)
:
gene_query
=
store
.
hgnc_genes
(
query
,
search
=
True
)
json_terms
=
[
{
'name'
:
"{} | {} ({})"
.
format
(
gene
[
'hgnc_id'
]
,
gene
[
'hgnc_symbol'
]
,
', '
.
join
(
gene
[
'aliases'
]
)
)
,
'id'
:
gene
[
'hgnc_id'
]
}
for
gene
in
gene_query
]
return
json_terms

Fetch matching genes and convert to JSON .


def index():
    """Display the Scout dashboard."""
    accessible_institutes = current_user.institutes
    if not 'admin' in current_user.roles:
        accessible_institutes = current_user.institutes
        if not accessible_institutes:
            flash('Not allowed to see information - please visit the dashboard later!')
            return redirect(url_for('cases.dahboard_general.html'))

    LOG.debug('User accessible institutes: {}'.format(accessible_institutes))
    institutes = [inst for inst in store.institutes(accessible_institutes)]

    # Insert a entry that displays all institutes in the beginning of the array
    institutes.insert(0, {'_id': None, 'display_name': 'All institutes'})

    institute_id = None
    slice_query = None
    panel=1
    if request.method=='POST':
        institute_id = request.form.get('institute')
        slice_query = request.form.get('query')
        panel=request.form.get('pane_id')

    elif request.method=='GET':
        institute_id = request.args.get('institute')
        slice_query = request.args.get('query')

    # User should be restricted to their own institute if:
    #1) Their default institute when the page is first loaded
    #2) if they ask for an institute that they don't belong to
    #3) if they want perform a query on all institutes

    if not institute_id:
        institute_id = accessible_institutes[0]
    elif (not current_user.is_admin) and (slice_query and institute_id == 'None'):
        institute_id = accessible_institutes[0]
    elif (not institute_id in accessible_institutes) and not (institute_id == 'None'):
        institute_id = accessible_institutes[0]

    LOG.info("Fetch all cases with institute: %s", institute_id)

    data = get_dashboard_info(store, institute_id, slice_query)
    data['institutes'] = institutes
    data['choice'] = institute_id
    total_cases = data['total_cases']

    LOG.info("Found %s cases", total_cases)
    if total_cases == 0:
        flash('no cases found for institute {} (with that query) - please visit the dashboard later!'.format(institute_id), 'info')
#        return redirect(url_for('cases.index'))

    return render_template(
        'dashboard/dashboard_general.html', institute=institute_id, query=slice_query, panel=panel, **data)


def transcripts(context, build, hgnc_id, json):
    """Show all transcripts in the database"""
    LOG.info("Running scout view transcripts")
    adapter = context.obj['adapter']

    if not json:
        click.echo("Chromosome\tstart\tend\ttranscript_id\thgnc_id\trefseq\tis_primary")
    for tx_obj in adapter.transcripts(build=build, hgnc_id=hgnc_id):
        if json:
            pp(tx_obj)
            continue
        click.echo("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}".format(
            tx_obj['chrom'],
            tx_obj['start'],
            tx_obj['end'],
            tx_obj['ensembl_transcript_id'],
            tx_obj['hgnc_id'],
            tx_obj.get('refseq_id', ''),
            tx_obj.get('is_primary') or '',
        ))


def day_display(year, month, all_month_events, day):
    """
    Returns the events that occur on the given day.
    Works by getting all occurrences for the month, then drilling
    down to only those occurring on the given day.
    """
    # Get a dict with all of the events for the month
    count = CountHandler(year, month, all_month_events).get_count()
    pks = [x[1] for x in count[day]]  # list of pks for events on given day
    # List enables sorting.
    # See the comments in EventMonthView in views.py for more info
    day_events = list(Event.objects.filter(pk__in=pks).order_by(
        'start_date').prefetch_related('cancellations'))
    day_events.sort(key=lambda x: x.l_start_date.hour)
    return day_events

def day_display ( year , month , all_month_events , day ) : count = CountHandler ( year , month , all_month_events ) . get_count ( ) pks = [ x [ 1 ] for x in count [ day ] ] day_events = list ( Event . objects . filter ( pk__in = pks ) . order_by ( 'start_date' ) . prefetch_related ( 'cancellations' ) ) day_events . sort ( key = lambda x : x . l_start_date . hour ) return day_events

Returns the events that occur on the given day . Works by getting all occurrences for the month then drilling down to only those occurring on the given day .


def sv_variants(store, institute_obj, case_obj, variants_query, page=1, per_page=50):
    """Pre-process list of SV variants."""
    skip_count = (per_page * max(page - 1, 0))
    more_variants = True if variants_query.count() > (skip_count + per_page) else False

    genome_build = case_obj.get('genome_build', '37')
    if genome_build not in ['37','38']:
        genome_build = '37'

    return {
        'variants': (parse_variant(store, institute_obj, case_obj, variant, genome_build=genome_build) for variant in
                     variants_query.skip(skip_count).limit(per_page)),
        'more_variants': more_variants,
    }

def sv_variants ( store , institute_obj , case_obj , variants_query , page = 1 , per_page = 50 ) : skip_count = ( per_page * max ( page - 1 , 0 ) ) more_variants = True if variants_query . count ( ) > ( skip_count + per_page ) else False genome_build = case_obj . get ( 'genome_build' , '37' ) if genome_build not in [ '37' , '38' ] : genome_build = '37' return { 'variants' : ( parse_variant ( store , institute_obj , case_obj , variant , genome_build = genome_build ) for variant in variants_query . skip ( skip_count ) . limit ( per_page ) ) , 'more_variants' : more_variants , }


def str_variants(store, institute_obj, case_obj, variants_query, page=1, per_page=50):
    """Pre-process list of STR variants."""
    # Nothing unique to STRs on this level. Inheritance?
    return variants(store, institute_obj, case_obj, variants_query, page, per_page)

def str_variants ( store , institute_obj , case_obj , variants_query , page = 1 , per_page = 50 ) : return variants ( store , institute_obj , case_obj , variants_query , page , per_page )


def str_variant(store, institute_id, case_name, variant_id):
    """Pre-process an STR variant entry for detail page.

    Adds information to display variant

    Args:
        store(scout.adapter.MongoAdapter)
        institute_id(str)
        case_name(str)
        variant_id(str)

    Returns:
        detailed_information(dict): {
            'institute': <institute_obj>,
            'case': <case_obj>,
            'variant': <variant_obj>,
            'overlapping_snvs': <overlapping_snvs>,
            'manual_rank_options': MANUAL_RANK_OPTIONS,
            'dismiss_variant_options': DISMISS_VARIANT_OPTIONS
        }
        """

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj =  store.variant(variant_id)

    # fill in information for pilup view
    variant_case(store, case_obj, variant_obj)

    variant_obj['callers'] = callers(variant_obj, category='str')

    # variant_obj['str_ru']
    # variant_obj['str_repid']
    # variant_obj['str_ref']

    variant_obj['comments'] = store.events(institute_obj, case=case_obj,
                                           variant_id=variant_obj['variant_id'], comments=True)

    return {
        'institute': institute_obj,
        'case': case_obj,
        'variant': variant_obj,
        'overlapping_snvs': overlapping_snvs,
        'manual_rank_options': MANUAL_RANK_OPTIONS,
        'dismiss_variant_options': DISMISS_VARIANT_OPTIONS
    }

def str_variant ( store , institute_id , case_name , variant_id ) : institute_obj , case_obj = institute_and_case ( store , institute_id , case_name ) variant_obj = store . variant ( variant_id ) variant_case ( store , case_obj , variant_obj ) variant_obj [ 'callers' ] = callers ( variant_obj , category = 'str' ) variant_obj [ 'comments' ] = store . events ( institute_obj , case = case_obj , variant_id = variant_obj [ 'variant_id' ] , comments = True ) return { 'institute' : institute_obj , 'case' : case_obj , 'variant' : variant_obj , 'overlapping_snvs' : overlapping_snvs , 'manual_rank_options' : MANUAL_RANK_OPTIONS , 'dismiss_variant_options' : DISMISS_VARIANT_OPTIONS }

Pre - process an STR variant entry for detail page .


def sv_variant(store, institute_id, case_name, variant_id=None, variant_obj=None, add_case=True,
               get_overlapping=True):
    """Pre-process an SV variant entry for detail page.

    Adds information to display variant

    Args:
        store(scout.adapter.MongoAdapter)
        institute_id(str)
        case_name(str)
        variant_id(str)
        variant_obj(dcit)
        add_case(bool): If information about case files should be added

    Returns:
        detailed_information(dict): {
            'institute': <institute_obj>,
            'case': <case_obj>,
            'variant': <variant_obj>,
            'overlapping_snvs': <overlapping_snvs>,
            'manual_rank_options': MANUAL_RANK_OPTIONS,
            'dismiss_variant_options': DISMISS_VARIANT_OPTIONS
        }
    """
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)

    if not variant_obj:
        variant_obj = store.variant(variant_id)

    if add_case:
        # fill in information for pilup view
        variant_case(store, case_obj, variant_obj)

    # frequencies
    variant_obj['frequencies'] = [
        ('1000G', variant_obj.get('thousand_genomes_frequency')),
        ('1000G (left)', variant_obj.get('thousand_genomes_frequency_left')),
        ('1000G (right)', variant_obj.get('thousand_genomes_frequency_right')),
        ('ClinGen CGH (benign)', variant_obj.get('clingen_cgh_benign')),
        ('ClinGen CGH (pathogenic)', variant_obj.get('clingen_cgh_pathogenic')),
        ('ClinGen NGI', variant_obj.get('clingen_ngi')),
        ('SweGen', variant_obj.get('swegen')),
        ('Decipher', variant_obj.get('decipher')),
    ]

    variant_obj['callers'] = callers(variant_obj, category='sv')

    overlapping_snvs = []
    if get_overlapping:
        overlapping_snvs = (parse_variant(store, institute_obj, case_obj, variant) for variant in
                            store.overlapping(variant_obj))

    # parse_gene function is not called for SVs, but a link to ensembl gene is required
    for gene_obj in variant_obj['genes']:
        if gene_obj.get('common'):
            ensembl_id = gene_obj['common']['ensembl_id']
            try:
                build = int(gene_obj['common'].get('build','37'))
            except Exception:
                build = 37
            gene_obj['ensembl_link'] = ensembl(ensembl_id, build=build)

    variant_obj['comments'] = store.events(institute_obj, case=case_obj,
                                           variant_id=variant_obj['variant_id'], comments=True)

    case_clinvars = store.case_to_clinVars(case_obj.get('display_name'))
    if variant_id in case_clinvars:
        variant_obj['clinvar_clinsig'] = case_clinvars.get(variant_id)['clinsig']

    if not 'end_chrom' in variant_obj:
        variant_obj['end_chrom'] = variant_obj['chromosome']

    return {
        'institute': institute_obj,
        'case': case_obj,
        'variant': variant_obj,
        'overlapping_snvs': overlapping_snvs,
        'manual_rank_options': MANUAL_RANK_OPTIONS,
        'dismiss_variant_options': DISMISS_VARIANT_OPTIONS
    }

def
sv_variant
(
store
,
institute_id
,
case_name
,
variant_id
=
,
variant_obj
=
,
add_case
=
True
,
get_overlapping
=
True
)
:
institute_obj
,
case_obj
=
institute_and_case
(
store
,
institute_id
,
case_name
)
if
not
variant_obj
:
variant_obj
=
store
.
variant
(
variant_id
)
if
add_case
:
variant_case
(
store
,
case_obj
,
variant_obj
)
variant_obj
[
'frequencies'
]
=
[
(
'1000G'
,
variant_obj
.
get
(
'thousand_genomes_frequency'
)
)
,
(
'1000G (left)'
,
variant_obj
.
get
(
'thousand_genomes_frequency_left'
)
)
,
(
'1000G (right)'
,
variant_obj
.
get
(
'thousand_genomes_frequency_right'
)
)
,
(
'ClinGen CGH (benign)'
,
variant_obj
.
get
(
'clingen_cgh_benign'
)
)
,
(
'ClinGen CGH (pathogenic)'
,
variant_obj
.
get
(
'clingen_cgh_pathogenic'
)
)
,
(
'ClinGen NGI'
,
variant_obj
.
get
(
'clingen_ngi'
)
)
,
(
'SweGen'
,
variant_obj
.
get
(
'swegen'
)
)
,
(
'Decipher'
,
variant_obj
.
get
(
'decipher'
)
)
,
]
variant_obj
[
'callers'
]
=
callers
(
variant_obj
,
category
=
'sv'
)
overlapping_snvs
=
[
]
if
get_overlapping
:
overlapping_snvs
=
(
parse_variant
(
store
,
institute_obj
,
case_obj
,
variant
)
for
variant
in
store
.
overlapping
(
variant_obj
)
)
for
gene_obj
in
variant_obj
[
'genes'
]
:
if
gene_obj
.
get
(
'common'
)
:
ensembl_id
=
gene_obj
[
'common'
]
[
'ensembl_id'
]
try
:
build
=
int
(
gene_obj
[
'common'
]
.
get
(
'build'
,
'37'
)
)
except
Exception
:
build
=
37
gene_obj
[
'ensembl_link'
]
=
ensembl
(
ensembl_id
,
build
=
build
)
variant_obj
[
'comments'
]
=
store
.
events
(
institute_obj
,
case
=
case_obj
,
variant_id
=
variant_obj
[
'variant_id'
]
,
comments
=
True
)
case_clinvars
=
store
.
case_to_clinVars
(
case_obj
.
get
(
'display_name'
)
)
if
variant_id
in
case_clinvars
:
variant_obj
[
'clinvar_clinsig'
]
=
case_clinvars
.
get
(
variant_id
)
[
'clinsig'
]
if
not
'end_chrom'
in
variant_obj
:
variant_obj
[
'end_chrom'
]
=
variant_obj
[
'chromosome'
]
return
{
'institute'
:
institute_obj
,
'case'
:
case_obj
,
'variant'
:
variant_obj
,
'overlapping_snvs'
:
overlapping_snvs
,
'manual_rank_options'
:
MANUAL_RANK_OPTIONS
,
'dismiss_variant_options'
:
DISMISS_VARIANT_OPTIONS
}

Pre - process an SV variant entry for detail page .


def parse_variant(store, institute_obj, case_obj, variant_obj, update=False, genome_build='37',
                  get_compounds = True):
    """Parse information about variants.

    - Adds information about compounds
    - Updates the information about compounds if necessary and 'update=True'

    Args:
        store(scout.adapter.MongoAdapter)
        institute_obj(scout.models.Institute)
        case_obj(scout.models.Case)
        variant_obj(scout.models.Variant)
        update(bool): If variant should be updated in database
        genome_build(str)

    """
    has_changed = False
    compounds = variant_obj.get('compounds', [])
    if compounds and get_compounds:
        # Check if we need to add compound information
        # If it is the first time the case is viewed we fill in some compound information
        if 'not_loaded' not in compounds[0]:
            new_compounds = store.update_variant_compounds(variant_obj)
            variant_obj['compounds'] = new_compounds
            has_changed = True

        # sort compounds on combined rank score
        variant_obj['compounds'] = sorted(variant_obj['compounds'],
                                          key=lambda compound: -compound['combined_score'])

    # Update the hgnc symbols if they are incorrect
    variant_genes = variant_obj.get('genes')
    if variant_genes is not None:
        for gene_obj in variant_genes:
            # If there is no hgnc id there is nothin we can do
            if not gene_obj['hgnc_id']:
                continue
            # Else we collect the gene object and check the id
            if gene_obj.get('hgnc_symbol') is None:
                hgnc_gene = store.hgnc_gene(gene_obj['hgnc_id'], build=genome_build)
                if not hgnc_gene:
                    continue
                has_changed = True
                gene_obj['hgnc_symbol'] = hgnc_gene['hgnc_symbol']

    # We update the variant if some information was missing from loading
    # Or if symbold in reference genes have changed
    if update and has_changed:
        variant_obj = store.update_variant(variant_obj)

    variant_obj['comments'] = store.events(institute_obj, case=case_obj,
                                           variant_id=variant_obj['variant_id'], comments=True)

    if variant_genes:
        variant_obj.update(get_predictions(variant_genes))
        if variant_obj.get('category') == 'cancer':
            variant_obj.update(get_variant_info(variant_genes))

    for compound_obj in compounds:
        compound_obj.update(get_predictions(compound_obj.get('genes', [])))

    if isinstance(variant_obj.get('acmg_classification'), int):
        acmg_code = ACMG_MAP[variant_obj['acmg_classification']]
        variant_obj['acmg_classification'] = ACMG_COMPLETE_MAP[acmg_code]

    # convert length for SV variants
    variant_length = variant_obj.get('length')
    variant_obj['length'] = {100000000000: 'inf', -1: 'n.d.'}.get(variant_length, variant_length)
    if not 'end_chrom' in variant_obj:
        variant_obj['end_chrom'] = variant_obj['chromosome']

    return variant_obj

def parse_variant ( store , institute_obj , case_obj , variant_obj , update = False , genome_build = '37' , get_compounds = True ) : has_changed = False compounds = variant_obj . get ( 'compounds' , [ ] ) if compounds and get_compounds : if 'not_loaded' not in compounds [ 0 ] : new_compounds = store . update_variant_compounds ( variant_obj ) variant_obj [ 'compounds' ] = new_compounds has_changed = True variant_obj [ 'compounds' ] = sorted ( variant_obj [ 'compounds' ] , key = lambda compound : - compound [ 'combined_score' ] ) variant_genes = variant_obj . get ( 'genes' ) if variant_genes is not None : for gene_obj in variant_genes : if not gene_obj [ 'hgnc_id' ] : continue if gene_obj . get ( 'hgnc_symbol' ) is None : hgnc_gene = store . hgnc_gene ( gene_obj [ 'hgnc_id' ] , build = genome_build ) if not hgnc_gene : continue has_changed = True gene_obj [ 'hgnc_symbol' ] = hgnc_gene [ 'hgnc_symbol' ] if update and has_changed : variant_obj = store . update_variant ( variant_obj ) variant_obj [ 'comments' ] = store . events ( institute_obj , case = case_obj , variant_id = variant_obj [ 'variant_id' ] , comments = True ) if variant_genes : variant_obj . update ( get_predictions ( variant_genes ) ) if variant_obj . get ( 'category' ) == 'cancer' : variant_obj . update ( get_variant_info ( variant_genes ) ) for compound_obj in compounds : compound_obj . update ( get_predictions ( compound_obj . get ( 'genes' , [ ] ) ) ) if isinstance ( variant_obj . get ( 'acmg_classification' ) , int ) : acmg_code = ACMG_MAP [ variant_obj [ 'acmg_classification' ] ] variant_obj [ 'acmg_classification' ] = ACMG_COMPLETE_MAP [ acmg_code ] variant_length = variant_obj . get ( 'length' ) variant_obj [ 'length' ] = { 100000000000 : 'inf' , - 1 : 'n.d.' } . get ( variant_length , variant_length ) if not 'end_chrom' in variant_obj : variant_obj [ 'end_chrom' ] = variant_obj [ 'chromosome' ] return variant_obj


def variants_export_header(case_obj):
    """Returns a header for the CSV file with the filtered variants to be exported.

        Args:
            case_obj(scout.models.Case)

        Returns:
            header: includes the fields defined in scout.constants.variants_export EXPORT_HEADER
                    + AD_reference, AD_alternate, GT_quality for each sample analysed for a case
    """
    header = []
    header = header + EXPORT_HEADER
    # Add fields specific for case samples
    for individual in case_obj['individuals']:
        display_name = str(individual['display_name'])
        header.append('AD_reference_'+display_name) # Add AD reference field for a sample
        header.append('AD_alternate_'+display_name) # Add AD alternate field for a sample
        header.append('GT_quality_'+display_name) # Add Genotype quality field for a sample
    return header

def variants_export_header ( case_obj ) : header = [ ] header = header + EXPORT_HEADER for individual in case_obj [ 'individuals' ] : display_name = str ( individual [ 'display_name' ] ) header . append ( 'AD_reference_' + display_name ) header . append ( 'AD_alternate_' + display_name ) header . append ( 'GT_quality_' + display_name ) return header

Returns a header for the CSV file with the filtered variants to be exported .


def get_variant_info(genes):
    """Get variant information"""
    data = {'canonical_transcripts': []}
    for gene_obj in genes:
        if not gene_obj.get('canonical_transcripts'):
            tx = gene_obj['transcripts'][0]
            tx_id = tx['transcript_id']
            exon = tx.get('exon', '-')
            c_seq = tx.get('coding_sequence_name', '-')
        else:
            tx_id = gene_obj['canonical_transcripts']
            exon = gene_obj.get('exon', '-')
            c_seq = gene_obj.get('hgvs_identifier', '-')

        if len(c_seq) > 20:
            c_seq = c_seq[:20] + '...'

        if len(genes) == 1:
            value = ':'.join([tx_id,exon,c_seq])
        else:
            gene_id = gene_obj.get('hgnc_symbol') or str(gene_obj['hgnc_id'])
            value = ':'.join([gene_id, tx_id,exon,c_seq])
        data['canonical_transcripts'].append(value)

    return data

def get_variant_info ( genes ) : data = { 'canonical_transcripts' : [ ] } for gene_obj in genes : if not gene_obj . get ( 'canonical_transcripts' ) : tx = gene_obj [ 'transcripts' ] [ 0 ] tx_id = tx [ 'transcript_id' ] exon = tx . get ( 'exon' , '-' ) c_seq = tx . get ( 'coding_sequence_name' , '-' ) else : tx_id = gene_obj [ 'canonical_transcripts' ] exon = gene_obj . get ( 'exon' , '-' ) c_seq = gene_obj . get ( 'hgvs_identifier' , '-' ) if len ( c_seq ) > 20 : c_seq = c_seq [ : 20 ] + '...' if len ( genes ) == 1 : value = ':' . join ( [ tx_id , exon , c_seq ] ) else : gene_id = gene_obj . get ( 'hgnc_symbol' ) or str ( gene_obj [ 'hgnc_id' ] ) value = ':' . join ( [ gene_id , tx_id , exon , c_seq ] ) data [ 'canonical_transcripts' ] . append ( value ) return data


def get_predictions(genes):
    """Get sift predictions from genes."""
    data = {
        'sift_predictions': [],
        'polyphen_predictions': [],
        'region_annotations': [],
        'functional_annotations': []
    }
    for gene_obj in genes:
        for pred_key in data:
            gene_key = pred_key[:-1]
            if len(genes) == 1:
                value = gene_obj.get(gene_key, '-')
            else:
                gene_id = gene_obj.get('hgnc_symbol') or str(gene_obj['hgnc_id'])
                value = ':'.join([gene_id, gene_obj.get(gene_key, '-')])
            data[pred_key].append(value)

    return data

def get_predictions ( genes ) : data = { 'sift_predictions' : [ ] , 'polyphen_predictions' : [ ] , 'region_annotations' : [ ] , 'functional_annotations' : [ ] } for gene_obj in genes : for pred_key in data : gene_key = pred_key [ : - 1 ] if len ( genes ) == 1 : value = gene_obj . get ( gene_key , '-' ) else : gene_id = gene_obj . get ( 'hgnc_symbol' ) or str ( gene_obj [ 'hgnc_id' ] ) value = ':' . join ( [ gene_id , gene_obj . get ( gene_key , '-' ) ] ) data [ pred_key ] . append ( value ) return data


def variant_case(store, case_obj, variant_obj):
    """Pre-process case for the variant view.

    Adds information about files from case obj to variant

    Args:
        store(scout.adapter.MongoAdapter)
        case_obj(scout.models.Case)
        variant_obj(scout.models.Variant)
    """
    case_obj['bam_files'] = []
    case_obj['mt_bams'] = []
    case_obj['bai_files'] = []
    case_obj['mt_bais'] = []
    case_obj['sample_names'] = []
    for individual in case_obj['individuals']:
        bam_path = individual.get('bam_file')
        mt_bam = individual.get('mt_bam')
        case_obj['sample_names'].append(individual.get('display_name'))
        if bam_path and os.path.exists(bam_path):
            case_obj['bam_files'].append(individual['bam_file'])
            case_obj['bai_files'].append(find_bai_file(individual['bam_file']))
        if mt_bam and os.path.exists(mt_bam):
            case_obj['mt_bams'].append(individual['mt_bam'])
            case_obj['mt_bais'].append(find_bai_file(individual['mt_bam']))

        else:
            LOG.debug("%s: no bam file found", individual['individual_id'])

    try:
        genes = variant_obj.get('genes', [])
        if len(genes) == 1:
            hgnc_gene_obj = store.hgnc_gene(variant_obj['genes'][0]['hgnc_id'])
            if hgnc_gene_obj:
                vcf_path = store.get_region_vcf(case_obj, gene_obj=hgnc_gene_obj)
                case_obj['region_vcf_file'] = vcf_path
            else:
                case_obj['region_vcf_file'] = None
        elif len(genes) > 1:
            chrom = variant_obj['genes'][0]['common']['chromosome']
            start = min(gene['common']['start'] for gene in variant_obj['genes'])
            end = max(gene['common']['end'] for gene in variant_obj['genes'])
            # Create a reduced VCF with variants in the region
            vcf_path = store.get_region_vcf(case_obj, chrom=chrom, start=start, end=end)
            case_obj['region_vcf_file'] = vcf_path
    except (SyntaxError, Exception):
        LOG.warning("skip VCF region for alignment view")

def
variant_case
(
store
,
case_obj
,
variant_obj
)
:
case_obj
[
'bam_files'
]
=
[
]
case_obj
[
'mt_bams'
]
=
[
]
case_obj
[
'bai_files'
]
=
[
]
case_obj
[
'mt_bais'
]
=
[
]
case_obj
[
'sample_names'
]
=
[
]
for
individual
in
case_obj
[
'individuals'
]
:
bam_path
=
individual
.
get
(
'bam_file'
)
mt_bam
=
individual
.
get
(
'mt_bam'
)
case_obj
[
'sample_names'
]
.
append
(
individual
.
get
(
'display_name'
)
)
if
bam_path
and
os
.
path
.
exists
(
bam_path
)
:
case_obj
[
'bam_files'
]
.
append
(
individual
[
'bam_file'
]
)
case_obj
[
'bai_files'
]
.
append
(
find_bai_file
(
individual
[
'bam_file'
]
)
)
if
mt_bam
and
os
.
path
.
exists
(
mt_bam
)
:
case_obj
[
'mt_bams'
]
.
append
(
individual
[
'mt_bam'
]
)
case_obj
[
'mt_bais'
]
.
append
(
find_bai_file
(
individual
[
'mt_bam'
]
)
)
else
:
LOG
.
debug
(
"%s: no bam file found"
,
individual
[
'individual_id'
]
)
try
:
genes
=
variant_obj
.
get
(
'genes'
,
[
]
)
if
len
(
genes
)
==
1
:
hgnc_gene_obj
=
store
.
hgnc_gene
(
variant_obj
[
'genes'
]
[
0
]
[
'hgnc_id'
]
)
if
hgnc_gene_obj
:
vcf_path
=
store
.
get_region_vcf
(
case_obj
,
gene_obj
=
hgnc_gene_obj
)
case_obj
[
'region_vcf_file'
]
=
vcf_path
else
:
case_obj
[
'region_vcf_file'
]
=
elif
len
(
genes
)
>
1
:
chrom
=
variant_obj
[
'genes'
]
[
0
]
[
'common'
]
[
'chromosome'
]
start
=
min
(
gene
[
'common'
]
[
'start'
]
for
gene
in
variant_obj
[
'genes'
]
)
end
=
max
(
gene
[
'common'
]
[
'end'
]
for
gene
in
variant_obj
[
'genes'
]
)
vcf_path
=
store
.
get_region_vcf
(
case_obj
,
chrom
=
chrom
,
start
=
start
,
end
=
end
)
case_obj
[
'region_vcf_file'
]
=
vcf_path
except
(
SyntaxError
,
Exception
)
:
LOG
.
warning
(
"skip VCF region for alignment view"
)

Pre - process case for the variant view .


def find_bai_file(bam_file):
    """Find out BAI file by extension given the BAM file."""
    bai_file = bam_file.replace('.bam', '.bai')
    if not os.path.exists(bai_file):
        # try the other convention
        bai_file = "{}.bai".format(bam_file)
    return bai_file

def find_bai_file ( bam_file ) : bai_file = bam_file . replace ( '.bam' , '.bai' ) if not os . path . exists ( bai_file ) : bai_file = "{}.bai" . format ( bam_file ) return bai_file

Find out BAI file by extension given the BAM file .


def observations(store, loqusdb, case_obj, variant_obj):
    """Query observations for a variant."""
    composite_id = ("{this[chromosome]}_{this[position]}_{this[reference]}_"
                    "{this[alternative]}".format(this=variant_obj))
    obs_data = loqusdb.get_variant({'_id': composite_id}) or {}
    obs_data['total'] = loqusdb.case_count()

    obs_data['cases'] = []
    institute_id = variant_obj['institute']
    for case_id in obs_data.get('families', []):
        if case_id != variant_obj['case_id'] and case_id.startswith(institute_id):
            other_variant = store.variant(variant_obj['variant_id'], case_id=case_id)
            other_case = store.case(case_id)
            obs_data['cases'].append(dict(case=other_case, variant=other_variant))

    return obs_data


def parse_gene(gene_obj, build=None):
    """Parse variant genes."""
    build = build or 37

    if gene_obj.get('common'):
        add_gene_links(gene_obj, build)
        refseq_transcripts = []
        for tx_obj in gene_obj['transcripts']:
            parse_transcript(gene_obj, tx_obj, build)

            # select refseq transcripts as "primary"
            if not tx_obj.get('refseq_id'):
                continue

            refseq_transcripts.append(tx_obj)

        gene_obj['primary_transcripts'] = (refseq_transcripts if refseq_transcripts else [])

def parse_gene ( gene_obj , build = None ) : build = build or 37 if gene_obj . get ( 'common' ) : add_gene_links ( gene_obj , build ) refseq_transcripts = [ ] for tx_obj in gene_obj [ 'transcripts' ] : parse_transcript ( gene_obj , tx_obj , build ) if not tx_obj . get ( 'refseq_id' ) : continue refseq_transcripts . append ( tx_obj ) gene_obj [ 'primary_transcripts' ] = ( refseq_transcripts if refseq_transcripts else [ ] )


def transcript_str(transcript_obj, gene_name=None):
    """Generate amino acid change as a string."""
    if transcript_obj.get('exon'):
        gene_part, part_count_raw = 'exon', transcript_obj['exon']
    elif transcript_obj.get('intron'):
        gene_part, part_count_raw = 'intron', transcript_obj['intron']
    else:
        # variant between genes
        gene_part, part_count_raw = 'intergenic', '0'

    part_count = part_count_raw.rpartition('/')[0]
    change_str = "{}:{}{}:{}:{}".format(
        transcript_obj.get('refseq_id', ''),
        gene_part,
        part_count,
        transcript_obj.get('coding_sequence_name', 'NA'),
        transcript_obj.get('protein_sequence_name', 'NA'),
    )
    if gene_name:
        change_str = "{}:".format(gene_name) + change_str
    return change_str

def transcript_str ( transcript_obj , gene_name = None ) : if transcript_obj . get ( 'exon' ) : gene_part , part_count_raw = 'exon' , transcript_obj [ 'exon' ] elif transcript_obj . get ( 'intron' ) : gene_part , part_count_raw = 'intron' , transcript_obj [ 'intron' ] else : gene_part , part_count_raw = 'intergenic' , '0' part_count = part_count_raw . rpartition ( '/' ) [ 0 ] change_str = "{}:{}{}:{}:{}" . format ( transcript_obj . get ( 'refseq_id' , '' ) , gene_part , part_count , transcript_obj . get ( 'coding_sequence_name' , 'NA' ) , transcript_obj . get ( 'protein_sequence_name' , 'NA' ) , ) if gene_name : change_str = "{}:" . format ( gene_name ) + change_str return change_str

Generate amino acid change as a string .


def end_position(variant_obj):
    """Calculate end position for a variant."""
    alt_bases = len(variant_obj['alternative'])
    num_bases = max(len(variant_obj['reference']), alt_bases)
    return variant_obj['position'] + (num_bases - 1)

def end_position ( variant_obj ) : alt_bases = len ( variant_obj [ 'alternative' ] ) num_bases = max ( len ( variant_obj [ 'reference' ] ) , alt_bases ) return variant_obj [ 'position' ] + ( num_bases - 1 )


def frequency(variant_obj):
    """Returns a judgement on the overall frequency of the variant.

    Combines multiple metrics into a single call.
    """
    most_common_frequency = max(variant_obj.get('thousand_genomes_frequency') or 0,
                                variant_obj.get('exac_frequency') or 0)
    if most_common_frequency > .05:
        return 'common'
    elif most_common_frequency > .01:
        return 'uncommon'
    else:
        return 'rare'

def frequency ( variant_obj ) : most_common_frequency = max ( variant_obj . get ( 'thousand_genomes_frequency' ) or 0 , variant_obj . get ( 'exac_frequency' ) or 0 ) if most_common_frequency > .05 : return 'common' elif most_common_frequency > .01 : return 'uncommon' else : return 'rare'

Returns a judgement on the overall frequency of the variant .


def clinsig_human(variant_obj):
    """Convert to human readable version of CLINSIG evaluation."""
    for clinsig_obj in variant_obj['clnsig']:
        # The clinsig objects allways have a accession
        if isinstance(clinsig_obj['accession'], int):
            # New version
            link = "https://www.ncbi.nlm.nih.gov/clinvar/variation/{}"
        else:
            # Old version
            link = "https://www.ncbi.nlm.nih.gov/clinvar/{}"

        human_str = 'not provided'
        if clinsig_obj.get('value'):
            try:
                # Old version
                int(clinsig_obj['value'])
                human_str = CLINSIG_MAP.get(clinsig_obj['value'], 'not provided')
            except ValueError:
                # New version
                human_str = clinsig_obj['value']

        clinsig_obj['human'] = human_str
        clinsig_obj['link'] = link.format(clinsig_obj['accession'])

        yield clinsig_obj

Convert to human readable version of CLINSIG evaluation .


def thousandg_link(variant_obj, build=None):
    """Compose link to 1000G page for detailed information."""
    dbsnp_id = variant_obj.get('dbsnp_id')
    build = build or 37

    if not dbsnp_id:
        return None

    if build == 37:
        url_template = ("http://grch37.ensembl.org/Homo_sapiens/Variation/Explore"
                        "?v={};vdb=variation")
    else:
        url_template = ("http://www.ensembl.org/Homo_sapiens/Variation/Explore"
                        "?v={};vdb=variation")

    return url_template.format(dbsnp_id)

Compose link to 1000G page for detailed information .


def cosmic_link(variant_obj):
    """Compose link to COSMIC Database.

    Args:
        variant_obj(scout.models.Variant)

    Returns:
        url_template(str): Link to COSMIIC database if cosmic id is present
    """

    cosmic_ids = variant_obj.get('cosmic_ids')

    if not cosmic_ids:
        return None
    else:
        cosmic_id = cosmic_ids[0]
        url_template = ("https://cancer.sanger.ac.uk/cosmic/mutation/overview?id={}")

    return url_template.format(cosmic_id)


def beacon_link(variant_obj, build=None):
    """Compose link to Beacon Network."""
    build = build or 37
    url_template = ("https://beacon-network.org/#/search?pos={this[position]}&"
                    "chrom={this[chromosome]}&allele={this[alternative]}&"
                    "ref={this[reference]}&rs=GRCh37")
    # beacon does not support build 38 at the moment
    # if build == '38':
    #     url_template = ("https://beacon-network.org/#/search?pos={this[position]}&"
    #                     "chrom={this[chromosome]}&allele={this[alternative]}&"
    #                     "ref={this[reference]}&rs=GRCh38")

    return url_template.format(this=variant_obj)


def ucsc_link(variant_obj, build=None):
    """Compose link to UCSC."""
    build = build or 37
    url_template = ("http://genome.ucsc.edu/cgi-bin/hgTracks?db=hg19&"
                        "position=chr{this[chromosome]}:{this[position]}"
                        "-{this[position]}&dgv=pack&knownGene=pack&omimGene=pack")
    if build == 38:
        url_template = ("http://genome.ucsc.edu/cgi-bin/hgTracks?db=hg20&"
                        "position=chr{this[chromosome]}:{this[position]}"
                        "-{this[position]}&dgv=pack&knownGene=pack&omimGene=pack")

    return url_template.format(this=variant_obj)


def spidex_human(variant_obj):
    """Translate SPIDEX annotation to human readable string."""
    if variant_obj.get('spidex') is None:
        return 'not_reported'
    elif abs(variant_obj['spidex']) < SPIDEX_HUMAN['low']['pos'][1]:
        return 'low'
    elif abs(variant_obj['spidex']) < SPIDEX_HUMAN['medium']['pos'][1]:
        return 'medium'
    else:
        return 'high'

def spidex_human ( variant_obj ) : if variant_obj . get ( 'spidex' ) is None : return 'not_reported' elif abs ( variant_obj [ 'spidex' ] ) < SPIDEX_HUMAN [ 'low' ] [ 'pos' ] [ 1 ] : return 'low' elif abs ( variant_obj [ 'spidex' ] ) < SPIDEX_HUMAN [ 'medium' ] [ 'pos' ] [ 1 ] : return 'medium' else : return 'high'

Translate SPIDEX annotation to human readable string .


def expected_inheritance(variant_obj):
    """Gather information from common gene information."""
    manual_models = set()
    for gene in variant_obj.get('genes', []):
        manual_models.update(gene.get('manual_inheritance', []))
    return list(manual_models)

def expected_inheritance ( variant_obj ) : manual_models = set ( ) for gene in variant_obj . get ( 'genes' , [ ] ) : manual_models . update ( gene . get ( 'manual_inheritance' , [ ] ) ) return list ( manual_models )

Gather information from common gene information .


def callers(variant_obj, category='snv'):
    """Return info about callers."""
    calls = set()
    for caller in CALLERS[category]:
        if variant_obj.get(caller['id']):
            calls.add((caller['name'], variant_obj[caller['id']]))

    return list(calls)

def callers ( variant_obj , category = 'snv' ) : calls = set ( ) for caller in CALLERS [ category ] : if variant_obj . get ( caller [ 'id' ] ) : calls . add ( ( caller [ 'name' ] , variant_obj [ caller [ 'id' ] ] ) ) return list ( calls )


def cancer_variants(store, request_args, institute_id, case_name):
    """Fetch data related to cancer variants for a case."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    form = CancerFiltersForm(request_args)
    variants_query = store.variants(case_obj['_id'], category='cancer', query=form.data).limit(50)
    data = dict(
        institute=institute_obj,
        case=case_obj,
        variants=(parse_variant(store, institute_obj, case_obj, variant, update=True) for
                  variant in variants_query),
        form=form,
        variant_type=request_args.get('variant_type', 'clinical'),
    )
    return data

def cancer_variants ( store , request_args , institute_id , case_name ) : institute_obj , case_obj = institute_and_case ( store , institute_id , case_name ) form = CancerFiltersForm ( request_args ) variants_query = store . variants ( case_obj [ '_id' ] , category = 'cancer' , query = form . data ) . limit ( 50 ) data = dict ( institute = institute_obj , case = case_obj , variants = ( parse_variant ( store , institute_obj , case_obj , variant , update = True ) for variant in variants_query ) , form = form , variant_type = request_args . get ( 'variant_type' , 'clinical' ) , ) return data

Fetch data related to cancer variants for a case .


def clinvar_export(store, institute_id, case_name, variant_id):
    """Gather the required data for creating the clinvar submission form

        Args:
            store(scout.adapter.MongoAdapter)
            institute_id(str): Institute ID
            case_name(str): case ID
            variant_id(str): variant._id

        Returns:
            a dictionary with all the required data (case and variant level) to pre-fill in fields in the clinvar submission form

    """

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    pinned = [store.variant(variant_id) or variant_id for variant_id in
                  case_obj.get('suspects', [])]
    variant_obj = store.variant(variant_id)
    return dict(
        today = str(date.today()),
        institute=institute_obj,
        case=case_obj,
        variant=variant_obj,
        pinned_vars=pinned
    )

def clinvar_export ( store , institute_id , case_name , variant_id ) : institute_obj , case_obj = institute_and_case ( store , institute_id , case_name ) pinned = [ store . variant ( variant_id ) or variant_id for variant_id in case_obj . get ( 'suspects' , [ ] ) ] variant_obj = store . variant ( variant_id ) return dict ( today = str ( date . today ( ) ) , institute = institute_obj , case = case_obj , variant = variant_obj , pinned_vars = pinned )


def get_clinvar_submission(store, institute_id, case_name, variant_id, submission_id):
    """Collects all variants from the clinvar submission collection with a specific submission_id

        Args:
            store(scout.adapter.MongoAdapter)
            institute_id(str): Institute ID
            case_name(str): case ID
            variant_id(str): variant._id
            submission_id(str): clinvar submission id, i.e. SUB76578

        Returns:
            A dictionary with all the data to display the clinvar_update.html template page
    """

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    pinned = [store.variant(variant_id) or variant_id for variant_id in
                  case_obj.get('suspects', [])]
    variant_obj = store.variant(variant_id)
    clinvar_submission_objs = store.clinvars(submission_id=submission_id)
    return dict(
        today = str(date.today()),
        institute=institute_obj,
        case=case_obj,
        variant=variant_obj,
        pinned_vars=pinned,
        clinvars = clinvar_submission_objs
    )

def get_clinvar_submission ( store , institute_id , case_name , variant_id , submission_id ) : institute_obj , case_obj = institute_and_case ( store , institute_id , case_name ) pinned = [ store . variant ( variant_id ) or variant_id for variant_id in case_obj . get ( 'suspects' , [ ] ) ] variant_obj = store . variant ( variant_id ) clinvar_submission_objs = store . clinvars ( submission_id = submission_id ) return dict ( today = str ( date . today ( ) ) , institute = institute_obj , case = case_obj , variant = variant_obj , pinned_vars = pinned , clinvars = clinvar_submission_objs )


def variant_acmg(store, institute_id, case_name, variant_id):
    """Collect data relevant for rendering ACMG classification form."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj = store.variant(variant_id)
    return dict(institute=institute_obj, case=case_obj, variant=variant_obj,
                CRITERIA=ACMG_CRITERIA, ACMG_OPTIONS=ACMG_OPTIONS)

def variant_acmg ( store , institute_id , case_name , variant_id ) : institute_obj , case_obj = institute_and_case ( store , institute_id , case_name ) variant_obj = store . variant ( variant_id ) return dict ( institute = institute_obj , case = case_obj , variant = variant_obj , CRITERIA = ACMG_CRITERIA , ACMG_OPTIONS = ACMG_OPTIONS )

Collect data relevant for rendering ACMG classification form .


def variant_acmg_post(store, institute_id, case_name, variant_id, user_email, criteria):
    """Calculate an ACMG classification based on a list of criteria."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj = store.variant(variant_id)
    user_obj = store.user(user_email)
    variant_link = url_for('variants.variant', institute_id=institute_id,
                           case_name=case_name, variant_id=variant_id)
    classification = store.submit_evaluation(
        institute_obj=institute_obj,
        case_obj=case_obj,
        variant_obj=variant_obj,
        user_obj=user_obj,
        link=variant_link,
        criteria=criteria,
    )
    return classification

Calculate an ACMG classification based on a list of criteria .


def evaluation(store, evaluation_obj):
    """Fetch and fill-in evaluation object."""
    evaluation_obj['institute'] = store.institute(evaluation_obj['institute_id'])
    evaluation_obj['case'] = store.case(evaluation_obj['case_id'])
    evaluation_obj['variant'] = store.variant(evaluation_obj['variant_specific'])
    evaluation_obj['criteria'] = {criterion['term']: criterion for criterion in
                                  evaluation_obj['criteria']}
    evaluation_obj['classification'] = ACMG_COMPLETE_MAP[evaluation_obj['classification']]
    return evaluation_obj

def evaluation ( store , evaluation_obj ) : evaluation_obj [ 'institute' ] = store . institute ( evaluation_obj [ 'institute_id' ] ) evaluation_obj [ 'case' ] = store . case ( evaluation_obj [ 'case_id' ] ) evaluation_obj [ 'variant' ] = store . variant ( evaluation_obj [ 'variant_specific' ] ) evaluation_obj [ 'criteria' ] = { criterion [ 'term' ] : criterion for criterion in evaluation_obj [ 'criteria' ] } evaluation_obj [ 'classification' ] = ACMG_COMPLETE_MAP [ evaluation_obj [ 'classification' ] ] return evaluation_obj


def upload_panel(store, institute_id, case_name, stream):
    """Parse out HGNC symbols from a stream."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    raw_symbols = [line.strip().split('\t')[0] for line in stream if
                   line and not line.startswith('#')]
    # check if supplied gene symbols exist
    hgnc_symbols = []
    for raw_symbol in raw_symbols:
        if store.hgnc_genes(raw_symbol).count() == 0:
            flash("HGNC symbol not found: {}".format(raw_symbol), 'warning')
        else:
            hgnc_symbols.append(raw_symbol)
    return hgnc_symbols

def
upload_panel
(
store
,
institute_id
,
case_name
,
stream
)
:
institute_obj
,
case_obj
=
institute_and_case
(
store
,
institute_id
,
case_name
)
raw_symbols
=
[
line
.
strip
(
)
.
split
(
'\t'
)
[
0
]
for
line
in
stream
if
line
and
not
line
.
startswith
(
'#'
)
]
hgnc_symbols
=
[
]
for
raw_symbol
in
raw_symbols
:
if
store
.
hgnc_genes
(
raw_symbol
)
.
count
(
)
==
0
:
flash
(
"HGNC symbol not found: {}"
.
format
(
raw_symbol
)
,
'warning'
)
else
:
hgnc_symbols
.
append
(
raw_symbol
)
return
hgnc_symbols


def verified_excel_file(store, institute_list, temp_excel_dir):
    """Collect all verified variants in a list on institutes and save them to file

    Args:
        store(adapter.MongoAdapter)
        institute_list(list): a list of institute ids
        temp_excel_dir(os.Path): folder where the temp excel files are written to

    Returns:
        written_files(int): the number of files written to temp_excel_dir
    """
    document_lines = []
    written_files = 0
    today = datetime.datetime.now().strftime('%Y-%m-%d')
    LOG.info('Creating verified variant document..')

    for cust in institute_list:
        verif_vars = store.verified(institute_id=cust)
        LOG.info('Found {} verified variants for customer {}'.format(len(verif_vars), cust))

        if not verif_vars:
            continue
        unique_callers = set()
        for var_type, var_callers in CALLERS.items():
            for caller in var_callers:
                unique_callers.add(caller.get('id'))
        cust_verified = export_verified_variants(verif_vars, unique_callers)

        document_name = '.'.join([cust, '_verified_variants', today]) + '.xlsx'
        workbook = Workbook(os.path.join(temp_excel_dir,document_name))
        Report_Sheet = workbook.add_worksheet()

        # Write the column header
        row = 0
        for col,field in enumerate(VERIFIED_VARIANTS_HEADER + list(unique_callers)):
            Report_Sheet.write(row,col,field)

        # Write variant lines, after header (start at line 1)
        for row, line in enumerate(cust_verified,1): # each line becomes a row in the document
            for col, field in enumerate(line): # each field in line becomes a cell
                Report_Sheet.write(row,col,field)
        workbook.close()

        if os.path.exists(os.path.join(temp_excel_dir,document_name)):
            written_files += 1

    return written_files


def export_genes(adapter, build='37'):
    """Export all genes from the database"""
    LOG.info("Exporting all genes to .bed format")

    for gene_obj in adapter.all_genes(build=build):
        yield gene_obj

def
export_genes
(
adapter
,
build
=
'37'
)
:
LOG
.
info
(
"Exporting all genes to .bed format"
)
for
gene_obj
in
adapter
.
all_genes
(
build
=
build
)
:
yield
gene_obj