def parse_genes(gene_lines):
    """Parse a file with genes and return the hgnc ids

    Args:
        gene_lines(iterable(str)): Stream with genes

    Returns:
        genes(list(dict)): Dictionaries with relevant gene info
    """
    genes = []
    header = []
    hgnc_identifiers = set()
    delimiter = '\t'
    # This can be '\t' or ';'
    delimiters = ['\t', ' ', ';']

    # There are files that have '#' to indicate headers
    # There are some files that start with a header line without
    # any special symbol
    for i,line in enumerate(gene_lines):
        line = line.rstrip()
        if not len(line) > 0:
            continue
        if line.startswith('#'):
            if not line.startswith('##'):
                # We need to try delimiters
                # We prefer ';' or '\t' byt should accept ' '
                line_length = 0
                delimiter = None
                for alt in delimiters:
                    head_line = line.split(alt)
                    if len(head_line) > line_length:
                        line_length = len(head_line)
                        delimiter = alt

                header = [word.lower() for word in line[1:].split(delimiter)]
        else:
            # If no header symbol(#) assume first line is header
            if i == 0:
                line_length = 0
                for alt in delimiters:
                    head_line = line.split(alt)
                    if len(head_line) > line_length:
                        line_length = len(head_line)
                        delimiter = alt

                if ('hgnc' in line or 'HGNC' in line):
                    header = [word.lower() for word in line.split(delimiter)]
                    continue
                # If first line is not a header try to sniff what the first
                # columns holds
                if line.split(delimiter)[0].isdigit():
                    header = ['hgnc_id']
                else:
                    header = ['hgnc_symbol']

            splitted_line = line.split(delimiter)
            gene_info = dict(zip(header, splitted_line))

            # There are cases when excel exports empty lines that looks like
            # ;;;;;;;. This is a exception to handle these
            info_found = False
            for key in gene_info:
                if gene_info[key]:
                    info_found = True
                    break
            # If no info was found we skip that line
            if not info_found:
                continue

            try:
                gene = parse_gene(gene_info)
            except Exception as e:
                LOG.warning(e)
                raise SyntaxError("Line {0} is malformed".format(i + 1))

            identifier = gene.pop('identifier')

            if not identifier in hgnc_identifiers:
                hgnc_identifiers.add(identifier)
                genes.append(gene)

    return genes

def
parse_genes
(
gene_lines
)
:
genes
=
[
]
header
=
[
]
hgnc_identifiers
=
set
(
)
delimiter
=
'\t'
delimiters
=
[
'\t'
,
' '
,
';'
]
for
i
,
line
in
enumerate
(
gene_lines
)
:
line
=
line
.
rstrip
(
)
if
not
len
(
line
)
>
0
:
continue
if
line
.
startswith
(
'#'
)
:
if
not
line
.
startswith
(
'##'
)
:
line_length
=
0
delimiter
=
for
alt
in
delimiters
:
head_line
=
line
.
split
(
alt
)
if
len
(
head_line
)
>
line_length
:
line_length
=
len
(
head_line
)
delimiter
=
alt
header
=
[
word
.
lower
(
)
for
word
in
line
[
1
:
]
.
split
(
delimiter
)
]
else
:
if
i
==
0
:
line_length
=
0
for
alt
in
delimiters
:
head_line
=
line
.
split
(
alt
)
if
len
(
head_line
)
>
line_length
:
line_length
=
len
(
head_line
)
delimiter
=
alt
if
(
'hgnc'
in
line
or
'HGNC'
in
line
)
:
header
=
[
word
.
lower
(
)
for
word
in
line
.
split
(
delimiter
)
]
continue
if
line
.
split
(
delimiter
)
[
0
]
.
isdigit
(
)
:
header
=
[
'hgnc_id'
]
else
:
header
=
[
'hgnc_symbol'
]
splitted_line
=
line
.
split
(
delimiter
)
gene_info
=
dict
(
zip
(
header
,
splitted_line
)
)
info_found
=
False
for
key
in
gene_info
:
if
gene_info
[
key
]
:
info_found
=
True
break
if
not
info_found
:
continue
try
:
gene
=
parse_gene
(
gene_info
)
except
Exception
as
e
:
LOG
.
warning
(
e
)
raise
SyntaxError
(
"Line {0} is malformed"
.
format
(
i
+
1
)
)
identifier
=
gene
.
pop
(
'identifier'
)
if
not
identifier
in
hgnc_identifiers
:
hgnc_identifiers
.
add
(
identifier
)
genes
.
append
(
gene
)
return
genes


def parse_gene_panel(path, institute='cust000', panel_id='test', panel_type='clinical', date=datetime.now(),
                     version=1.0, display_name=None, genes = None):
    """Parse the panel info and return a gene panel

        Args:
            path(str): Path to panel file
            institute(str): Name of institute that owns the panel
            panel_id(str): Panel id
            date(datetime.datetime): Date of creation
            version(float)
            full_name(str): Option to have a long name

        Returns:
            gene_panel(dict)
    """
    LOG.info("Parsing gene panel %s", panel_id)
    gene_panel = {}

    gene_panel['path'] = path
    gene_panel['type'] = panel_type
    gene_panel['date'] = date
    gene_panel['panel_id'] = panel_id
    gene_panel['institute'] = institute
    version = version or 1.0
    gene_panel['version'] = float(version)
    gene_panel['display_name'] = display_name or panel_id

    if not path:
        panel_handle = genes
    else:
        panel_handle = get_file_handle(gene_panel['path'])
    gene_panel['genes'] = parse_genes(gene_lines=panel_handle)

    return gene_panel

def
parse_gene_panel
(
path
,
institute
=
'cust000'
,
panel_id
=
'test'
,
panel_type
=
'clinical'
,
date
=
datetime
.
now
(
)
,
version
=
1.0
,
display_name
=
,
genes
=
)
:
LOG
.
info
(
"Parsing gene panel %s"
,
panel_id
)
gene_panel
=
{
}
gene_panel
[
'path'
]
=
path
gene_panel
[
'type'
]
=
panel_type
gene_panel
[
'date'
]
=
date
gene_panel
[
'panel_id'
]
=
panel_id
gene_panel
[
'institute'
]
=
institute
version
=
version
or
1.0
gene_panel
[
'version'
]
=
float
(
version
)
gene_panel
[
'display_name'
]
=
display_name
or
panel_id
if
not
path
:
panel_handle
=
genes
else
:
panel_handle
=
get_file_handle
(
gene_panel
[
'path'
]
)
gene_panel
[
'genes'
]
=
parse_genes
(
gene_lines
=
panel_handle
)
return
gene_panel


def diseases(context):
    """Show all diseases in the database"""
    LOG.info("Running scout view diseases")
    adapter = context.obj['adapter']

    disease_objs = adapter.disease_terms()

    nr_diseases = disease_objs.count()
    if nr_diseases == 0:
        click.echo("No diseases found")
    else:
        click.echo("Disease")
        for disease_obj in adapter.disease_terms():
            click.echo("{0}".format(disease_obj['_id']))
        LOG.info("{0} diseases found".format(nr_diseases))

def
diseases
(
context
)
:
LOG
.
info
(
"Running scout view diseases"
)
adapter
=
context
.
obj
[
'adapter'
]
disease_objs
=
adapter
.
disease_terms
(
)
nr_diseases
=
disease_objs
.
count
(
)
if
nr_diseases
==
0
:
click
.
echo
(
"No diseases found"
)
else
:
click
.
echo
(
"Disease"
)
for
disease_obj
in
adapter
.
disease_terms
(
)
:
click
.
echo
(
"{0}"
.
format
(
disease_obj
[
'_id'
]
)
)
LOG
.
info
(
"{0} diseases found"
.
format
(
nr_diseases
)
)


def hpo(context):
    """
    Update the hpo terms in the database. Fetch the latest release and update terms.
    """
    LOG.info("Running scout update hpo")
    adapter = context.obj['adapter']

    LOG.info("Dropping HPO terms")
    adapter.hpo_term_collection.drop()
    LOG.debug("HPO terms dropped")

    load_hpo_terms(adapter)

def
hpo
(
context
)
:
LOG
.
info
(
"Running scout update hpo"
)
adapter
=
context
.
obj
[
'adapter'
]
LOG
.
info
(
"Dropping HPO terms"
)
adapter
.
hpo_term_collection
.
drop
(
)
LOG
.
debug
(
"HPO terms dropped"
)
load_hpo_terms
(
adapter
)

Update the hpo terms in the database . Fetch the latest release and update terms .


def users(store):
    """Display a list of all users and which institutes they belong to."""
    user_objs = list(store.users())
    total_events = store.user_events().count()
    for user_obj in user_objs:
        if user_obj.get('institutes'):
            user_obj['institutes'] = [store.institute(inst_id) for inst_id in user_obj.get('institutes')]
        else:
            user_obj['institutes'] = []
        user_obj['events'] = store.user_events(user_obj).count()
        user_obj['events_rank'] = event_rank(user_obj['events'])
    return dict(
        users=sorted(user_objs, key=lambda user: -user['events']),
        total_events=total_events,
    )

def users ( store ) : user_objs = list ( store . users ( ) ) total_events = store . user_events ( ) . count ( ) for user_obj in user_objs : if user_obj . get ( 'institutes' ) : user_obj [ 'institutes' ] = [ store . institute ( inst_id ) for inst_id in user_obj . get ( 'institutes' ) ] else : user_obj [ 'institutes' ] = [ ] user_obj [ 'events' ] = store . user_events ( user_obj ) . count ( ) user_obj [ 'events_rank' ] = event_rank ( user_obj [ 'events' ] ) return dict ( users = sorted ( user_objs , key = lambda user : - user [ 'events' ] ) , total_events = total_events , )

Display a list of all users and which institutes they belong to .


def parse_conservations(variant):
    """Parse the conservation predictors

        Args:
            variant(dict): A variant dictionary

        Returns:
            conservations(dict): A dictionary with the conservations
    """
    conservations = {}

    conservations['gerp'] = parse_conservation(
                                            variant,
                                            'dbNSFP_GERP___RS'
                                        )
    conservations['phast'] = parse_conservation(
                                            variant,
                                            'dbNSFP_phastCons100way_vertebrate'
                                        )
    conservations['phylop'] = parse_conservation(
                                            variant,
                                            'dbNSFP_phyloP100way_vertebrate'
                                        )
    return conservations

def parse_conservations ( variant ) : conservations = { } conservations [ 'gerp' ] = parse_conservation ( variant , 'dbNSFP_GERP RS' ) conservations [ 'phast' ] = parse_conservation ( variant , 'dbNSFP_phastCons100way_vertebrate' ) conservations [ 'phylop' ] = parse_conservation ( variant , 'dbNSFP_phyloP100way_vertebrate' ) return conservations


def parse_conservation(variant, info_key):
    """Get the conservation prediction

        Args:
            variant(dict): A variant dictionary
            info_key(str)

        Returns:
            conservations(list): List of censervation terms
    """
    raw_score = variant.INFO.get(info_key)
    conservations = []

    if raw_score:
        if isinstance(raw_score, numbers.Number):
            raw_score = (raw_score,)

        for score in raw_score:
            if score >= CONSERVATION[info_key]['conserved_min']:
                conservations.append('Conserved')
            else:
                conservations.append('NotConserved')

    return conservations

def parse_conservation ( variant , info_key ) : raw_score = variant . INFO . get ( info_key ) conservations = [ ] if raw_score : if isinstance ( raw_score , numbers . Number ) : raw_score = ( raw_score , ) for score in raw_score : if score >= CONSERVATION [ info_key ] [ 'conserved_min' ] : conservations . append ( 'Conserved' ) else : conservations . append ( 'NotConserved' ) return conservations


def get_general_case_info(adapter, institute_id=None, slice_query=None):
    """Return general information about cases

    Args:
        adapter(adapter.MongoAdapter)
        institute_id(str)
        slice_query(str):   Query to filter cases to obtain statistics for.

    Returns:
        general(dict)
    """
    general = {}

    # Potentially sensitive slice queries are assumed allowed if we have got this far
    name_query = slice_query

    cases = adapter.cases(owner=institute_id, name_query=name_query)

    phenotype_cases = 0
    causative_cases = 0
    pinned_cases = 0
    cohort_cases = 0

    pedigree = {
        1: {
            'title': 'Single',
            'count': 0
        },
        2: {
            'title': 'Duo',
            'count': 0
        },
        3: {
            'title': 'Trio',
            'count': 0
        },
        'many': {
            'title': 'Many',
            'count': 0
        },
    }

    case_ids = set()

    total_cases = 0
    for total_cases,case in enumerate(cases,1):
        # If only looking at one institute we need to save the case ids
        if institute_id:
            case_ids.add(case['_id'])
        if case.get('phenotype_terms'):
            phenotype_cases += 1
        if case.get('causatives'):
            causative_cases += 1
        if case.get('suspects'):
            pinned_cases += 1
        if case.get('cohorts'):
            cohort_cases += 1

        nr_individuals = len(case.get('individuals',[]))
        if nr_individuals == 0:
            continue
        if nr_individuals > 3:
            pedigree['many']['count'] += 1
        else:
            pedigree[nr_individuals]['count'] += 1

    general['total_cases'] = total_cases
    general['phenotype_cases'] = phenotype_cases
    general['causative_cases'] = causative_cases
    general['pinned_cases'] = pinned_cases
    general['cohort_cases'] = cohort_cases
    general['pedigree'] = pedigree
    general['case_ids'] = case_ids

    return general

def get_general_case_info ( adapter , institute_id = None , slice_query = None ) : general = { } name_query = slice_query cases = adapter . cases ( owner = institute_id , name_query = name_query ) phenotype_cases = 0 causative_cases = 0 pinned_cases = 0 cohort_cases = 0 pedigree = { 1 : { 'title' : 'Single' , 'count' : 0 } , 2 : { 'title' : 'Duo' , 'count' : 0 } , 3 : { 'title' : 'Trio' , 'count' : 0 } , 'many' : { 'title' : 'Many' , 'count' : 0 } , } case_ids = set ( ) total_cases = 0 for total_cases , case in enumerate ( cases , 1 ) : if institute_id : case_ids . add ( case [ '_id' ] ) if case . get ( 'phenotype_terms' ) : phenotype_cases += 1 if case . get ( 'causatives' ) : causative_cases += 1 if case . get ( 'suspects' ) : pinned_cases += 1 if case . get ( 'cohorts' ) : cohort_cases += 1 nr_individuals = len ( case . get ( 'individuals' , [ ] ) ) if nr_individuals == 0 : continue if nr_individuals > 3 : pedigree [ 'many' ] [ 'count' ] += 1 else : pedigree [ nr_individuals ] [ 'count' ] += 1 general [ 'total_cases' ] = total_cases general [ 'phenotype_cases' ] = phenotype_cases general [ 'causative_cases' ] = causative_cases general [ 'pinned_cases' ] = pinned_cases general [ 'cohort_cases' ] = cohort_cases general [ 'pedigree' ] = pedigree general [ 'case_ids' ] = case_ids return general


def get_case_groups(adapter, total_cases, institute_id=None, slice_query=None):
    """Return the information about case groups

    Args:
        store(adapter.MongoAdapter)
        total_cases(int): Total number of cases
        slice_query(str): Query to filter cases to obtain statistics for.

    Returns:
        cases(dict):
    """
    # Create a group with all cases in the database
    cases = [{'status': 'all', 'count': total_cases, 'percent': 1}]
    # Group the cases based on their status
    pipeline = []
    group = {'$group' : {'_id': '$status', 'count': {'$sum': 1}}}

    subquery = {}
    if institute_id and slice_query:
        subquery = adapter.cases(owner=institute_id, name_query=slice_query,
                              yield_query=True)
    elif institute_id:
        subquery = adapter.cases(owner=institute_id, yield_query=True)
    elif slice_query:
        subquery = adapter.cases(name_query=slice_query, yield_query=True)

    query = {'$match': subquery} if subquery else {}

    if query:
        pipeline.append(query)

    pipeline.append(group)
    res = adapter.case_collection.aggregate(pipeline)

    for status_group in res:
        cases.append({'status': status_group['_id'],
                      'count': status_group['count'],
                      'percent': status_group['count'] / total_cases})

    return cases

def get_case_groups ( adapter , total_cases , institute_id = None , slice_query = None ) : cases = [ { 'status' : 'all' , 'count' : total_cases , 'percent' : 1 } ] pipeline = [ ] group = { '$group' : { '_id' : '$status' , 'count' : { '$sum' : 1 } } } subquery = { } if institute_id and slice_query : subquery = adapter . cases ( owner = institute_id , name_query = slice_query , yield_query = True ) elif institute_id : subquery = adapter . cases ( owner = institute_id , yield_query = True ) elif slice_query : subquery = adapter . cases ( name_query = slice_query , yield_query = True ) query = { '$match' : subquery } if subquery else { } if query : pipeline . append ( query ) pipeline . append ( group ) res = adapter . case_collection . aggregate ( pipeline ) for status_group in res : cases . append ( { 'status' : status_group [ '_id' ] , 'count' : status_group [ 'count' ] , 'percent' : status_group [ 'count' ] / total_cases } ) return cases


def render_to_json_response(self, context, **kwargs):
        """
        Returns a JSON response, transforming 'context' to make the payload.
        """
        return HttpResponse(
            self.convert_context_to_json(context),
            content_type='application/json',
            **kwargs
        )

def render_to_json_response ( self , context , ** kwargs ) : return HttpResponse ( self . convert_context_to_json ( context ) , content_type = 'application/json' , ** kwargs )

Returns a JSON response transforming context to make the payload .


def get_year_and_month(self, net, qs, **kwargs):
        """
        Get the year and month. First tries from kwargs, then from
        querystrings. If none, or if cal_ignore qs is specified,
        sets year and month to this year and this month.
        """
        now = c.get_now()
        year = now.year
        month = now.month + net
        month_orig = None

        if 'cal_ignore=true' not in qs:
            if 'year' and 'month' in self.kwargs:  # try kwargs
                year, month_orig = map(
                    int, (self.kwargs['year'], self.kwargs['month'])
                )
                month = month_orig + net
            else:
                try:  # try querystring
                    year = int(self.request.GET['cal_year'])
                    month_orig = int(self.request.GET['cal_month'])
                    month = month_orig + net
                except Exception:
                    pass
        # return the year and month, and any errors that may have occurred do
        # to an invalid month/year being given.
        return c.clean_year_month(year, month, month_orig)

def get_year_and_month ( self , net , qs , ** kwargs ) : now = c . get_now ( ) year = now . year month = now . month + net month_orig = None if 'cal_ignore=true' not in qs : if 'year' and 'month' in self . kwargs : year , month_orig = map ( int , ( self . kwargs [ 'year' ] , self . kwargs [ 'month' ] ) ) month = month_orig + net else : try : year = int ( self . request . GET [ 'cal_year' ] ) month_orig = int ( self . request . GET [ 'cal_month' ] ) month = month_orig + net except Exception : pass return c . clean_year_month ( year , month , month_orig )

Get the year and month . First tries from kwargs then from querystrings . If none or if cal_ignore qs is specified sets year and month to this year and this month .


def check_for_cancelled_events(self, d):
        """Check if any events are cancelled on the given date 'd'."""
        for event in self.events:
            for cn in event.cancellations.all():
                if cn.date == d:
                    event.title += ' (CANCELLED)'

def
check_for_cancelled_events
(
self
,
d
)
:
for
event
in
self
.
events
:
for
cn
in
event
.
cancellations
.
all
(
)
:
if
cn
.
date
==
d
:
event
.
title
+=
' (CANCELLED)'

Check if any events are cancelled on the given date d .


def hpo_term(self, hpo_id):
        """Fetch a hpo term

        Args:
            hpo_id(str)

        Returns:
            hpo_obj(dict)
        """
        LOG.debug("Fetching hpo term %s", hpo_id)

        return self.hpo_term_collection.find_one({'_id': hpo_id})

def
hpo_term
(
self
,
hpo_id
)
:
LOG
.
debug
(
"Fetching hpo term %s"
,
hpo_id
)
return
self
.
hpo_term_collection
.
find_one
(
{
'_id'
:
hpo_id
}
)


def hpo_terms(self, query=None, hpo_term=None, text=None, limit=None):
        """Return all HPO terms

        If a query is sent hpo_terms will try to match with regex on term or
        description.

        Args:
            query(str): Part of a hpoterm or description
            hpo_term(str): Search for a specific hpo term
            limit(int): the number of desired results

        Returns:
            result(pymongo.Cursor): A cursor with hpo terms
        """
        query_dict = {}
        search_term = None
        if query:
            query_dict = {'$or':
                [
                    {'hpo_id': {'$regex': query, '$options':'i'}},
                    {'description': {'$regex': query, '$options':'i'}},
                ]
            }
            search_term = query
        elif text:
            new_string = ''
            for i,word in enumerate(text.split(' ')):
                if i == 0:
                    new_string += word
                else:
                    new_string += ' \"{0}\"'.format(word)
            LOG.info("Search HPO terms with %s", new_string)
            query_dict['$text'] = {'$search': new_string}
            search_term = text
        elif hpo_term:
            query_dict['hpo_id'] = hpo_term
            search_term = hpo_term

        limit = limit or int(10e10)
        res = self.hpo_term_collection.find(query_dict).limit(limit).sort('hpo_number',ASCENDING)

        LOG.info("Found {0} terms with search word {1}".format(res.count(), search_term))
        return res

def
hpo_terms
(
self
,
query
=
,
hpo_term
=
,
text
=
,
limit
=
)
:
query_dict
=
{
}
search_term
=
if
query
:
query_dict
=
{
'$or'
:
[
{
'hpo_id'
:
{
'$regex'
:
query
,
'$options'
:
'i'
}
}
,
{
'description'
:
{
'$regex'
:
query
,
'$options'
:
'i'
}
}
,
]
}
search_term
=
query
elif
text
:
new_string
=
''
for
i
,
word
in
enumerate
(
text
.
split
(
' '
)
)
:
if
i
==
0
:
new_string
+=
word
else
:
new_string
+=
' \"{0}\"'
.
format
(
word
)
LOG
.
info
(
"Search HPO terms with %s"
,
new_string
)
query_dict
[
'$text'
]
=
{
'$search'
:
new_string
}
search_term
=
text
elif
hpo_term
:
query_dict
[
'hpo_id'
]
=
hpo_term
search_term
=
hpo_term
limit
=
limit
or
int
(
10e10
)
res
=
self
.
hpo_term_collection
.
find
(
query_dict
)
.
limit
(
limit
)
.
sort
(
'hpo_number'
,
ASCENDING
)
LOG
.
info
(
"Found {0} terms with search word {1}"
.
format
(
res
.
count
(
)
,
search_term
)
)
return
res


def disease_term(self, disease_identifier):
        """Return a disease term

        Checks if the identifier is a disease number or a id

        Args:
            disease_identifier(str)

        Returns:
            disease_obj(dict)
        """
        query = {}
        try:
            disease_identifier = int(disease_identifier)
            query['disease_nr'] = disease_identifier
        except ValueError:
            query['_id'] = disease_identifier

        return self.disease_term_collection.find_one(query)

def disease_term ( self , disease_identifier ) : query = { } try : disease_identifier = int ( disease_identifier ) query [ 'disease_nr' ] = disease_identifier except ValueError : query [ '_id' ] = disease_identifier return self . disease_term_collection . find_one ( query )


def disease_terms(self, hgnc_id=None):
        """Return all disease terms that overlaps a gene

            If no gene, return all disease terms

        Args:
            hgnc_id(int)

        Returns:
            iterable(dict): A list with all disease terms that match
        """
        query = {}
        if hgnc_id:
            LOG.debug("Fetching all diseases for gene %s", hgnc_id)
            query['genes'] = hgnc_id
        else:
            LOG.info("Fetching all disease terms")

        return list(self.disease_term_collection.find(query))

def
disease_terms
(
self
,
hgnc_id
=
)
:
query
=
{
}
if
hgnc_id
:
LOG
.
debug
(
"Fetching all diseases for gene %s"
,
hgnc_id
)
query
[
'genes'
]
=
hgnc_id
else
:
LOG
.
info
(
"Fetching all disease terms"
)
return
list
(
self
.
disease_term_collection
.
find
(
query
)
)


def load_disease_term(self, disease_obj):
        """Load a disease term into the database

        Args:
            disease_obj(dict)
        """
        LOG.debug("Loading disease term %s into database", disease_obj['_id'])
        try:
            self.disease_term_collection.insert_one(disease_obj)
        except DuplicateKeyError as err:
            raise IntegrityError("Disease term %s already exists in database".format(disease_obj['_id']))

        LOG.debug("Disease term saved")


def generate_hpo_gene_list(self, *hpo_terms):
        """Generate a sorted list with namedtuples of hpogenes

            Each namedtuple of the list looks like (hgnc_id, count)

            Args:
                hpo_terms(iterable(str))

            Returns:
                hpo_genes(list(HpoGene))
        """
        genes = {}
        for term in hpo_terms:
            hpo_obj = self.hpo_term(term)
            if hpo_obj:
                for hgnc_id in hpo_obj['genes']:
                    if hgnc_id in genes:
                        genes[hgnc_id] += 1
                    else:
                        genes[hgnc_id] = 1
            else:
                LOG.warning("Term %s could not be found", term)

        sorted_genes = sorted(genes.items(), key=operator.itemgetter(1), reverse=True)
        return sorted_genes

def
generate_hpo_gene_list
(
self
,
*
hpo_terms
)
:
genes
=
{
}
for
term
in
hpo_terms
:
hpo_obj
=
self
.
hpo_term
(
term
)
if
hpo_obj
:
for
hgnc_id
in
hpo_obj
[
'genes'
]
:
if
hgnc_id
in
genes
:
genes
[
hgnc_id
]
+=
1
else
:
genes
[
hgnc_id
]
=
1
else
:
LOG
.
warning
(
"Term %s could not be found"
,
term
)
sorted_genes
=
sorted
(
genes
.
items
(
)
,
key
=
operator
.
itemgetter
(
1
)
,
reverse
=
True
)
return
sorted_genes


def read_hdf5(self, filename, f_start=None, f_stop=None,
                        t_start=None, t_stop=None, load_data=True):
        """ Populate Filterbank instance with data from HDF5 file

        Note:
            This is to be deprecated in future, please use Waterfall() to open files.
        """
        print("Warning: this function will be deprecated in the future. Please use Waterfall to open HDF5 files.")
#        raise DeprecationWarning('Please use Waterfall to open HDF5 files.')

        self.header = {}
        self.filename = filename
        self.h5 = h5py.File(filename)
        for key, val in self.h5[b'data'].attrs.items():
            if six.PY3:
                key = bytes(key, 'ascii')
            if key == b'src_raj':
                self.header[key] = Angle(val, unit='hr')
            elif key == b'src_dej':
                self.header[key] = Angle(val, unit='deg')
            else:
                self.header[key] = val

        self.n_ints_in_file = self.h5[b"data"].shape[0]
        i_start, i_stop, chan_start_idx, chan_stop_idx = self._setup_freqs(f_start=f_start, f_stop=f_stop)
        ii_start, ii_stop, n_ints = self._setup_time_axis(t_start=t_start, t_stop=t_stop)

        if load_data:
            self.data = self.h5[b"data"][ii_start:ii_stop, :, chan_start_idx:chan_stop_idx]

            self.file_size_bytes = os.path.getsize(self.filename)

#         if self.header[b'foff'] < 0:
#             self.data = self.data[..., ::-1] # Reverse data

        else:
            print("Skipping data load...")
            self.data = np.array([0])
            self.n_ints_in_file  = 0
            self.file_size_bytes = os.path.getsize(self.filename)


def _setup_freqs(self, f_start=None, f_stop=None):
        """ Setup frequency axis """
        ## Setup frequency axis
        f0 = self.header[b'fch1']
        f_delt = self.header[b'foff']

        i_start, i_stop = 0, self.header[b'nchans']
        if f_start:
            i_start = int((f_start - f0) / f_delt)
        if f_stop:
            i_stop  = int((f_stop - f0)  / f_delt)

        #calculate closest true index value
        chan_start_idx = np.int(i_start)
        chan_stop_idx  = np.int(i_stop)

        #create freq array
        if i_start < i_stop:
            i_vals = np.arange(chan_start_idx, chan_stop_idx)
        else:
            i_vals = np.arange(chan_stop_idx, chan_start_idx)

        self.freqs = f_delt * i_vals + f0

#         if f_delt < 0:
#             self.freqs = self.freqs[::-1]

        if chan_stop_idx < chan_start_idx:
            chan_stop_idx, chan_start_idx = chan_start_idx,chan_stop_idx

        return i_start, i_stop, chan_start_idx, chan_stop_idx

def _setup_freqs ( self , f_start = None , f_stop = None ) : f0 = self . header [ b'fch1' ] f_delt = self . header [ b'foff' ] i_start , i_stop = 0 , self . header [ b'nchans' ] if f_start : i_start = int ( ( f_start - f0 ) / f_delt ) if f_stop : i_stop = int ( ( f_stop - f0 ) / f_delt ) chan_start_idx = np . int ( i_start ) chan_stop_idx = np . int ( i_stop ) if i_start < i_stop : i_vals = np . arange ( chan_start_idx , chan_stop_idx ) else : i_vals = np . arange ( chan_stop_idx , chan_start_idx ) self . freqs = f_delt * i_vals + f0 if chan_stop_idx < chan_start_idx : chan_stop_idx , chan_start_idx = chan_start_idx , chan_stop_idx return i_start , i_stop , chan_start_idx , chan_stop_idx


def _setup_time_axis(self, t_start=None, t_stop=None):
        """  Setup time axis. """

        # now check to see how many integrations requested
        ii_start, ii_stop = 0, self.n_ints_in_file
        if t_start:
            ii_start = t_start
        if t_stop:
            ii_stop = t_stop
        n_ints = ii_stop - ii_start

        ## Setup time axis
        t0 = self.header[b'tstart']
        t_delt = self.header[b'tsamp']

        self.timestamps = np.arange(0, n_ints) * t_delt / 24./60./60 + t0

        return ii_start, ii_stop, n_ints


def read_filterbank(self, filename=None, f_start=None, f_stop=None,
                        t_start=None, t_stop=None, load_data=True):
        """ Populate Filterbank instance with data from Filterbank file

        Note:
            This is to be deprecated in future, please use Waterfall() to open files.
        """
        if filename is None:
            filename = self.filename
        else:
            self.filename = filename

        self.header = read_header(filename)

        #convert input frequencies into what their corresponding index would be
        i_start, i_stop, chan_start_idx, chan_stop_idx = self._setup_freqs(f_start=f_start, f_stop=f_stop)

        n_bits  = self.header[b'nbits']
        n_bytes  = int(self.header[b'nbits'] / 8)
        n_chans = self.header[b'nchans']
        n_chans_selected = self.freqs.shape[0]
        n_ifs   = self.header[b'nifs']

        # Load binary data
        self.idx_data = len_header(filename)
        f = open(filename, 'rb')
        f.seek(self.idx_data)
        filesize = os.path.getsize(self.filename)
        n_bytes_data = filesize - self.idx_data

        # Finally add some other info to the class as objects
        self.n_ints_in_file  = calc_n_ints_in_file(self.filename)
        self.file_size_bytes = filesize

        ## Setup time axis
        ii_start, ii_stop, n_ints = self._setup_time_axis(t_start=t_start, t_stop=t_stop)

        # Seek to first integration
        f.seek(int(ii_start * n_bits * n_ifs * n_chans / 8), 1)

        # Set up indexes used in file read (taken out of loop for speed)
        i0 = np.min((chan_start_idx, chan_stop_idx))
        i1 = np.max((chan_start_idx, chan_stop_idx))

        #Set up the data type (taken out of loop for speed)
        if n_bits == 2:
            dd_type = b'uint8'
            n_chans_selected = int(n_chans_selected/4)
        elif n_bytes == 4:
            dd_type = b'float32'
        elif n_bytes == 2:
            dd_type = b'uint16'
        elif n_bytes == 1:
            dd_type = b'uint8'

        if load_data:

            if n_ints * n_ifs * n_chans_selected > MAX_DATA_ARRAY_SIZE:
                print("[Filterbank]  Error: data array is too large to load. Either select fewer points or manually increase MAX_DATA_ARRAY_SIZE. Large files are now handle with Waterfall .")
                sys.exit()

            if n_bits == 2:
                self.data = np.zeros((n_ints, n_ifs, n_chans_selected*4), dtype=dd_type)
            else:
                self.data = np.zeros((n_ints, n_ifs, n_chans_selected), dtype=dd_type)

            for ii in range(n_ints):
                """d = f.read(n_bytes * n_chans * n_ifs)
                """

                for jj in range(n_ifs):

                    f.seek(n_bytes * i0, 1) # 1 = from current location
                    #d = f.read(n_bytes * n_chans_selected)
                    #bytes_to_read = n_bytes * n_chans_selected

                    dd = np.fromfile(f, count=n_chans_selected, dtype=dd_type)

                    # Reverse array if frequency axis is flipped
#                     if f_delt < 0:
#                         dd = dd[::-1]

                    if n_bits == 2:
                        dd = unpack_2to8(dd)
                    self.data[ii, jj] = dd

                    f.seek(n_bytes * (n_chans - i1), 1)  # Seek to start of next block
        else:
            print("Skipping data load...")
            self.data = np.array([0], dtype=dd_type)


def compute_lst(self):
        """ Compute LST for observation """
        if self.header[b'telescope_id'] == 6:
            self.coords = gbt_coords
        elif self.header[b'telescope_id'] == 4:
            self.coords = parkes_coords
        else:
            raise RuntimeError("Currently only Parkes and GBT supported")
        if HAS_SLALIB:
            # dut1 = (0.2 /3600.0) * np.pi/12.0
            dut1 = 0.0
            mjd = self.header[b'tstart']
            tellong = np.deg2rad(self.coords[1])
            last = s.sla_gmst(mjd) - tellong + s.sla_eqeqx(mjd) + dut1
            # lmst = s.sla_gmst(mjd) - tellong
            if last < 0.0 : last = last + 2.0*np.pi
            return last
        else:
            raise RuntimeError("This method requires pySLALIB")


def blank_dc(self, n_coarse_chan):
        """ Blank DC bins in coarse channels.

        Note: currently only works if entire file is read
        """

        if n_coarse_chan < 1:
            logger.warning('Coarse channel number < 1, unable to blank DC bin.')
            return None

        if not n_coarse_chan % int(n_coarse_chan) == 0:
            logger.warning('Selection does not contain an interger number of coarse channels, unable to blank DC bin.')
            return None

        n_coarse_chan = int(n_coarse_chan)

        n_chan = self.data.shape[-1]
        n_chan_per_coarse = int(n_chan / n_coarse_chan)

        mid_chan = int(n_chan_per_coarse / 2)

        for ii in range(n_coarse_chan):
            ss = ii*n_chan_per_coarse
            self.data[..., ss+mid_chan] = np.median(self.data[..., ss+mid_chan+5:ss+mid_chan+10])


def info(self):
        """ Print header information """

        for key, val in self.header.items():
            if key == b'src_raj':
                val = val.to_string(unit=u.hour, sep=':')
            if key == b'src_dej':
                val = val.to_string(unit=u.deg, sep=':')
            if key == b'tsamp':
                val *= u.second
            if key in ('foff', 'fch1'):
                val *= u.MHz
            if key == b'tstart':
                print("%16s : %32s" % ("tstart (ISOT)", Time(val, format='mjd').isot))
                key = "tstart (MJD)"
            print("%16s : %32s" % (key, val))

        print("\n%16s : %32s" % ("Num ints in file", self.n_ints_in_file))
        print("%16s : %32s" % ("Data shape", self.data.shape))
        print("%16s : %32s" % ("Start freq (MHz)", self.freqs[0]))
        print("%16s : %32s" % ("Stop freq (MHz)", self.freqs[-1]))

def
info
(
self
)
:
for
key
,
val
in
self
.
header
.
items
(
)
:
if
key
==
b'src_raj'
:
val
=
val
.
to_string
(
unit
=
u
.
hour
,
sep
=
':'
)
if
key
==
b'src_dej'
:
val
=
val
.
to_string
(
unit
=
u
.
deg
,
sep
=
':'
)
if
key
==
b'tsamp'
:
val
*=
u
.
second
if
key
in
(
'foff'
,
'fch1'
)
:
val
*=
u
.
MHz
if
key
==
b'tstart'
:
print
(
"%16s : %32s"
%
(
"tstart (ISOT)"
,
Time
(
val
,
format
=
'mjd'
)
.
isot
)
)
key
=
"tstart (MJD)"
print
(
"%16s : %32s"
%
(
key
,
val
)
)
print
(
"\n%16s : %32s"
%
(
"Num ints in file"
,
self
.
n_ints_in_file
)
)
print
(
"%16s : %32s"
%
(
"Data shape"
,
self
.
data
.
shape
)
)
print
(
"%16s : %32s"
%
(
"Start freq (MHz)"
,
self
.
freqs
[
0
]
)
)
print
(
"%16s : %32s"
%
(
"Stop freq (MHz)"
,
self
.
freqs
[
1
]
)
)


def _calc_extent(self,plot_f=None,plot_t=None,MJD_time=False):
        """ Setup ploting edges.
        """

        plot_f_begin = plot_f[0]
        plot_f_end = plot_f[-1] + (plot_f[1]-plot_f[0])

        plot_t_begin = self.timestamps[0]
        plot_t_end  = self.timestamps[-1] + (self.timestamps[1] - self.timestamps[0])

        if MJD_time:
            extent=(plot_f_begin, plot_f_begin_end, plot_t_begin, plot_t_end)
        else:
            extent=(plot_f_begin, plot_f_end, 0.0,(plot_t_end-plot_t_begin)*24.*60.*60)

        return extent

def _calc_extent ( self , plot_f = None , plot_t = None , MJD_time = False ) : plot_f_begin = plot_f [ 0 ] plot_f_end = plot_f [ - 1 ] + ( plot_f [ 1 ] - plot_f [ 0 ] ) plot_t_begin = self . timestamps [ 0 ] plot_t_end = self . timestamps [ - 1 ] + ( self . timestamps [ 1 ] - self . timestamps [ 0 ] ) if MJD_time : extent = ( plot_f_begin , plot_f_begin_end , plot_t_begin , plot_t_end ) else : extent = ( plot_f_begin , plot_f_end , 0.0 , ( plot_t_end - plot_t_begin ) * 24. * 60. * 60 ) return extent


def plot_waterfall(self, f_start=None, f_stop=None, if_id=0, logged=True, cb=True, MJD_time=False, **kwargs):
        """ Plot waterfall of data

        Args:
            f_start (float): start frequency, in MHz
            f_stop (float): stop frequency, in MHz
            logged (bool): Plot in linear (False) or dB units (True),
            cb (bool): for plotting the colorbar
            kwargs: keyword args to be passed to matplotlib imshow()
        """

        plot_f, plot_data = self.grab_data(f_start, f_stop, if_id)

        #Using accending frequency for all plots.
        if self.header[b'foff'] < 0:
            plot_data = plot_data[..., ::-1] # Reverse data
            plot_f = plot_f[::-1]

        if logged:
            plot_data = db(plot_data)

        # Make sure waterfall plot is under 4k*4k
        dec_fac_x, dec_fac_y = 1, 1
        if plot_data.shape[0] > MAX_IMSHOW_POINTS[0]:
            dec_fac_x = int(plot_data.shape[0] / MAX_IMSHOW_POINTS[0])

        if plot_data.shape[1] > MAX_IMSHOW_POINTS[1]:
            dec_fac_y =  int(plot_data.shape[1] /  MAX_IMSHOW_POINTS[1])

        plot_data = rebin(plot_data, dec_fac_x, dec_fac_y)

        try:
            plt.title(self.header[b'source_name'])
        except KeyError:
            plt.title(self.filename)

        extent = self._calc_extent(plot_f=plot_f,plot_t=self.timestamps,MJD_time=MJD_time)

        plt.imshow(plot_data,
            aspect='auto',
            origin='lower',
            rasterized=True,
            interpolation='nearest',
            extent=extent,
            cmap='viridis',
            **kwargs
        )
        if cb:
            plt.colorbar()
        plt.xlabel("Frequency [MHz]")
        if MJD_time:
            plt.ylabel("Time [MJD]")
        else:
            plt.ylabel("Time [s]")

def
plot_waterfall
(
self
,
f_start
=
,
f_stop
=
,
if_id
=
0
,
logged
=
True
,
cb
=
True
,
MJD_time
=
False
,
**
kwargs
)
:
plot_f
,
plot_data
=
self
.
grab_data
(
f_start
,
f_stop
,
if_id
)
if
self
.
header
[
b'foff'
]
<
0
:
plot_data
=
plot_data
[
...
,
:
:
1
]
plot_f
=
plot_f
[
:
:
1
]
if
logged
:
plot_data
=
db
(
plot_data
)
dec_fac_x
,
dec_fac_y
=
1
,
1
if
plot_data
.
shape
[
0
]
>
MAX_IMSHOW_POINTS
[
0
]
:
dec_fac_x
=
int
(
plot_data
.
shape
[
0
]
/
MAX_IMSHOW_POINTS
[
0
]
)
if
plot_data
.
shape
[
1
]
>
MAX_IMSHOW_POINTS
[
1
]
:
dec_fac_y
=
int
(
plot_data
.
shape
[
1
]
/
MAX_IMSHOW_POINTS
[
1
]
)
plot_data
=
rebin
(
plot_data
,
dec_fac_x
,
dec_fac_y
)
try
:
plt
.
title
(
self
.
header
[
b'source_name'
]
)
except
KeyError
:
plt
.
title
(
self
.
filename
)
extent
=
self
.
_calc_extent
(
plot_f
=
plot_f
,
plot_t
=
self
.
timestamps
,
MJD_time
=
MJD_time
)
plt
.
imshow
(
plot_data
,
aspect
=
'auto'
,
origin
=
'lower'
,
rasterized
=
True
,
interpolation
=
'nearest'
,
extent
=
extent
,
cmap
=
'viridis'
,
**
kwargs
)
if
cb
:
plt
.
colorbar
(
)
plt
.
xlabel
(
"Frequency [MHz]"
)
if
MJD_time
:
plt
.
ylabel
(
"Time [MJD]"
)
else
:
plt
.
ylabel
(
"Time [s]"
)


def plot_time_series(self, f_start=None, f_stop=None, if_id=0, logged=True, orientation='h', MJD_time=False, **kwargs):
        """ Plot the time series.

         Args:
            f_start (float): start frequency, in MHz
            f_stop (float): stop frequency, in MHz
            logged (bool): Plot in linear (False) or dB units (True),
            kwargs: keyword args to be passed to matplotlib imshow()
        """

        ax = plt.gca()
        plot_f, plot_data = self.grab_data(f_start, f_stop, if_id)

        if logged and self.header[b'nbits'] >= 8:
            plot_data = db(plot_data)

        #Since the data has been squeezed, the axis for time goes away if only one bin, causing a bug with axis=1
        if len(plot_data.shape) > 1:
            plot_data = plot_data.mean(axis=1)
        else:
            plot_data = plot_data.mean()

        #Make proper time axis for plotting (but only for plotting!). Note that this makes the values inclusive.
        extent = self._calc_extent(plot_f=plot_f,plot_t=self.timestamps,MJD_time=MJD_time)
        plot_t = np.linspace(extent[2],extent[3],len(self.timestamps))

        if MJD_time:
            tlabel = "Time [MJD]"
        else:
            tlabel = "Time [s]"

        if logged:
            plabel = "Power [dB]"
        else:
            plabel = "Power [counts]"

        # Reverse oder if vertical orientation.
        if 'v' in orientation:
            plt.plot(plot_data, plot_t, **kwargs)
            plt.xlabel(plabel)

        else:
            plt.plot(plot_t, plot_data, **kwargs)
            plt.xlabel(tlabel)
            plt.ylabel(plabel)

        ax.autoscale(axis='both',tight=True)

def
plot_time_series
(
self
,
f_start
=
,
f_stop
=
,
if_id
=
0
,
logged
=
True
,
orientation
=
'h'
,
MJD_time
=
False
,
**
kwargs
)
:
ax
=
plt
.
gca
(
)
plot_f
,
plot_data
=
self
.
grab_data
(
f_start
,
f_stop
,
if_id
)
if
logged
and
self
.
header
[
b'nbits'
]
>=
8
:
plot_data
=
db
(
plot_data
)
if
len
(
plot_data
.
shape
)
>
1
:
plot_data
=
plot_data
.
mean
(
axis
=
1
)
else
:
plot_data
=
plot_data
.
mean
(
)
extent
=
self
.
_calc_extent
(
plot_f
=
plot_f
,
plot_t
=
self
.
timestamps
,
MJD_time
=
MJD_time
)
plot_t
=
np
.
linspace
(
extent
[
2
]
,
extent
[
3
]
,
len
(
self
.
timestamps
)
)
if
MJD_time
:
tlabel
=
"Time [MJD]"
else
:
tlabel
=
"Time [s]"
if
logged
:
plabel
=
"Power [dB]"
else
:
plabel
=
"Power [counts]"
if
'v'
in
orientation
:
plt
.
plot
(
plot_data
,
plot_t
,
**
kwargs
)
plt
.
xlabel
(
plabel
)
else
:
plt
.
plot
(
plot_t
,
plot_data
,
**
kwargs
)
plt
.
xlabel
(
tlabel
)
plt
.
ylabel
(
plabel
)
ax
.
autoscale
(
axis
=
'both'
,
tight
=
True
)


def write_to_filterbank(self, filename_out):
        """ Write data to blimpy file.

        Args:
            filename_out (str): Name of output file
        """

        print("[Filterbank] Warning: Non-standard function to write in filterbank (.fil) format. Please use Waterfall.")

        n_bytes  = int(self.header[b'nbits'] / 8)
        with open(filename_out, "wb") as fileh:
            fileh.write(generate_sigproc_header(self))
            j = self.data
            if n_bytes == 4:
                np.float32(j.ravel()).tofile(fileh)
            elif n_bytes == 2:
                np.int16(j.ravel()).tofile(fileh)
            elif n_bytes == 1:
                np.int8(j.ravel()).tofile(fileh)


def calibrate_band_pass_N1(self):
        """ One way to calibrate the band pass is to take the median value
            for every frequency fine channel, and divide by it.
        """

        band_pass = np.median(self.data.squeeze(),axis=0)
        self.data = self.data/band_pass

def calibrate_band_pass_N1 ( self ) : band_pass = np . median ( self . data . squeeze ( ) , axis = 0 ) self . data = self . data / band_pass

One way to calibrate the band pass is to take the median value for every frequency fine channel and divide by it .


def convert_to_coarse(data,chan_per_coarse):
    '''
    Converts a data array with length n_chans to an array of length n_coarse_chans
    by averaging over the coarse channels
    '''
    #find number of coarse channels and reshape array
    num_coarse = data.size/chan_per_coarse
    data_shaped = np.array(np.reshape(data,(num_coarse,chan_per_coarse)))

    #Return the average over each coarse channel
    return np.mean(data_shaped[:,2:-1],axis=1)

def convert_to_coarse ( data , chan_per_coarse ) : num_coarse = data . size / chan_per_coarse data_shaped = np . array ( np . reshape ( data , ( num_coarse , chan_per_coarse ) ) ) return np . mean ( data_shaped [ : , 2 : - 1 ] , axis = 1 )


def apply_Mueller(I,Q,U,V, gain_offsets, phase_offsets, chan_per_coarse, feedtype='l'):
    '''
    Returns calibrated Stokes parameters for an observation given an array
    of differential gains and phase differences.
    '''

    #Find shape of data arrays and calculate number of coarse channels
    shape = I.shape
    ax0 = I.shape[0]
    ax1 = I.shape[1]
    nchans = I.shape[2]
    ncoarse = nchans/chan_per_coarse

    #Reshape data arrays to separate coarse channels
    I = np.reshape(I,(ax0,ax1,ncoarse,chan_per_coarse))
    Q = np.reshape(Q,(ax0,ax1,ncoarse,chan_per_coarse))
    U = np.reshape(U,(ax0,ax1,ncoarse,chan_per_coarse))
    V = np.reshape(V,(ax0,ax1,ncoarse,chan_per_coarse))

    #Swap axes 2 and 3 to in order for broadcasting to work correctly
    I = np.swapaxes(I,2,3)
    Q = np.swapaxes(Q,2,3)
    U = np.swapaxes(U,2,3)
    V = np.swapaxes(V,2,3)

    #Apply top left corner of electronics chain inverse Mueller matrix
    a = 1/(1-gain_offsets**2)
    if feedtype=='l':
        Icorr = a*(I-gain_offsets*Q)
        Qcorr = a*(-1*gain_offsets*I+Q)
        I = None
        Q = None
    if feedtype=='c':
        Icorr = a*(I-gain_offsets*V)
        Vcorr = a*(-1*gain_offsets*I+V)
        I = None
        V = None

    #Apply bottom right corner of electronics chain inverse Mueller matrix
    if feedtype=='l':
        Ucorr = U*np.cos(phase_offsets)-V*np.sin(phase_offsets)
        Vcorr = U*np.sin(phase_offsets)+V*np.cos(phase_offsets)
        U = None
        V = None
    if feedtype=='c':
        Qcorr = Q*np.cos(phase_offsets)+U*np.sin(phase_offsets)
        Ucorr = -1*Q*np.sin(phase_offsets)+U*np.cos(phase_offsets)
        Q = None
        U = None

    #Reshape arrays to original shape
    Icorr = np.reshape(np.swapaxes(Icorr,2,3),shape)
    Qcorr = np.reshape(np.swapaxes(Qcorr,2,3),shape)
    Ucorr = np.reshape(np.swapaxes(Ucorr,2,3),shape)
    Vcorr = np.reshape(np.swapaxes(Vcorr,2,3),shape)

    #Return corrected data arrays
    return Icorr,Qcorr,Ucorr,Vcorr

def apply_Mueller ( I , Q , U , V , gain_offsets , phase_offsets , chan_per_coarse , feedtype = 'l' ) : shape = I . shape ax0 = I . shape [ 0 ] ax1 = I . shape [ 1 ] nchans = I . shape [ 2 ] ncoarse = nchans / chan_per_coarse I = np . reshape ( I , ( ax0 , ax1 , ncoarse , chan_per_coarse ) ) Q = np . reshape ( Q , ( ax0 , ax1 , ncoarse , chan_per_coarse ) ) U = np . reshape ( U , ( ax0 , ax1 , ncoarse , chan_per_coarse ) ) V = np . reshape ( V , ( ax0 , ax1 , ncoarse , chan_per_coarse ) ) I = np . swapaxes ( I , 2 , 3 ) Q = np . swapaxes ( Q , 2 , 3 ) U = np . swapaxes ( U , 2 , 3 ) V = np . swapaxes ( V , 2 , 3 ) a = 1 / ( 1 - gain_offsets ** 2 ) if feedtype == 'l' : Icorr = a * ( I - gain_offsets * Q ) Qcorr = a * ( - 1 * gain_offsets * I + Q ) I = None Q = None if feedtype == 'c' : Icorr = a * ( I - gain_offsets * V ) Vcorr = a * ( - 1 * gain_offsets * I + V ) I = None V = None if feedtype == 'l' : Ucorr = U * np . cos ( phase_offsets ) - V * np . sin ( phase_offsets ) Vcorr = U * np . sin ( phase_offsets ) + V * np . cos ( phase_offsets ) U = None V = None if feedtype == 'c' : Qcorr = Q * np . cos ( phase_offsets ) + U * np . sin ( phase_offsets ) Ucorr = - 1 * Q * np . sin ( phase_offsets ) + U * np . cos ( phase_offsets ) Q = None U = None Icorr = np . reshape ( np . swapaxes ( Icorr , 2 , 3 ) , shape ) Qcorr = np . reshape ( np . swapaxes ( Qcorr , 2 , 3 ) , shape ) Ucorr = np . reshape ( np . swapaxes ( Ucorr , 2 , 3 ) , shape ) Vcorr = np . reshape ( np . swapaxes ( Vcorr , 2 , 3 ) , shape ) return Icorr , Qcorr , Ucorr , Vcorr

Returns calibrated Stokes parameters for an observation given an array of differential gains and phase differences .


def calibrate_pols(cross_pols,diode_cross,obsI=None,onefile=True,feedtype='l',**kwargs):
    '''
    Write Stokes-calibrated filterbank file for a given observation
    with a calibrator noise diode measurement on the source

    Parameters
    ----------
    cross_pols : string
        Path to cross polarization filterbank file (rawspec output) for observation to be calibrated
    diode_cross : string
        Path to cross polarization filterbank file of noise diode measurement ON the target
    obsI : string
        Path to Stokes I filterbank file of main observation (only needed if onefile=False)
    onefile : boolean
        True writes all calibrated Stokes parameters to a single filterbank file,
        False writes four separate files
    feedtype : 'l' or 'c'
        Basis of antenna dipoles. 'c' for circular, 'l' for linear
    '''
    #Obtain time sample length, frequencies, and noise diode data
    obs = Waterfall(diode_cross,max_load=150)
    cross_dat = obs.data
    tsamp = obs.header['tsamp']

    #Calculate number of coarse channels in the noise diode measurement (usually 8)
    dio_ncoarse = obs.calc_n_coarse_chan()
    dio_nchans = obs.header['nchans']
    dio_chan_per_coarse = dio_nchans/dio_ncoarse
    obs = None
    Idat,Qdat,Udat,Vdat = get_stokes(cross_dat,feedtype)
    cross_dat = None
    #Calculate differential gain and phase from noise diode measurements
    print('Calculating Mueller Matrix variables')
    gams = gain_offsets(Idat,Qdat,Udat,Vdat,tsamp,dio_chan_per_coarse,feedtype,**kwargs)
    psis = phase_offsets(Idat,Qdat,Udat,Vdat,tsamp,dio_chan_per_coarse,feedtype,**kwargs)

    #Clear data arrays to save memory
    Idat = None
    Qdat = None
    Udat = None
    Vdat = None

    #Get corrected Stokes parameters
    print('Opening '+cross_pols)
    cross_obs = Waterfall(cross_pols,max_load=150)
    obs_ncoarse = cross_obs.calc_n_coarse_chan()
    obs_nchans = cross_obs.header['nchans']
    obs_chan_per_coarse = obs_nchans/obs_ncoarse

    print('Grabbing Stokes parameters')
    I,Q,U,V = get_stokes(cross_obs.data,feedtype)

    print('Applying Mueller Matrix')
    I,Q,U,V = apply_Mueller(I,Q,U,V,gams,psis,obs_chan_per_coarse,feedtype)

    #Use onefile (default) to produce one filterbank file containing all Stokes information
    if onefile==True:
        cross_obs.data[:,0,:] = np.squeeze(I)
        cross_obs.data[:,1,:] = np.squeeze(Q)
        cross_obs.data[:,2,:] = np.squeeze(U)
        cross_obs.data[:,3,:] = np.squeeze(V)
        cross_obs.write_to_fil(cross_pols[:-15]+'.SIQUV.polcal.fil')
        print('Calibrated Stokes parameters written to '+cross_pols[:-15]+'.SIQUV.polcal.fil')
        return

    #Write corrected Stokes parameters to four filterbank files if onefile==False
    obs = Waterfall(obs_I,max_load=150)
    obs.data = I
    obs.write_to_fil(cross_pols[:-15]+'.SI.polcal.fil')   #assuming file is named *.cross_pols.fil
    print('Calibrated Stokes I written to '+cross_pols[:-15]+'.SI.polcal.fil')

    obs.data = Q
    obs.write_to_fil(cross_pols[:-15]+'.Q.polcal.fil')   #assuming file is named *.cross_pols.fil
    print('Calibrated Stokes Q written to '+cross_pols[:-15]+'.Q.polcal.fil')

    obs.data = U
    obs.write_to_fil(cross_pols[:-15]+'.U.polcal.fil')   #assuming file is named *.cross_pols.fil
    print('Calibrated Stokes U written to '+cross_pols[:-15]+'.U.polcal.fil')

    obs.data = V
    obs.write_to_fil(cross_pols[:-15]+'.V.polcal.fil')   #assuming file is named *.cross_pols.fil
    print('Calibrated Stokes V written to '+cross_pols[:-15]+'.V.polcal.fil')

Write Stokes - calibrated filterbank file for a given observation with a calibrator noise diode measurement on the source


def fracpols(str, **kwargs):
    '''Output fractional linear and circular polarizations for a
    rawspec cross polarization .fil file. NOT STANDARD USE'''

    I,Q,U,V,L=get_stokes(str, **kwargs)
    return L/I,V/I

def fracpols ( str , ** kwargs ) : I , Q , U , V , L = get_stokes ( str , ** kwargs ) return L / I , V / I

Output fractional linear and circular polarizations for a rawspec cross polarization . fil file . NOT STANDARD USE


def write_polfils(str, str_I, **kwargs):
    '''Writes two new filterbank files containing fractional linear and
    circular polarization data'''

    lin,circ=fracpols(str, **kwargs)
    obs = Waterfall(str_I, max_load=150)

    obs.data = lin
    obs.write_to_fil(str[:-15]+'.linpol.fil')   #assuming file is named *.cross_pols.fil

    obs.data = circ
    obs.write_to_fil(str[:-15]+'.circpol.fil')

def write_polfils ( str , str_I , ** kwargs ) : lin , circ = fracpols ( str , ** kwargs ) obs = Waterfall ( str_I , max_load = 150 ) obs . data = lin obs . write_to_fil ( str [ : - 15 ] + '.linpol.fil' ) obs . data = circ obs . write_to_fil ( str [ : - 15 ] + '.circpol.fil' )


def closest(xarr, val):
    """ Return the index of the closest in xarr to value val """
    idx_closest = np.argmin(np.abs(np.array(xarr) - val))
    return idx_closest

def closest ( xarr , val ) : idx_closest = np . argmin ( np . abs ( np . array ( xarr ) - val ) ) return idx_closest


def rebin(d, n_x, n_y=None):
    """ Rebin data by averaging bins together

    Args:
    d (np.array): data
    n_x (int): number of bins in x dir to rebin into one
    n_y (int): number of bins in y dir to rebin into one

    Returns:
    d: rebinned data with shape (n_x, n_y)
    """

    if d.ndim == 2:
        if n_y is None:
            n_y = 1
        if n_x is None:
            n_x = 1
        d = d[:int(d.shape[0] // n_x) * n_x, :int(d.shape[1] // n_y) * n_y]
        d = d.reshape((d.shape[0] // n_x, n_x, d.shape[1] // n_y, n_y))
        d = d.mean(axis=3)
        d = d.mean(axis=1)
    elif d.ndim == 1:
        d = d[:int(d.shape[0] // n_x) * n_x]
        d = d.reshape((d.shape[0] // n_x, n_x))
        d = d.mean(axis=1)
    else:
        raise RuntimeError("Only NDIM <= 2 supported")
    return d

def
rebin
(
d
,
n_x
,
n_y
=
)
:
if
d
.
ndim
==
2
:
if
n_y
is
:
n_y
=
1
if
n_x
is
:
n_x
=
1
d
=
d
[
:
int
(
d
.
shape
[
0
]
//
n_x
)
*
n_x
,
:
int
(
d
.
shape
[
1
]
//
n_y
)
*
n_y
]
d
=
d
.
reshape
(
(
d
.
shape
[
0
]
//
n_x
,
n_x
,
d
.
shape
[
1
]
//
n_y
,
n_y
)
)
d
=
d
.
mean
(
axis
=
3
)
d
=
d
.
mean
(
axis
=
1
)
elif
d
.
ndim
==
1
:
d
=
d
[
:
int
(
d
.
shape
[
0
]
//
n_x
)
*
n_x
]
d
=
d
.
reshape
(
(
d
.
shape
[
0
]
//
n_x
,
n_x
)
)
d
=
d
.
mean
(
axis
=
1
)
else
:
raise
RuntimeError
(
"Only NDIM <= 2 supported"
)
return
d


def unpack(data, nbit):
    """upgrade data from nbits to 8bits

    Notes: Pretty sure this function is a little broken!
    """
    if nbit > 8:
        raise ValueError("unpack: nbit must be <= 8")
    if 8 % nbit != 0:
        raise ValueError("unpack: nbit must divide into 8")
    if data.dtype not in (np.uint8, np.int8):
        raise TypeError("unpack: dtype must be 8-bit")
    if nbit == 8:
        return data
    elif nbit == 4:
        data = unpack_4to8(data)
        return data
    elif nbit == 2:
        data = unpack_2to8(data)
        return data
    elif nbit == 1:
        data = unpack_1to8(data)
        return data

def
unpack
(
data
,
nbit
)
:
if
nbit
>
8
:
raise
ValueError
(
"unpack: nbit must be <= 8"
)
if
8
%
nbit
!=
0
:
raise
ValueError
(
"unpack: nbit must divide into 8"
)
if
data
.
dtype
not
in
(
np
.
uint8
,
np
.
int8
)
:
raise
TypeError
(
"unpack: dtype must be 8-bit"
)
if
nbit
==
8
:
return
data
elif
nbit
==
4
:
data
=
unpack_4to8
(
data
)
return
data
elif
nbit
==
2
:
data
=
unpack_2to8
(
data
)
return
data
elif
nbit
==
1
:
data
=
unpack_1to8
(
data
)
return
data


def get_diff(dio_cross,feedtype,**kwargs):
    '''
    Returns ON-OFF for all Stokes parameters given a cross_pols noise diode measurement
    '''
    #Get Stokes parameters, frequencies, and time sample length
    obs = Waterfall(dio_cross,max_load=150)
    freqs = obs.populate_freqs()
    tsamp = obs.header['tsamp']
    data = obs.data
    obs = None

    I,Q,U,V = get_stokes(data,feedtype)

    #Fold noise diode data
    I_OFF,I_ON = foldcal(I,tsamp,**kwargs)
    Q_OFF,Q_ON = foldcal(Q,tsamp,**kwargs)
    U_OFF,U_ON = foldcal(U,tsamp,**kwargs)
    V_OFF,V_ON = foldcal(V,tsamp,**kwargs)

    #Do ON-OFF subtraction
    Idiff = I_ON-I_OFF
    Qdiff = Q_ON-Q_OFF
    Udiff = U_ON-U_OFF
    Vdiff = V_ON-V_OFF

    return Idiff,Qdiff,Udiff,Vdiff,freqs

def get_diff ( dio_cross , feedtype , ** kwargs ) : obs = Waterfall ( dio_cross , max_load = 150 ) freqs = obs . populate_freqs ( ) tsamp = obs . header [ 'tsamp' ] data = obs . data obs = None I , Q , U , V = get_stokes ( data , feedtype ) I_OFF , I_ON = foldcal ( I , tsamp , ** kwargs ) Q_OFF , Q_ON = foldcal ( Q , tsamp , ** kwargs ) U_OFF , U_ON = foldcal ( U , tsamp , ** kwargs ) V_OFF , V_ON = foldcal ( V , tsamp , ** kwargs ) Idiff = I_ON - I_OFF Qdiff = Q_ON - Q_OFF Udiff = U_ON - U_OFF Vdiff = V_ON - V_OFF return Idiff , Qdiff , Udiff , Vdiff , freqs

Returns ON - OFF for all Stokes parameters given a cross_pols noise diode measurement


def plot_Stokes_diode(dio_cross,diff=True,feedtype='l',**kwargs):
    '''
    Plots the uncalibrated full stokes spectrum of the noise diode.
    Use diff=False to plot both ON and OFF, or diff=True for ON-OFF
    '''

    #If diff=True, get ON-OFF. If not get ON and OFF separately
    if diff==True:
        Idiff,Qdiff,Udiff,Vdiff,freqs = get_diff(dio_cross,feedtype,**kwargs)
    else:
        obs = Waterfall(dio_cross,max_load=150)
        freqs = obs.populate_freqs()
        tsamp = obs.header['tsamp']
        data = obs.data
        I,Q,U,V = get_stokes(data,feedtype)

        I_OFF,I_ON = foldcal(I,tsamp,**kwargs)
        Q_OFF,Q_ON = foldcal(Q,tsamp,**kwargs)
        U_OFF,U_ON = foldcal(U,tsamp,**kwargs)
        V_OFF,V_ON = foldcal(V,tsamp,**kwargs)

    #Plot spectra
    if diff==True:
        plt.plot(freqs,Idiff,'k-',label='I')
        plt.plot(freqs,Qdiff,'r-',label='Q')
        plt.plot(freqs,Udiff,'g-',label='U')
        plt.plot(freqs,Vdiff,'m-',label='V')
    else:
        plt.plot(freqs,I_ON,'k-',label='I ON')
        plt.plot(freqs,I_OFF,'k--',label='I OFF')
        plt.plot(freqs,Q_ON,'r-',label='Q ON')
        plt.plot(freqs,Q_OFF,'r--',label='Q OFF')
        plt.plot(freqs,U_ON,'g-',label='U ON')
        plt.plot(freqs,U_OFF,'g--',label='U OFF')
        plt.plot(freqs,V_ON,'m-',label='V ON')
        plt.plot(freqs,V_OFF,'m--',label='V OFF')

    plt.legend()
    plt.xlabel('Frequency (MHz)')
    plt.title('Uncalibrated Full Stokes Noise Diode Spectrum')
    plt.ylabel('Power (Counts)')

Plots the uncalibrated full stokes spectrum of the noise diode . Use diff = False to plot both ON and OFF or diff = True for ON - OFF


def plot_calibrated_diode(dio_cross,chan_per_coarse=8,feedtype='l',**kwargs):
    '''
    Plots the corrected noise diode spectrum for a given noise diode measurement
    after application of the inverse Mueller matrix for the electronics chain.
    '''
    #Get full stokes data for the ND observation
    obs = Waterfall(dio_cross,max_load=150)
    freqs = obs.populate_freqs()
    tsamp = obs.header['tsamp']
    data = obs.data
    obs = None
    I,Q,U,V = get_stokes(data,feedtype)
    data = None

    #Calculate Mueller Matrix variables for each coarse channel
    psis = phase_offsets(I,Q,U,V,tsamp,chan_per_coarse,feedtype,**kwargs)
    G = gain_offsets(I,Q,U,V,tsamp,chan_per_coarse,feedtype,**kwargs)

    #Apply the Mueller matrix to original noise diode data and refold
    I,Q,U,V = apply_Mueller(I,Q,U,V,G,psis,chan_per_coarse,feedtype)
    I_OFF,I_ON = foldcal(I,tsamp,**kwargs)
    Q_OFF,Q_ON = foldcal(Q,tsamp,**kwargs)
    U_OFF,U_ON = foldcal(U,tsamp,**kwargs)
    V_OFF,V_ON = foldcal(V,tsamp,**kwargs)

    #Delete data arrays for space
    I = None
    Q = None
    U = None
    V = None

    #Plot new ON-OFF spectra
    plt.plot(freqs,I_ON-I_OFF,'k-',label='I')
    plt.plot(freqs,Q_ON-Q_OFF,'r-',label='Q')
    plt.plot(freqs,U_ON-U_OFF,'g-',label='U')
    plt.plot(freqs,V_ON-V_OFF,'m-',label='V')

    plt.legend()
    plt.xlabel('Frequency (MHz)')
    plt.title('Calibrated Full Stokes Noise Diode Spectrum')
    plt.ylabel('Power (Counts)')

Plots the corrected noise diode spectrum for a given noise diode measurement after application of the inverse Mueller matrix for the electronics chain .


def plot_gain_offsets(dio_cross,dio_chan_per_coarse=8,feedtype='l',ax1=None,ax2=None,legend=True,**kwargs):
    '''
    Plots the calculated gain offsets of each coarse channel along with
    the time averaged power spectra of the X and Y feeds
    '''
    #Get ON-OFF ND spectra
    Idiff,Qdiff,Udiff,Vdiff,freqs = get_diff(dio_cross,feedtype,**kwargs)
    obs = Waterfall(dio_cross,max_load=150)
    tsamp = obs.header['tsamp']
    data = obs.data
    obs = None
    I,Q,U,V = get_stokes(data,feedtype)

    #Get phase offsets and convert to degrees
    coarse_G = gain_offsets(I,Q,U,V,tsamp,dio_chan_per_coarse,feedtype,**kwargs)
    coarse_freqs = convert_to_coarse(freqs,dio_chan_per_coarse)

    #Get X and Y spectra for the noise diode ON and OFF
    #If using circular feeds these correspond to LL and RR
    XX_OFF,XX_ON = foldcal(np.expand_dims(data[:,0,:],axis=1),tsamp,**kwargs)
    YY_OFF,YY_ON = foldcal(np.expand_dims(data[:,1,:],axis=1),tsamp,**kwargs)

    if ax1==None:
        plt.subplot(211)
    else:
        axG = plt.axes(ax1)
        plt.setp(axG.get_xticklabels(),visible=False)
    plt.plot(coarse_freqs,coarse_G,'ko',markersize=2)
    plt.ylabel(r'$\frac{\Delta G}{2}$',rotation=90)
    if feedtype=='l':
        plt.title('XY Gain Difference')
    if feedtype=='c':
        plt.title('LR Gain Difference')
    plt.grid(True)

    if ax2==None:
        plt.subplot(212)
    else:
        axXY = plt.axes(ax2,sharex=axG)
    if feedtype=='l':
        plt.plot(freqs,XX_OFF,'b-',label='XX')
        plt.plot(freqs,YY_OFF,'r-',label='YY')
    if feedtype=='c':
        plt.plot(freqs,XX_OFF,'b-',label='LL')
        plt.plot(freqs,YY_OFF,'r-',label='RR')
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Power (Counts)')
    if legend==True:
        plt.legend()

def
plot_gain_offsets
(
dio_cross
,
dio_chan_per_coarse
=
8
,
feedtype
=
'l'
,
ax1
=
,
ax2
=
,
legend
=
True
,
**
kwargs
)
:
Idiff
,
Qdiff
,
Udiff
,
Vdiff
,
freqs
=
get_diff
(
dio_cross
,
feedtype
,
**
kwargs
)
obs
=
Waterfall
(
dio_cross
,
max_load
=
150
)
tsamp
=
obs
.
header
[
'tsamp'
]
data
=
obs
.
data
obs
=
I
,
Q
,
U
,
V
=
get_stokes
(
data
,
feedtype
)
coarse_G
=
gain_offsets
(
I
,
Q
,
U
,
V
,
tsamp
,
dio_chan_per_coarse
,
feedtype
,
**
kwargs
)
coarse_freqs
=
convert_to_coarse
(
freqs
,
dio_chan_per_coarse
)
XX_OFF
,
XX_ON
=
foldcal
(
np
.
expand_dims
(
data
[
:
,
0
,
:
]
,
axis
=
1
)
,
tsamp
,
**
kwargs
)
YY_OFF
,
YY_ON
=
foldcal
(
np
.
expand_dims
(
data
[
:
,
1
,
:
]
,
axis
=
1
)
,
tsamp
,
**
kwargs
)
if
ax1
==
:
plt
.
subplot
(
211
)
else
:
axG
=
plt
.
axes
(
ax1
)
plt
.
setp
(
axG
.
get_xticklabels
(
)
,
visible
=
False
)
plt
.
plot
(
coarse_freqs
,
coarse_G
,
'ko'
,
markersize
=
2
)
plt
.
ylabel
(
r'$\frac{\Delta G}{2}$'
,
rotation
=
90
)
if
feedtype
==
'l'
:
plt
.
title
(
'XY Gain Difference'
)
if
feedtype
==
'c'
:
plt
.
title
(
'LR Gain Difference'
)
plt
.
grid
(
True
)
if
ax2
==
:
plt
.
subplot
(
212
)
else
:
axXY
=
plt
.
axes
(
ax2
,
sharex
=
axG
)
if
feedtype
==
'l'
:
plt
.
plot
(
freqs
,
XX_OFF
,
'b-'
,
label
=
'XX'
)
plt
.
plot
(
freqs
,
YY_OFF
,
'r-'
,
label
=
'YY'
)
if
feedtype
==
'c'
:
plt
.
plot
(
freqs
,
XX_OFF
,
'b-'
,
label
=
'LL'
)
plt
.
plot
(
freqs
,
YY_OFF
,
'r-'
,
label
=
'RR'
)
plt
.
xlabel
(
'Frequency (MHz)'
)
plt
.
ylabel
(
'Power (Counts)'
)
if
legend
==
True
:
plt
.
legend
(
)


def open_file(filename, f_start=None, f_stop=None,t_start=None, t_stop=None,load_data=True,max_load=1.):
    """Open a HDF5 or filterbank file

    Returns instance of a Reader to read data from file.

    ================== ==================================================
    Filename extension File type
    ================== ==================================================
    h5, hdf5           HDF5 format
    fil                fil format
    *other*            Will raise NotImplementedError
    ================== ==================================================

    """
    if not os.path.isfile(filename):
        type(filename)
        print(filename)
        raise IOError("No such file or directory: " + filename)

    filename = os.path.expandvars(os.path.expanduser(filename))
    # Get file extension to determine type
    ext = filename.split(".")[-1].strip().lower()

    if six.PY3:
        ext = bytes(ext, 'ascii')

    if h5py.is_hdf5(filename):
        # Open HDF5 file
        return H5Reader(filename, f_start=f_start, f_stop=f_stop, t_start=t_start, t_stop=t_stop,
                        load_data=load_data, max_load=max_load)
    elif sigproc.is_filterbank(filename):
        # Open FIL file
        return FilReader(filename, f_start=f_start, f_stop=f_stop, t_start=t_start, t_stop=t_stop, load_data=load_data, max_load=max_load)
    else:
        raise NotImplementedError('Cannot open this type of file with Waterfall')


def _setup_selection_range(self, f_start=None, f_stop=None, t_start=None, t_stop=None, init=False):
        """Making sure the selection if time and frequency are within the file limits.

        Args:
            init (bool): If call during __init__
        """

        # This avoids resetting values
        if init is True:
            if t_start is None:
                t_start = self.t_begin
            if t_stop is None:
                t_stop = self.t_end
            if f_start is None:
                f_start = self.f_begin
            if f_stop is None:
                f_stop = self.f_end
        else:
            if f_start is None:
                f_start = self.f_start
            if f_stop is None:
                f_stop = self.f_stop
            if t_start is None:
                t_start = self.t_start
            if t_stop is None:
                t_stop = self.t_stop

        # By now, all values start/stop are populated.

        if t_stop >= 0 and t_start >= 0 and t_stop < t_start:
            t_stop, t_start = t_start,t_stop
            logger.warning('Given t_stop < t_start, assuming reversed values.')
        if f_stop and f_start and f_stop < f_start:
            f_stop, f_start = f_start,f_stop
            logger.warning('Given f_stop < f_start, assuming reversed values.')

        if t_start >= self.t_begin and t_start < self.t_end:
            self.t_start = int(t_start)
        else:
            if init is False or t_start != None:
                logger.warning('Setting t_start = %f, since t_start not given or not valid.'%self.t_begin)
            self.t_start = self.t_begin

        if t_stop <= self.t_end  and t_stop > self.t_begin:
            self.t_stop = int(t_stop)
        else:
            if init is False or t_stop:
                logger.warning('Setting t_stop = %f, since t_stop not given or not valid.'%self.t_end)
            self.t_stop = self.t_end

        if f_start >= self.f_begin and f_start < self.f_end:
            self.f_start = f_start
        else:
            if init is False or f_start:
                logger.warning('Setting f_start = %f, since f_start not given or not valid.'%self.f_begin)
            self.f_start = self.f_begin

        if f_stop <= self.f_end and f_stop > self.f_begin:
            self.f_stop = f_stop
        else:
            if init is False or f_stop:
                logger.warning('Setting f_stop = %f, since f_stop not given or not valid.'%self.f_end)
            self.f_stop = self.f_end

        # Now we have setup bounds, we can calculate shape of selection
        self.selection_shape = self._calc_selection_shape()

Making sure the selection if time and frequency are within the file limits .


def _calc_selection_size(self):
        """Calculate size of data of interest.
        """

        #Check to see how many integrations requested
        n_ints = self.t_stop - self.t_start
        #Check to see how many frequency channels requested
        n_chan = (self.f_stop - self.f_start) / abs(self.header[b'foff'])

        n_bytes  = self._n_bytes
        selection_size = int(n_ints*n_chan*n_bytes)

        return selection_size

def _calc_selection_size ( self ) : n_ints = self . t_stop - self . t_start n_chan = ( self . f_stop - self . f_start ) / abs ( self . header [ b'foff' ] ) n_bytes = self . _n_bytes selection_size = int ( n_ints * n_chan * n_bytes ) return selection_size


def _calc_selection_shape(self):
        """Calculate shape of data of interest.
        """

        #Check how many integrations requested
        n_ints = int(self.t_stop - self.t_start)
        #Check how many frequency channels requested
        n_chan = int(np.round((self.f_stop - self.f_start) / abs(self.header[b'foff'])))

        selection_shape = (n_ints,int(self.header[b'nifs']),n_chan)

        return selection_shape

def _calc_selection_shape ( self ) : n_ints = int ( self . t_stop - self . t_start ) n_chan = int ( np . round ( ( self . f_stop - self . f_start ) / abs ( self . header [ b'foff' ] ) ) ) selection_shape = ( n_ints , int ( self . header [ b'nifs' ] ) , n_chan ) return selection_shape


def _setup_chans(self):
        """Setup channel borders
        """

        if self.header[b'foff'] < 0:
            f0 = self.f_end
        else:
            f0 = self.f_begin

        i_start, i_stop = 0, self.n_channels_in_file
        if self.f_start:
            i_start = np.round((self.f_start - f0) / self.header[b'foff'])
        if self.f_stop:
            i_stop  = np.round((self.f_stop - f0)  / self.header[b'foff'])

        #calculate closest true index value
        chan_start_idx = np.int(i_start)
        chan_stop_idx  = np.int(i_stop)

        if chan_stop_idx < chan_start_idx:
            chan_stop_idx, chan_start_idx = chan_start_idx,chan_stop_idx

        self.chan_start_idx =  chan_start_idx
        self.chan_stop_idx = chan_stop_idx

def _setup_chans ( self ) : if self . header [ b'foff' ] < 0 : f0 = self . f_end else : f0 = self . f_begin i_start , i_stop = 0 , self . n_channels_in_file if self . f_start : i_start = np . round ( ( self . f_start - f0 ) / self . header [ b'foff' ] ) if self . f_stop : i_stop = np . round ( ( self . f_stop - f0 ) / self . header [ b'foff' ] ) chan_start_idx = np . int ( i_start ) chan_stop_idx = np . int ( i_stop ) if chan_stop_idx < chan_start_idx : chan_stop_idx , chan_start_idx = chan_start_idx , chan_stop_idx self . chan_start_idx = chan_start_idx self . chan_stop_idx = chan_stop_idx


def _setup_freqs(self):
        """Updating frequency borders from channel values
        """

        if self.header[b'foff'] > 0:
            self.f_start = self.f_begin + self.chan_start_idx*abs(self.header[b'foff'])
            self.f_stop = self.f_begin + self.chan_stop_idx*abs(self.header[b'foff'])
        else:
            self.f_start = self.f_end - self.chan_stop_idx*abs(self.header[b'foff'])
            self.f_stop = self.f_end - self.chan_start_idx*abs(self.header[b'foff'])

def _setup_freqs ( self ) : if self . header [ b'foff' ] > 0 : self . f_start = self . f_begin + self . chan_start_idx * abs ( self . header [ b'foff' ] ) self . f_stop = self . f_begin + self . chan_stop_idx * abs ( self . header [ b'foff' ] ) else : self . f_start = self . f_end - self . chan_stop_idx * abs ( self . header [ b'foff' ] ) self . f_stop = self . f_end - self . chan_start_idx * abs ( self . header [ b'foff' ] )


def populate_timestamps(self,update_header=False):
        """  Populate time axis.
            IF update_header then only return tstart
        """

        #Check to see how many integrations requested
        ii_start, ii_stop = 0, self.n_ints_in_file
        if self.t_start:
            ii_start = self.t_start
        if self.t_stop:
            ii_stop = self.t_stop

        ## Setup time axis
        t0 = self.header[b'tstart']
        t_delt = self.header[b'tsamp']

        if update_header:
            timestamps = ii_start * t_delt / 24./60./60. + t0
        else:
            timestamps = np.arange(ii_start, ii_stop) * t_delt / 24./60./60. + t0

        return timestamps

def populate_timestamps ( self , update_header = False ) : ii_start , ii_stop = 0 , self . n_ints_in_file if self . t_start : ii_start = self . t_start if self . t_stop : ii_stop = self . t_stop t0 = self . header [ b'tstart' ] t_delt = self . header [ b'tsamp' ] if update_header : timestamps = ii_start * t_delt / 24. / 60. / 60. + t0 else : timestamps = np . arange ( ii_start , ii_stop ) * t_delt / 24. / 60. / 60. + t0 return timestamps

Populate time axis . IF update_header then only return tstart


def populate_freqs(self):
        """
         Populate frequency axis
        """

        if self.header[b'foff'] < 0:
            f0 = self.f_end
        else:
            f0 = self.f_begin

        self._setup_chans()

        #create freq array
        i_vals = np.arange(self.chan_start_idx, self.chan_stop_idx)
        freqs = self.header[b'foff'] * i_vals + f0

        return freqs

def populate_freqs ( self ) : if self . header [ b'foff' ] < 0 : f0 = self . f_end else : f0 = self . f_begin self . _setup_chans ( ) i_vals = np . arange ( self . chan_start_idx , self . chan_stop_idx ) freqs = self . header [ b'foff' ] * i_vals + f0 return freqs


def calc_n_coarse_chan(self, chan_bw=None):
        """ This makes an attempt to calculate the number of coarse channels in a given file.

            Note:
                This is unlikely to work on non-Breakthrough Listen data, as a-priori knowledge of
                the digitizer system is required.

        """
        nchans = int(self.header[b'nchans'])
        # Do we have a file with enough channels that it has coarse channelization?
        if chan_bw is not None:
            bandwidth = abs(self.f_stop - self.f_start)
            n_coarse_chan = int(bandwidth / chan_bw)
            return n_coarse_chan
        elif nchans >= 2**20:
            # Does the common FFT length of 2^20 divide through without a remainder?
            # This should work for most GBT and all Parkes hires data
            if nchans % 2**20 == 0:
                n_coarse_chan = nchans // 2**20
                return n_coarse_chan
            # Early GBT data has non-2^N FFT length, check if it is GBT data
            elif self.header[b'telescope_id'] == 6:
                coarse_chan_bw = 2.9296875
                bandwidth = abs(self.f_stop - self.f_start)
                n_coarse_chan = int(bandwidth / coarse_chan_bw)
                return n_coarse_chan
            else:
                logger.warning("Couldn't figure out n_coarse_chan")
        elif self.header[b'telescope_id'] == 6 and nchans < 2**20:
            #For GBT non-hires data
            coarse_chan_bw = 2.9296875
            bandwidth = abs(self.f_stop - self.f_start)
            n_coarse_chan = int(bandwidth / coarse_chan_bw)
            return n_coarse_chan
        else:
            logger.warning("This function currently only works for hires BL Parkes or GBT data.")

This makes an attempt to calculate the number of coarse channels in a given file .


def calc_n_blobs(self, blob_dim):
        """ Given the blob dimensions, calculate how many fit in the data selection.
        """

        n_blobs = int(np.ceil(1.0 * np.prod(self.selection_shape) / np.prod(blob_dim)))

        return n_blobs

def calc_n_blobs ( self , blob_dim ) : n_blobs = int ( np . ceil ( 1.0 * np . prod ( self . selection_shape ) / np . prod ( blob_dim ) ) ) return n_blobs

Given the blob dimensions calculate how many fit in the data selection .


def isheavy(self):
        """ Check if the current selection is too large.
        """

        selection_size_bytes = self._calc_selection_size()

        if selection_size_bytes > self.MAX_DATA_ARRAY_SIZE:
            return True
        else:
            return False

def isheavy ( self ) : selection_size_bytes = self . _calc_selection_size ( ) if selection_size_bytes > self . MAX_DATA_ARRAY_SIZE : return True else : return False

Check if the current selection is too large .


def read_data(self, f_start=None, f_stop=None,t_start=None, t_stop=None):
        """ Read data.
        """

        self._setup_selection_range(f_start=f_start, f_stop=f_stop, t_start=t_start, t_stop=t_stop)

        #check if selection is small enough.
        if self.isheavy():
            logger.warning("Selection size of %.2f GB, exceeding our size limit %.2f GB. Instance created, "
                           "header loaded, but data not loaded, please try another (t,v) selection." % (self._calc_selection_size() / (1024. ** 3), self.MAX_DATA_ARRAY_SIZE / (1024. ** 3)))
            self.data = np.array([0],dtype=self._d_type)
            return None

        #Convert input frequencies into what their corresponding channel number would be.
        self._setup_chans()
        #Update frequencies ranges from channel number.
        self._setup_freqs()

        n_chans = self.header[b'nchans']
        n_chans_selected = self.selection_shape[self.freq_axis]
        n_ifs   = self.header[b'nifs']

        # Load binary data
        f = open(self.filename, 'rb')
        f.seek(int(self.idx_data))

        # now check to see how many integrations requested
        n_ints = self.t_stop - self.t_start

        # Seek to first integration
        f.seek(int(self.t_start * self._n_bytes  * n_ifs * n_chans), 1)

        #Loading  data
        self.data = np.zeros((n_ints, n_ifs, n_chans_selected), dtype=self._d_type)

        for ii in range(n_ints):
            for jj in range(n_ifs):
                f.seek(int(self._n_bytes  * self.chan_start_idx), 1) # 1 = from current location
                dd = np.fromfile(f, count=n_chans_selected, dtype=self._d_type)

                # Reverse array if frequency axis is flipped
#                     if self.header[b'foff'] < 0:
#                         dd = dd[::-1]

                self.data[ii, jj] = dd

                f.seek(int(self._n_bytes  * (n_chans - self.chan_stop_idx)), 1)


def read_all(self,reverse=True):
        """ read all the data.
            If reverse=True the x axis is flipped.
        """
        raise NotImplementedError('To be implemented')

        # go to start of the data
        self.filfile.seek(int(self.datastart))
        # read data into 2-D numpy array
#        data=np.fromfile(self.filfile,dtype=self.dtype).reshape(self.channels,self.blocksize,order='F')
        data=np.fromfile(self.filfile,dtype=self.dtype).reshape(self.blocksize, self.channels)
        if reverse:
            data = data[:,::-1]
        return data

def
read_all
(
self
,
reverse
=
True
)
:
raise
NotImplementedError
(
'To be implemented'
)
self
.
filfile
.
seek
(
int
(
self
.
datastart
)
)
data
=
np
.
fromfile
(
self
.
filfile
,
dtype
=
self
.
dtype
)
.
reshape
(
self
.
blocksize
,
self
.
channels
)
if
reverse
:
data
=
data
[
:
,
:
:
1
]
return
data

read all the data . If reverse = True the x axis is flipped .


def read_row(self,rownumber,reverse=True):
        """ Read a block of data. The number of samples per row is set in self.channels
            If reverse=True the x axis is flipped.
        """
        raise NotImplementedError('To be implemented')

        # go to start of the row
        self.filfile.seek(int(self.datastart+self.channels*rownumber*(int(self.nbits/8))))
        # read data into 2-D numpy array
        data=np.fromfile(self.filfile,count=self.channels,dtype=self.dtype).reshape(1, self.channels)
        if reverse:
            data = data[:,::-1]
        return data

def
read_row
(
self
,
rownumber
,
reverse
=
True
)
:
raise
NotImplementedError
(
'To be implemented'
)
self
.
filfile
.
seek
(
int
(
self
.
datastart
+
self
.
channels
*
rownumber
*
(
int
(
self
.
nbits
/
8
)
)
)
)
data
=
np
.
fromfile
(
self
.
filfile
,
count
=
self
.
channels
,
dtype
=
self
.
dtype
)
.
reshape
(
1
,
self
.
channels
)
if
reverse
:
data
=
data
[
:
,
:
:
1
]
return
data

Read a block of data . The number of samples per row is set in self . channels If reverse = True the x axis is flipped .


def read_data(self, f_start=None, f_stop=None,t_start=None, t_stop=None):
        """ Reads data selection if small enough.
        """

        self.container.read_data(f_start=f_start, f_stop=f_stop,t_start=t_start, t_stop=t_stop)

        self.__load_data()

def read_data ( self , f_start = None , f_stop = None , t_start = None , t_stop = None ) : self . container . read_data ( f_start = f_start , f_stop = f_stop , t_start = t_start , t_stop = t_stop ) self . __load_data ( )


def __update_header(self):
        """ Updates the header information from the original file to the selection.
        """

        #Updating frequency of first channel from selection
        if self.header[b'foff'] < 0:
            self.header[b'fch1'] = self.container.f_stop
        else:
            self.header[b'fch1'] = self.container.f_start

        #Updating number of coarse channels.
        self.header[b'nchans'] = self.container.selection_shape[self.freq_axis]

        #Updating time stamp for first time bin from selection
        self.header[b'tstart'] = self.container.populate_timestamps(update_header=True)

def __update_header ( self ) : if self . header [ b'foff' ] < 0 : self . header [ b'fch1' ] = self . container . f_stop else : self . header [ b'fch1' ] = self . container . f_start self . header [ b'nchans' ] = self . container . selection_shape [ self . freq_axis ] self . header [ b'tstart' ] = self . container . populate_timestamps ( update_header = True )

Updates the header information from the original file to the selection .


def info(self):
        """ Print header information and other derived information. """

        print("\n--- File Info ---")

        for key, val in self.file_header.items():
            if key == 'src_raj':
                val = val.to_string(unit=u.hour, sep=':')
            if key == 'src_dej':
                val = val.to_string(unit=u.deg, sep=':')
            print("%16s : %32s" % (key, val))

        print("\n%16s : %32s" % ("Num ints in file", self.n_ints_in_file))
        print("%16s : %32s" % ("File shape", self.file_shape))
        print("--- Selection Info ---")
        print("%16s : %32s" % ("Data selection shape", self.selection_shape))
        print("%16s : %32s" % ("Minimum freq (MHz)", self.container.f_start))
        print("%16s : %32s" % ("Maximum freq (MHz)", self.container.f_stop))

def
info
(
self
)
:
print
(
"\n--- File Info ---"
)
for
key
,
val
in
self
.
file_header
.
items
(
)
:
if
key
==
'src_raj'
:
val
=
val
.
to_string
(
unit
=
u
.
hour
,
sep
=
':'
)
if
key
==
'src_dej'
:
val
=
val
.
to_string
(
unit
=
u
.
deg
,
sep
=
':'
)
print
(
"%16s : %32s"
%
(
key
,
val
)
)
print
(
"\n%16s : %32s"
%
(
"Num ints in file"
,
self
.
n_ints_in_file
)
)
print
(
"%16s : %32s"
%
(
"File shape"
,
self
.
file_shape
)
)
print
(
"--- Selection Info ---"
)
print
(
"%16s : %32s"
%
(
"Data selection shape"
,
self
.
selection_shape
)
)
print
(
"%16s : %32s"
%
(
"Minimum freq (MHz)"
,
self
.
container
.
f_start
)
)
print
(
"%16s : %32s"
%
(
"Maximum freq (MHz)"
,
self
.
container
.
f_stop
)
)

Print header information and other derived information .


def write_to_fil(self, filename_out, *args, **kwargs):
        """ Write data to .fil file.
            It check the file size then decides how to write the file.

        Args:
            filename_out (str): Name of output file
        """

        #For timing how long it takes to write a file.
        t0 = time.time()

        #Update header
        self.__update_header()

        if self.container.isheavy():
            self.__write_to_fil_heavy(filename_out)
        else:
            self.__write_to_fil_light(filename_out)

        t1 = time.time()
        logger.info('Conversion time: %2.2fsec' % (t1- t0))

def
write_to_fil
(
self
,
filename_out
,
*
args
,
**
kwargs
)
:
t0
=
time
.
time
(
)
self
.
__update_header
(
)
if
self
.
container
.
isheavy
(
)
:
self
.
__write_to_fil_heavy
(
filename_out
)
else
:
self
.
__write_to_fil_light
(
filename_out
)
t1
=
time
.
time
(
)
logger
.
info
(
'Conversion time: %2.2fsec'
%
(
t1
t0
)
)

Write data to . fil file . It check the file size then decides how to write the file .


def write_to_hdf5(self, filename_out, *args, **kwargs):
        """ Write data to HDF5 file.
            It check the file size then decides how to write the file.

        Args:
            filename_out (str): Name of output file
        """

        #For timing how long it takes to write a file.
        t0 = time.time()

        #Update header
        self.__update_header()

        if self.container.isheavy():
            self.__write_to_hdf5_heavy(filename_out)
        else:
            self.__write_to_hdf5_light(filename_out)

        t1 = time.time()
        logger.info('Conversion time: %2.2fsec' % (t1- t0))

def
write_to_hdf5
(
self
,
filename_out
,
*
args
,
**
kwargs
)
:
t0
=
time
.
time
(
)
self
.
__update_header
(
)
if
self
.
container
.
isheavy
(
)
:
self
.
__write_to_hdf5_heavy
(
filename_out
)
else
:
self
.
__write_to_hdf5_light
(
filename_out
)
t1
=
time
.
time
(
)
logger
.
info
(
'Conversion time: %2.2fsec'
%
(
t1
t0
)
)

Write data to HDF5 file . It check the file size then decides how to write the file .


def __write_to_hdf5_light(self, filename_out, *args, **kwargs):
        """ Write data to HDF5 file in one go.

        Args:
            filename_out (str): Name of output file
        """

        block_size = 0

        with h5py.File(filename_out, 'w') as h5:

            h5.attrs[b'CLASS']   = b'FILTERBANK'
            h5.attrs[b'VERSION'] = b'1.0'

            if HAS_BITSHUFFLE:
                bs_compression = bitshuffle.h5.H5FILTER
                bs_compression_opts = (block_size, bitshuffle.h5.H5_COMPRESS_LZ4)
            else:
                bs_compression = None
                bs_compression_opts = None
                logger.warning("Warning: bitshuffle not found. No compression applied.")

            dset = h5.create_dataset('data',
                        data=self.data,
#                          compression='lzf')
                        compression=bs_compression,
                        compression_opts=bs_compression_opts)

            dset_mask = h5.create_dataset('mask',
                        shape=self.file_shape,
#                                 compression='lzf',
                        compression=bs_compression,
                        compression_opts=bs_compression_opts,
                        dtype='uint8')

            dset.dims[0].label = b"frequency"
            dset.dims[1].label = b"feed_id"
            dset.dims[2].label = b"time"

            dset_mask.dims[0].label = b"frequency"
            dset_mask.dims[1].label = b"feed_id"
            dset_mask.dims[2].label = b"time"

            # Copy over header information as attributes
            for key, value in self.header.items():
                dset.attrs[key] = value


def __get_blob_dimensions(self, chunk_dim):
        """ Sets the blob dimmentions, trying to read around 1024 MiB at a time.
            This is assuming a chunk is about 1 MiB.
        """

        #Taking the size into consideration, but avoiding having multiple blobs within a single time bin.
        if self.selection_shape[self.freq_axis] > chunk_dim[self.freq_axis]*MAX_BLOB_MB:
            freq_axis_size = self.selection_shape[self.freq_axis]
#             while freq_axis_size > chunk_dim[self.freq_axis]*MAX_BLOB_MB:
#                 freq_axis_size /= 2
            time_axis_size = 1
        else:
            freq_axis_size = self.selection_shape[self.freq_axis]
            time_axis_size = np.min([chunk_dim[self.time_axis] * MAX_BLOB_MB * chunk_dim[self.freq_axis] / freq_axis_size, self.selection_shape[self.time_axis]])

        blob_dim = (int(time_axis_size), 1, freq_axis_size)

        return blob_dim

def __get_blob_dimensions ( self , chunk_dim ) : if self . selection_shape [ self . freq_axis ] > chunk_dim [ self . freq_axis ] * MAX_BLOB_MB : freq_axis_size = self . selection_shape [ self . freq_axis ] time_axis_size = 1 else : freq_axis_size = self . selection_shape [ self . freq_axis ] time_axis_size = np . min ( [ chunk_dim [ self . time_axis ] * MAX_BLOB_MB * chunk_dim [ self . freq_axis ] / freq_axis_size , self . selection_shape [ self . time_axis ] ] ) blob_dim = ( int ( time_axis_size ) , 1 , freq_axis_size ) return blob_dim

Sets the blob dimmentions trying to read around 1024 MiB at a time . This is assuming a chunk is about 1 MiB .


def __get_chunk_dimensions(self):
        """ Sets the chunking dimmentions depending on the file type.
        """

        #Usually '.0000.' is in self.filename
        if np.abs(self.header[b'foff']) < 1e-5:
            logger.info('Detecting high frequency resolution data.')
            chunk_dim = (1,1,1048576) #1048576 is the number of channels in a coarse channel.
            return chunk_dim
        #Usually '.0001.' is in self.filename
        elif np.abs(self.header[b'tsamp']) < 1e-3:
            logger.info('Detecting high time resolution data.')
            chunk_dim = (2048,1,512) #512 is the total number of channels per single band (ie. blc00)
            return chunk_dim
        #Usually '.0002.' is in self.filename
        elif np.abs(self.header[b'foff']) < 1e-2 and np.abs(self.header[b'foff'])  >= 1e-5:
            logger.info('Detecting intermediate frequency and time resolution data.')
            chunk_dim = (10,1,65536)  #65536 is the total number of channels per single band (ie. blc00)
#            chunk_dim = (1,1,65536/4)
            return chunk_dim
        else:
            logger.warning('File format not known. Will use minimum chunking. NOT OPTIMAL.')
            chunk_dim = (1,1,512)
            return chunk_dim

Sets the chunking dimmentions depending on the file type .


def grab_data(self, f_start=None, f_stop=None,t_start=None, t_stop=None, if_id=0):
        """ Extract a portion of data by frequency range.

        Args:
            f_start (float): start frequency in MHz
            f_stop (float): stop frequency in MHz
            if_id (int): IF input identification (req. when multiple IFs in file)

        Returns:
            (freqs, data) (np.arrays): frequency axis in MHz and data subset
        """

        self.freqs = self.populate_freqs()
        self.timestamps = self.populate_timestamps()

        if f_start is None:
            f_start = self.freqs[0]
        if f_stop is None:
            f_stop = self.freqs[-1]

        i0 = np.argmin(np.abs(self.freqs - f_start))
        i1 = np.argmin(np.abs(self.freqs - f_stop))

        if i0 < i1:
            plot_f    = self.freqs[i0:i1 + 1]
            plot_data = np.squeeze(self.data[t_start:t_stop, ..., i0:i1 + 1])
        else:
            plot_f    = self.freqs[i1:i0 + 1]
            plot_data = np.squeeze(self.data[t_start:t_stop, ..., i1:i0 + 1])

        return plot_f, plot_data

def grab_data ( self , f_start = None , f_stop = None , t_start = None , t_stop = None , if_id = 0 ) : self . freqs = self . populate_freqs ( ) self . timestamps = self . populate_timestamps ( ) if f_start is None : f_start = self . freqs [ 0 ] if f_stop is None : f_stop = self . freqs [ - 1 ] i0 = np . argmin ( np . abs ( self . freqs - f_start ) ) i1 = np . argmin ( np . abs ( self . freqs - f_stop ) ) if i0 < i1 : plot_f = self . freqs [ i0 : i1 + 1 ] plot_data = np . squeeze ( self . data [ t_start : t_stop , ... , i0 : i1 + 1 ] ) else : plot_f = self . freqs [ i1 : i0 + 1 ] plot_data = np . squeeze ( self . data [ t_start : t_stop , ... , i1 : i0 + 1 ] ) return plot_f , plot_data

Extract a portion of data by frequency range .


def cmd_tool(args=None):
    """ Command line tool for plotting and viewing info on guppi raw files """

    from argparse import ArgumentParser

    parser = ArgumentParser(description="Command line utility for creating spectra from GuppiRaw files.")

    parser.add_argument('filename', type=str, help='Name of file to read')
    parser.add_argument('-o', dest='outdir', type=str, default='./', help='output directory for PNG files')
    args = parser.parse_args()

    r = GuppiRaw(args.filename)

    r.print_stats()
    bname = os.path.splitext(os.path.basename(args.filename))[0]
    bname = os.path.join(args.outdir, bname)
    r.plot_histogram(filename="%s_hist.png" % bname)
    r.plot_spectrum(filename="%s_spec.png" % bname)


def read_first_header(self):
        """ Read first header in file

        Returns:
            header (dict): keyword:value pairs of header metadata
        """
        self.file_obj.seek(0)
        header_dict, pos = self.read_header()
        self.file_obj.seek(0)
        return header_dict

def read_first_header ( self ) : self . file_obj . seek ( 0 ) header_dict , pos = self . read_header ( ) self . file_obj . seek ( 0 ) return header_dict


def find_n_data_blocks(self):
        """ Seek through the file to find how many data blocks there are in the file

        Returns:
            n_blocks (int): number of data blocks in the file
        """
        self.file_obj.seek(0)
        header0, data_idx0 = self.read_header()

        self.file_obj.seek(data_idx0)
        block_size = int(header0['BLOCSIZE'])
        n_bits = int(header0['NBITS'])
        self.file_obj.seek(int(header0['BLOCSIZE']), 1)
        n_blocks = 1
        end_found = False
        while not end_found:
            try:
                header, data_idx = self.read_header()
                self.file_obj.seek(data_idx)
                self.file_obj.seek(header['BLOCSIZE'], 1)
                n_blocks += 1
            except EndOfFileError:
                end_found = True
                break

        self.file_obj.seek(0)
        return n_blocks

def find_n_data_blocks ( self ) : self . file_obj . seek ( 0 ) header0 , data_idx0 = self . read_header ( ) self . file_obj . seek ( data_idx0 ) block_size = int ( header0 [ 'BLOCSIZE' ] ) n_bits = int ( header0 [ 'NBITS' ] ) self . file_obj . seek ( int ( header0 [ 'BLOCSIZE' ] ) , 1 ) n_blocks = 1 end_found = False while not end_found : try : header , data_idx = self . read_header ( ) self . file_obj . seek ( data_idx ) self . file_obj . seek ( header [ 'BLOCSIZE' ] , 1 ) n_blocks += 1 except EndOfFileError : end_found = True break self . file_obj . seek ( 0 ) return n_blocks


def print_stats(self):
        """ Compute some basic stats on the next block of data """

        header, data = self.read_next_data_block()
        data = data.view('float32')

        print("AVG: %2.3f" % data.mean())
        print("STD: %2.3f" % data.std())
        print("MAX: %2.3f" % data.max())
        print("MIN: %2.3f" % data.min())

        import pylab as plt

def
print_stats
(
self
)
:
header
,
data
=
self
.
read_next_data_block
(
)
data
=
data
.
view
(
'float32'
)
print
(
"AVG: %2.3f"
%
data
.
mean
(
)
)
print
(
"STD: %2.3f"
%
data
.
std
(
)
)
print
(
"MAX: %2.3f"
%
data
.
max
(
)
)
print
(
"MIN: %2.3f"
%
data
.
min
(
)
)
import
pylab
as
plt


def plot_histogram(self, filename=None):
        """ Plot a histogram of data values """
        header, data = self.read_next_data_block()
        data = data.view('float32')

        plt.figure("Histogram")
        plt.hist(data.flatten(), 65, facecolor='#cc0000')
        if filename:
            plt.savefig(filename)
        plt.show()

def plot_histogram ( self , filename = None ) : header , data = self . read_next_data_block ( ) data = data . view ( 'float32' ) plt . figure ( "Histogram" ) plt . hist ( data . flatten ( ) , 65 , facecolor = '#cc0000' ) if filename : plt . savefig ( filename ) plt . show ( )


def generate_filterbank_header(self, nchans=1, ):
        """ Generate a blimpy header dictionary """
        gp_head = self.read_first_header()
        fb_head = {}

        telescope_str = gp_head.get("TELESCOP", "unknown")
        if telescope_str in ('GBT', 'GREENBANK'):
            fb_head["telescope_id"] = 6
        elif telescope_str in ('PKS', 'PARKES'):
            fb_head["telescop_id"] = 7
        else:
            fb_head["telescop_id"] = 0

        # Using .get() method allows us to fill in default values if not present
        fb_head["source_name"] = gp_head.get("SRC_NAME", "unknown")
        fb_head["az_start"] = gp_head.get("AZ", 0)
        fb_head["za_start"] = gp_head.get("ZA", 0)

        fb_head["src_raj"] = Angle(str(gp_head.get("RA", 0.0)) + "hr")
        fb_head["src_dej"] = Angle(str(gp_head.get("DEC", 0.0)) + "deg")
        fb_head["rawdatafile"] = self.filename

        # hardcoded
        fb_head["machine_id"] = 20
        fb_head["data_type"] = 1  # blio datatype
        fb_head["barycentric"] = 0
        fb_head["pulsarcentric"] = 0
        fb_head["nbits"] = 32

        # TODO - compute these values. Need to figure out the correct calcs
        fb_head["tstart"] = 0.0
        fb_head["tsamp"] = 1.0
        fb_head["fch1"] = 0.0
        fb_head["foff"] = 187.5 / nchans

        # Need to be updated based on output specs
        fb_head["nchans"] = nchans
        fb_head["nifs"] = 1
        fb_head["nbeams"] = 1

        return fb_head

def generate_filterbank_header ( self , nchans = 1 , ) : gp_head = self . read_first_header ( ) fb_head = { } telescope_str = gp_head . get ( "TELESCOP" , "unknown" ) if telescope_str in ( 'GBT' , 'GREENBANK' ) : fb_head [ "telescope_id" ] = 6 elif telescope_str in ( 'PKS' , 'PARKES' ) : fb_head [ "telescop_id" ] = 7 else : fb_head [ "telescop_id" ] = 0 fb_head [ "source_name" ] = gp_head . get ( "SRC_NAME" , "unknown" ) fb_head [ "az_start" ] = gp_head . get ( "AZ" , 0 ) fb_head [ "za_start" ] = gp_head . get ( "ZA" , 0 ) fb_head [ "src_raj" ] = Angle ( str ( gp_head . get ( "RA" , 0.0 ) ) + "hr" ) fb_head [ "src_dej" ] = Angle ( str ( gp_head . get ( "DEC" , 0.0 ) ) + "deg" ) fb_head [ "rawdatafile" ] = self . filename fb_head [ "machine_id" ] = 20 fb_head [ "data_type" ] = 1 fb_head [ "barycentric" ] = 0 fb_head [ "pulsarcentric" ] = 0 fb_head [ "nbits" ] = 32 fb_head [ "tstart" ] = 0.0 fb_head [ "tsamp" ] = 1.0 fb_head [ "fch1" ] = 0.0 fb_head [ "foff" ] = 187.5 / nchans fb_head [ "nchans" ] = nchans fb_head [ "nifs" ] = 1 fb_head [ "nbeams" ] = 1 return fb_head