def find_header_size(filename):
    ''' Script to find the header size of a filterbank file'''

    # open datafile
    filfile=open(filename,'rb')
    # go to the start of the file
    filfile.seek(0)
    #read some region larger than the header.
    round1 = filfile.read(1000)
    headersize = round1.find('HEADER_END')+len('HEADER_END')

    return headersize

def find_header_size ( filename ) : filfile = open ( filename , 'rb' ) filfile . seek ( 0 ) round1 = filfile . read ( 1000 ) headersize = round1 . find ( 'HEADER_END' ) + len ( 'HEADER_END' ) return headersize


def cmd_tool(args=None):
    """ Command line tool to make a md5sum comparison of two .fil files. """

    if 'bl' in local_host:
        header_loc = '/usr/local/sigproc/bin/header' #Current location of header command in GBT.
    else:
        raise IOError('Script only able to run in BL systems.')

    p = OptionParser()
    p.set_usage('matchfils <FIL_FILE1> <FIL_FILE2>')
    opts, args = p.parse_args(sys.argv[1:])

    file1 = args[0]
    file2 = args[1]

    #------------------------------------
    #Create batch script

    make_batch_script()

    #------------------------------------
    #First checksum

    headersize1 = find_header_size(file1)
    file_size1 = os.path.getsize(file1)

    #Strip header from file, and calculate the md5sum of the rest.
    #command=['tail','-c',str(file_size1-headersize1),file1,'|','md5sum']
    command=['./tail_sum.sh',file1,str(file_size1-headersize1)]
    print('[matchfils] '+' '.join(command))

    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (out, err) = proc.communicate()

    check_sum1 = out.split()[0]
    print('[matchfils] Checksum is:', check_sum1)

    if err:
        raise Error('There is an error.')

    #---
    out,err = reset_outs()

    command=[header_loc,file1]
    print('[matchfils] Header information:')

    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (out, err) = proc.communicate()

    header1 = out
    print(header1)

    #------------------------------------
    #Second checksum

    out,err = reset_outs()

    headersize2 = find_header_size(file2)
    file_size2 = os.path.getsize(file2)

    #Strip header from file, and calculate the md5sum of the rest.
    command=['./tail_sum.sh',file2,str(file_size2-headersize2)]
    print('[matchfils] '+' '.join(command))

    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (out, err) = proc.communicate()

    check_sum2 = out.split()[0]
    print('[matchfils] Checksum is:', check_sum2)

    if err:
        raise Error('There is an error.')

    #---
    out,err = reset_outs()

    command=[header_loc,file2]
    print('[matchfils] Header information:')

    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (out, err) = proc.communicate()

    header2 = out
    print(header2)

    #------------------------------------
    #check the checksums

    if check_sum1 != check_sum2:
        print('[matchfils] Booo! Checksum does not match between files.')
    else:
        print('[matchfils] Hooray! Checksum matches between files.')

    #------------------------------------
    #Remove batch script

    os.remove('tail_sum.sh')

Command line tool to make a md5sum comparison of two . fil files .


def cmd_tool(args=None):
    """ Command line tool for converting guppi raw into HDF5 versions of guppi raw """
    from argparse import ArgumentParser

    if not HAS_BITSHUFFLE:
        print("Error: the bitshuffle library is required to run this script.")
        exit()

    parser = ArgumentParser(description="Command line utility for creating HDF5 Raw files.")
    parser.add_argument('filename', type=str, help='Name of filename to read')
    args = parser.parse_args()

    fileroot = args.filename.split('.0000.raw')[0]

    filelist = glob.glob(fileroot + '*.raw')
    filelist = sorted(filelist)

    # Read first file
    r = GuppiRaw(filelist[0])
    header, data = r.read_next_data_block()
    dshape = data.shape #r.read_next_data_block_shape()
    print(dshape)

    n_blocks_total = 0
    for filename in filelist:
        print(filename)
        r = GuppiRaw(filename)
        n_blocks_total += r.n_blocks
    print(n_blocks_total)

    full_dshape = np.concatenate(((n_blocks_total,), dshape))

    # Create h5py file
    h5 = h5py.File(fileroot + '.h5', 'w')
    h5.attrs['CLASS'] = 'GUPPIRAW'
    block_size = 0      # This is chunk block size
    dset = h5.create_dataset('data',
                  shape=full_dshape,
                  #compression=bitshuffle.h5.H5FILTER,
                  #compression_opts=(block_size, bitshuffle.h5.H5_COMPRESS_LZ4),
                  dtype=data.dtype)

    h5_idx = 0
    for filename in filelist:
        print("\nReading %s header..." % filename)
        r = GuppiRaw(filename)
        h5 = h5py.File(filename + '.h5', 'w')

        header, data = r.read_next_data_block()

        for ii in range(0, r.n_blocks):
            t0 = time.time()
            print("Reading block %i of %i" % (h5_idx+1, full_dshape[0]))
            header, data = r.read_next_data_block()
            t1 = time.time()

            t2 = time.time()
            print("Writing block %i of %i" % (h5_idx+1, full_dshape[0]))
            dset[h5_idx, :] = data
            t3 = time.time()
            print("Read: %2.2fs, Write %2.2fs" % ((t1-t0), (t3-t2)))

            h5_idx += 1

            # Copy over header information as attributes
            for key, value in header.items():
                dset.attrs[key] = value

        h5.close()

        t1 = time.time()
        print("Conversion time: %2.2fs" % (t1- t0))


def foldcal(data,tsamp, diode_p=0.04,numsamps=1000,switch=False,inds=False):
    '''
    Returns time-averaged spectra of the ON and OFF measurements in a
    calibrator measurement with flickering noise diode

    Parameters
    ----------
    data : 2D Array object (float)
        2D dynamic spectrum for data (any Stokes parameter) with flickering noise diode.
    tsamp : float
        Sampling time of data in seconds
    diode_p : float
        Period of the flickering noise diode in seconds
    numsamps : int
        Number of samples over which to average noise diode ON and OFF
    switch : boolean
        Use switch=True if the noise diode "skips" turning from OFF to ON once or vice versa
    inds : boolean
        Use inds=True to also return the indexes of the time series where the ND is ON and OFF
    '''

    halfper = diode_p/2.0

    foldt = halfper/tsamp   #number of time samples per diode switch

    onesec = 1/tsamp    #number of time samples in the first second

    #Find diode switches in units of time samples and round down to the nearest int
    ints = np.arange(0,numsamps)
    t_switch = (onesec+ints*foldt)
    t_switch = t_switch.astype('int')

    ONints = np.array(np.reshape(t_switch[:],(numsamps/2,2)))
    ONints[:,0] = ONints[:,0]+1   #Find index ranges of ON time samples

    OFFints = np.array(np.reshape(t_switch[1:-1],(numsamps/2-1,2)))
    OFFints[:,0] = OFFints[:,0]+1   #Find index ranges of OFF time samples

    av_ON = []
    av_OFF = []

    #Average ON and OFF spectra separately with respect to time
    for i in ONints:
        if i[1]!=i[0]:
            av_ON.append(np.sum(data[i[0]:i[1],:,:],axis=0)/(i[1]-i[0]))

    for i in OFFints:
        if i[1]!=i[0]:
            av_OFF.append(np.sum(data[i[0]:i[1],:,:],axis=0)/(i[1]-i[0]))

    #If switch=True, flip the return statement since ON is actually OFF
    if switch==False:
        if inds==False:
            return np.squeeze(np.mean(av_ON,axis=0)), np.squeeze(np.mean(av_OFF,axis=0))
        else:
            return np.squeeze(np.mean(av_ON,axis=0)), np.squeeze(np.mean(av_OFF,axis=0)),ONints,OFFints
    if switch==True:
        if inds==False:
            return np.squeeze(np.mean(av_OFF,axis=0)), np.squeeze(np.mean(av_ON,axis=0))
        else:
            return np.squeeze(np.mean(av_OFF,axis=0)), np.squeeze(np.mean(av_ON,axis=0)),OFFints,ONints

def foldcal ( data , tsamp , diode_p = 0.04 , numsamps = 1000 , switch = False , inds = False ) : halfper = diode_p / 2.0 foldt = halfper / tsamp onesec = 1 / tsamp ints = np . arange ( 0 , numsamps ) t_switch = ( onesec + ints * foldt ) t_switch = t_switch . astype ( 'int' ) ONints = np . array ( np . reshape ( t_switch [ : ] , ( numsamps / 2 , 2 ) ) ) ONints [ : , 0 ] = ONints [ : , 0 ] + 1 OFFints = np . array ( np . reshape ( t_switch [ 1 : - 1 ] , ( numsamps / 2 - 1 , 2 ) ) ) OFFints [ : , 0 ] = OFFints [ : , 0 ] + 1 av_ON = [ ] av_OFF = [ ] for i in ONints : if i [ 1 ] != i [ 0 ] : av_ON . append ( np . sum ( data [ i [ 0 ] : i [ 1 ] , : , : ] , axis = 0 ) / ( i [ 1 ] - i [ 0 ] ) ) for i in OFFints : if i [ 1 ] != i [ 0 ] : av_OFF . append ( np . sum ( data [ i [ 0 ] : i [ 1 ] , : , : ] , axis = 0 ) / ( i [ 1 ] - i [ 0 ] ) ) if switch == False : if inds == False : return np . squeeze ( np . mean ( av_ON , axis = 0 ) ) , np . squeeze ( np . mean ( av_OFF , axis = 0 ) ) else : return np . squeeze ( np . mean ( av_ON , axis = 0 ) ) , np . squeeze ( np . mean ( av_OFF , axis = 0 ) ) , ONints , OFFints if switch == True : if inds == False : return np . squeeze ( np . mean ( av_OFF , axis = 0 ) ) , np . squeeze ( np . mean ( av_ON , axis = 0 ) ) else : return np . squeeze ( np . mean ( av_OFF , axis = 0 ) ) , np . squeeze ( np . mean ( av_ON , axis = 0 ) ) , OFFints , ONints

Returns time - averaged spectra of the ON and OFF measurements in a calibrator measurement with flickering noise diode


def integrate_calib(name,chan_per_coarse,fullstokes=False,**kwargs):
    '''
    Folds Stokes I noise diode data and integrates along coarse channels

    Parameters
    ----------
    name : str
        Path to noise diode filterbank file
    chan_per_coarse : int
        Number of frequency bins per coarse channel
    fullstokes : boolean
        Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if
        it is in cross_pols format
    '''
    #Load data
    obs = Waterfall(name,max_load=150)
    data = obs.data

    #If the data has cross_pols format calculate Stokes I
    if fullstokes==False and data.shape[1]>1:
        data = data[:,0,:]+data[:,1,:]
        data = np.expand_dims(data,axis=1)
    #If the data has IQUV format get Stokes I
    if fullstokes==True:
        data = data[:,0,:]
        data = np.expand_dims(data,axis=1)

    tsamp = obs.header['tsamp']

    #Calculate ON and OFF values
    OFF,ON = foldcal(data,tsamp,**kwargs)

    freqs = obs.populate_freqs()

    #Find ON and OFF spectra by coarse channel
    ON_int = integrate_chans(ON,freqs,chan_per_coarse)
    OFF_int = integrate_chans(OFF,freqs,chan_per_coarse)

    #If "ON" is actually "OFF" switch them
    if np.sum(ON_int)<np.sum(OFF_int):
        temp = ON_int
        ON_int = OFF_int
        OFF_int = temp

    #Return coarse channel spectrum of OFF and ON
    return OFF_int,ON_int

def integrate_calib ( name , chan_per_coarse , fullstokes = False , ** kwargs ) : obs = Waterfall ( name , max_load = 150 ) data = obs . data if fullstokes == False and data . shape [ 1 ] > 1 : data = data [ : , 0 , : ] + data [ : , 1 , : ] data = np . expand_dims ( data , axis = 1 ) if fullstokes == True : data = data [ : , 0 , : ] data = np . expand_dims ( data , axis = 1 ) tsamp = obs . header [ 'tsamp' ] OFF , ON = foldcal ( data , tsamp , ** kwargs ) freqs = obs . populate_freqs ( ) ON_int = integrate_chans ( ON , freqs , chan_per_coarse ) OFF_int = integrate_chans ( OFF , freqs , chan_per_coarse ) if np . sum ( ON_int ) < np . sum ( OFF_int ) : temp = ON_int ON_int = OFF_int OFF_int = temp return OFF_int , ON_int


def get_calfluxes(calflux,calfreq,spec_in,centerfreqs,oneflux):
    '''
    Given properties of the calibrator source, calculate fluxes of the source
    in a particular frequency range

    Parameters
    ----------
    calflux : float
        Known flux of calibrator source at a particular frequency
    calfreq : float
        Frequency where calibrator source has flux calflux (see above)
    spec_in : float
        Known power-law spectral index of calibrator source. Use convention flux(frequency) = constant * frequency^(spec_in)
    centerfreqs : 1D Array (float)
        Central frequency values of each coarse channel
    oneflux : boolean
        Use oneflux to choose between calculating the flux for each core channel (False)
        or using one value for the entire frequency range (True)
    '''

    const = calflux/np.power(calfreq,spec_in)
    if oneflux==False:
        return const*np.power(centerfreqs,spec_in)
    else:
        return const*np.power(np.mean(centerfreqs),spec_in)

def get_calfluxes ( calflux , calfreq , spec_in , centerfreqs , oneflux ) : const = calflux / np . power ( calfreq , spec_in ) if oneflux == False : return const * np . power ( centerfreqs , spec_in ) else : return const * np . power ( np . mean ( centerfreqs ) , spec_in )

Given properties of the calibrator source calculate fluxes of the source in a particular frequency range


def get_centerfreqs(freqs,chan_per_coarse):
    '''
    Returns central frequency of each coarse channel

    Parameters
    ----------
    freqs : 1D Array (float)
        Frequency values for each bin of the spectrum
    chan_per_coarse: int
        Number of frequency bins per coarse channel
    '''

    num_coarse = freqs.size/chan_per_coarse
    freqs = np.reshape(freqs,(num_coarse,chan_per_coarse))
    return np.mean(freqs,axis=1)

def get_centerfreqs ( freqs , chan_per_coarse ) : num_coarse = freqs . size / chan_per_coarse freqs = np . reshape ( freqs , ( num_coarse , chan_per_coarse ) ) return np . mean ( freqs , axis = 1 )


def f_ratios(calON_obs,calOFF_obs,chan_per_coarse,**kwargs):
    '''
    Calculate f_ON, and f_OFF as defined in van Straten et al. 2012 equations 2 and 3

    Parameters
    ----------
    calON_obs : str
        Path to filterbank file (any format) for observation ON the calibrator source
    calOFF_obs : str
        Path to filterbank file (any format) for observation OFF the calibrator source
    '''
    #Calculate noise diode ON and noise diode OFF spectra (H and L) for both observations
    L_ON,H_ON = integrate_calib(calON_obs,chan_per_coarse,**kwargs)
    L_OFF,H_OFF = integrate_calib(calOFF_obs,chan_per_coarse,**kwargs)

    f_ON = H_ON/L_ON-1
    f_OFF = H_OFF/L_OFF-1

    return f_ON, f_OFF

Calculate f_ON and f_OFF as defined in van Straten et al . 2012 equations 2 and 3


def diode_spec(calON_obs,calOFF_obs,calflux,calfreq,spec_in,average=True,oneflux=False,**kwargs):
    '''
    Calculate the coarse channel spectrum and system temperature of the noise diode in Jy given two noise diode
    measurements ON and OFF the calibrator source with the same frequency and time resolution

    Parameters
    ----------
    calON_obs : str
        (see f_ratios() above)
    calOFF_obs : str
        (see f_ratios() above)
    calflux : float
        Known flux of calibrator source at a particular frequency
    calfreq : float
        Frequency where calibrator source has flux calflux (see above)
    spec_in : float
        Known power-law spectral index of calibrator source. Use convention flux(frequency) = constant * frequency^(spec_in)
    average : boolean
        Use average=True to return noise diode and Tsys spectra averaged over frequencies
    '''
    #Load frequencies and calculate number of channels per coarse channel
    obs = Waterfall(calON_obs,max_load=150)
    freqs = obs.populate_freqs()
    ncoarse = obs.calc_n_coarse_chan()
    nchans = obs.header['nchans']
    chan_per_coarse = nchans/ncoarse

    f_ON, f_OFF = f_ratios(calON_obs,calOFF_obs,chan_per_coarse,**kwargs)

    #Obtain spectrum of the calibrator source for the given frequency range
    centerfreqs = get_centerfreqs(freqs,chan_per_coarse)
    calfluxes = get_calfluxes(calflux,calfreq,spec_in,centerfreqs,oneflux)

    #C_o and Tsys as defined in van Straten et al. 2012
    C_o = calfluxes/(1/f_ON-1/f_OFF)
    Tsys = C_o/f_OFF

    #return coarse channel diode spectrum
    if average==True:
        return np.mean(C_o),np.mean(Tsys)
    else:
        return C_o,Tsys

def diode_spec ( calON_obs , calOFF_obs , calflux , calfreq , spec_in , average = True , oneflux = False , ** kwargs ) : obs = Waterfall ( calON_obs , max_load = 150 ) freqs = obs . populate_freqs ( ) ncoarse = obs . calc_n_coarse_chan ( ) nchans = obs . header [ 'nchans' ] chan_per_coarse = nchans / ncoarse f_ON , f_OFF = f_ratios ( calON_obs , calOFF_obs , chan_per_coarse , ** kwargs ) centerfreqs = get_centerfreqs ( freqs , chan_per_coarse ) calfluxes = get_calfluxes ( calflux , calfreq , spec_in , centerfreqs , oneflux ) C_o = calfluxes / ( 1 / f_ON - 1 / f_OFF ) Tsys = C_o / f_OFF if average == True : return np . mean ( C_o ) , np . mean ( Tsys ) else : return C_o , Tsys


def get_Tsys(calON_obs,calOFF_obs,calflux,calfreq,spec_in,oneflux=False,**kwargs):
    '''
    Returns frequency dependent system temperature given observations on and off a calibrator source

    Parameters
    ----------
    (See diode_spec())
    '''
    return diode_spec(calON_obs,calOFF_obs,calflux,calfreq,spec_in,average=False,oneflux=False,**kwargs)[1]

def get_Tsys ( calON_obs , calOFF_obs , calflux , calfreq , spec_in , oneflux = False , ** kwargs ) : return diode_spec ( calON_obs , calOFF_obs , calflux , calfreq , spec_in , average = False , oneflux = False , ** kwargs ) [ 1 ]


def calibrate_fluxes(main_obs_name,dio_name,dspec,Tsys,fullstokes=False,**kwargs):
    '''
    Produce calibrated Stokes I for an observation given a noise diode
    measurement on the source and a diode spectrum with the same number of
    coarse channels

    Parameters
    ----------
    main_obs_name : str
        Path to filterbank file containing final data to be calibrated
    dio_name : str
        Path to filterbank file for observation on the target source with flickering noise diode
    dspec : 1D Array (float) or float
        Coarse channel spectrum (or average) of the noise diode in Jy (obtained from diode_spec())
    Tsys : 1D Array (float) or float
        Coarse channel spectrum (or average) of the system temperature in Jy
    fullstokes: boolean
        Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if
        it is in cross_pols format
    '''

    #Find folded spectra of the target source with the noise diode ON and OFF
    main_obs = Waterfall(main_obs_name,max_load=150)
    ncoarse = main_obs.calc_n_coarse_chan()
    dio_obs = Waterfall(dio_name,max_load=150)
    dio_chan_per_coarse = dio_obs.header['nchans']/ncoarse
    dOFF,dON = integrate_calib(dio_name,dio_chan_per_coarse,fullstokes,**kwargs)

    #Find Jy/count for each coarse channel using the diode spectrum
    main_dat = main_obs.data
    scale_facs = dspec/(dON-dOFF)
    print(scale_facs)

    nchans = main_obs.header['nchans']
    obs_chan_per_coarse = nchans/ncoarse

    ax0_size = np.size(main_dat,0)
    ax1_size = np.size(main_dat,1)

    #Reshape data array of target observation and multiply coarse channels by the scale factors
    main_dat = np.reshape(main_dat,(ax0_size,ax1_size,ncoarse,obs_chan_per_coarse))
    main_dat = np.swapaxes(main_dat,2,3)

    main_dat = main_dat*scale_facs
    main_dat = main_dat-Tsys
    main_dat = np.swapaxes(main_dat,2,3)
    main_dat = np.reshape(main_dat,(ax0_size,ax1_size,nchans))

    #Write calibrated data to a new filterbank file with ".fluxcal" extension

    main_obs.data = main_dat
    main_obs.write_to_filterbank(main_obs_name[:-4]+'.fluxcal.fil')
    print('Finished: calibrated product written to ' + main_obs_name[:-4]+'.fluxcal.fil')


def len_header(filename):
    """ Return the length of the blimpy header, in bytes

    Args:
        filename (str): name of file to open

    Returns:
        idx_end (int): length of header, in bytes
    """
    with  open(filename, 'rb') as f:
        header_sub_count = 0
        eoh_found = False
        while not eoh_found:
            header_sub = f.read(512)
            header_sub_count += 1
            if b'HEADER_END' in header_sub:
                idx_end = header_sub.index(b'HEADER_END') + len(b'HEADER_END')
                eoh_found = True
                break

        idx_end = (header_sub_count -1) * 512 + idx_end
    return idx_end

def len_header ( filename ) : with open ( filename , 'rb' ) as f : header_sub_count = 0 eoh_found = False while not eoh_found : header_sub = f . read ( 512 ) header_sub_count += 1 if b'HEADER_END' in header_sub : idx_end = header_sub . index ( b'HEADER_END' ) + len ( b'HEADER_END' ) eoh_found = True break idx_end = ( header_sub_count - 1 ) * 512 + idx_end return idx_end

Return the length of the blimpy header in bytes


def is_filterbank(filename):
    """ Open file and confirm if it is a filterbank file or not. """
    with open(filename, 'rb') as fh:
        is_fil = True

        # Check this is a blimpy file
        try:
            keyword, value, idx = read_next_header_keyword(fh)
            try:
                assert keyword == b'HEADER_START'
            except AssertionError:
                is_fil = False
        except KeyError:
            is_fil = False
        return is_fil

def is_filterbank ( filename ) : with open ( filename , 'rb' ) as fh : is_fil = True try : keyword , value , idx = read_next_header_keyword ( fh ) try : assert keyword == b'HEADER_START' except AssertionError : is_fil = False except KeyError : is_fil = False return is_fil

Open file and confirm if it is a filterbank file or not .


def fix_header(filename, keyword, new_value):
    """ Apply a quick patch-up to a Filterbank header by overwriting a header value

    Args:
        filename (str): name of file to open and fix. WILL BE MODIFIED.
        keyword (stt):  header keyword to update
        new_value (long, double, angle or string): New value to write.

    Notes:
        This will overwrite the current value of the blimpy with a desired
        'fixed' version. Note that this has limited support for patching
        string-type values - if the length of the string changes, all hell will
        break loose.

    """

    # Read header data and return indexes of data offsets in file
    hd = read_header(filename)
    hi = read_header(filename, return_idxs=True)
    idx = hi[keyword]

    # Find out the datatype for the given keyword
    dtype = header_keyword_types[keyword]
    dtype_to_type = {b'<l'  : np.int32,
                     b'str' : bytes,
                     b'<d'  : np.float64,
                     b'angle' : to_sigproc_angle}
    value_dtype = dtype_to_type[dtype]

    # Generate the new string
    if isinstance(value_dtype, bytes):
        if len(hd[keyword]) == len(new_value):
            val_str = np.int32(len(new_value)).tostring() + new_value
        else:
            raise RuntimeError("String size mismatch. Cannot update without rewriting entire file.")
    else:
        val_str = value_dtype(new_value).tostring()

    # Write the new string to file
    with open(filename, 'rb+') as fh:
        fh.seek(idx)
        fh.write(val_str)

Apply a quick patch - up to a Filterbank header by overwriting a header value


def generate_sigproc_header(f):
    """ Generate a serialzed sigproc header which can be written to disk.

    Args:
        f (Filterbank object): Filterbank object for which to generate header

    Returns:
        header_str (str): Serialized string corresponding to header
    """

    header_string = b''
    header_string += to_sigproc_keyword(b'HEADER_START')

    for keyword in f.header.keys():
        if keyword == b'src_raj':
            header_string += to_sigproc_keyword(b'src_raj')  + to_sigproc_angle(f.header[b'src_raj'])
        elif keyword == b'src_dej':
            header_string += to_sigproc_keyword(b'src_dej')  + to_sigproc_angle(f.header[b'src_dej'])
        elif keyword == b'az_start' or keyword == b'za_start':
            header_string += to_sigproc_keyword(keyword)  + np.float64(f.header[keyword]).tostring()
        elif keyword not in header_keyword_types.keys():
            pass
        else:
            header_string += to_sigproc_keyword(keyword, f.header[keyword])

    header_string += to_sigproc_keyword(b'HEADER_END')
    return header_string

def generate_sigproc_header ( f ) : header_string = b'' header_string += to_sigproc_keyword ( b'HEADER_START' ) for keyword in f . header . keys ( ) : if keyword == b'src_raj' : header_string += to_sigproc_keyword ( b'src_raj' ) + to_sigproc_angle ( f . header [ b'src_raj' ] ) elif keyword == b'src_dej' : header_string += to_sigproc_keyword ( b'src_dej' ) + to_sigproc_angle ( f . header [ b'src_dej' ] ) elif keyword == b'az_start' or keyword == b'za_start' : header_string += to_sigproc_keyword ( keyword ) + np . float64 ( f . header [ keyword ] ) . tostring ( ) elif keyword not in header_keyword_types . keys ( ) : pass else : header_string += to_sigproc_keyword ( keyword , f . header [ keyword ] ) header_string += to_sigproc_keyword ( b'HEADER_END' ) return header_string

Generate a serialzed sigproc header which can be written to disk .


def to_sigproc_angle(angle_val):
    """ Convert an astropy.Angle to the ridiculous sigproc angle format string. """
    x = str(angle_val)

    if '.' in x:
        if 'h' in x:
                d, m, s, ss = int(x[0:x.index('h')]), int(x[x.index('h')+1:x.index('m')]), \
                int(x[x.index('m')+1:x.index('.')]), float(x[x.index('.'):x.index('s')])
        if 'd' in x:
            d, m, s, ss = int(x[0:x.index('d')]), int(x[x.index('d')+1:x.index('m')]), \
            int(x[x.index('m')+1:x.index('.')]), float(x[x.index('.'):x.index('s')])
    else:
        if 'h' in x:
            d, m, s = int(x[0:x.index('h')]), int(x[x.index('h')+1:x.index('m')]), \
            int(x[x.index('m')+1:x.index('s')])
        if 'd' in x:
            d, m, s = int(x[0:x.index('d')]), int(x[x.index('d')+1:x.index('m')]), \
            int(x[x.index('m')+1:x.index('s')])
        ss = 0
    num = str(d).zfill(2) + str(m).zfill(2) + str(s).zfill(2)+ '.' + str(ss).split(".")[-1]
    return np.float64(num).tostring()

def to_sigproc_angle ( angle_val ) : x = str ( angle_val ) if '.' in x : if 'h' in x : d , m , s , ss = int ( x [ 0 : x . index ( 'h' ) ] ) , int ( x [ x . index ( 'h' ) + 1 : x . index ( 'm' ) ] ) , int ( x [ x . index ( 'm' ) + 1 : x . index ( '.' ) ] ) , float ( x [ x . index ( '.' ) : x . index ( 's' ) ] ) if 'd' in x : d , m , s , ss = int ( x [ 0 : x . index ( 'd' ) ] ) , int ( x [ x . index ( 'd' ) + 1 : x . index ( 'm' ) ] ) , int ( x [ x . index ( 'm' ) + 1 : x . index ( '.' ) ] ) , float ( x [ x . index ( '.' ) : x . index ( 's' ) ] ) else : if 'h' in x : d , m , s = int ( x [ 0 : x . index ( 'h' ) ] ) , int ( x [ x . index ( 'h' ) + 1 : x . index ( 'm' ) ] ) , int ( x [ x . index ( 'm' ) + 1 : x . index ( 's' ) ] ) if 'd' in x : d , m , s = int ( x [ 0 : x . index ( 'd' ) ] ) , int ( x [ x . index ( 'd' ) + 1 : x . index ( 'm' ) ] ) , int ( x [ x . index ( 'm' ) + 1 : x . index ( 's' ) ] ) ss = 0 num = str ( d ) . zfill ( 2 ) + str ( m ) . zfill ( 2 ) + str ( s ) . zfill ( 2 ) + '.' + str ( ss ) . split ( "." ) [ - 1 ] return np . float64 ( num ) . tostring ( )

Convert an astropy . Angle to the ridiculous sigproc angle format string .


def calc_n_ints_in_file(filename):
    """ Calculate number of integrations in a given file """
    # Load binary data
    h = read_header(filename)
    n_bytes  = int(h[b'nbits'] / 8)

    n_chans = h[b'nchans']
    n_ifs   = h[b'nifs']
    idx_data = len_header(filename)
    f = open(filename, 'rb')
    f.seek(idx_data)
    filesize = os.path.getsize(filename)
    n_bytes_data = filesize - idx_data

    if h[b'nbits'] == 2:
        n_ints = int(4 * n_bytes_data / (n_chans * n_ifs))
    else:
        n_ints = int(n_bytes_data / (n_bytes * n_chans * n_ifs))

    return n_ints

def calc_n_ints_in_file ( filename ) : h = read_header ( filename ) n_bytes = int ( h [ b'nbits' ] / 8 ) n_chans = h [ b'nchans' ] n_ifs = h [ b'nifs' ] idx_data = len_header ( filename ) f = open ( filename , 'rb' ) f . seek ( idx_data ) filesize = os . path . getsize ( filename ) n_bytes_data = filesize - idx_data if h [ b'nbits' ] == 2 : n_ints = int ( 4 * n_bytes_data / ( n_chans * n_ifs ) ) else : n_ints = int ( n_bytes_data / ( n_bytes * n_chans * n_ifs ) ) return n_ints


def to_dict(self):
        """Convert a Traceback into a dictionary representation"""
        if self.tb_next is None:
            tb_next = None
        else:
            tb_next = self.tb_next.to_dict()

        code = {
            'co_filename': self.tb_frame.f_code.co_filename,
            'co_name': self.tb_frame.f_code.co_name,
        }
        frame = {
            'f_globals': self.tb_frame.f_globals,
            'f_code': code,
        }
        return {
            'tb_frame': frame,
            'tb_lineno': self.tb_lineno,
            'tb_next': tb_next,
        }

def to_dict ( self ) : if self . tb_next is None : tb_next = None else : tb_next = self . tb_next . to_dict ( ) code = { 'co_filename' : self . tb_frame . f_code . co_filename , 'co_name' : self . tb_frame . f_code . co_name , } frame = { 'f_globals' : self . tb_frame . f_globals , 'f_code' : code , } return { 'tb_frame' : frame , 'tb_lineno' : self . tb_lineno , 'tb_next' : tb_next , }


def make_rr_subparser(subparsers, rec_type, args_and_types):
    """
    Make a subparser for a given type of DNS record
    """
    sp = subparsers.add_parser(rec_type)

    sp.add_argument("name", type=str)
    sp.add_argument("ttl", type=int, nargs='?')
    sp.add_argument(rec_type, type=str)

    for my_spec in args_and_types:
        (argname, argtype) = my_spec[:2]
        if len(my_spec) > 2:
            nargs = my_spec[2]
            sp.add_argument(argname, type=argtype, nargs=nargs)
        else:
            sp.add_argument(argname, type=argtype)
    return sp

def make_rr_subparser ( subparsers , rec_type , args_and_types ) : sp = subparsers . add_parser ( rec_type ) sp . add_argument ( "name" , type = str ) sp . add_argument ( "ttl" , type = int , nargs = '?' ) sp . add_argument ( rec_type , type = str ) for my_spec in args_and_types : ( argname , argtype ) = my_spec [ : 2 ] if len ( my_spec ) > 2 : nargs = my_spec [ 2 ] sp . add_argument ( argname , type = argtype , nargs = nargs ) else : sp . add_argument ( argname , type = argtype ) return sp


def make_parser():
    """
    Make an ArgumentParser that accepts DNS RRs
    """
    line_parser = ZonefileLineParser()
    subparsers = line_parser.add_subparsers()

    # parse $ORIGIN
    sp = subparsers.add_parser("$ORIGIN")
    sp.add_argument("$ORIGIN", type=str)

    # parse $TTL
    sp = subparsers.add_parser("$TTL")
    sp.add_argument("$TTL", type=int)

    # parse each RR
    args_and_types = [
        ("mname", str), ("rname", str), ("serial", int), ("refresh", int),
        ("retry", int), ("expire", int), ("minimum", int)
    ]
    make_rr_subparser(subparsers, "SOA", args_and_types)

    make_rr_subparser(subparsers, "NS", [("host", str)])
    make_rr_subparser(subparsers, "A", [("ip", str)])
    make_rr_subparser(subparsers, "AAAA", [("ip", str)])
    make_rr_subparser(subparsers, "CNAME", [("alias", str)])
    make_rr_subparser(subparsers, "ALIAS", [("host", str)])
    make_rr_subparser(subparsers, "MX", [("preference", str), ("host", str)])
    make_txt_subparser(subparsers)
    make_rr_subparser(subparsers, "PTR", [("host", str)])
    make_rr_subparser(subparsers, "SRV", [("priority", int), ("weight", int), ("port", int), ("target", str)])
    make_rr_subparser(subparsers, "SPF", [("data", str)])
    make_rr_subparser(subparsers, "URI", [("priority", int), ("weight", int), ("target", str)])

    return line_parser

def make_parser ( ) : line_parser = ZonefileLineParser ( ) subparsers = line_parser . add_subparsers ( ) sp = subparsers . add_parser ( "$ORIGIN" ) sp . add_argument ( "$ORIGIN" , type = str ) sp = subparsers . add_parser ( "$TTL" ) sp . add_argument ( "$TTL" , type = int ) args_and_types = [ ( "mname" , str ) , ( "rname" , str ) , ( "serial" , int ) , ( "refresh" , int ) , ( "retry" , int ) , ( "expire" , int ) , ( "minimum" , int ) ] make_rr_subparser ( subparsers , "SOA" , args_and_types ) make_rr_subparser ( subparsers , "NS" , [ ( "host" , str ) ] ) make_rr_subparser ( subparsers , "A" , [ ( "ip" , str ) ] ) make_rr_subparser ( subparsers , "AAAA" , [ ( "ip" , str ) ] ) make_rr_subparser ( subparsers , "CNAME" , [ ( "alias" , str ) ] ) make_rr_subparser ( subparsers , "ALIAS" , [ ( "host" , str ) ] ) make_rr_subparser ( subparsers , "MX" , [ ( "preference" , str ) , ( "host" , str ) ] ) make_txt_subparser ( subparsers ) make_rr_subparser ( subparsers , "PTR" , [ ( "host" , str ) ] ) make_rr_subparser ( subparsers , "SRV" , [ ( "priority" , int ) , ( "weight" , int ) , ( "port" , int ) , ( "target" , str ) ] ) make_rr_subparser ( subparsers , "SPF" , [ ( "data" , str ) ] ) make_rr_subparser ( subparsers , "URI" , [ ( "priority" , int ) , ( "weight" , int ) , ( "target" , str ) ] ) return line_parser


def remove_comments(text):
    """
    Remove comments from a zonefile
    """
    ret = []
    lines = text.split("\n")
    for line in lines:
        if len(line) == 0:
            continue

        line = serialize(tokenize_line(line))
        ret.append(line)

    return "\n".join(ret)

def remove_comments ( text ) : ret = [ ] lines = text . split ( "\n" ) for line in lines : if len ( line ) == 0 : continue line = serialize ( tokenize_line ( line ) ) ret . append ( line ) return "\n" . join ( ret )


def add_default_name(text):
    """
    Go through each line of the text and ensure that
    a name is defined.  Use '@' if there is none.
    """
    global SUPPORTED_RECORDS

    lines = text.split("\n")
    ret = []
    for line in lines:
        tokens = tokenize_line(line)
        if len(tokens) == 0:
            continue

        if tokens[0] in SUPPORTED_RECORDS and not tokens[0].startswith("$"):
            # add back the name
            tokens = ['@'] + tokens

        ret.append(serialize(tokens))

    return "\n".join(ret)

def add_default_name ( text ) : global SUPPORTED_RECORDS lines = text . split ( "\n" ) ret = [ ] for line in lines : tokens = tokenize_line ( line ) if len ( tokens ) == 0 : continue if tokens [ 0 ] in SUPPORTED_RECORDS and not tokens [ 0 ] . startswith ( "$" ) : tokens = [ '@' ] + tokens ret . append ( serialize ( tokens ) ) return "\n" . join ( ret )

Go through each line of the text and ensure that a name is defined . Use


def parse_line(parser, record_token, parsed_records):
    """
    Given the parser, capitalized list of a line's tokens, and the current set of records
    parsed so far, parse it into a dictionary.

    Return the new set of parsed records.
    Raise an exception on error.
    """

    global SUPPORTED_RECORDS

    line = " ".join(record_token)

    # match parser to record type
    if len(record_token) >= 2 and record_token[1] in SUPPORTED_RECORDS:
        # with no ttl
        record_token = [record_token[1]] + record_token
    elif len(record_token) >= 3 and record_token[2] in SUPPORTED_RECORDS:
        # with ttl
        record_token = [record_token[2]] + record_token
        if record_token[0] == "TXT":
            record_token = record_token[:2] + ["--ttl"] + record_token[2:]
    try:
        rr, unmatched = parser.parse_known_args(record_token)
        assert len(unmatched) == 0, "Unmatched fields: %s" % unmatched
    except (SystemExit, AssertionError, InvalidLineException):
        # invalid argument
        raise InvalidLineException(line)

    record_dict = rr.__dict__
    if record_token[0] == "TXT" and len(record_dict['txt']) == 1:
        record_dict['txt'] = record_dict['txt'][0]

    # what kind of record? including origin and ttl
    record_type = None
    for key in record_dict.keys():
        if key in SUPPORTED_RECORDS and (key.startswith("$") or record_dict[key] == key):
            record_type = key
            if record_dict[key] == key:
                del record_dict[key]
            break

    assert record_type is not None, "Unknown record type in %s" % rr

    # clean fields
    for field in record_dict.keys():
        if record_dict[field] is None:
            del record_dict[field]

    current_origin = record_dict.get('$ORIGIN', parsed_records.get('$ORIGIN', None))

    # special record-specific fix-ups
    if record_type == 'PTR':
        record_dict['fullname'] = record_dict['name'] + '.' + current_origin

    if len(record_dict) > 0:
        if record_type.startswith("$"):
            # put the value directly
            record_dict_key = record_type.lower()
            parsed_records[record_dict_key] = record_dict[record_type]
        else:
            record_dict_key = record_type.lower()
            parsed_records[record_dict_key].append(record_dict)

    return parsed_records

def
parse_line
(
parser
,
record_token
,
parsed_records
)
:
global
SUPPORTED_RECORDS
line
=
" "
.
join
(
record_token
)
if
len
(
record_token
)
>=
2
and
record_token
[
1
]
in
SUPPORTED_RECORDS
:
record_token
=
[
record_token
[
1
]
]
+
record_token
elif
len
(
record_token
)
>=
3
and
record_token
[
2
]
in
SUPPORTED_RECORDS
:
record_token
=
[
record_token
[
2
]
]
+
record_token
if
record_token
[
0
]
==
"TXT"
:
record_token
=
record_token
[
:
2
]
+
[
"--ttl"
]
+
record_token
[
2
:
]
try
:
rr
,
unmatched
=
parser
.
parse_known_args
(
record_token
)
assert
len
(
unmatched
)
==
0
,
"Unmatched fields: %s"
%
unmatched
except
(
SystemExit
,
AssertionError
,
InvalidLineException
)
:
raise
InvalidLineException
(
line
)
record_dict
=
rr
.
__dict__
if
record_token
[
0
]
==
"TXT"
and
len
(
record_dict
[
'txt'
]
)
==
1
:
record_dict
[
'txt'
]
=
record_dict
[
'txt'
]
[
0
]
record_type
=
for
key
in
record_dict
.
keys
(
)
:
if
key
in
SUPPORTED_RECORDS
and
(
key
.
startswith
(
"$"
)
or
record_dict
[
key
]
==
key
)
:
record_type
=
key
if
record_dict
[
key
]
==
key
:
del
record_dict
[
key
]
break
assert
record_type
is
not
,
"Unknown record type in %s"
%
rr
for
field
in
record_dict
.
keys
(
)
:
if
record_dict
[
field
]
is
:
del
record_dict
[
field
]
current_origin
=
record_dict
.
get
(
'$ORIGIN'
,
parsed_records
.
get
(
'$ORIGIN'
,
)
)
if
record_type
==
'PTR'
:
record_dict
[
'fullname'
]
=
record_dict
[
'name'
]
+
'.'
+
current_origin
if
len
(
record_dict
)
>
0
:
if
record_type
.
startswith
(
"$"
)
:
record_dict_key
=
record_type
.
lower
(
)
parsed_records
[
record_dict_key
]
=
record_dict
[
record_type
]
else
:
record_dict_key
=
record_type
.
lower
(
)
parsed_records
[
record_dict_key
]
.
append
(
record_dict
)
return
parsed_records

Given the parser capitalized list of a line s tokens and the current set of records parsed so far parse it into a dictionary .


def parse_lines(text, ignore_invalid=False):
    """
    Parse a zonefile into a dict.
    @text must be flattened--each record must be on one line.
    Also, all comments must be removed.
    """
    json_zone_file = defaultdict(list)
    record_lines = text.split("\n")
    parser = make_parser()

    for record_line in record_lines:
        record_token = tokenize_line(record_line)
        try:
            json_zone_file = parse_line(parser, record_token, json_zone_file)
        except InvalidLineException:
            if ignore_invalid:
                continue
            else:
                raise

    return json_zone_file

def parse_lines ( text , ignore_invalid = False ) : json_zone_file = defaultdict ( list ) record_lines = text . split ( "\n" ) parser = make_parser ( ) for record_line in record_lines : record_token = tokenize_line ( record_line ) try : json_zone_file = parse_line ( parser , record_token , json_zone_file ) except InvalidLineException : if ignore_invalid : continue else : raise return json_zone_file


def parse_zone_file(text, ignore_invalid=False):
    """
    Parse a zonefile into a dict
    """
    text = remove_comments(text)
    text = flatten(text)
    text = remove_class(text)
    text = add_default_name(text)
    json_zone_file = parse_lines(text, ignore_invalid=ignore_invalid)
    return json_zone_file

def parse_zone_file ( text , ignore_invalid = False ) : text = remove_comments ( text ) text = flatten ( text ) text = remove_class ( text ) text = add_default_name ( text ) json_zone_file = parse_lines ( text , ignore_invalid = ignore_invalid ) return json_zone_file


def quote_field(data, field):
    """
    Quote a field in a list of DNS records.
    Return the new data records.
    """
    if data is None:
        return None

    data_dup = copy.deepcopy(data)
    for i in xrange(0, len(data_dup)):
        data_dup[i][field] = '"%s"' % data_dup[i][field]
        data_dup[i][field] = data_dup[i][field].replace(";", "\;")

    return data_dup

def quote_field ( data , field ) : if data is None : return None data_dup = copy . deepcopy ( data ) for i in xrange ( 0 , len ( data_dup ) ) : data_dup [ i ] [ field ] = '"%s"' % data_dup [ i ] [ field ] data_dup [ i ] [ field ] = data_dup [ i ] [ field ] . replace ( ";" , "\;" ) return data_dup

Quote a field in a list of DNS records . Return the new data records .


def parse_schema_string(schema_string):
    """
    Load and return a PySchema class from an avsc string
    """
    if isinstance(schema_string, str):
        schema_string = schema_string.decode("utf8")
    schema_struct = json.loads(schema_string)

    return AvroSchemaParser().parse_schema_struct(schema_struct)

def parse_schema_string ( schema_string ) : if isinstance ( schema_string , str ) : schema_string = schema_string . decode ( "utf8" ) schema_struct = json . loads ( schema_string ) return AvroSchemaParser ( ) . parse_schema_struct ( schema_struct )


def to_python_package(classes, target_folder, parent_package=None, indent=DEFAULT_INDENT):
    '''
    This function can be used to build a python package representation of pyschema classes.
    One module is created per namespace in a package matching the namespace hierarchy.

    Args:
        classes: A collection of classes to build the package from
        target_folder: Root folder of the package
        parent_package: Prepended on all import statements in order to support absolute imports.
            parent_package is not used when building the package file structure
        indent: Indent level. Defaults to 4 spaces
    '''
    PackageBuilder(target_folder, parent_package, indent).from_classes_with_refs(classes)

def to_python_package ( classes , target_folder , parent_package = None , indent = DEFAULT_INDENT ) : PackageBuilder ( target_folder , parent_package , indent ) . from_classes_with_refs ( classes )

This function can be used to build a python package representation of pyschema classes . One module is created per namespace in a package matching the namespace hierarchy .


def _class_source(schema, indent):
    """Generate Python source code for one specific class

    Doesn't include or take into account any dependencies between record types
    """

    def_pattern = (
        "class {class_name}(pyschema.Record):\n"
        "{indent}# WARNING: This class was generated by pyschema.to_python_source\n"
        "{indent}# there is a risk that any modification made to this class will be overwritten\n"
        "{optional_namespace_def}"
        "{field_defs}\n"
    )
    if hasattr(schema, '_namespace'):
        optional_namespace_def = "{indent}_namespace = {namespace!r}\n".format(
            namespace=schema._namespace, indent=indent)
    else:
        optional_namespace_def = ""

    field_defs = [
        "{indent}{field_name} = {field!r}".format(field_name=field_name, field=field, indent=indent)
        for field_name, field in schema._fields.iteritems()
    ]
    if not field_defs:
        field_defs = ["{indent}pass".format(indent=indent)]

    return def_pattern.format(
        class_name=schema._schema_name,
        optional_namespace_def=optional_namespace_def,
        field_defs="\n".join(field_defs),
        indent=indent
    )


def no_auto_store():
    """ Temporarily disable automatic registration of records in the auto_store

    Decorator factory. This is _NOT_ thread safe

    >>> @no_auto_store()
    ... class BarRecord(Record):
    ...     pass
    >>> BarRecord in auto_store
    False

    """
    original_auto_register_value = PySchema.auto_register
    disable_auto_register()

    def decorator(cls):
        PySchema.auto_register = original_auto_register_value
        return cls

    return decorator

def no_auto_store ( ) : original_auto_register_value = PySchema . auto_register disable_auto_register ( ) def decorator ( cls ) : PySchema . auto_register = original_auto_register_value return cls return decorator


def to_json_compatible(record):
    "Dump record in json-encodable object format"
    d = {}
    for fname, f in record._fields.iteritems():
        val = getattr(record, fname)
        if val is not None:
            d[fname] = f.dump(val)
    return d

Dump record in json - encodable object format


def load_json_dct(
        dct,
        record_store=None,
        schema=None,
        loader=from_json_compatible
):
    """ Create a Record instance from a json-compatible dictionary

    The dictionary values should have types that are json compatible,
    as if just loaded from a json serialized record string.

    :param dct:
        Python dictionary with key/value pairs for the record

    :param record_store:
        Record store to use for schema lookups (when $schema field is present)

    :param schema:
        PySchema Record class for the record to load.
        This will override any $schema fields specified in `dct`

    """
    if schema is None:
        if record_store is None:
            record_store = auto_store
        try:
            schema_name = dct.pop(SCHEMA_FIELD_NAME)
        except KeyError:
            raise ParseError((
                "Serialized record missing '{0}' "
                "record identifier and no schema supplied")
                .format(SCHEMA_FIELD_NAME)
            )
        try:
            schema = record_store.get(schema_name)
        except KeyError:
            raise ParseError(
                "Can't recognize record type %r"
                % (schema_name,), schema_name)

    # if schema is explicit, use that instead of SCHEMA_FIELD_NAME
    elif SCHEMA_FIELD_NAME in dct:
        dct.pop(SCHEMA_FIELD_NAME)

    record = loader(schema, dct)
    return record

Create a Record instance from a json - compatible dictionary


def loads(
        s,
        record_store=None,
        schema=None,
        loader=from_json_compatible,
        record_class=None  # deprecated in favor of schema
):
    """ Create a Record instance from a json serialized dictionary

    :param s:
        String with a json-serialized dictionary

    :param record_store:
        Record store to use for schema lookups (when $schema field is present)

    :param loader:
        Function called to fetch attributes from json. Typically shouldn't be used by end users

    :param schema:
        PySchema Record class for the record to load.
        This will override any $schema fields specified in `s`

    :param record_class:
        DEPRECATED option, old name for the `schema` parameter

    """
    if record_class is not None:
        warnings.warn(
            "The record_class parameter is deprecated in favour of schema",
            DeprecationWarning,
            stacklevel=2
        )
        schema = record_class
    if not isinstance(s, unicode):
        s = s.decode('utf8')
    if s.startswith(u"{"):
        json_dct = json.loads(s)
        return load_json_dct(json_dct, record_store, schema, loader)
    else:
        raise ParseError("Not a json record")


def add_record(self, schema, _bump_stack_level=False):
        """ Add record class to record store for retrieval at record load time.

            Can be used as a class decorator
        """
        full_name = get_full_name(schema)
        has_namespace = '.' in full_name
        self._force_add(full_name, schema, _bump_stack_level, _raise_on_existing=has_namespace)
        if has_namespace and schema.__name__ not in self._schema_map:
            self._force_add(schema.__name__, schema, _bump_stack_level)
        return schema

def add_record ( self , schema , _bump_stack_level = False ) : full_name = get_full_name ( schema ) has_namespace = '.' in full_name self . _force_add ( full_name , schema , _bump_stack_level , _raise_on_existing = has_namespace ) if has_namespace and schema . __name__ not in self . _schema_map : self . _force_add ( schema . __name__ , schema , _bump_stack_level ) return schema

Add record class to record store for retrieval at record load time .


def get(self, record_name):
        """
        Will return a matching record or raise KeyError is no record is found.

        If the record name is a full name we will first check for a record matching the full name.
        If no such record is found any record matching the last part of the full name (without the namespace) will
        be returned.
        """
        if record_name in self._schema_map:
            return self._schema_map[record_name]
        else:
            last_name = record_name.split('.')[-1]
            return self._schema_map[last_name]

def get ( self , record_name ) : if record_name in self . _schema_map : return self . _schema_map [ record_name ] else : last_name = record_name . split ( '.' ) [ - 1 ] return self . _schema_map [ last_name ]

Will return a matching record or raise KeyError is no record is found .


def repr_vars(self):
        """Return a dictionary the field definition

        Should contain all fields that are required for the definition of this field in a pyschema class"""
        d = OrderedDict()
        d["nullable"] = repr(self.nullable)
        d["default"] = repr(self.default)
        if self.description is not None:
            d["description"] = repr(self.description)
        return d

def repr_vars ( self ) : d = OrderedDict ( ) d [ "nullable" ] = repr ( self . nullable ) d [ "default" ] = repr ( self . default ) if self . description is not None : d [ "description" ] = repr ( self . description ) return d


def mixin(cls, mixin_cls):
        """Decorator for mixing in additional functionality into field type

        Example:

        >>> @Integer.mixin
        ... class IntegerPostgresExtensions:
        ...     postgres_type = 'INT'
        ...
        ...     def postgres_dump(self, obj):
        ...         self.dump(obj) + "::integer"

        Is roughly equivalent to:

        >>> Integer.postgres_type = 'INT'
        ...
        ... def postgres_dump(self, obj):
        ...     self.dump(obj) + "::integer"
        ...
        ... Integer.postgres_dump = postgres_dump

        """
        for item_name in dir(mixin_cls):
            if item_name.startswith("__"):
                # don't copy magic properties
                continue
            item = getattr(mixin_cls, item_name)

            if isinstance(item, types.MethodType):
                # unbound method will cause problems
                # so get the underlying function instead
                item = item.im_func

            setattr(cls, item_name, item)
        return mixin_cls

def mixin ( cls , mixin_cls ) : for item_name in dir ( mixin_cls ) : if item_name . startswith ( "__" ) : continue item = getattr ( mixin_cls , item_name ) if isinstance ( item , types . MethodType ) : item = item . im_func setattr ( cls , item_name , item ) return mixin_cls


def from_class(metacls, cls, auto_store=True):
        """Create proper PySchema class from cls

        Any methods and attributes will be transferred to the
        new object
        """
        if auto_store:
            def wrap(cls):
                return cls
        else:
            wrap = no_auto_store()

        return wrap(metacls.__new__(
            metacls,
            cls.__name__,
            (Record,),
            dict(cls.__dict__)
        ))

def from_class ( metacls , cls , auto_store = True ) : if auto_store : def wrap ( cls ) : return cls else : wrap = no_auto_store ( ) return wrap ( metacls . __new__ ( metacls , cls . __name__ , ( Record , ) , dict ( cls . __dict__ ) ) )


def get_schema_dict(record, state=None):
    """Return a python dict representing the jsonschema of a record

    Any references to sub-schemas will be URI fragments that won't be
    resolvable without a root schema, available from get_root_schema_dict.
    """
    state = state or SchemaGeneratorState()
    schema = OrderedDict([
        ('type', 'object'),
        ('id', record._schema_name),
    ])
    fields = dict()
    for field_name, field_type in record._fields.iteritems():
        fields[field_name] = field_type.jsonschema_type_schema(state)

    required = set(fields.keys())
    schema['properties'] = fields
    schema['required'] = sorted(list(required))
    schema['additionalProperties'] = False

    state.record_schemas[record._schema_name] = schema
    return schema

def get_schema_dict ( record , state = None ) : state = state or SchemaGeneratorState ( ) schema = OrderedDict ( [ ( 'type' , 'object' ) , ( 'id' , record . _schema_name ) , ] ) fields = dict ( ) for field_name , field_type in record . _fields . iteritems ( ) : fields [ field_name ] = field_type . jsonschema_type_schema ( state ) required = set ( fields . keys ( ) ) schema [ 'properties' ] = fields schema [ 'required' ] = sorted ( list ( required ) ) schema [ 'additionalProperties' ] = False state . record_schemas [ record . _schema_name ] = schema return schema


def get_root_schema_dict(record):
    """Return a root jsonschema for a given record

    A root schema includes the $schema attribute and all sub-record
    schemas and definitions.
    """
    state = SchemaGeneratorState()
    schema = get_schema_dict(record, state)
    del state.record_schemas[record._schema_name]
    if state.record_schemas:
        schema['definitions'] = dict()
        for name, sub_schema in state.record_schemas.iteritems():
            schema['definitions'][name] = sub_schema
    return schema

def get_root_schema_dict ( record ) : state = SchemaGeneratorState ( ) schema = get_schema_dict ( record , state ) del state . record_schemas [ record . _schema_name ] if state . record_schemas : schema [ 'definitions' ] = dict ( ) for name , sub_schema in state . record_schemas . iteritems ( ) : schema [ 'definitions' ] [ name ] = sub_schema return schema


def mr_reader(job, input_stream, loads=core.loads):
    """ Converts a file object with json serialised pyschema records
        to a stream of pyschema objects

    Can be used as job.reader in luigi.hadoop.JobTask
    """
    for line in input_stream:
        yield loads(line),

def mr_reader ( job , input_stream , loads = core . loads ) : for line in input_stream : yield loads ( line ) ,


def mr_writer(job, outputs, output_stream,
              stderr=sys.stderr, dumps=core.dumps):
    """ Writes a stream of json serialised pyschema Records to a file object

    Can be used as job.writer in luigi.hadoop.JobTask
    """
    for output in outputs:
        try:
            print >> output_stream, dumps(output)
        except core.ParseError, e:
            print >> stderr, e
            raise

def mr_writer ( job , outputs , output_stream , stderr = sys . stderr , dumps = core . dumps ) : for output in outputs : try : print >> output_stream , dumps ( output ) except core . ParseError , e : print >> stderr , e raise


def ordereddict_push_front(dct, key, value):
    """Set a value at the front of an OrderedDict

    The original dict isn't modified, instead a copy is returned
    """
    d = OrderedDict()
    d[key] = value
    d.update(dct)
    return d

def ordereddict_push_front ( dct , key , value ) : d = OrderedDict ( ) d [ key ] = value d . update ( dct ) return d


def query_string(self, **params):
        """Specify query string to use with the collection.

        Returns: :py:class:`SearchResult`
        """
        return SearchResult(self, self._api.get(self._href, **params))

def query_string ( self , ** params ) : return SearchResult ( self , self . _api . get ( self . _href , ** params ) )

Specify query string to use with the collection .


def raw_filter(self, filters):
        """Sends all filters to the API.

        No fancy, just a wrapper. Any advanced functionality shall be implemented as another method.

        Args:
            filters: List of filters (strings)

        Returns: :py:class:`SearchResult`
        """
        return SearchResult(self, self._api.get(self._href, **{"filter[]": filters}))

def raw_filter ( self , filters ) : return SearchResult ( self , self . _api . get ( self . _href , ** { "filter[]" : filters } ) )


def all_include_attributes(self, attributes):
        """Returns all entities present in the collection with ``attributes`` included."""
        self.reload(expand=True, attributes=attributes)
        entities = [Entity(self, r, attributes=attributes) for r in self._resources]
        self.reload()
        return entities

def all_include_attributes ( self , attributes ) : self . reload ( expand = True , attributes = attributes ) entities = [ Entity ( self , r , attributes = attributes ) for r in self . _resources ] self . reload ( ) return entities

Returns all entities present in the collection with attributes included .


def _get_entity_from_href(self, result):
        """Returns entity in correct collection.

        If the "href" value in result doesn't match the current collection,
        try to find the collection that the "href" refers to.
        """
        href_result = result['href']

        if self.collection._href.startswith(href_result):
            return Entity(self.collection, result, incomplete=True)

        href_match = re.match(r"(https?://.+/api[^?]*)/([a-z_-]+)", href_result)
        if not href_match:
            raise ValueError("Malformed href: {}".format(href_result))
        collection_name = href_match.group(2)
        entry_point = href_match.group(1)
        new_collection = Collection(
            self.collection.api,
            "{}/{}".format(entry_point, collection_name),
            collection_name
        )
        return Entity(new_collection, result, incomplete=True)

def
_get_entity_from_href
(
self
,
result
)
:
href_result
=
result
[
'href'
]
if
self
.
collection
.
_href
.
startswith
(
href_result
)
:
return
Entity
(
self
.
collection
,
result
,
incomplete
=
True
)
href_match
=
re
.
match
(
r"(https?://.+/api[^?]*)/([a-z_-]+)"
,
href_result
)
if
not
href_match
:
raise
ValueError
(
"Malformed href: {}"
.
format
(
href_result
)
)
collection_name
=
href_match
.
group
(
2
)
entry_point
=
href_match
.
group
(
1
)
new_collection
=
Collection
(
self
.
collection
.
api
,
"{}/{}"
.
format
(
entry_point
,
collection_name
)
,
collection_name
)
return
Entity
(
new_collection
,
result
,
incomplete
=
True
)


def give_another_quote(q):
    """When you pass a quote character, returns you an another one if possible"""
    for qc in QUOTES:
        if qc != q:
            return qc
    else:
        raise ValueError(u'Could not find a different quote for {}'.format(q))

When you pass a quote character returns you an another one if possible


def escape_filter(o):
    """Tries to escape the values that are passed to filter as correctly as possible.

    No standard way is followed, but at least it is simple.
    """
    if o is None:
        return u'NULL'
    if isinstance(o, int):
        return str(o)
    if not isinstance(o, six.string_types):
        raise ValueError('Filters take only None, int or a string type')
    if not o:
        # Empty string
        return u"''"
    # Now enforce unicode
    o = unicode_process(o)
    if u'"' not in o:
        # Simple case, just put the quote that does not exist in the string
        return u'"' + o + u'"'
    elif u"'" not in o:
        # Simple case, just put the quote that does not exist in the string
        return u"'" + o + u"'"
    else:
        # Both are there, so start guessing
        # Empty strings are sorted out, so the string must contain something.
        # String with length == 1 are sorted out because if they have a quote, they would be quoted
        # with the another quote in preceeding branch. Therefore the string is at least 2 chars long
        # here which allows us to NOT check the length here.
        first_char = o[0]
        last_char = o[-1]
        if first_char in QUOTES and last_char in QUOTES:
            # The first and last chars definitely are quotes
            if first_char == last_char:
                # Simple, just put another ones around them
                quote = give_another_quote(first_char)
                return quote + o + quote
            else:
                # I don't like this but the nature of the escape is like that ...
                # Since now it uses both of the quotes, just pick the simple ones and surround it
                return u"'" + o + u"'"
        elif first_char not in QUOTES and last_char not in QUOTES:
            # First and last chars are not quotes, so a simple solution
            return u"'" + o + u"'"
        else:
            # One of the first or last chars is not a quote
            if first_char in QUOTES:
                quote = give_another_quote(first_char)
            else:
                # last_char
                quote = give_another_quote(last_char)
            return quote + o + quote

Tries to escape the values that are passed to filter as correctly as possible .


def elementaryRotationMatrix(axis, rotationAngle):
  """
  Construct an elementary rotation matrix describing a rotation around the x, y, or z-axis.

  Parameters
  ----------

  axis          - Axis around which to rotate ("x", "y", or "z")
  rotationAngle - the rotation angle in radians

  Returns
  -------

  The rotation matrix

  Example usage
  -------------

  rotmat = elementaryRotationMatrix("y", pi/6.0)
  """
  if (axis=="x" or axis=="X"):
    return array([[1.0, 0.0, 0.0], [0.0, cos(rotationAngle), sin(rotationAngle)], [0.0,
      -sin(rotationAngle), cos(rotationAngle)]])
  elif (axis=="y" or axis=="Y"):
    return array([[cos(rotationAngle), 0.0, -sin(rotationAngle)], [0.0, 1.0, 0.0], [sin(rotationAngle),
      0.0, cos(rotationAngle)]])
  elif (axis=="z" or axis=="Z"):
    return array([[cos(rotationAngle), sin(rotationAngle), 0.0], [-sin(rotationAngle),
      cos(rotationAngle), 0.0], [0.0, 0.0, 1.0]])
  else:
    raise Exception("Unknown rotation axis "+axis+"!")

def
elementaryRotationMatrix
(
axis
,
rotationAngle
)
:
if
(
axis
==
"x"
or
axis
==
"X"
)
:
return
array
(
[
[
1.0
,
0.0
,
0.0
]
,
[
0.0
,
cos
(
rotationAngle
)
,
sin
(
rotationAngle
)
]
,
[
0.0
,
sin
(
rotationAngle
)
,
cos
(
rotationAngle
)
]
]
)
elif
(
axis
==
"y"
or
axis
==
"Y"
)
:
return
array
(
[
[
cos
(
rotationAngle
)
,
0.0
,
sin
(
rotationAngle
)
]
,
[
0.0
,
1.0
,
0.0
]
,
[
sin
(
rotationAngle
)
,
0.0
,
cos
(
rotationAngle
)
]
]
)
elif
(
axis
==
"z"
or
axis
==
"Z"
)
:
return
array
(
[
[
cos
(
rotationAngle
)
,
sin
(
rotationAngle
)
,
0.0
]
,
[
sin
(
rotationAngle
)
,
cos
(
rotationAngle
)
,
0.0
]
,
[
0.0
,
0.0
,
1.0
]
]
)
else
:
raise
Exception
(
"Unknown rotation axis "
+
axis
+
"!"
)

Construct an elementary rotation matrix describing a rotation around the x y or z - axis .


def construct_covariance_matrix(cvec, parallax, radial_velocity, radial_velocity_error):
    """
    Take the astrometric parameter standard uncertainties and the uncertainty correlations as quoted in
    the Gaia catalogue and construct the covariance matrix.

    Parameters
    ----------

    cvec : array_like
        Array of shape (15,) (1 source) or (n,15) (n sources) for the astrometric parameter standard
        uncertainties and their correlations, as listed in the Gaia catalogue [ra_error, dec_error,
        parallax_error, pmra_error, pmdec_error, ra_dec_corr, ra_parallax_corr, ra_pmra_corr,
        ra_pmdec_corr, dec_parallax_corr, dec_pmra_corr, dec_pmdec_corr, parallax_pmra_corr,
        parallax_pmdec_corr, pmra_pmdec_corr]. Units are (mas^2, mas^2/yr, mas^2/yr^2).

    parallax : array_like (n elements)
        Source parallax (mas).

    radial_velocity : array_like (n elements)
        Source radial velocity (km/s, does not have to be from Gaia RVS!). If the radial velocity is not
        known it can be set to zero.

    radial_velocity_error : array_like (n elements)
        Source radial velocity  uncertainty (km/s). If the radial velocity is not know this can be set to
        the radial velocity dispersion for the population the source was drawn from.

    Returns
    -------

    Covariance matrix as a 6x6 array.
    """

    if np.ndim(cvec)==1:
        cmat = np.zeros((1,6,6))
        nsources = 1
        cv = np.atleast_2d(cvec)
    else:
        nsources = cvec.shape[0]
        cmat = np.zeros((nsources,6,6))
        cv = cvec
    for k in range(nsources):
        cmat[k,0:5,0:5] = cv[k,0:5]**2

    iu = np.triu_indices(5,k=1)
    for k in range(10):
        i = iu[0][k]
        j = iu[1][k]
        cmat[:,i,j] = cv[:,i]*cv[:,j]*cv[:,k+5]
        cmat[:,j,i] = cmat[:,i,j]

    for k in range(nsources):
        cmat[k,0:5,5] = cmat[k,0:5,2]*np.atleast_1d(radial_velocity)[k]/auKmYearPerSec
    cmat[:,5,0:5] = cmat[:,0:5,5]
    cmat[:,5,5] = cmat[:,2,2]*(radial_velocity**2 + radial_velocity_error**2)/auKmYearPerSec**2 + \
            (parallax*radial_velocity_error/auKmYearPerSec)**2

    return np.squeeze(cmat)

def construct_covariance_matrix ( cvec , parallax , radial_velocity , radial_velocity_error ) : if np . ndim ( cvec ) == 1 : cmat = np . zeros ( ( 1 , 6 , 6 ) ) nsources = 1 cv = np . atleast_2d ( cvec ) else : nsources = cvec . shape [ 0 ] cmat = np . zeros ( ( nsources , 6 , 6 ) ) cv = cvec for k in range ( nsources ) : cmat [ k , 0 : 5 , 0 : 5 ] = cv [ k , 0 : 5 ] ** 2 iu = np . triu_indices ( 5 , k = 1 ) for k in range ( 10 ) : i = iu [ 0 ] [ k ] j = iu [ 1 ] [ k ] cmat [ : , i , j ] = cv [ : , i ] * cv [ : , j ] * cv [ : , k + 5 ] cmat [ : , j , i ] = cmat [ : , i , j ] for k in range ( nsources ) : cmat [ k , 0 : 5 , 5 ] = cmat [ k , 0 : 5 , 2 ] * np . atleast_1d ( radial_velocity ) [ k ] / auKmYearPerSec cmat [ : , 5 , 0 : 5 ] = cmat [ : , 0 : 5 , 5 ] cmat [ : , 5 , 5 ] = cmat [ : , 2 , 2 ] * ( radial_velocity ** 2 + radial_velocity_error ** 2 ) / auKmYearPerSec ** 2 + ( parallax * radial_velocity_error / auKmYearPerSec ) ** 2 return np . squeeze ( cmat )

Take the astrometric parameter standard uncertainties and the uncertainty correlations as quoted in the Gaia catalogue and construct the covariance matrix .


def vradErrorSkyAvg(vmag, spt):
  """
  Calculate radial velocity error from V and the spectral type. The value of the error is an average over
  the sky.

  Parameters
  ----------

  vmag - Value of V-band magnitude.
  spt  - String representing the spectral type of the star.

  Returns
  -------

  The radial velocity error in km/s.
  """
  return _vradCalibrationFloor + _vradErrorBCoeff[spt]*exp(_vradErrorACoeff[spt]*(vmag-_vradMagnitudeZeroPoint))

def vradErrorSkyAvg ( vmag , spt ) : return _vradCalibrationFloor + _vradErrorBCoeff [ spt ] * exp ( _vradErrorACoeff [ spt ] * ( vmag - _vradMagnitudeZeroPoint ) )

Calculate radial velocity error from V and the spectral type . The value of the error is an average over the sky .


def calcParallaxError(args):
  """
  Calculate the parallax error for the given input source magnitude and colour.

  :argument args: command line arguments
  """
  gmag=float(args['gmag'])
  vmini=float(args['vmini'])
  sigmaPar=parallaxErrorSkyAvg(gmag, vmini)
  gminv=gminvFromVmini(vmini)
  print("G = {0}".format(gmag))
  print("V = {0}".format(gmag-gminv))
  print("(V-I) = {0}".format(vmini))
  print("(G-V) = {0}".format(gminv))
  print("standard error = {0} muas".format(sigmaPar))

def
calcParallaxError
(
args
)
:
gmag
=
float
(
args
[
'gmag'
]
)
vmini
=
float
(
args
[
'vmini'
]
)
sigmaPar
=
parallaxErrorSkyAvg
(
gmag
,
vmini
)
gminv
=
gminvFromVmini
(
vmini
)
print
(
"G = {0}"
.
format
(
gmag
)
)
print
(
"V = {0}"
.
format
(
gmag
gminv
)
)
print
(
"(V-I) = {0}"
.
format
(
vmini
)
)
print
(
"(G-V) = {0}"
.
format
(
gminv
)
)
print
(
"standard error = {0} muas"
.
format
(
sigmaPar
)
)

Calculate the parallax error for the given input source magnitude and colour .


def gMagnitudeError(G):
  """
  Calculate the single-field-of-view-transit photometric standard error in the G band as a function
  of G. A 20% margin is included.

  Parameters
  ----------

  G     - Value(s) of G-band magnitude.

  Returns
  -------

  The G band photometric standard error in units of magnitude.
  """
  z=calcZ(G)
  return 1.0e-3*sqrt(0.04895*z*z + 1.8633*z + 0.0001985) * _scienceMargin

def gMagnitudeError ( G ) : z = calcZ ( G ) return 1.0e-3 * sqrt ( 0.04895 * z * z + 1.8633 * z + 0.0001985 ) * _scienceMargin

Calculate the single - field - of - view - transit photometric standard error in the G band as a function of G . A 20% margin is included .


def gMagnitudeErrorEoM(G, nobs=70):
  """
  Calculate the end of mission photometric standard error in the G band as a function
  of G. A 20% margin is included.

  Parameters
  ----------

  G     - Value(s) of G-band magnitude.

  Keywords
  --------

  nobs  - Number of observations collected (default 70).

  Returns
  -------

  The G band photometric standard error in units of magnitude.
  """
  return sqrt( (power(gMagnitudeError(G)/_scienceMargin,2) +
      _eomCalibrationFloorG*_eomCalibrationFloorG)/nobs ) * _scienceMargin

def gMagnitudeErrorEoM ( G , nobs = 70 ) : return sqrt ( ( power ( gMagnitudeError ( G ) / _scienceMargin , 2 ) + _eomCalibrationFloorG * _eomCalibrationFloorG ) / nobs ) * _scienceMargin

Calculate the end of mission photometric standard error in the G band as a function of G . A 20% margin is included .


def makePlot(args):
  """
  Make the plot with photometry performance predictions.

  :argument args: command line arguments
  """
  gmag=np.linspace(3.0,20.0,171)

  vmini = args['vmini']

  vmag=gmag-gminvFromVmini(vmini)

  if args['eom']:
      sigmaG = gMagnitudeErrorEoM(gmag)
      sigmaGBp = bpMagnitudeErrorEoM(gmag, vmini)
      sigmaGRp = rpMagnitudeErrorEoM(gmag, vmini)
      yminmax = (1.0-4,0.1)
  else:
      sigmaG = gMagnitudeError(gmag)
      sigmaGBp = bpMagnitudeError(gmag, vmini)
      sigmaGRp = rpMagnitudeError(gmag, vmini)
      yminmax = (1.0-4,1)

  fig=plt.figure(figsize=(10,6.5))

  if (args['vmagAbscissa']):
    plt.semilogy(vmag, sigmaG, 'k', label='$\\sigma_G$')
    plt.semilogy(vmag, sigmaGBp, 'b', label='$\\sigma_{G_\\mathrm{BP}}$'+' for $(V-I)={0}$'.format(vmini))
    plt.semilogy(vmag, sigmaGRp, 'r', label='$\\sigma_{G_\\mathrm{RP}}$'+' for $(V-I)={0}$'.format(vmini))
    plt.xlim((6,20))
    #plt.ylim(yminmax)
    plt.legend(loc=0)
    plt.xlabel('$V$ [mag]')
  else:
    ax=fig.add_subplot(111)
    plt.semilogy(gmag, sigmaG, 'k', label='$\\sigma_G$')
    plt.semilogy(gmag, sigmaGBp, 'b', label='$\\sigma_{G_\\mathrm{BP}}$'+' for $(V-I)={0}$'.format(vmini))
    plt.semilogy(gmag, sigmaGRp, 'r', label='$\\sigma_{G_\\mathrm{RP}}$'+' for $(V-I)={0}$'.format(vmini))
    plt.xlim((6,20))
    #plt.ylim(yminmax)
    plt.legend(loc=0)
    plt.xlabel('$G$ [mag]')

  plt.xticks(np.arange(6,20,2))
  ax = plt.gca().yaxis
  #ax.set_major_formatter(matplotlib.ticker.ScalarFormatter())
  #plt.ticklabel_format(axis='y',style='plain')
  plt.grid(which='both')
  plt.ylabel('Photometric error [mag]')
  if args['eom']:
      plt.title('End-of-mission mean photometry: sky averaged errors for $(V-I)={0}$'.format(vmini), fontsize=14)
  else:
      plt.title('Single-FoV-transit photometry: sky averaged errors for $(V-I)={0}$'.format(vmini), fontsize=14)

  basename = 'PhotometricErrors'
  if (args['pdfOutput']):
    plt.savefig(basename+'.pdf')
  elif (args['pngOutput']):
    plt.savefig(basename+'.png')
  else:
    plt.show()

Make the plot with photometry performance predictions .


def averageNumberOfTransits(beta):
  """
  Returns the number of transits across the Gaia focal plane averaged over ecliptic longitude.

  Parameters
  ----------

  beta - Value(s) of the Ecliptic latitude.

  Returns
  -------

  Average number of transits for the input values of beta.
  """
  indices = array(floor(abs(sin(beta))*_numStepsSinBeta), dtype=int)
  indices[(indices==_numStepsSinBeta)] = _numStepsSinBeta-1
  return _averageTransitNumber[indices]

def averageNumberOfTransits ( beta ) : indices = array ( floor ( abs ( sin ( beta ) ) * _numStepsSinBeta ) , dtype = int ) indices [ ( indices == _numStepsSinBeta ) ] = _numStepsSinBeta - 1 return _averageTransitNumber [ indices ]

Returns the number of transits across the Gaia focal plane averaged over ecliptic longitude .


def angularDistance(phi1, theta1, phi2, theta2):
    """
    Calculate the angular distance between pairs of sky coordinates.

    Parameters
    ----------

    phi1 : float
        Longitude of first coordinate (radians).
    theta1 : float
        Latitude of first coordinate (radians).
    phi2 : float
        Longitude of second coordinate (radians).
    theta2 : float
        Latitude of second coordinate (radians).

    Returns
    -------

    Angular distance in radians.
    """
    # Formula below is more numerically stable than arccos( sin(theta1)*sin(theta2) +
    # cos(phi2-phi1)*cos(theta1)*cos(theta2) )
    # See: https://en.wikipedia.org/wiki/Great-circle_distance
    return arctan( sqrt((cos(theta2)*sin(phi2-phi1))**2 +
        (cos(theta1)*sin(theta2)-sin(theta1)*cos(theta2)*cos(phi2-phi1))**2) / (sin(theta1)*sin(theta2) +
            cos(phi2-phi1)*cos(theta1)*cos(theta2)) )

def angularDistance ( phi1 , theta1 , phi2 , theta2 ) : return arctan ( sqrt ( ( cos ( theta2 ) * sin ( phi2 - phi1 ) ) ** 2 + ( cos ( theta1 ) * sin ( theta2 ) - sin ( theta1 ) * cos ( theta2 ) * cos ( phi2 - phi1 ) ) ** 2 ) / ( sin ( theta1 ) * sin ( theta2 ) + cos ( phi2 - phi1 ) * cos ( theta1 ) * cos ( theta2 ) ) )

Calculate the angular distance between pairs of sky coordinates .


def transformCartesianCoordinates(self, x, y, z):
        """
        Rotates Cartesian coordinates from one reference system to another using the rotation matrix with
        which the class was initialized. The inputs  can be scalars or 1-dimensional numpy arrays.

        Parameters
        ----------

        x - Value of X-coordinate in original reference system
        y - Value of Y-coordinate in original reference system
        z - Value of Z-coordinate in original reference system

        Returns
        -------

        xrot - Value of X-coordinate after rotation
        yrot - Value of Y-coordinate after rotation
        zrot - Value of Z-coordinate after rotation
        """
        xrot, yrot, zrot = dot(self.rotationMatrix,[x,y,z])
        return xrot, yrot, zrot

def transformCartesianCoordinates ( self , x , y , z ) : xrot , yrot , zrot = dot ( self . rotationMatrix , [ x , y , z ] ) return xrot , yrot , zrot

Rotates Cartesian coordinates from one reference system to another using the rotation matrix with which the class was initialized . The inputs can be scalars or 1 - dimensional numpy arrays .


def transformSkyCoordinates(self, phi, theta):
        """
        Converts sky coordinates from one reference system to another, making use of the rotation matrix with
        which the class was initialized. Inputs can be scalars or 1-dimensional numpy arrays.

        Parameters
        ----------

        phi   - Value of the azimuthal angle (right ascension, longitude) in radians.
        theta - Value of the elevation angle (declination, latitude) in radians.

        Returns
        -------

        phirot   - Value of the transformed azimuthal angle in radians.
        thetarot - Value of the transformed elevation angle in radians.
        """
        r=ones_like(phi)
        x, y, z = sphericalToCartesian(r, phi, theta)
        xrot, yrot, zrot = self.transformCartesianCoordinates(x, y, z)
        r, phirot, thetarot = cartesianToSpherical(xrot, yrot, zrot)
        return phirot, thetarot

def transformSkyCoordinates ( self , phi , theta ) : r = ones_like ( phi ) x , y , z = sphericalToCartesian ( r , phi , theta ) xrot , yrot , zrot = self . transformCartesianCoordinates ( x , y , z ) r , phirot , thetarot = cartesianToSpherical ( xrot , yrot , zrot ) return phirot , thetarot

Converts sky coordinates from one reference system to another making use of the rotation matrix with which the class was initialized . Inputs can be scalars or 1 - dimensional numpy arrays .


def transformCovarianceMatrix(self, phi, theta, covmat):
        """
        Transform the astrometric covariance matrix to its representation in the new coordinate system.

        Parameters
        ----------

        phi       - The longitude-like angle of the position of the source (radians).
        theta     - The latitude-like angle of the position of the source (radians).
        covmat    - Covariance matrix (5x5) of the astrometric parameters.

        Returns
        -------

        covmat_rot - Covariance matrix in its representation in the new coordinate system.
        """

        c, s = self._getJacobian(phi,theta)
        jacobian = identity(5)
        jacobian[0][0]=c
        jacobian[1][1]=c
        jacobian[3][3]=c
        jacobian[4][4]=c
        jacobian[0][1]=s
        jacobian[1][0]=-s
        jacobian[3][4]=s
        jacobian[4][3]=-s

        return dot( dot(jacobian, covmat), jacobian.T )

def transformCovarianceMatrix ( self , phi , theta , covmat ) : c , s = self . _getJacobian ( phi , theta ) jacobian = identity ( 5 ) jacobian [ 0 ] [ 0 ] = c jacobian [ 1 ] [ 1 ] = c jacobian [ 3 ] [ 3 ] = c jacobian [ 4 ] [ 4 ] = c jacobian [ 0 ] [ 1 ] = s jacobian [ 1 ] [ 0 ] = - s jacobian [ 3 ] [ 4 ] = s jacobian [ 4 ] [ 3 ] = - s return dot ( dot ( jacobian , covmat ) , jacobian . T )

Transform the astrometric covariance matrix to its representation in the new coordinate system .


def errorScalingFactor(observable, beta):
  """
  Look up the numerical factors to apply to the sky averaged parallax error in order to obtain error
  values for a given astrometric parameter, taking the Ecliptic latitude and the number of transits into
  account.

  Parameters
  ----------

  observable - Name of astrometric observable (one of: alphaStar, delta, parallax, muAlphaStar, muDelta)
  beta       - Values(s) of the Ecliptic latitude.

  Returns
  -------

  Numerical factors to apply to the errors of the given observable.
  """
  if isscalar(beta):
    index=int(floor(abs(sin(beta))*_numStepsSinBeta))
    if index == _numStepsSinBeta:
      return _astrometricErrorFactors[observable][_numStepsSinBeta-1]
    else:
      return _astrometricErrorFactors[observable][index]
  else:
    indices = array(floor(abs(sin(beta))*_numStepsSinBeta), dtype=int)
    indices[(indices==_numStepsSinBeta)] = _numStepsSinBeta-1
    return _astrometricErrorFactors[observable][indices]

def errorScalingFactor ( observable , beta ) : if isscalar ( beta ) : index = int ( floor ( abs ( sin ( beta ) ) * _numStepsSinBeta ) ) if index == _numStepsSinBeta : return _astrometricErrorFactors [ observable ] [ _numStepsSinBeta - 1 ] else : return _astrometricErrorFactors [ observable ] [ index ] else : indices = array ( floor ( abs ( sin ( beta ) ) * _numStepsSinBeta ) , dtype = int ) indices [ ( indices == _numStepsSinBeta ) ] = _numStepsSinBeta - 1 return _astrometricErrorFactors [ observable ] [ indices ]

Look up the numerical factors to apply to the sky averaged parallax error in order to obtain error values for a given astrometric parameter taking the Ecliptic latitude and the number of transits into account .


def makePlot(pdf=False, png=False):
  """
  Plot relative parallax errors as a function of distance for stars of a given spectral type.

  Parameters
  ----------

  args - command line arguments
  """
  logdistancekpc = np.linspace(-1,np.log10(20.0),100)
  sptVabsAndVmini=OrderedDict([('K0V',(5.58,0.87)), ('G5V',(4.78,0.74)), ('G0V',(4.24,0.67)),
    ('F5V',(3.50,0.50)), ('F0V',(2.98,0.38)), ('RC',(0.8,1.0))])
  lines={}

  fig=plt.figure(figsize=(10,6.5))
  currentAxis=plt.gca()

  for spt in sptVabsAndVmini.keys():
    vmag=sptVabsAndVmini[spt][0]+5.0*logdistancekpc+10.0
    indices=(vmag>14) & (vmag<16)
    gmag=vmag+gminvFromVmini(sptVabsAndVmini[spt][1])
    parerrors=parallaxErrorSkyAvg(gmag,sptVabsAndVmini[spt][1])
    relparerrors=parerrors*10**logdistancekpc/1000.0
    plt.loglog(10**logdistancekpc, relparerrors,'--k',lw=1)
    plt.loglog(10**logdistancekpc[indices], relparerrors[indices],'-',label=spt)
  plt.xlim(0.1,20.0)
  plt.ylim(0.001,0.5)
  plt.text(0.9, 0.05,'Colours indicate $14<V<16$',
     horizontalalignment='right',
     verticalalignment='bottom',
     transform = currentAxis.transAxes)
  plt.legend(loc=2)
  plt.xlabel('distance [kpc]')
  plt.ylabel('$\\sigma_\\varpi/\\varpi$')
  plt.grid(which='both')

  if (args['pdfOutput']):
    plt.savefig('RelativeParallaxErrorsVsDist.pdf')
  elif (args['pngOutput']):
    plt.savefig('RelativeParallaxErrorsVsDist.png')
  else:
    plt.show()

def
makePlot
(
pdf
=
False
,
png
=
False
)
:
logdistancekpc
=
np
.
linspace
(
1
,
np
.
log10
(
20.0
)
,
100
)
sptVabsAndVmini
=
OrderedDict
(
[
(
'K0V'
,
(
5.58
,
0.87
)
)
,
(
'G5V'
,
(
4.78
,
0.74
)
)
,
(
'G0V'
,
(
4.24
,
0.67
)
)
,
(
'F5V'
,
(
3.50
,
0.50
)
)
,
(
'F0V'
,
(
2.98
,
0.38
)
)
,
(
'RC'
,
(
0.8
,
1.0
)
)
]
)
lines
=
{
}
fig
=
plt
.
figure
(
figsize
=
(
10
,
6.5
)
)
currentAxis
=
plt
.
gca
(
)
for
spt
in
sptVabsAndVmini
.
keys
(
)
:
vmag
=
sptVabsAndVmini
[
spt
]
[
0
]
+
5.0
*
logdistancekpc
+
10.0
indices
=
(
vmag
>
14
)
&
(
vmag
<
16
)
gmag
=
vmag
+
gminvFromVmini
(
sptVabsAndVmini
[
spt
]
[
1
]
)
parerrors
=
parallaxErrorSkyAvg
(
gmag
,
sptVabsAndVmini
[
spt
]
[
1
]
)
relparerrors
=
parerrors
*
10
**
logdistancekpc
/
1000.0
plt
.
loglog
(
10
**
logdistancekpc
,
relparerrors
,
'--k'
,
lw
=
1
)
plt
.
loglog
(
10
**
logdistancekpc
[
indices
]
,
relparerrors
[
indices
]
,
'-'
,
label
=
spt
)
plt
.
xlim
(
0.1
,
20.0
)
plt
.
ylim
(
0.001
,
0.5
)
plt
.
text
(
0.9
,
0.05
,
'Colours indicate $14<V<16$'
,
horizontalalignment
=
'right'
,
verticalalignment
=
'bottom'
,
transform
=
currentAxis
.
transAxes
)
plt
.
legend
(
loc
=
2
)
plt
.
xlabel
(
'distance [kpc]'
)
plt
.
ylabel
(
'$\\sigma_\\varpi/\\varpi$'
)
plt
.
grid
(
which
=
'both'
)
if
(
args
[
'pdfOutput'
]
)
:
plt
.
savefig
(
'RelativeParallaxErrorsVsDist.pdf'
)
elif
(
args
[
'pngOutput'
]
)
:
plt
.
savefig
(
'RelativeParallaxErrorsVsDist.png'
)
else
:
plt
.
show
(
)

Plot relative parallax errors as a function of distance for stars of a given spectral type .


def makePlot(args):
  """
  Make the plot with radial velocity performance predictions.

  :argument args: command line arguments
  """
  gRvs=np.linspace(5.7,16.1,101)

  spts=['B0V', 'B5V', 'A0V', 'A5V', 'F0V', 'G0V',
        'G5V', 'K0V', 'K1IIIMP', 'K4V', 'K1III']

  fig=plt.figure(figsize=(10,6.5))
  deltaHue = 240.0/(len(spts)-1)
  hsv=np.zeros((1,1,3))
  hsv[0,0,1]=1.0
  hsv[0,0,2]=0.9
  count=0
  for spt in spts:
    hsv[0,0,0]=(240-count*deltaHue)/360.0
    vmag = vminGrvsFromVmini(vminiFromSpt(spt)) + gRvs
    vradErrors = vradErrorSkyAvg(vmag, spt)
    plt.plot(vmag, vradErrors, '-', label=spt, color=hsv_to_rgb(hsv)[0,0,:])
    count+=1
  plt.grid(which='both')
  plt.xlim(9,17.5)
  plt.ylim(0,20)
  plt.xticks(np.arange(9,18,1))
  plt.yticks(np.arange(0,20.5,5))
  plt.xlabel('$V$ [mag]')
  plt.ylabel('End-of-mission radial velocity error [km s$^{-1}$]')
  leg=plt.legend(loc=0,  handlelength=2.0, labelspacing=0.10)
  for t in leg.get_texts():
    t.set_fontsize(12)

  if (args['pdfOutput']):
    plt.savefig('RadialVelocityErrors.pdf')
  elif (args['pngOutput']):
    plt.savefig('RadialVelocityErrors.png')
  else:
    plt.show()

Make the plot with radial velocity performance predictions .


def either(*funcs):
    """
    A utility function for selecting the first non-null query.

    Parameters:

      funcs: One or more functions

    Returns:

       A function that, when called with a :class:`Node`, will
       pass the input to each `func`, and return the first non-Falsey
       result.

    Examples:

       >>> s = Soupy("
hi
")
       >>> s.apply(either(Q.find('a'), Q.find('p').text))
       Scalar('hi')
    """
    def either(val):
        for func in funcs:
            result = val.apply(func)
            if result:
                return result
        return Null()

    return either

def either ( * funcs ) : def either ( val ) : for func in funcs : result = val . apply ( func ) if result : return result return Null ( ) return either

A utility function for selecting the first non - null query .


def _helpful_failure(method):
    """
    Decorator for eval_ that prints a helpful error message
    if an exception is generated in a Q expression
    """

    @wraps(method)
    def wrapper(self, val):
        try:
            return method(self, val)
        except:
            exc_cls, inst, tb = sys.exc_info()

            if hasattr(inst, '_RERAISE'):
                _, expr, _, inner_val = Q.__debug_info__
                Q.__debug_info__ = QDebug(self, expr, val, inner_val)
                raise

            if issubclass(exc_cls, KeyError):  # Overrides formatting
                exc_cls = QKeyError

            # Show val, unless it's too long
            prettyval = repr(val)
            if len(prettyval) > 150:
                prettyval = "<%s instance>" % (type(val).__name__)

            msg = "{0}\n\n\tEncountered when evaluating {1}{2}".format(
                inst, prettyval, self)

            new_exc = exc_cls(msg)
            new_exc._RERAISE = True
            Q.__debug_info__ = QDebug(self, self, val, val)

            six.reraise(exc_cls, new_exc, tb)

    return wrapper


def _uniquote(value):
    """
    Convert to unicode, and add quotes if initially a string
    """
    if isinstance(value, six.binary_type):
        try:
            value = value.decode('utf-8')
        except UnicodeDecodeError:  # Not utf-8. Show the repr
            value = six.text_type(_dequote(repr(value)))  # trim quotes

    result = six.text_type(value)

    if isinstance(value, six.text_type):
        result = "'%s'" % result
    return result

def _uniquote ( value ) : if isinstance ( value , six . binary_type ) : try : value = value . decode ( 'utf-8' ) except UnicodeDecodeError : value = six . text_type ( _dequote ( repr ( value ) ) ) result = six . text_type ( value ) if isinstance ( value , six . text_type ) : result = "'%s'" % result return result

Convert to unicode and add quotes if initially a string


def each(self, *funcs):
        """
        Call `func` on each element in the collection.

        If multiple functions are provided, each item
        in the output will be a tuple of each
        func(item) in self.

        Returns a new Collection.

        Example:

            >>> col = Collection([Scalar(1), Scalar(2)])
            >>> col.each(Q * 10)
            Collection([Scalar(10), Scalar(20)])
            >>> col.each(Q * 10, Q - 1)
            Collection([Scalar((10, 0)), Scalar((20, 1))])
        """

        funcs = list(map(_make_callable, funcs))

        if len(funcs) == 1:
            return Collection(map(funcs[0], self._items))

        tupler = lambda item: Scalar(
            tuple(_unwrap(func(item)) for func in funcs))
        return Collection(map(tupler, self._items))

def each ( self , * funcs ) : funcs = list ( map ( _make_callable , funcs ) ) if len ( funcs ) == 1 : return Collection ( map ( funcs [ 0 ] , self . _items ) ) tupler = lambda item : Scalar ( tuple ( _unwrap ( func ( item ) ) for func in funcs ) ) return Collection ( map ( tupler , self . _items ) )

Call func on each element in the collection .


def exclude(self, func=None):
        """
        Return a new Collection excluding some items

        Parameters:

            func : function(Node) -> Scalar

                A function that, when called on each item
                in the collection, returns a boolean-like
                value. If no function is provided, then
                truthy items will be removed.

        Returns:

            A new Collection consisting of the items
            where bool(func(item)) == False
        """
        func = _make_callable(func)
        inverse = lambda x: not func(x)
        return self.filter(inverse)

def exclude ( self , func = None ) : func = _make_callable ( func ) inverse = lambda x : not func ( x ) return self . filter ( inverse )


def filter(self, func=None):
        """
        Return a new Collection with some items removed.

        Parameters:

            func : function(Node) -> Scalar

                A function that, when called on each item
                in the collection, returns a boolean-like
                value. If no function is provided, then
                false-y items will be removed.

        Returns:

            A new Collection consisting of the items
            where bool(func(item)) == True

        Examples:

            node.find_all('a').filter(Q['href'].startswith('http'))
        """
        func = _make_callable(func)
        return Collection(filter(func, self._items))

def filter ( self , func = None ) : func = _make_callable ( func ) return Collection ( filter ( func , self . _items ) )

Return a new Collection with some items removed .


def takewhile(self, func=None):
        """
        Return a new Collection with the last few items removed.

        Parameters:

            func : function(Node) -> Node

        Returns:

            A new Collection, discarding all items
            at and after the first item where bool(func(item)) == False

        Examples:

            node.find_all('tr').takewhile(Q.find_all('td').count() > 3)

        """
        func = _make_callable(func)
        return Collection(takewhile(func, self._items))

def takewhile ( self , func = None ) : func = _make_callable ( func ) return Collection ( takewhile ( func , self . _items ) )

Return a new Collection with the last few items removed .


def dropwhile(self, func=None):
        """
        Return a new Collection with the first few items removed.

        Parameters:

            func : function(Node) -> Node

        Returns:

            A new Collection, discarding all items
            before the first item where bool(func(item)) == True
        """
        func = _make_callable(func)
        return Collection(dropwhile(func, self._items))

def dropwhile ( self , func = None ) : func = _make_callable ( func ) return Collection ( dropwhile ( func , self . _items ) )

Return a new Collection with the first few items removed .


def zip(self, *others):
        """
        Zip the items of this collection with one or more
        other sequences, and wrap the result.

        Unlike Python's zip, all sequences must be the same length.

        Parameters:

            others: One or more iterables or Collections

        Returns:

            A new collection.

        Examples:

            >>> c1 = Collection([Scalar(1), Scalar(2)])
            >>> c2 = Collection([Scalar(3), Scalar(4)])
            >>> c1.zip(c2).val()
            [(1, 3), (2, 4)]
        """
        args = [_unwrap(item) for item in (self,) + others]
        ct = self.count()
        if not all(len(arg) == ct for arg in args):
            raise ValueError("Arguments are not all the same length")
        return Collection(map(Wrapper.wrap, zip(*args)))

def
zip
(
self
,
*
others
)
:
args
=
[
_unwrap
(
item
)
for
item
in
(
self
,
)
+
others
]
ct
=
self
.
count
(
)
if
not
all
(
len
(
arg
)
==
ct
for
arg
in
args
)
:
raise
ValueError
(
"Arguments are not all the same length"
)
return
Collection
(
map
(
Wrapper
.
wrap
,
zip
(
*
args
)
)
)

Zip the items of this collection with one or more other sequences and wrap the result .


def find(self, *args, **kwargs):
        """
        Find a single Node among this Node's descendants.

        Returns :class:`NullNode` if nothing matches.

        This inputs to this function follow the same semantics
        as BeautifulSoup. See http://bit.ly/bs4doc for more info.

        Examples:

         - node.find('a')  # look for `a` tags
         - node.find('a', 'foo') # look for `a` tags with class=`foo`
         - node.find(func) # find tag where func(tag) is True
         - node.find(val=3)  # look for tag like <a, val=3>
        """
        op = operator.methodcaller('find', *args, **kwargs)
        return self._wrap_node(op)

def find ( self , * args , ** kwargs ) : op = operator . methodcaller ( 'find' , * args , ** kwargs ) return self . _wrap_node ( op )

Find a single Node among this Node s descendants .


def serach_path():
    """Return potential locations of IACA installation."""
    operating_system = get_os()
    # 1st choice: in ~/.kerncraft/iaca-{}
    # 2nd choice: in package directory / iaca-{}
    return [os.path.expanduser("~/.kerncraft/iaca/{}/".format(operating_system)),
            os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/iaca/{}/'.format(
                operating_system)]

def serach_path ( ) : operating_system = get_os ( ) return [ os . path . expanduser ( "~/.kerncraft/iaca/{}/" . format ( operating_system ) ) , os . path . abspath ( os . path . dirname ( os . path . realpath ( __file__ ) ) ) + '/iaca/{}/' . format ( operating_system ) ]

Return potential locations of IACA installation .


def group_iterator(group):
    """
    Yild all groups of simple regex-like expression.

    The only special character is a dash (-), which take the preceding and the following chars to
    compute a range. If the range is non-sensical (e.g., b-a) it will be empty

    Example:
    >>> list(group_iterator('a-f'))
    ['a', 'b', 'c', 'd', 'e', 'f']
    >>> list(group_iterator('148'))
    ['1', '4', '8']
    >>> list(group_iterator('7-9ab'))
    ['7', '8', '9', 'a', 'b']
    >>> list(group_iterator('0B-A1'))
    ['0', '1']
    """
    ordered_chars = string.ascii_letters + string.digits
    tokenizer = ('(?P<seq>[a-zA-Z0-9]-[a-zA-Z0-9])|'
                 '(?P<chr>.)')
    for m in re.finditer(tokenizer, group):
        if m.group('seq'):
            start, sep, end = m.group('seq')
            for i in range(ordered_chars.index(start), ordered_chars.index(end)+1):
                yield ordered_chars[i]
        else:
            yield m.group('chr')

def group_iterator ( group ) : ordered_chars = string . ascii_letters + string . digits tokenizer = ( '(?P<seq>[a-zA-Z0-9]-[a-zA-Z0-9])|' '(?P<chr>.)' ) for m in re . finditer ( tokenizer , group ) : if m . group ( 'seq' ) : start , sep , end = m . group ( 'seq' ) for i in range ( ordered_chars . index ( start ) , ordered_chars . index ( end ) + 1 ) : yield ordered_chars [ i ] else : yield m . group ( 'chr' )

Yild all groups of simple regex - like expression .


def register_options(regdescr):
    """
    Very reduced regular expressions for describing a group of registers.

    Only groups in square bracktes and unions with pipes (|) are supported.

    Examples:
    >>> list(register_options('PMC[0-3]'))
    ['PMC0', 'PMC1', 'PMC2', 'PMC3']
    >>> list(register_options('MBOX0C[01]'))
    ['MBOX0C0', 'MBOX0C1']
    >>> list(register_options('CBOX2C1'))
    ['CBOX2C1']
    >>> list(register_options('CBOX[0-3]C[01]'))
    ['CBOX0C0', 'CBOX0C1', 'CBOX1C0', 'CBOX1C1', 'CBOX2C0', 'CBOX2C1', 'CBOX3C0', 'CBOX3C1']
    >>> list(register_options('PMC[0-1]|PMC[23]'))
    ['PMC0', 'PMC1', 'PMC2', 'PMC3']

    """
    if not regdescr:
        yield None
    tokenizer = ('\[(?P<grp>[^]]+)\]|'
                 '(?P<chr>.)')
    for u in regdescr.split('|'):
        m = re.match(tokenizer, u)

        if m.group('grp'):
            current = group_iterator(m.group('grp'))
        else:
            current = [m.group('chr')]

        for c in current:
            if u[m.end():]:
                for r in register_options(u[m.end():]):
                    yield c + r
            else:
                yield c

def register_options ( regdescr ) : if not regdescr : yield None tokenizer = ( '\[(?P<grp>[^]]+)\]|' '(?P<chr>.)' ) for u in regdescr . split ( '|' ) : m = re . match ( tokenizer , u ) if m . group ( 'grp' ) : current = group_iterator ( m . group ( 'grp' ) ) else : current = [ m . group ( 'chr' ) ] for c in current : if u [ m . end ( ) : ] : for r in register_options ( u [ m . end ( ) : ] ) : yield c + r else : yield c

Very reduced regular expressions for describing a group of registers .


def eventstr(event_tuple=None, event=None, register=None, parameters=None):
    """
    Return a LIKWID event string from an event tuple or keyword arguments.

    *event_tuple* may have two or three arguments: (event, register) or
    (event, register, parameters)

    Keyword arguments will be overwritten by *event_tuple*.

    >>> eventstr(('L1D_REPLACEMENT', 'PMC0', None))
    'L1D_REPLACEMENT:PMC0'
    >>> eventstr(('L1D_REPLACEMENT', 'PMC0'))
    'L1D_REPLACEMENT:PMC0'
    >>> eventstr(('MEM_UOPS_RETIRED_LOADS', 'PMC3', {'EDGEDETECT': None, 'THRESHOLD': 2342}))
    'MEM_UOPS_RETIRED_LOADS:PMC3:EDGEDETECT:THRESHOLD=0x926'
    >>> eventstr(event='DTLB_LOAD_MISSES_WALK_DURATION', register='PMC3')
    'DTLB_LOAD_MISSES_WALK_DURATION:PMC3'
    """
    if len(event_tuple) == 3:
        event, register, parameters = event_tuple
    elif len(event_tuple) == 2:
        event, register = event_tuple
    event_dscr = [event, register]

    if parameters:
        for k, v in sorted(event_tuple[2].items()):  # sorted for reproducability
            if type(v) is int:
                k += "={}".format(hex(v))
            event_dscr.append(k)
    return ":".join(event_dscr)

def eventstr ( event_tuple = None , event = None , register = None , parameters = None ) : if len ( event_tuple ) == 3 : event , register , parameters = event_tuple elif len ( event_tuple ) == 2 : event , register = event_tuple event_dscr = [ event , register ] if parameters : for k , v in sorted ( event_tuple [ 2 ] . items ( ) ) : if type ( v ) is int : k += "={}" . format ( hex ( v ) ) event_dscr . append ( k ) return ":" . join ( event_dscr )

Return a LIKWID event string from an event tuple or keyword arguments .