def gcs_put_file(local_file,
                 bucketname,
                 service_account_json=None,
                 client=None,
                 raiseonfail=False):
    """This puts a single file into a Google Cloud Storage bucket.

    Parameters
    ----------

    local_file : str
        Path to the file to upload to GCS.

    bucket : str
        The GCS bucket to upload the file to.

    service_account_json : str
        Path to a downloaded GCS credentials JSON file.

    client : google.cloud.storage.Client instance
        The instance of the Client to use to perform the download operation. If
        this is None, a new Client will be used. If this is None and
        `service_account_json` points to a downloaded JSON file with GCS
        credentials, a new Client with the provided credentials will be used. If
        this is not None, the existing Client instance will be used.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    str or None
        If the file upload is successful, returns the gs:// URL of the uploaded
        file. If it failed, will return None.

    """

    if not client:

        if (service_account_json is not None and
            os.path.exists(service_account_json)):
            client = storage.Client.from_service_account_json(
                service_account_json
            )
        else:
            client = storage.Client()

    try:

        bucket = client.get_bucket(bucketname)
        remote_blob = bucket.blob(local_file)
        remote_blob.upload_from_filename(local_file)
        return 'gs://%s/%s' % (bucketname, local_file.lstrip('/'))

    except Exception as e:

        LOGEXCEPTION('could not upload %s to bucket %s' % (local_file,
                                                           bucket))

        if raiseonfail:
            raise

        return None

def
gcs_put_file
(
local_file
,
bucketname
,
service_account_json
=
,
client
=
,
raiseonfail
=
False
)
:
if
not
client
:
if
(
service_account_json
is
not
and
os
.
path
.
exists
(
service_account_json
)
)
:
client
=
storage
.
Client
.
from_service_account_json
(
service_account_json
)
else
:
client
=
storage
.
Client
(
)
try
:
bucket
=
client
.
get_bucket
(
bucketname
)
remote_blob
=
bucket
.
blob
(
local_file
)
remote_blob
.
upload_from_filename
(
local_file
)
return
'gs://%s/%s'
%
(
bucketname
,
local_file
.
lstrip
(
'/'
)
)
except
Exception
as
e
:
LOGEXCEPTION
(
'could not upload %s to bucket %s'
%
(
local_file
,
bucket
)
)
if
raiseonfail
:
raise
return

This puts a single file into a Google Cloud Storage bucket .


def read_fakelc(fakelcfile):
    '''
    This just reads a pickled fake LC.

    Parameters
    ----------

    fakelcfile : str
        The fake LC file to read.

    Returns
    -------

    dict
        This returns an lcdict.

    '''

    try:
        with open(fakelcfile,'rb') as infd:
            lcdict = pickle.load(infd)
    except UnicodeDecodeError:
        with open(fakelcfile,'rb') as infd:
            lcdict = pickle.load(infd, encoding='latin1')

    return lcdict

def read_fakelc ( fakelcfile ) : try : with open ( fakelcfile , 'rb' ) as infd : lcdict = pickle . load ( infd ) except UnicodeDecodeError : with open ( fakelcfile , 'rb' ) as infd : lcdict = pickle . load ( infd , encoding = 'latin1' ) return lcdict


def get_varfeatures(simbasedir,
                    mindet=1000,
                    nworkers=None):
    '''This runs `lcproc.lcvfeatures.parallel_varfeatures` on fake LCs in
    `simbasedir`.

    Parameters
    ----------

    simbasedir : str
        The directory containing the fake LCs to process.

    mindet : int
        The minimum number of detections needed to accept an LC and process it.

    nworkers : int or None
        The number of parallel workers to use when extracting variability
        features from the input light curves.

    Returns
    -------

    str
        The path to the `varfeatures` pickle created after running the
        `lcproc.lcvfeatures.parallel_varfeatures` function.

    '''

    # get the info from the simbasedir
    with open(os.path.join(simbasedir, 'fakelcs-info.pkl'),'rb') as infd:
        siminfo = pickle.load(infd)

    lcfpaths = siminfo['lcfpath']
    varfeaturedir = os.path.join(simbasedir,'varfeatures')

    # get the column defs for the fakelcs
    timecols = siminfo['timecols']
    magcols = siminfo['magcols']
    errcols = siminfo['errcols']

    # get the column defs for the fakelcs
    timecols = siminfo['timecols']
    magcols = siminfo['magcols']
    errcols = siminfo['errcols']

    # register the fakelc pklc as a custom lcproc format
    # now we should be able to use all lcproc functions correctly
    fakelc_formatkey = 'fake-%s' % siminfo['lcformat']
    lcproc.register_lcformat(
        fakelc_formatkey,
        '*-fakelc.pkl',
        timecols,
        magcols,
        errcols,
        'astrobase.lcproc',
        '_read_pklc',
        magsarefluxes=siminfo['magsarefluxes']
    )

    # now we can use lcproc.parallel_varfeatures directly
    varinfo = lcvfeatures.parallel_varfeatures(lcfpaths,
                                               varfeaturedir,
                                               lcformat=fakelc_formatkey,
                                               mindet=mindet,
                                               nworkers=nworkers)

    with open(os.path.join(simbasedir,'fakelc-varfeatures.pkl'),'wb') as outfd:
        pickle.dump(varinfo, outfd, pickle.HIGHEST_PROTOCOL)

    return os.path.join(simbasedir,'fakelc-varfeatures.pkl')

def get_varfeatures ( simbasedir , mindet = 1000 , nworkers = None ) : with open ( os . path . join ( simbasedir , 'fakelcs-info.pkl' ) , 'rb' ) as infd : siminfo = pickle . load ( infd ) lcfpaths = siminfo [ 'lcfpath' ] varfeaturedir = os . path . join ( simbasedir , 'varfeatures' ) timecols = siminfo [ 'timecols' ] magcols = siminfo [ 'magcols' ] errcols = siminfo [ 'errcols' ] timecols = siminfo [ 'timecols' ] magcols = siminfo [ 'magcols' ] errcols = siminfo [ 'errcols' ] fakelc_formatkey = 'fake-%s' % siminfo [ 'lcformat' ] lcproc . register_lcformat ( fakelc_formatkey , '*-fakelc.pkl' , timecols , magcols , errcols , 'astrobase.lcproc' , '_read_pklc' , magsarefluxes = siminfo [ 'magsarefluxes' ] ) varinfo = lcvfeatures . parallel_varfeatures ( lcfpaths , varfeaturedir , lcformat = fakelc_formatkey , mindet = mindet , nworkers = nworkers ) with open ( os . path . join ( simbasedir , 'fakelc-varfeatures.pkl' ) , 'wb' ) as outfd : pickle . dump ( varinfo , outfd , pickle . HIGHEST_PROTOCOL ) return os . path . join ( simbasedir , 'fakelc-varfeatures.pkl' )

This runs lcproc . lcvfeatures . parallel_varfeatures on fake LCs in simbasedir .


def precision(ntp, nfp):
    '''
    This calculates precision.

    https://en.wikipedia.org/wiki/Precision_and_recall

    Parameters
    ----------

    ntp : int
        The number of true positives.

    nfp : int
        The number of false positives.

    Returns
    -------

    float
        The precision calculated using `ntp/(ntp + nfp)`.

    '''

    if (ntp+nfp) > 0:
        return ntp/(ntp+nfp)
    else:
        return np.nan

def precision ( ntp , nfp ) : if ( ntp + nfp ) > 0 : return ntp / ( ntp + nfp ) else : return np . nan


def recall(ntp, nfn):
    '''
    This calculates recall.

    https://en.wikipedia.org/wiki/Precision_and_recall

    Parameters
    ----------

    ntp : int
        The number of true positives.

    nfn : int
        The number of false negatives.

    Returns
    -------

    float
        The precision calculated using `ntp/(ntp + nfn)`.

    '''

    if (ntp+nfn) > 0:
        return ntp/(ntp+nfn)
    else:
        return np.nan

def recall ( ntp , nfn ) : if ( ntp + nfn ) > 0 : return ntp / ( ntp + nfn ) else : return np . nan


def matthews_correl_coeff(ntp, ntn, nfp, nfn):
    '''
    This calculates the Matthews correlation coefficent.

    https://en.wikipedia.org/wiki/Matthews_correlation_coefficient

    Parameters
    ----------

    ntp : int
        The number of true positives.

    ntn : int
        The number of true negatives

    nfp : int
        The number of false positives.

    nfn : int
        The number of false negatives.

    Returns
    -------

    float
        The Matthews correlation coefficient.

    '''

    mcc_top = (ntp*ntn - nfp*nfn)
    mcc_bot = msqrt((ntp + nfp)*(ntp + nfn)*(ntn + nfp)*(ntn + nfn))

    if mcc_bot > 0:
        return mcc_top/mcc_bot
    else:
        return np.nan

def matthews_correl_coeff ( ntp , ntn , nfp , nfn ) : mcc_top = ( ntp * ntn - nfp * nfn ) mcc_bot = msqrt ( ( ntp + nfp ) * ( ntp + nfn ) * ( ntn + nfp ) * ( ntn + nfn ) ) if mcc_bot > 0 : return mcc_top / mcc_bot else : return np . nan

This calculates the Matthews correlation coefficent .


def magbin_varind_gridsearch_worker(task):
    '''
    This is a parallel grid search worker for the function below.

    '''

    simbasedir, gridpoint, magbinmedian = task

    try:
        res = get_recovered_variables_for_magbin(simbasedir,
                                                 magbinmedian,
                                                 stetson_stdev_min=gridpoint[0],
                                                 inveta_stdev_min=gridpoint[1],
                                                 iqr_stdev_min=gridpoint[2],
                                                 statsonly=True)
        return res
    except Exception as e:
        LOGEXCEPTION('failed to get info for %s' % gridpoint)
        return None

def
magbin_varind_gridsearch_worker
(
task
)
:
simbasedir
,
gridpoint
,
magbinmedian
=
task
try
:
res
=
get_recovered_variables_for_magbin
(
simbasedir
,
magbinmedian
,
stetson_stdev_min
=
gridpoint
[
0
]
,
inveta_stdev_min
=
gridpoint
[
1
]
,
iqr_stdev_min
=
gridpoint
[
2
]
,
statsonly
=
True
)
return
res
except
Exception
as
e
:
LOGEXCEPTION
(
'failed to get info for %s'
%
gridpoint
)
return

This is a parallel grid search worker for the function below .


def variable_index_gridsearch_magbin(simbasedir,
                                     stetson_stdev_range=(1.0,20.0),
                                     inveta_stdev_range=(1.0,20.0),
                                     iqr_stdev_range=(1.0,20.0),
                                     ngridpoints=32,
                                     ngridworkers=None):
    '''This runs a variable index grid search per magbin.

    For each magbin, this does a grid search using the stetson and inveta ranges
    provided and tries to optimize the Matthews Correlation Coefficient (best
    value is +1.0), indicating the best possible separation of variables
    vs. nonvariables. The thresholds on these two variable indexes that produce
    the largest coeff for the collection of fake LCs will probably be the ones
    that work best for actual variable classification on the real LCs.

    https://en.wikipedia.org/wiki/Matthews_correlation_coefficient

    For each grid-point, calculates the true positives, false positives, true
    negatives, false negatives. Then gets the precision and recall, confusion
    matrix, and the ROC curve for variable vs. nonvariable.

    Once we've identified the best thresholds to use, we can then calculate
    variable object numbers:

    - as a function of magnitude
    - as a function of period
    - as a function of number of detections
    - as a function of amplitude of variability

    Writes everything back to `simbasedir/fakevar-recovery.pkl`. Use the
    plotting function below to make plots for the results.

    Parameters
    ----------

    simbasedir : str
        The directory where the fake LCs are located.

    stetson_stdev_range : sequence of 2 floats
        The min and max values of the Stetson J variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    inveta_stdev_range : sequence of 2 floats
        The min and max values of the 1/eta variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    iqr_stdev_range : sequence of 2 floats
        The min and max values of the IQR variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    ngridpoints : int
        The number of grid points for each variability index grid. Remember that
        this function will be searching in 3D and will require lots of time to
        run if ngridpoints is too large.

        For the default number of grid points and 25000 simulated light curves,
        this takes about 3 days to run on a 40 (effective) core machine with 2 x
        Xeon E5-2650v3 CPUs.

    ngridworkers : int or None
        The number of parallel grid search workers that will be launched.

    Returns
    -------

    dict
        The returned dict contains a list of recovery stats for each magbin and
        each grid point in the variability index grids that were used. This dict
        can be passed to the plotting function below to plot the results.

    '''

    # make the output directory where all the pkls from the variability
    # threshold runs will go
    outdir = os.path.join(simbasedir,'recvar-threshold-pkls')
    if not os.path.exists(outdir):
        os.mkdir(outdir)

    # get the info from the simbasedir
    with open(os.path.join(simbasedir, 'fakelcs-info.pkl'),'rb') as infd:
        siminfo = pickle.load(infd)

    # get the column defs for the fakelcs
    timecols = siminfo['timecols']
    magcols = siminfo['magcols']
    errcols = siminfo['errcols']

    # get the magbinmedians to use for the recovery processing
    magbinmedians = siminfo['magrms'][magcols[0]]['binned_sdssr_median']

    # generate the grids for stetson and inveta
    stetson_grid = np.linspace(stetson_stdev_range[0],
                               stetson_stdev_range[1],
                               num=ngridpoints)
    inveta_grid = np.linspace(inveta_stdev_range[0],
                              inveta_stdev_range[1],
                              num=ngridpoints)
    iqr_grid = np.linspace(iqr_stdev_range[0],
                           iqr_stdev_range[1],
                           num=ngridpoints)

    # generate the grid
    stet_inveta_iqr_grid = []
    for stet in stetson_grid:
        for inveta in inveta_grid:
            for iqr in iqr_grid:
                grid_point = [stet, inveta, iqr]
                stet_inveta_iqr_grid.append(grid_point)

    # the output dict
    grid_results = {'stetson_grid':stetson_grid,
                    'inveta_grid':inveta_grid,
                    'iqr_grid':iqr_grid,
                    'stet_inveta_iqr_grid':stet_inveta_iqr_grid,
                    'magbinmedians':magbinmedians,
                    'timecols':timecols,
                    'magcols':magcols,
                    'errcols':errcols,
                    'simbasedir':os.path.abspath(simbasedir),
                    'recovery':[]}

    # set up the pool
    pool = mp.Pool(ngridworkers)

    # run the grid search per magbinmedian
    for magbinmedian in magbinmedians:

        LOGINFO('running stetson J-inveta grid-search '
                'for magbinmedian = %.3f...' % magbinmedian)

        tasks = [(simbasedir, gp, magbinmedian) for gp in stet_inveta_iqr_grid]
        thisbin_results = pool.map(magbin_varind_gridsearch_worker, tasks)
        grid_results['recovery'].append(thisbin_results)

    pool.close()
    pool.join()

    LOGINFO('done.')
    with open(os.path.join(simbasedir,
                           'fakevar-recovery-per-magbin.pkl'),'wb') as outfd:
        pickle.dump(grid_results,outfd,pickle.HIGHEST_PROTOCOL)

    return grid_results

def
variable_index_gridsearch_magbin
(
simbasedir
,
stetson_stdev_range
=
(
1.0
,
20.0
)
,
inveta_stdev_range
=
(
1.0
,
20.0
)
,
iqr_stdev_range
=
(
1.0
,
20.0
)
,
ngridpoints
=
32
,
ngridworkers
=
)
:
outdir
=
os
.
path
.
join
(
simbasedir
,
'recvar-threshold-pkls'
)
if
not
os
.
path
.
exists
(
outdir
)
:
os
.
mkdir
(
outdir
)
with
open
(
os
.
path
.
join
(
simbasedir
,
'fakelcs-info.pkl'
)
,
'rb'
)
as
infd
:
siminfo
=
pickle
.
load
(
infd
)
timecols
=
siminfo
[
'timecols'
]
magcols
=
siminfo
[
'magcols'
]
errcols
=
siminfo
[
'errcols'
]
magbinmedians
=
siminfo
[
'magrms'
]
[
magcols
[
0
]
]
[
'binned_sdssr_median'
]
stetson_grid
=
np
.
linspace
(
stetson_stdev_range
[
0
]
,
stetson_stdev_range
[
1
]
,
num
=
ngridpoints
)
inveta_grid
=
np
.
linspace
(
inveta_stdev_range
[
0
]
,
inveta_stdev_range
[
1
]
,
num
=
ngridpoints
)
iqr_grid
=
np
.
linspace
(
iqr_stdev_range
[
0
]
,
iqr_stdev_range
[
1
]
,
num
=
ngridpoints
)
stet_inveta_iqr_grid
=
[
]
for
stet
in
stetson_grid
:
for
inveta
in
inveta_grid
:
for
iqr
in
iqr_grid
:
grid_point
=
[
stet
,
inveta
,
iqr
]
stet_inveta_iqr_grid
.
append
(
grid_point
)
grid_results
=
{
'stetson_grid'
:
stetson_grid
,
'inveta_grid'
:
inveta_grid
,
'iqr_grid'
:
iqr_grid
,
'stet_inveta_iqr_grid'
:
stet_inveta_iqr_grid
,
'magbinmedians'
:
magbinmedians
,
'timecols'
:
timecols
,
'magcols'
:
magcols
,
'errcols'
:
errcols
,
'simbasedir'
:
os
.
path
.
abspath
(
simbasedir
)
,
'recovery'
:
[
]
}
pool
=
mp
.
Pool
(
ngridworkers
)
for
magbinmedian
in
magbinmedians
:
LOGINFO
(
'running stetson J-inveta grid-search '
'for magbinmedian = %.3f...'
%
magbinmedian
)
tasks
=
[
(
simbasedir
,
gp
,
magbinmedian
)
for
gp
in
stet_inveta_iqr_grid
]
thisbin_results
=
pool
.
map
(
magbin_varind_gridsearch_worker
,
tasks
)
grid_results
[
'recovery'
]
.
append
(
thisbin_results
)
pool
.
close
(
)
pool
.
join
(
)
LOGINFO
(
'done.'
)
with
open
(
os
.
path
.
join
(
simbasedir
,
'fakevar-recovery-per-magbin.pkl'
)
,
'wb'
)
as
outfd
:
pickle
.
dump
(
grid_results
,
outfd
,
pickle
.
HIGHEST_PROTOCOL
)
return
grid_results

This runs a variable index grid search per magbin .


def run_periodfinding(simbasedir,
                      pfmethods=('gls','pdm','bls'),
                      pfkwargs=({},{},{'startp':1.0,'maxtransitduration':0.3}),
                      getblssnr=False,
                      sigclip=5.0,
                      nperiodworkers=10,
                      ncontrolworkers=4,
                      liststartindex=None,
                      listmaxobjects=None):
    '''This runs periodfinding using several period-finders on a collection of
    fake LCs.

    As a rough benchmark, 25000 fake LCs with 10000--50000 points per LC take
    about 26 days in total to run on an invocation of this function using
    GLS+PDM+BLS and 10 periodworkers and 4 controlworkers (so all 40 'cores') on
    a 2 x Xeon E5-2660v3 machine.

    Parameters
    ----------

    pfmethods : sequence of str
        This is used to specify which periodfinders to run. These must be in the
        `lcproc.periodsearch.PFMETHODS` dict.

    pfkwargs : sequence of dict
        This is used to provide optional kwargs to the period-finders.

    getblssnr : bool
        If this is True, will run BLS SNR calculations for each object and
        magcol. This takes a while to run, so it's disabled (False) by default.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    nperiodworkers : int
        This is the number of parallel period-finding worker processes to use.

    ncontrolworkers : int
        This is the number of parallel period-finding control workers to
        use. Each control worker will launch `nperiodworkers` worker processes.

    liststartindex : int
        The starting index of processing. This refers to the filename list
        generated by running `glob.glob` on the fake LCs in `simbasedir`.

    maxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input light curves over several sessions or machines.

    Returns
    -------

    str
        The path to the output summary pickle produced by
        `lcproc.periodsearch.parallel_pf`

    '''

    # get the info from the simbasedir
    with open(os.path.join(simbasedir, 'fakelcs-info.pkl'),'rb') as infd:
        siminfo = pickle.load(infd)

    lcfpaths = siminfo['lcfpath']
    pfdir = os.path.join(simbasedir,'periodfinding')

    # get the column defs for the fakelcs
    timecols = siminfo['timecols']
    magcols = siminfo['magcols']
    errcols = siminfo['errcols']

    # register the fakelc pklc as a custom lcproc format
    # now we should be able to use all lcproc functions correctly
    fakelc_formatkey = 'fake-%s' % siminfo['lcformat']
    lcproc.register_lcformat(
        fakelc_formatkey,
        '*-fakelc.pkl',
        timecols,
        magcols,
        errcols,
        'astrobase.lcproc',
        '_read_pklc',
        magsarefluxes=siminfo['magsarefluxes']
    )

    if liststartindex:
        lcfpaths = lcfpaths[liststartindex:]

    if listmaxobjects:
        lcfpaths = lcfpaths[:listmaxobjects]

    pfinfo = periodsearch.parallel_pf(lcfpaths,
                                      pfdir,
                                      lcformat=fakelc_formatkey,
                                      pfmethods=pfmethods,
                                      pfkwargs=pfkwargs,
                                      getblssnr=getblssnr,
                                      sigclip=sigclip,
                                      nperiodworkers=nperiodworkers,
                                       ncontrolworkers=ncontrolworkers)

    with open(os.path.join(simbasedir,
                           'fakelc-periodsearch.pkl'),'wb') as outfd:
        pickle.dump(pfinfo, outfd, pickle.HIGHEST_PROTOCOL)

    return os.path.join(simbasedir,'fakelc-periodsearch.pkl')

def run_periodfinding ( simbasedir , pfmethods = ( 'gls' , 'pdm' , 'bls' ) , pfkwargs = ( { } , { } , { 'startp' : 1.0 , 'maxtransitduration' : 0.3 } ) , getblssnr = False , sigclip = 5.0 , nperiodworkers = 10 , ncontrolworkers = 4 , liststartindex = None , listmaxobjects = None ) : with open ( os . path . join ( simbasedir , 'fakelcs-info.pkl' ) , 'rb' ) as infd : siminfo = pickle . load ( infd ) lcfpaths = siminfo [ 'lcfpath' ] pfdir = os . path . join ( simbasedir , 'periodfinding' ) timecols = siminfo [ 'timecols' ] magcols = siminfo [ 'magcols' ] errcols = siminfo [ 'errcols' ] fakelc_formatkey = 'fake-%s' % siminfo [ 'lcformat' ] lcproc . register_lcformat ( fakelc_formatkey , '*-fakelc.pkl' , timecols , magcols , errcols , 'astrobase.lcproc' , '_read_pklc' , magsarefluxes = siminfo [ 'magsarefluxes' ] ) if liststartindex : lcfpaths = lcfpaths [ liststartindex : ] if listmaxobjects : lcfpaths = lcfpaths [ : listmaxobjects ] pfinfo = periodsearch . parallel_pf ( lcfpaths , pfdir , lcformat = fakelc_formatkey , pfmethods = pfmethods , pfkwargs = pfkwargs , getblssnr = getblssnr , sigclip = sigclip , nperiodworkers = nperiodworkers , ncontrolworkers = ncontrolworkers ) with open ( os . path . join ( simbasedir , 'fakelc-periodsearch.pkl' ) , 'wb' ) as outfd : pickle . dump ( pfinfo , outfd , pickle . HIGHEST_PROTOCOL ) return os . path . join ( simbasedir , 'fakelc-periodsearch.pkl' )

This runs periodfinding using several period - finders on a collection of fake LCs .


def periodrec_worker(task):
    '''This is a parallel worker for running period-recovery.

    Parameters
    ----------

    task : tuple
        This is used to pass args to the `periodicvar_recovery` function::

            task[0] = period-finding result pickle to work on
            task[1] = simbasedir
            task[2] = period_tolerance

    Returns
    -------

    dict
        This is the dict produced by the `periodicvar_recovery` function for the
        input period-finding result pickle.

    '''

    pfpkl, simbasedir, period_tolerance = task

    try:
        return periodicvar_recovery(pfpkl,
                                    simbasedir,
                                    period_tolerance=period_tolerance)

    except Exception as e:
        LOGEXCEPTION('periodic var recovery failed for %s' % repr(task))
        return None

def
periodrec_worker
(
task
)
:
pfpkl
,
simbasedir
,
period_tolerance
=
task
try
:
return
periodicvar_recovery
(
pfpkl
,
simbasedir
,
period_tolerance
=
period_tolerance
)
except
Exception
as
e
:
LOGEXCEPTION
(
'periodic var recovery failed for %s'
%
repr
(
task
)
)
return

This is a parallel worker for running period - recovery .


def parallel_periodicvar_recovery(simbasedir,
                                  period_tolerance=1.0e-3,
                                  liststartind=None,
                                  listmaxobjects=None,
                                  nworkers=None):
    '''This is a parallel driver for `periodicvar_recovery`.

    Parameters
    ----------

    simbasedir : str
        The base directory where all of the fake LCs and period-finding results
        are.

    period_tolerance : float
        The maximum difference that this function will consider between an
        actual period (or its aliases) and a recovered period to consider it as
        as a 'recovered' period.

    liststartindex : int
        The starting index of processing. This refers to the filename list
        generated by running `glob.glob` on the period-finding result pickles in
        `simbasedir/periodfinding`.

    listmaxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input period-finding result pickles over several sessions or machines.

    nperiodworkers : int
        This is the number of parallel period-finding worker processes to use.

    Returns
    -------

    str
        Returns the filename of the pickle produced containing all of the period
        recovery results.

    '''

    # figure out the periodfinding pickles directory
    pfpkldir = os.path.join(simbasedir,'periodfinding')

    if not os.path.exists(pfpkldir):
        LOGERROR('no "periodfinding" subdirectory in %s, can\'t continue' %
                 simbasedir)
        return None

    # find all the periodfinding pickles
    pfpkl_list = glob.glob(os.path.join(pfpkldir,'*periodfinding*pkl*'))

    if len(pfpkl_list) > 0:

        if liststartind:
            pfpkl_list = pfpkl_list[liststartind:]

        if listmaxobjects:
            pfpkl_list = pfpkl_list[:listmaxobjects]

        tasks = [(x, simbasedir, period_tolerance) for x in pfpkl_list]

        pool = mp.Pool(nworkers)
        results = pool.map(periodrec_worker, tasks)
        pool.close()
        pool.join()

        resdict = {x['objectid']:x for x in results if x is not None}

        actual_periodicvars = np.array(
            [x['objectid'] for x in results
             if (x is not None and x['actual_vartype'] in PERIODIC_VARTYPES)],
            dtype=np.unicode_
        )

        recovered_periodicvars = np.array(
            [x['objectid'] for x in results
             if (x is not None and 'actual' in x['best_recovered_status'])],
            dtype=np.unicode_
        )
        alias_twice_periodicvars = np.array(
            [x['objectid'] for x in results
             if (x is not None and 'twice' in x['best_recovered_status'])],
            dtype=np.unicode_
        )
        alias_half_periodicvars = np.array(
            [x['objectid'] for x in results
             if (x is not None and 'half' in x['best_recovered_status'])],
            dtype=np.unicode_
        )

        all_objectids = [x['objectid'] for x in results]

        outdict = {'simbasedir':os.path.abspath(simbasedir),
                   'objectids':all_objectids,
                   'period_tolerance':period_tolerance,
                   'actual_periodicvars':actual_periodicvars,
                   'recovered_periodicvars':recovered_periodicvars,
                   'alias_twice_periodicvars':alias_twice_periodicvars,
                   'alias_half_periodicvars':alias_half_periodicvars,
                   'details':resdict}

        outfile = os.path.join(simbasedir,'periodicvar-recovery.pkl')
        with open(outfile, 'wb') as outfd:
            pickle.dump(outdict, outfd, pickle.HIGHEST_PROTOCOL)

        return outdict

    else:

        LOGERROR(
            'no periodfinding result pickles found in %s, can\'t continue' %
            pfpkldir
        )
        return None

This is a parallel driver for periodicvar_recovery .


def tic_conesearch(
        ra,
        decl,
        radius_arcmin=5.0,
        apiversion='v0',
        forcefetch=False,
        cachedir='~/.astrobase/mast-cache',
        verbose=True,
        timeout=10.0,
        refresh=5.0,
        maxtimeout=90.0,
        maxtries=3,
        jitter=5.0,
        raiseonfail=False
):
    '''This runs a TESS Input Catalog cone search on MAST.

    If you use this, please cite the TIC paper (Stassun et al 2018;
    http://adsabs.harvard.edu/abs/2018AJ....156..102S). Also see the "living"
    TESS input catalog docs:

    https://docs.google.com/document/d/1zdiKMs4Ld4cXZ2DW4lMX-fuxAF6hPHTjqjIwGqnfjqI

    Also see: https://mast.stsci.edu/api/v0/_t_i_cfields.html for the fields
    returned by the service and present in the result JSON file.

    Parameters
    ----------

    ra,decl : float
        The center coordinates of the cone-search in decimal degrees.

    radius_arcmin : float
        The cone-search radius in arcminutes.

    apiversion : str
        The API version of the MAST service to use. This sets the URL that this
        function will call, using `apiversion` as key into the `MAST_URLS` dict
        above.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    raiseonfail : bool
        If this is True, the function will raise an Exception if something goes
        wrong, instead of returning None.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    params = {'ra':ra,
              'dec':decl,
              'radius':radius_arcmin/60.0}
    service = 'Mast.Catalogs.Tic.Cone'

    return mast_query(service,
                      params,
                      jitter=jitter,
                      apiversion=apiversion,
                      forcefetch=forcefetch,
                      cachedir=cachedir,
                      verbose=verbose,
                      timeout=timeout,
                      refresh=refresh,
                      maxtimeout=maxtimeout,
                      maxtries=maxtries,
                      raiseonfail=raiseonfail)

def tic_conesearch ( ra , decl , radius_arcmin = 5.0 , apiversion = 'v0' , forcefetch = False , cachedir = '~/.astrobase/mast-cache' , verbose = True , timeout = 10.0 , refresh = 5.0 , maxtimeout = 90.0 , maxtries = 3 , jitter = 5.0 , raiseonfail = False ) : params = { 'ra' : ra , 'dec' : decl , 'radius' : radius_arcmin / 60.0 } service = 'Mast.Catalogs.Tic.Cone' return mast_query ( service , params , jitter = jitter , apiversion = apiversion , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , raiseonfail = raiseonfail )

This runs a TESS Input Catalog cone search on MAST .


def tic_xmatch(
        ra,
        decl,
        radius_arcsec=5.0,
        apiversion='v0',
        forcefetch=False,
        cachedir='~/.astrobase/mast-cache',
        verbose=True,
        timeout=90.0,
        refresh=5.0,
        maxtimeout=180.0,
        maxtries=3,
        jitter=5.0,
        raiseonfail=False
):
    '''This does a cross-match with TIC.

    Parameters
    ----------

    ra,decl : np.arrays or lists of floats
        The coordinates that will be cross-matched against the TIC.

    radius_arcsec : float
        The cross-match radius in arcseconds.

    apiversion : str
        The API version of the MAST service to use. This sets the URL that this
        function will call, using `apiversion` as key into the `MAST_URLS` dict
        above.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    raiseonfail : bool
        If this is True, the function will raise an Exception if something goes
        wrong, instead of returning None.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    service = 'Mast.Tic.Crossmatch'

    xmatch_input = {'fields':[{'name':'ra','type':'float'},
                              {'name':'dec','type':'float'}]}
    xmatch_input['data'] = [{'ra':x, 'dec':y} for (x,y) in zip(ra, decl)]

    params = {'raColumn':'ra',
              'decColumn':'dec',
              'radius':radius_arcsec/3600.0}

    return mast_query(service,
                      params,
                      data=xmatch_input,
                      jitter=jitter,
                      apiversion=apiversion,
                      forcefetch=forcefetch,
                      cachedir=cachedir,
                      verbose=verbose,
                      timeout=timeout,
                      refresh=refresh,
                      maxtimeout=maxtimeout,
                      maxtries=maxtries,
                      raiseonfail=raiseonfail)

def tic_xmatch ( ra , decl , radius_arcsec = 5.0 , apiversion = 'v0' , forcefetch = False , cachedir = '~/.astrobase/mast-cache' , verbose = True , timeout = 90.0 , refresh = 5.0 , maxtimeout = 180.0 , maxtries = 3 , jitter = 5.0 , raiseonfail = False ) : service = 'Mast.Tic.Crossmatch' xmatch_input = { 'fields' : [ { 'name' : 'ra' , 'type' : 'float' } , { 'name' : 'dec' , 'type' : 'float' } ] } xmatch_input [ 'data' ] = [ { 'ra' : x , 'dec' : y } for ( x , y ) in zip ( ra , decl ) ] params = { 'raColumn' : 'ra' , 'decColumn' : 'dec' , 'radius' : radius_arcsec / 3600.0 } return mast_query ( service , params , data = xmatch_input , jitter = jitter , apiversion = apiversion , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , raiseonfail = raiseonfail )


def tic_objectsearch(
        objectid,
        idcol_to_use="ID",
        apiversion='v0',
        forcefetch=False,
        cachedir='~/.astrobase/mast-cache',
        verbose=True,
        timeout=90.0,
        refresh=5.0,
        maxtimeout=180.0,
        maxtries=3,
        jitter=5.0,
        raiseonfail=False
):
    '''
    This runs a TIC search for a specified TIC ID.

    Parameters
    ----------

    objectid : str
        The object ID to look up information for.

    idcol_to_use : str
        This is the name of the object ID column to use when looking up the
        provided `objectid`. This is one of {'ID', 'HIP', 'TYC', 'UCAC',
        'TWOMASS', 'ALLWISE', 'SDSS', 'GAIA', 'APASS', 'KIC'}.

    apiversion : str
        The API version of the MAST service to use. This sets the URL that this
        function will call, using `apiversion` as key into the `MAST_URLS` dict
        above.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    raiseonfail : bool
        If this is True, the function will raise an Exception if something goes
        wrong, instead of returning None.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}
    '''

    params = {
        'columns':'*',
        'filters':[
            {"paramName": idcol_to_use,
             "values":[str(objectid)]}
        ]
    }
    service = 'Mast.Catalogs.Filtered.Tic'

    return mast_query(service,
                      params,
                      jitter=jitter,
                      apiversion=apiversion,
                      forcefetch=forcefetch,
                      cachedir=cachedir,
                      verbose=verbose,
                      timeout=timeout,
                      refresh=refresh,
                      maxtimeout=maxtimeout,
                      maxtries=maxtries,
                      raiseonfail=raiseonfail)

def tic_objectsearch ( objectid , idcol_to_use = "ID" , apiversion = 'v0' , forcefetch = False , cachedir = '~/.astrobase/mast-cache' , verbose = True , timeout = 90.0 , refresh = 5.0 , maxtimeout = 180.0 , maxtries = 3 , jitter = 5.0 , raiseonfail = False ) : params = { 'columns' : '*' , 'filters' : [ { "paramName" : idcol_to_use , "values" : [ str ( objectid ) ] } ] } service = 'Mast.Catalogs.Filtered.Tic' return mast_query ( service , params , jitter = jitter , apiversion = apiversion , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , raiseonfail = raiseonfail )

This runs a TIC search for a specified TIC ID .


def send_email(sender,
               subject,
               content,
               email_recipient_list,
               email_address_list,
               email_user=None,
               email_pass=None,
               email_server=None):
    '''This sends an email to addresses, informing them about events.

    The email account settings are retrieved from the settings file as described
    above.

    Parameters
    ----------

    sender : str
        The name of the sender to use in the email header.

    subject : str
        Subject of the email.

    content : str
        Content of the email.

    email_recipient list : list of str
        This is a list of email recipient names of the form:
        `['Example Person 1', 'Example Person 1', ...]`

    email_recipient list : list of str
        This is a list of email recipient addresses of the form:
        `['example1@example.com', 'example2@example.org', ...]`

    email_user : str
        The username of the email server account that will send the emails. If
        this is None, the value of EMAIL_USER from the
        ~/.astrobase/.emailsettings file will be used. If that is None as well,
        this function won't work.

    email_pass : str
        The password of the email server account that will send the emails. If
        this is None, the value of EMAIL_PASS from the
        ~/.astrobase/.emailsettings file will be used. If that is None as well,
        this function won't work.

    email_server : str
        The address of the email server that will send the emails. If this is
        None, the value of EMAIL_USER from the ~/.astrobase/.emailsettings file
        will be used. If that is None as well, this function won't work.

    Returns
    -------

    bool
        True if email sending succeeded. False if email sending failed.

    '''

    if not email_user:
        email_user = EMAIL_USER

    if not email_pass:
        email_pass = EMAIL_PASSWORD

    if not email_server:
        email_server = EMAIL_SERVER

    if not email_server and email_user and email_pass:
        raise ValueError("no email server address and "
                         "credentials available, can't continue")

    msg_text = EMAIL_TEMPLATE.format(
        sender=sender,
        hostname=socket.gethostname(),
        activity_time='%sZ' % datetime.utcnow().isoformat(),
        activity_report=content
    )

    email_sender = '%s <%s>' % (sender, EMAIL_USER)

    # put together the recipient and email lists
    email_recipients = [('%s <%s>' % (x,y))
                        for (x,y) in zip(email_recipient_list,
                                         email_address_list)]

    # put together the rest of the message
    email_msg = MIMEText(msg_text)
    email_msg['From'] = email_sender
    email_msg['To'] = ', '.join(email_recipients)
    email_msg['Message-Id'] = make_msgid()
    email_msg['Subject'] = '[%s on %s] %s' % (
        sender,
        socket.gethostname(),
        subject
    )
    email_msg['Date'] = formatdate(time.time())

    # start the email process

    try:
        server = smtplib.SMTP(EMAIL_SERVER, 587)
        server_ehlo_response = server.ehlo()

        if server.has_extn('STARTTLS'):

            try:

                tls_start_response = server.starttls()
                tls_ehlo_response = server.ehlo()

                login_response = server.login(EMAIL_USER, EMAIL_PASSWORD)

                send_response = (
                    server.sendmail(email_sender,
                                    email_address_list,
                                    email_msg.as_string())
                )

            except Exception as e:

                print('script email sending failed with error: %s'
                      % e)
                send_response = None

            if send_response is not None:

                print('script email sent successfully')
                quit_response = server.quit()
                return True

            else:

                quit_response = server.quit()
                return False

        else:

            print('email server does not support STARTTLS,'
                  ' bailing out...')
            quit_response = server.quit()
            return False

    except Exception as e:
        print('sending email failed with error: %s' % e)
        returnval = False

    quit_response = server.quit()
    return returnval

This sends an email to addresses informing them about events .


def fourier_sinusoidal_func(fourierparams, times, mags, errs):
    '''This generates a sinusoidal light curve using a Fourier cosine series.

    Parameters
    ----------

    fourierparams : list
        This MUST be a list of the following form like so::

            [period,
             epoch,
             [amplitude_1, amplitude_2, amplitude_3, ..., amplitude_X],
             [phase_1, phase_2, phase_3, ..., phase_X]]

        where X is the Fourier order.

    times,mags,errs : np.array
        The input time-series of measurements and associated errors for which
        the model will be generated. The times will be used to generate model
        mags, and the input `times`, `mags`, and `errs` will be resorted by
        model phase and returned.

    Returns
    -------

    (modelmags, phase, ptimes, pmags, perrs) : tuple
        Returns the model mags and phase values. Also returns the input `times`,
        `mags`, and `errs` sorted by the model's phase.

    '''

    period, epoch, famps, fphases = fourierparams

    # figure out the order from the length of the Fourier param list
    forder = len(famps)

    # phase the times with this period
    iphase = (times - epoch)/period
    iphase = iphase - np.floor(iphase)

    phasesortind = np.argsort(iphase)
    phase = iphase[phasesortind]
    ptimes = times[phasesortind]
    pmags = mags[phasesortind]
    perrs = errs[phasesortind]

    # calculate all the individual terms of the series
    fseries = [famps[x]*np.cos(2.0*np.pi*x*phase + fphases[x])
               for x in range(forder)]

    # this is the zeroth order coefficient - a constant equal to median mag
    modelmags = np.median(mags)

    # sum the series
    for fo in fseries:
        modelmags += fo

    return modelmags, phase, ptimes, pmags, perrs

def fourier_sinusoidal_func ( fourierparams , times , mags , errs ) : period , epoch , famps , fphases = fourierparams forder = len ( famps ) iphase = ( times - epoch ) / period iphase = iphase - np . floor ( iphase ) phasesortind = np . argsort ( iphase ) phase = iphase [ phasesortind ] ptimes = times [ phasesortind ] pmags = mags [ phasesortind ] perrs = errs [ phasesortind ] fseries = [ famps [ x ] * np . cos ( 2.0 * np . pi * x * phase + fphases [ x ] ) for x in range ( forder ) ] modelmags = np . median ( mags ) for fo in fseries : modelmags += fo return modelmags , phase , ptimes , pmags , perrs

This generates a sinusoidal light curve using a Fourier cosine series .


def fourier_sinusoidal_residual(fourierparams, times, mags, errs):
    '''
    This returns the residual between the model mags and the actual mags.

    Parameters
    ----------

    fourierparams : list
        This MUST be a list of the following form like so::

            [period,
             epoch,
             [amplitude_1, amplitude_2, amplitude_3, ..., amplitude_X],
             [phase_1, phase_2, phase_3, ..., phase_X]]

        where X is the Fourier order.

    times,mags,errs : np.array
        The input time-series of measurements and associated errors for which
        the model will be generated. The times will be used to generate model
        mags, and the input `times`, `mags`, and `errs` will be resorted by
        model phase and returned.

    Returns
    -------

    np.array
        The residuals between the input `mags` and generated `modelmags`,
        weighted by the measurement errors in `errs`.

    '''
    modelmags, phase, ptimes, pmags, perrs = (
        fourier_sinusoidal_func(fourierparams, times, mags, errs)
    )

    # this is now a weighted residual taking into account the measurement err
    return (pmags - modelmags)/perrs

def fourier_sinusoidal_residual ( fourierparams , times , mags , errs ) : modelmags , phase , ptimes , pmags , perrs = ( fourier_sinusoidal_func ( fourierparams , times , mags , errs ) ) return ( pmags - modelmags ) / perrs

This returns the residual between the model mags and the actual mags .


def _make_magseries_plot(axes,
                         stimes,
                         smags,
                         serrs,
                         magsarefluxes=False,
                         ms=2.0):
    '''Makes the mag-series plot tile for `checkplot_png` and
    `twolsp_checkplot_png`.

    axes : matplotlib.axes.Axes object
        The Axes object where the generated plot will go.

    stimes,smags,serrs : np.array
        The mag/flux time-series arrays along with associated errors. These
        should all have been run through nan-stripping and sigma-clipping
        beforehand.

    magsarefluxes : bool
        If True, indicates the input time-series is fluxes and not mags so the
        plot y-axis direction and range can be set appropriately.

    ms : float
        The `markersize` kwarg to use when making the mag-series plot.

    Returns
    -------

    Does not return anything, works on the input Axes object directly.

    '''

    scaledplottime = stimes - npmin(stimes)

    axes.plot(scaledplottime,
              smags,
              marker='o',
              ms=ms, ls='None',mew=0,
              color='green',
              rasterized=True)

    # flip y axis for mags
    if not magsarefluxes:
        plot_ylim = axes.get_ylim()
        axes.set_ylim((plot_ylim[1], plot_ylim[0]))

    # set the x axis limit
    axes.set_xlim((npmin(scaledplottime)-1.0,
                   npmax(scaledplottime)+1.0))

    # make a grid
    axes.grid(color='#a9a9a9',
              alpha=0.9,
              zorder=0,
              linewidth=1.0,
              linestyle=':')

    # make the x and y axis labels
    plot_xlabel = 'JD - %.3f' % npmin(stimes)
    if magsarefluxes:
        plot_ylabel = 'flux'
    else:
        plot_ylabel = 'magnitude'

    axes.set_xlabel(plot_xlabel)
    axes.set_ylabel(plot_ylabel)

    # fix the yaxis ticks (turns off offset and uses the full
    # value of the yaxis tick)
    axes.get_yaxis().get_major_formatter().set_useOffset(False)
    axes.get_xaxis().get_major_formatter().set_useOffset(False)

def
_make_magseries_plot
(
axes
,
stimes
,
smags
,
serrs
,
magsarefluxes
=
False
,
ms
=
2.0
)
:
scaledplottime
=
stimes
npmin
(
stimes
)
axes
.
plot
(
scaledplottime
,
smags
,
marker
=
'o'
,
ms
=
ms
,
ls
=
'None'
,
mew
=
0
,
color
=
'green'
,
rasterized
=
True
)
if
not
magsarefluxes
:
plot_ylim
=
axes
.
get_ylim
(
)
axes
.
set_ylim
(
(
plot_ylim
[
1
]
,
plot_ylim
[
0
]
)
)
axes
.
set_xlim
(
(
npmin
(
scaledplottime
)
1.0
,
npmax
(
scaledplottime
)
+
1.0
)
)
axes
.
grid
(
color
=
'#a9a9a9'
,
alpha
=
0.9
,
zorder
=
0
,
linewidth
=
1.0
,
linestyle
=
':'
)
plot_xlabel
=
'JD - %.3f'
%
npmin
(
stimes
)
if
magsarefluxes
:
plot_ylabel
=
'flux'
else
:
plot_ylabel
=
'magnitude'
axes
.
set_xlabel
(
plot_xlabel
)
axes
.
set_ylabel
(
plot_ylabel
)
axes
.
get_yaxis
(
)
.
get_major_formatter
(
)
.
set_useOffset
(
False
)
axes
.
get_xaxis
(
)
.
get_major_formatter
(
)
.
set_useOffset
(
False
)

Makes the mag - series plot tile for checkplot_png and twolsp_checkplot_png .


def precess_coordinates(ra, dec,
                        epoch_one, epoch_two,
                        jd=None,
                        mu_ra=0.0,
                        mu_dec=0.0,
                        outscalar=False):
    '''Precesses target coordinates `ra`, `dec` from `epoch_one` to `epoch_two`.

    This takes into account the jd of the observations, as well as the proper
    motion of the target mu_ra, mu_dec. Adapted from J. D. Hartman's
    VARTOOLS/converttime.c [coordprecess].

    Parameters
    ----------

    ra,dec : float
        The equatorial coordinates of the object at `epoch_one` to precess in
        decimal degrees.

    epoch_one : float
        Origin epoch to precess from to target epoch. This is a float, like:
        1985.0, 2000.0, etc.

    epoch_two : float
        Target epoch to precess from origin epoch. This is a float, like:
        2000.0, 2018.0, etc.

    jd : float
        The full Julian date to use along with the propermotions in `mu_ra`, and
        `mu_dec` to handle proper motion along with the coordinate frame
        precession. If one of `jd`, `mu_ra`, or `mu_dec` is missing, the proper
        motion will not be used to calculate the final precessed coordinates.

    mu_ra,mu_dec : float
        The proper motion in mas/yr in right ascension and declination. If these
        are provided along with `jd`, the total proper motion of the object will
        be taken into account to calculate the final precessed coordinates.

    outscalar : bool
        If True, converts the output coordinates from one-element np.arrays to
        scalars.

    Returns
    -------

    precessed_ra, precessed_dec : float
        A tuple of precessed equatorial coordinates in decimal degrees at
        `epoch_two` taking into account proper motion if `jd`, `mu_ra`, and
        `mu_dec` are provided.

    '''

    raproc, decproc = np.radians(ra), np.radians(dec)

    if ((mu_ra != 0.0) and (mu_dec != 0.0) and jd):

        jd_epoch_one = JD2000 + (epoch_one - epoch_two)*365.25
        raproc = (
            raproc +
            (jd - jd_epoch_one)*mu_ra*MAS_P_YR_TO_RAD_P_DAY/np.cos(decproc)
        )
        decproc = decproc + (jd - jd_epoch_one)*mu_dec*MAS_P_YR_TO_RAD_P_DAY

    ca = np.cos(raproc)
    cd = np.cos(decproc)
    sa = np.sin(raproc)
    sd = np.sin(decproc)

    if epoch_one != epoch_two:

        t1 = 1.0e-3 * (epoch_two - epoch_one)
        t2 = 1.0e-3 * (epoch_one - 2000.0)

        a = ( t1*ARCSEC_TO_RADIANS * (23062.181 + t2*(139.656 + 0.0139*t2) +
                                      t1*(30.188 - 0.344*t2+17.998*t1)) )
        b = t1*t1*ARCSEC_TO_RADIANS*(79.280 + 0.410*t2 + 0.205*t1) + a
        c = (
            ARCSEC_TO_RADIANS*t1*(20043.109 - t2*(85.33 + 0.217*t2) +
                                  t1*(-42.665 - 0.217*t2 - 41.833*t2))
        )
        sina, sinb, sinc = np.sin(a), np.sin(b), np.sin(c)
        cosa, cosb, cosc = np.cos(a), np.cos(b), np.cos(c)

        precmatrix = np.matrix([[cosa*cosb*cosc - sina*sinb,
                                 sina*cosb + cosa*sinb*cosc,
                                 cosa*sinc],
                                [-cosa*sinb - sina*cosb*cosc,
                                 cosa*cosb - sina*sinb*cosc,
                                 -sina*sinc],
                                [-cosb*sinc,
                                 -sinb*sinc,
                                 cosc]])

        precmatrix = precmatrix.transpose()

        x = (np.matrix([cd*ca, cd*sa, sd])).transpose()

        x2 = precmatrix * x

        outra = np.arctan2(x2[1],x2[0])
        outdec = np.arcsin(x2[2])

        outradeg = np.rad2deg(outra)
        outdecdeg = np.rad2deg(outdec)

        if outradeg < 0.0:
            outradeg = outradeg + 360.0

        if outscalar:
            return float(outradeg), float(outdecdeg)
        else:
            return outradeg, outdecdeg

    else:

        # if the epochs are the same and no proper motion, this will be the same
        # as the input values. if the epochs are the same, but there IS proper
        # motion (and a given JD), then these will be perturbed from the input
        # values of ra, dec by the appropriate amount of motion
        return np.degrees(raproc), np.degrees(decproc)

def precess_coordinates ( ra , dec , epoch_one , epoch_two , jd = None , mu_ra = 0.0 , mu_dec = 0.0 , outscalar = False ) : raproc , decproc = np . radians ( ra ) , np . radians ( dec ) if ( ( mu_ra != 0.0 ) and ( mu_dec != 0.0 ) and jd ) : jd_epoch_one = JD2000 + ( epoch_one - epoch_two ) * 365.25 raproc = ( raproc + ( jd - jd_epoch_one ) * mu_ra * MAS_P_YR_TO_RAD_P_DAY / np . cos ( decproc ) ) decproc = decproc + ( jd - jd_epoch_one ) * mu_dec * MAS_P_YR_TO_RAD_P_DAY ca = np . cos ( raproc ) cd = np . cos ( decproc ) sa = np . sin ( raproc ) sd = np . sin ( decproc ) if epoch_one != epoch_two : t1 = 1.0e-3 * ( epoch_two - epoch_one ) t2 = 1.0e-3 * ( epoch_one - 2000.0 ) a = ( t1 * ARCSEC_TO_RADIANS * ( 23062.181 + t2 * ( 139.656 + 0.0139 * t2 ) + t1 * ( 30.188 - 0.344 * t2 + 17.998 * t1 ) ) ) b = t1 * t1 * ARCSEC_TO_RADIANS * ( 79.280 + 0.410 * t2 + 0.205 * t1 ) + a c = ( ARCSEC_TO_RADIANS * t1 * ( 20043.109 - t2 * ( 85.33 + 0.217 * t2 ) + t1 * ( - 42.665 - 0.217 * t2 - 41.833 * t2 ) ) ) sina , sinb , sinc = np . sin ( a ) , np . sin ( b ) , np . sin ( c ) cosa , cosb , cosc = np . cos ( a ) , np . cos ( b ) , np . cos ( c ) precmatrix = np . matrix ( [ [ cosa * cosb * cosc - sina * sinb , sina * cosb + cosa * sinb * cosc , cosa * sinc ] , [ - cosa * sinb - sina * cosb * cosc , cosa * cosb - sina * sinb * cosc , - sina * sinc ] , [ - cosb * sinc , - sinb * sinc , cosc ] ] ) precmatrix = precmatrix . transpose ( ) x = ( np . matrix ( [ cd * ca , cd * sa , sd ] ) ) . transpose ( ) x2 = precmatrix * x outra = np . arctan2 ( x2 [ 1 ] , x2 [ 0 ] ) outdec = np . arcsin ( x2 [ 2 ] ) outradeg = np . rad2deg ( outra ) outdecdeg = np . rad2deg ( outdec ) if outradeg < 0.0 : outradeg = outradeg + 360.0 if outscalar : return float ( outradeg ) , float ( outdecdeg ) else : return outradeg , outdecdeg else : return np . degrees ( raproc ) , np . degrees ( decproc )

Precesses target coordinates ra dec from epoch_one to epoch_two .


def _single_true(iterable):
    '''This returns True if only one True-ish element exists in `iterable`.

    Parameters
    ----------

    iterable : iterable

    Returns
    -------

    bool
        True if only one True-ish element exists in `iterable`. False otherwise.

    '''

    # return True if exactly one true found
    iterator = iter(iterable)

    # consume from "i" until first true or it's exhausted
    has_true = any(iterator)

    # carry on consuming until another true value / exhausted
    has_another_true = any(iterator)

    return has_true and not has_another_true

def _single_true ( iterable ) : iterator = iter ( iterable ) has_true = any ( iterator ) has_another_true = any ( iterator ) return has_true and not has_another_true

This returns True if only one True - ish element exists in iterable .


def get_epochs_given_midtimes_and_period(
        t_mid,
        period,
        err_t_mid=None,
        t0_fixed=None,
        t0_percentile=None,
        verbose=False
):
    '''This calculates the future epochs for a transit, given a period and a
    starting epoch

    The equation used is::

        t_mid = period*epoch + t0

    Default behavior if no kwargs are used is to define `t0` as the median
    finite time of the passed `t_mid` array.

    Only one of `err_t_mid` or `t0_fixed` should be passed.

    Parameters
    ----------

    t_mid : np.array
        A np.array of transit mid-time measurements

    period : float
        The period used to calculate epochs, per the equation above. For typical
        use cases, a period precise to ~1e-5 days is sufficient to get correct
        epochs.

    err_t_mid : None or np.array
        If provided, contains the errors of the transit mid-time
        measurements. The zero-point epoch is then set equal to the average of
        the transit times, weighted as `1/err_t_mid^2` . This minimizes the
        covariance between the transit epoch and the period (e.g., Gibson et
        al. 2013). For standard O-C analysis this is the best method.

    t0_fixed : None or float:
        If provided, use this t0 as the starting epoch. (Overrides all others).

    t0_percentile : None or float
        If provided, use this percentile of `t_mid` to define `t0`.

    Returns
    -------

    tuple
        This is the of the form `(integer_epoch_array, t0)`.
        `integer_epoch_array` is an array of integer epochs (float-type),
        of length equal to the number of *finite* mid-times passed.

    '''

    kwargarr = np.array([isinstance(err_t_mid,np.ndarray),
                         t0_fixed,
                         t0_percentile])
    if not _single_true(kwargarr) and not np.all(~kwargarr.astype(bool)):
        raise AssertionError(
            'can have at most one of err_t_mid, t0_fixed, t0_percentile')

    t_mid = t_mid[np.isfinite(t_mid)]
    N_midtimes = len(t_mid)

    if t0_fixed:
        t0 = t0_fixed
    elif isinstance(err_t_mid,np.ndarray):
        # get the weighted average. then round it to the nearest transit epoch.
        t0_avg = np.average(t_mid, weights=1/err_t_mid**2)
        t0_options = np.arange(min(t_mid), max(t_mid)+period, period)
        t0 = t0_options[np.argmin(np.abs(t0_options - t0_avg))]
    else:
        if not t0_percentile:
            # if there are an odd number of times, take the median time as
            # epoch=0.  elif there are an even number of times, take the lower
            # of the two middle times as epoch=0.
            if N_midtimes % 2 == 1:
                t0 = np.median(t_mid)
            else:
                t0 = t_mid[int(N_midtimes/2)]
        else:
            t0 = np.sort(t_mid)[int(N_midtimes*t0_percentile/100)]

    epoch = (t_mid - t0)/period

    # do not convert numpy entries to actual ints, because np.nan is float type
    int_epoch = np.round(epoch, 0)

    if verbose:
        LOGINFO('epochs before rounding')
        LOGINFO('\n{:s}'.format(repr(epoch)))
        LOGINFO('epochs after rounding')
        LOGINFO('\n{:s}'.format(repr(int_epoch)))

    return int_epoch, t0

This calculates the future epochs for a transit given a period and a starting epoch


def jd_to_datetime(jd, returniso=False):
    '''This converts a UTC JD to a Python `datetime` object or ISO date string.

    Parameters
    ----------

    jd : float
        The Julian date measured at UTC.

    returniso : bool
        If False, returns a naive Python `datetime` object corresponding to
        `jd`. If True, returns the ISO format string corresponding to the date
        and time at UTC from `jd`.

    Returns
    -------

    datetime or str
        Depending on the value of `returniso`.

    '''

    tt = astime.Time(jd, format='jd', scale='utc')

    if returniso:
        return tt.iso
    else:
        return tt.datetime

def jd_to_datetime ( jd , returniso = False ) : tt = astime . Time ( jd , format = 'jd' , scale = 'utc' ) if returniso : return tt . iso else : return tt . datetime

This converts a UTC JD to a Python datetime object or ISO date string .


def jd_corr(jd,
            ra, dec,
            obslon=None,
            obslat=None,
            obsalt=None,
            jd_type='bjd'):
    '''Returns BJD_TDB or HJD_TDB for input JD_UTC.

    The equation used is::

        BJD_TDB = JD_UTC + JD_to_TDB_corr + romer_delay

    where:

    - JD_to_TDB_corr is the difference between UTC and TDB JDs
    - romer_delay is the delay caused by finite speed of light from Earth-Sun

    This is based on the code at:

    https://mail.scipy.org/pipermail/astropy/2014-April/003148.html

    Note that this does not correct for:

    1. precession of coordinates if the epoch is not 2000.0
    2. precession of coordinates if the target has a proper motion
    3. Shapiro delay
    4. Einstein delay

    Parameters
    ----------

    jd : float or array-like
        The Julian date(s) measured at UTC.

    ra,dec : float
        The equatorial coordinates of the object in decimal degrees.

    obslon,obslat,obsalt : float or None
        The longitude, latitude of the observatory in decimal degrees and
        altitude of the observatory in meters. If these are not provided, the
        corrected JD will be calculated with respect to the center of the Earth.

    jd_type : {'bjd','hjd'}
        Conversion type to perform, either to Baryocentric Julian Date ('bjd')
        or to Heliocenter Julian Date ('hjd').

    Returns
    -------

    float or np.array
        The converted BJD or HJD.

    '''

    if not HAVEKERNEL:
        LOGERROR('no JPL kernel available, can\'t continue!')
        return

    # Source unit-vector
    ## Assume coordinates in ICRS
    ## Set distance to unit (kilometers)

    # convert the angles to degrees
    rarad = np.radians(ra)
    decrad = np.radians(dec)
    cosra = np.cos(rarad)
    sinra = np.sin(rarad)
    cosdec = np.cos(decrad)
    sindec = np.sin(decrad)

    # this assumes that the target is very far away
    src_unitvector = np.array([cosdec*cosra,cosdec*sinra,sindec])

    # Convert epochs to astropy.time.Time
    ## Assume JD(UTC)
    if (obslon is None) or (obslat is None) or (obsalt is None):
        t = astime.Time(jd, scale='utc', format='jd')
    else:
        t = astime.Time(jd, scale='utc', format='jd',
                        location=('%.5fd' % obslon,
                                  '%.5fd' % obslat,
                                  obsalt))

    # Get Earth-Moon barycenter position
    ## NB: jplephem uses Barycentric Dynamical Time, e.g. JD(TDB)
    ## and gives positions relative to solar system barycenter
    barycenter_earthmoon = jplkernel[0,3].compute(t.tdb.jd)

    # Get Moon position vectors from the center of Earth to the Moon
    # this means we get the following vectors from the ephemerides
    # Earth Barycenter (3) -> Moon (301)
    # Earth Barycenter (3) -> Earth (399)
    # so the final vector is [3,301] - [3,399]
    # units are in km
    moonvector = (jplkernel[3,301].compute(t.tdb.jd) -
                  jplkernel[3,399].compute(t.tdb.jd))

    # Compute Earth position vectors (this is for the center of the earth with
    # respect to the solar system barycenter)
    # all these units are in km
    pos_earth = (barycenter_earthmoon - moonvector * 1.0/(1.0+EMRAT))

    if jd_type == 'bjd':

        # Compute BJD correction
        ## Assume source vectors parallel at Earth and Solar System
        ## Barycenter
        ## i.e. source is at infinity
        # the romer_delay correction is (r.dot.n)/c where:
        # r is the vector from SSB to earth center
        # n is the unit vector from
        correction_seconds = np.dot(pos_earth.T, src_unitvector)/CLIGHT_KPS
        correction_days = correction_seconds/SEC_P_DAY

    elif jd_type == 'hjd':

        # Compute HJD correction via Sun ephemeris

        # this is the position vector of the center of the sun in km
        # Solar System Barycenter (0) -> Sun (10)
        pos_sun = jplkernel[0,10].compute(t.tdb.jd)

        # this is the vector from the center of the sun to the center of the
        # earth
        sun_earth_vec = pos_earth - pos_sun

        # calculate the heliocentric correction
        correction_seconds = np.dot(sun_earth_vec.T, src_unitvector)/CLIGHT_KPS
        correction_days = correction_seconds/SEC_P_DAY

    # TDB is the appropriate time scale for these ephemerides
    new_jd = t.tdb.jd + correction_days

    return new_jd


def _lclist_parallel_worker(task):
    '''This is a parallel worker for makelclist.

    Parameters
    ----------

    task : tuple
        This is a tuple containing the following items:

        task[0] = lcf
        task[1] = columns
        task[2] = lcformat
        task[3] = lcformatdir
        task[4] = lcndetkey

    Returns
    -------

    dict or None
        This contains all of the info for the object processed in this LC read
        operation. If this fails, returns None

    '''

    lcf, columns, lcformat, lcformatdir, lcndetkey = task

    # get the bits needed for lcformat handling
    # NOTE: we re-import things in this worker function because sometimes
    # functions can't be pickled correctly for passing them to worker functions
    # in a processing pool
    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    # we store the full path of the light curve
    lcobjdict = {'lcfname':os.path.abspath(lcf)}

    try:

        # read the light curve in
        lcdict = readerfunc(lcf)

        # this should handle lists/tuples being returned by readerfunc
        # we assume that the first element is the actual lcdict
        # FIXME: figure out how to not need this assumption
        if ( (isinstance(lcdict, (list, tuple))) and
             (isinstance(lcdict[0], dict)) ):
            lcdict = lcdict[0]

        # insert all of the columns
        for colkey in columns:

            if '.' in colkey:
                getkey = colkey.split('.')
            else:
                getkey = [colkey]

            try:
                thiscolval = _dict_get(lcdict, getkey)
            except Exception as e:
                LOGWARNING('column %s does not exist for %s' %
                           (colkey, lcf))
                thiscolval = np.nan

            # update the lcobjdict with this value
            lcobjdict[getkey[-1]] = thiscolval

    except Exception as e:

        LOGEXCEPTION('could not figure out columns for %s' % lcf)

        # insert all of the columns as nans
        for colkey in columns:

            if '.' in colkey:
                getkey = colkey.split('.')
            else:
                getkey = [colkey]

            thiscolval = np.nan

            # update the lclistdict with this value
            lcobjdict[getkey[-1]] = thiscolval

    # now get the actual ndets; this excludes nans and infs
    for dk in lcndetkey:

        try:

            if '.' in dk:
                getdk = dk.split('.')
            else:
                getdk = [dk]

            ndetcol = _dict_get(lcdict, getdk)
            actualndets = ndetcol[np.isfinite(ndetcol)].size
            lcobjdict['%s.ndet' % getdk[-1]] = actualndets

        except Exception as e:
            lcobjdict['%s.ndet' % getdk[-1]] = np.nan

    return lcobjdict

This is a parallel worker for makelclist .


def _cpinfo_key_worker(task):
    '''This wraps `checkplotlist.checkplot_infokey_worker`.

    This is used to get the correct dtype for each element in retrieved results.

    Parameters
    ----------

    task : tuple
        task[0] = cpfile
        task[1] = keyspeclist (infokeys kwarg from `add_cpinfo_to_lclist`)

    Returns
    -------

    dict
        All of the requested keys from the checkplot are returned along with
        their values in a dict.

    '''

    cpfile, keyspeclist = task

    keystoget = [x[0] for x in keyspeclist]
    nonesubs = [x[-2] for x in keyspeclist]
    nansubs = [x[-1] for x in keyspeclist]

    # reform the keystoget into a list of lists
    for i, k in enumerate(keystoget):

        thisk = k.split('.')
        if sys.version_info[:2] < (3,4):
            thisk = [(int(x) if x.isdigit() else x) for x in thisk]
        else:
            thisk = [(int(x) if x.isdecimal() else x) for x in thisk]

        keystoget[i] = thisk

    # add in the objectid as well to match to the object catalog later
    keystoget.insert(0,['objectid'])
    nonesubs.insert(0, '')
    nansubs.insert(0,'')

    # get all the keys we need
    vals = checkplot_infokey_worker((cpfile, keystoget))

    # if they have some Nones, nans, etc., reform them as expected
    for val, nonesub, nansub, valind in zip(vals, nonesubs,
                                            nansubs, range(len(vals))):

        if val is None:
            outval = nonesub
        elif isinstance(val, float) and not np.isfinite(val):
            outval = nansub
        elif isinstance(val, (list, tuple)):
            outval = ', '.join(val)
        else:
            outval = val

        vals[valind] = outval

    return vals

def
_cpinfo_key_worker
(
task
)
:
cpfile
,
keyspeclist
=
task
keystoget
=
[
x
[
0
]
for
x
in
keyspeclist
]
nonesubs
=
[
x
[
2
]
for
x
in
keyspeclist
]
nansubs
=
[
x
[
1
]
for
x
in
keyspeclist
]
for
i
,
k
in
enumerate
(
keystoget
)
:
thisk
=
k
.
split
(
'.'
)
if
sys
.
version_info
[
:
2
]
<
(
3
,
4
)
:
thisk
=
[
(
int
(
x
)
if
x
.
isdigit
(
)
else
x
)
for
x
in
thisk
]
else
:
thisk
=
[
(
int
(
x
)
if
x
.
isdecimal
(
)
else
x
)
for
x
in
thisk
]
keystoget
[
i
]
=
thisk
keystoget
.
insert
(
0
,
[
'objectid'
]
)
nonesubs
.
insert
(
0
,
''
)
nansubs
.
insert
(
0
,
''
)
vals
=
checkplot_infokey_worker
(
(
cpfile
,
keystoget
)
)
for
val
,
nonesub
,
nansub
,
valind
in
zip
(
vals
,
nonesubs
,
nansubs
,
range
(
len
(
vals
)
)
)
:
if
val
is
:
outval
=
nonesub
elif
isinstance
(
val
,
float
)
and
not
np
.
isfinite
(
val
)
:
outval
=
nansub
elif
isinstance
(
val
,
(
list
,
tuple
)
)
:
outval
=
', '
.
join
(
val
)
else
:
outval
=
val
vals
[
valind
]
=
outval
return
vals

This wraps checkplotlist . checkplot_infokey_worker .


def handle_change(self, change):
        """ Handle changes from atom ContainerLists """
        op = change['operation']
        if op in 'append':
            self.add(len(change['value']), LatLng(*change['item']))
        elif op == 'insert':
            self.add(change['index'], LatLng(*change['item']))
        elif op == 'extend':
            points = [LatLng(*p) for p in change['items']]
            self.addAll([bridge.encode(c) for c in points])
        elif op == '__setitem__':
            self.set(change['index'], LatLng(*change['newitem']))
        elif op == 'pop':
            self.remove(change['index'])
        else:
            raise NotImplementedError(
                "Unsupported change operation {}".format(op))

def
handle_change
(
self
,
change
)
:
op
=
change
[
'operation'
]
if
op
in
'append'
:
self
.
add
(
len
(
change
[
'value'
]
)
,
LatLng
(
*
change
[
'item'
]
)
)
elif
op
==
'insert'
:
self
.
add
(
change
[
'index'
]
,
LatLng
(
*
change
[
'item'
]
)
)
elif
op
==
'extend'
:
points
=
[
LatLng
(
*
p
)
for
p
in
change
[
'items'
]
]
self
.
addAll
(
[
bridge
.
encode
(
c
)
for
c
in
points
]
)
elif
op
==
'__setitem__'
:
self
.
set
(
change
[
'index'
]
,
LatLng
(
*
change
[
'newitem'
]
)
)
elif
op
==
'pop'
:
self
.
remove
(
change
[
'index'
]
)
else
:
raise
NotImplementedError
(
"Unsupported change operation {}"
.
format
(
op
)
)


def create_widget(self):
        """ Create the underlying widget.

        """
        self.init_options()

        #: Retrieve the actual map
        MapFragment.newInstance(self.options).then(
            self.on_map_fragment_created)

        # Holder for the fragment
        self.widget = FrameLayout(self.get_context())

        # I wrote this a few days ago and already forget how this hack works...
        # lol We can't simply get a map reference using getMapAsync in the
        # return value like we normally do with a normal call function return
        # value. The bridge design was modified to store an object that cannot
        # be decoded normally (via a standard Bridge.Packer) by saving the new
        # object in the cache returning the id of the handler or proxy that
        # invoked it. This way we can manually create a new id and pass that
        # "future reference-able" object as our listener. At which point the
        # bridge will create a reference entry in the cache for us with the of
        # the object we gave it. Once in the cache we can use it like any
        # bridge object we created.
        self.map = GoogleMap(__id__=bridge.generate_id())

def create_widget ( self ) : self . init_options ( ) MapFragment . newInstance ( self . options ) . then ( self . on_map_fragment_created ) self . widget = FrameLayout ( self . get_context ( ) ) self . map = GoogleMap ( __id__ = bridge . generate_id ( ) )


def init_options(self):
        """ Initialize the underlying map options.

        """
        self.options = GoogleMapOptions()
        d = self.declaration
        self.set_map_type(d.map_type)
        if d.ambient_mode:
            self.set_ambient_mode(d.ambient_mode)
        if (d.camera_position or d.camera_zoom or
                d.camera_tilt or d.camera_bearing):
            self.update_camera()
        if d.map_bounds:
            self.set_map_bounds(d.map_bounds)
        if not d.show_compass:
            self.set_show_compass(d.show_compass)
        if not d.show_zoom_controls:
            self.set_show_zoom_controls(d.show_zoom_controls)
        if not d.show_toolbar:
            self.set_show_toolbar(d.show_toolbar)
        if d.lite_mode:
            self.set_lite_mode(d.lite_mode)
        if not d.rotate_gestures:
            self.set_rotate_gestures(d.rotate_gestures)
        if not d.scroll_gestures:
            self.set_scroll_gestures(d.scroll_gestures)
        if not d.tilt_gestures:
            self.set_tilt_gestures(d.tilt_gestures)
        if not d.zoom_gestures:
            self.set_zoom_gestures(d.zoom_gestures)
        if d.min_zoom:
            self.set_min_zoom(d.min_zoom)
        if d.max_zoom:
            self.set_max_zoom(d.max_zoom)

def init_options ( self ) : self . options = GoogleMapOptions ( ) d = self . declaration self . set_map_type ( d . map_type ) if d . ambient_mode : self . set_ambient_mode ( d . ambient_mode ) if ( d . camera_position or d . camera_zoom or d . camera_tilt or d . camera_bearing ) : self . update_camera ( ) if d . map_bounds : self . set_map_bounds ( d . map_bounds ) if not d . show_compass : self . set_show_compass ( d . show_compass ) if not d . show_zoom_controls : self . set_show_zoom_controls ( d . show_zoom_controls ) if not d . show_toolbar : self . set_show_toolbar ( d . show_toolbar ) if d . lite_mode : self . set_lite_mode ( d . lite_mode ) if not d . rotate_gestures : self . set_rotate_gestures ( d . rotate_gestures ) if not d . scroll_gestures : self . set_scroll_gestures ( d . scroll_gestures ) if not d . tilt_gestures : self . set_tilt_gestures ( d . tilt_gestures ) if not d . zoom_gestures : self . set_zoom_gestures ( d . zoom_gestures ) if d . min_zoom : self . set_min_zoom ( d . min_zoom ) if d . max_zoom : self . set_max_zoom ( d . max_zoom )


def init_map(self):
        """ Add markers, polys, callouts, etc.."""
        d = self.declaration
        if d.show_location:
            self.set_show_location(d.show_location)
        if d.show_traffic:
            self.set_show_traffic(d.show_traffic)
        if d.show_indoors:
            self.set_show_indoors(d.show_indoors)
        if d.show_buildings:
            self.set_show_buildings(d.show_buildings)

        #: Local ref access is faster
        mapview = self.map
        mid = mapview.getId()

        #: Connect signals
        #: Camera
        mapview.onCameraChange.connect(self.on_camera_changed)
        mapview.onCameraMoveStarted.connect(self.on_camera_move_started)
        mapview.onCameraMoveCanceled.connect(self.on_camera_move_stopped)
        mapview.onCameraIdle.connect(self.on_camera_move_stopped)
        mapview.setOnCameraChangeListener(mid)
        mapview.setOnCameraMoveStartedListener(mid)
        mapview.setOnCameraMoveCanceledListener(mid)
        mapview.setOnCameraIdleListener(mid)

        #: Clicks
        mapview.onMapClick.connect(self.on_map_clicked)
        mapview.setOnMapClickListener(mid)
        mapview.onMapLongClick.connect(self.on_map_long_clicked)
        mapview.setOnMapLongClickListener(mid)

        #: Markers
        mapview.onMarkerClick.connect(self.on_marker_clicked)
        mapview.setOnMarkerClickListener(self.map.getId())
        mapview.onMarkerDragStart.connect(self.on_marker_drag_start)
        mapview.onMarkerDrag.connect(self.on_marker_drag)
        mapview.onMarkerDragEnd.connect(self.on_marker_drag_end)
        mapview.setOnMarkerDragListener(mid)

        #: Info window
        mapview.onInfoWindowClick.connect(self.on_info_window_clicked)
        mapview.onInfoWindowLongClick.connect(self.on_info_window_long_clicked)
        mapview.onInfoWindowClose.connect(self.on_info_window_closed)
        mapview.setOnInfoWindowClickListener(mid)
        mapview.setOnInfoWindowCloseListener(mid)
        mapview.setOnInfoWindowLongClickListener(mid)

        #: Polys
        mapview.onPolygonClick.connect(self.on_poly_clicked)
        mapview.onPolylineClick.connect(self.on_poly_clicked)
        mapview.setOnPolygonClickListener(mid)
        mapview.setOnPolylineClickListener(mid)

        #: Circle
        mapview.onCircleClick.connect(self.on_circle_clicked)
        mapview.setOnCircleClickListener(mid)

def init_map ( self ) : d = self . declaration if d . show_location : self . set_show_location ( d . show_location ) if d . show_traffic : self . set_show_traffic ( d . show_traffic ) if d . show_indoors : self . set_show_indoors ( d . show_indoors ) if d . show_buildings : self . set_show_buildings ( d . show_buildings ) mapview = self . map mid = mapview . getId ( ) mapview . onCameraChange . connect ( self . on_camera_changed ) mapview . onCameraMoveStarted . connect ( self . on_camera_move_started ) mapview . onCameraMoveCanceled . connect ( self . on_camera_move_stopped ) mapview . onCameraIdle . connect ( self . on_camera_move_stopped ) mapview . setOnCameraChangeListener ( mid ) mapview . setOnCameraMoveStartedListener ( mid ) mapview . setOnCameraMoveCanceledListener ( mid ) mapview . setOnCameraIdleListener ( mid ) mapview . onMapClick . connect ( self . on_map_clicked ) mapview . setOnMapClickListener ( mid ) mapview . onMapLongClick . connect ( self . on_map_long_clicked ) mapview . setOnMapLongClickListener ( mid ) mapview . onMarkerClick . connect ( self . on_marker_clicked ) mapview . setOnMarkerClickListener ( self . map . getId ( ) ) mapview . onMarkerDragStart . connect ( self . on_marker_drag_start ) mapview . onMarkerDrag . connect ( self . on_marker_drag ) mapview . onMarkerDragEnd . connect ( self . on_marker_drag_end ) mapview . setOnMarkerDragListener ( mid ) mapview . onInfoWindowClick . connect ( self . on_info_window_clicked ) mapview . onInfoWindowLongClick . connect ( self . on_info_window_long_clicked ) mapview . onInfoWindowClose . connect ( self . on_info_window_closed ) mapview . setOnInfoWindowClickListener ( mid ) mapview . setOnInfoWindowCloseListener ( mid ) mapview . setOnInfoWindowLongClickListener ( mid ) mapview . onPolygonClick . connect ( self . on_poly_clicked ) mapview . onPolylineClick . connect ( self . on_poly_clicked ) mapview . setOnPolygonClickListener ( mid ) mapview . setOnPolylineClickListener ( mid ) mapview . onCircleClick . connect ( self . on_circle_clicked ) mapview . setOnCircleClickListener ( mid )


def init_info_window_adapter(self):
        """ Initialize the info window adapter. Should only be done if one of
        the markers defines a custom view.
        """
        adapter = self.adapter
        if adapter:
            return  #: Already initialized
        adapter = GoogleMap.InfoWindowAdapter()
        adapter.getInfoContents.connect(self.on_info_window_contents_requested)
        adapter.getInfoWindow.connect(self.on_info_window_requested)
        self.map.setInfoWindowAdapter(adapter)

def init_info_window_adapter ( self ) : adapter = self . adapter if adapter : return adapter = GoogleMap . InfoWindowAdapter ( ) adapter . getInfoContents . connect ( self . on_info_window_contents_requested ) adapter . getInfoWindow . connect ( self . on_info_window_requested ) self . map . setInfoWindowAdapter ( adapter )

Initialize the info window adapter . Should only be done if one of the markers defines a custom view .


def on_map_fragment_created(self, obj_id):
        """ Create the fragment and pull the map reference when it's loaded.
        """
        self.fragment = MapFragment(__id__=obj_id)

        #: Setup callback so we know when the map is ready
        self.map.onMapReady.connect(self.on_map_ready)
        self.fragment.getMapAsync(self.map.getId())

        context = self.get_context()

        def on_transaction(id):
            trans = FragmentTransaction(__id__=id)
            trans.add(self.widget.getId(), self.fragment)
            trans.commit()

        def on_fragment_manager(id):
            fm = FragmentManager(__id__=id)
            fm.beginTransaction().then(on_transaction)

        context.widget.getSupportFragmentManager().then(on_fragment_manager)

def on_map_fragment_created ( self , obj_id ) : self . fragment = MapFragment ( __id__ = obj_id ) self . map . onMapReady . connect ( self . on_map_ready ) self . fragment . getMapAsync ( self . map . getId ( ) ) context = self . get_context ( ) def on_transaction ( id ) : trans = FragmentTransaction ( __id__ = id ) trans . add ( self . widget . getId ( ) , self . fragment ) trans . commit ( ) def on_fragment_manager ( id ) : fm = FragmentManager ( __id__ = id ) fm . beginTransaction ( ) . then ( on_transaction ) context . widget . getSupportFragmentManager ( ) . then ( on_fragment_manager )

Create the fragment and pull the map reference when it s loaded .


def destroy(self):
        """ Remove the marker if it was added to the map when destroying"""
        marker = self.marker
        parent = self.parent()
        if marker:
            if parent:
                del parent.markers[marker.__id__]
            marker.remove()
        super(AndroidMapItemBase, self).destroy()

def destroy ( self ) : marker = self . marker parent = self . parent ( ) if marker : if parent : del parent . markers [ marker . __id__ ] marker . remove ( ) super ( AndroidMapItemBase , self ) . destroy ( )


def child_added(self, child):
        """ If a child is added we have to make sure the map adapter exists """
        if child.widget:
            # TODO: Should we keep count and remove the adapter if not all
            # markers request it?
            self.parent().init_info_window_adapter()
        super(AndroidMapMarker, self).child_added(child)

def child_added ( self , child ) : if child . widget : self . parent ( ) . init_info_window_adapter ( ) super ( AndroidMapMarker , self ) . child_added ( child )


def on_marker(self, marker):
        """ Convert our options into the actual marker object"""
        mid, pos = marker
        self.marker = Marker(__id__=mid)
        mapview = self.parent()
        # Save ref
        mapview.markers[mid] = self

        # Required so the packer can pass the id
        self.marker.setTag(mid)

        # If we have a child widget we must configure the map to use the
        # custom adapter
        for w in self.child_widgets():
            mapview.init_info_window_adapter()
            break

        d = self.declaration
        if d.show_info:
            self.set_show_info(d.show_info)

        #: Can free the options now
        del self.options

def on_marker ( self , marker ) : mid , pos = marker self . marker = Marker ( __id__ = mid ) mapview = self . parent ( ) mapview . markers [ mid ] = self self . marker . setTag ( mid ) for w in self . child_widgets ( ) : mapview . init_info_window_adapter ( ) break d = self . declaration if d . show_info : self . set_show_info ( d . show_info ) del self . options


def on_marker(self, mid):
        """ Convert our options into the actual circle object"""
        self.marker = Circle(__id__=mid)
        self.parent().markers[mid] = self

        #: Required so the packer can pass the id
        self.marker.setTag(mid)

        d = self.declaration
        if d.clickable:
            self.set_clickable(d.clickable)

        #: Can free the options now
        del self.options

def on_marker ( self , mid ) : self . marker = Circle ( __id__ = mid ) self . parent ( ) . markers [ mid ] = self self . marker . setTag ( mid ) d = self . declaration if d . clickable : self . set_clickable ( d . clickable ) del self . options


def fit_transform(self, raw_documents, y=None):
        """ Learn the vocabulary dictionary and return term-document matrix.
        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        raw_documents : iterable
            An iterable which yields either str, unicode or file objects.

        Returns
        -------
        X : array, [n_samples, n_features]
            Document-term matrix.
        """
        documents = super(CountVectorizer, self).fit_transform(
            raw_documents=raw_documents, y=y)
        self.n = len(raw_documents)
        m = (self.transform(raw_documents) > 0).astype(int)
        m = m.sum(axis=0).A1
        self.period_ = m
        self.df_ = m / self.n
        return documents

def fit_transform ( self , raw_documents , y = None ) : documents = super ( CountVectorizer , self ) . fit_transform ( raw_documents = raw_documents , y = y ) self . n = len ( raw_documents ) m = ( self . transform ( raw_documents ) > 0 ) . astype ( int ) m = m . sum ( axis = 0 ) . A1 self . period_ = m self . df_ = m / self . n return documents

Learn the vocabulary dictionary and return term - document matrix . This is equivalent to fit followed by transform but more efficiently implemented .


def data(self, X=None, y=None, sentences=None):
        """
        Add data to flow

        """
        self.X = X
        self.y = y
        self.sentences = sentences

def data ( self , X = None , y = None , sentences = None ) : self . X = X self . y = y self . sentences = sentences


def transform(self, transformer):
        """
        Add transformer to flow and apply transformer to data in flow

        Parameters
        ----------
        transformer : Transformer
            a transformer to transform data
        """
        self.transformers.append(transformer)
        from languageflow.transformer.tagged import TaggedTransformer

        if isinstance(transformer, TaggedTransformer):
            self.X, self.y = transformer.transform(self.sentences)
        if isinstance(transformer, TfidfVectorizer):
            self.X = transformer.fit_transform(self.X)
        if isinstance(transformer, CountVectorizer):
            self.X = transformer.fit_transform(self.X)
        if isinstance(transformer, NumberRemover):
            self.X = transformer.transform(self.X)

        if isinstance(transformer, MultiLabelBinarizer):
            self.y = transformer.fit_transform(self.y)

def transform ( self , transformer ) : self . transformers . append ( transformer ) from languageflow . transformer . tagged import TaggedTransformer if isinstance ( transformer , TaggedTransformer ) : self . X , self . y = transformer . transform ( self . sentences ) if isinstance ( transformer , TfidfVectorizer ) : self . X = transformer . fit_transform ( self . X ) if isinstance ( transformer , CountVectorizer ) : self . X = transformer . fit_transform ( self . X ) if isinstance ( transformer , NumberRemover ) : self . X = transformer . transform ( self . X ) if isinstance ( transformer , MultiLabelBinarizer ) : self . y = transformer . fit_transform ( self . y )


def train(self):
        """
        Train model with transformed data
        """
        for i, model in enumerate(self.models):
            N = [int(i * len(self.y)) for i in self.lc_range]
            for n in N:
                X = self.X[:n]
                y = self.y[:n]
                e = Experiment(X, y, model.estimator, self.scores,
                               self.validation_method)
                e.log_folder = self.log_folder
                e.train()

def train ( self ) : for i , model in enumerate ( self . models ) : N = [ int ( i * len ( self . y ) ) for i in self . lc_range ] for n in N : X = self . X [ : n ] y = self . y [ : n ] e = Experiment ( X , y , model . estimator , self . scores , self . validation_method ) e . log_folder = self . log_folder e . train ( )


def export(self, model_name, export_folder):
        """
        Export model and transformers to export_folder

        Parameters
        ----------
        model_name: string
            name of model to export
        export_folder: string
            folder to store exported model and transformers
        """
        for transformer in self.transformers:
            if isinstance(transformer, MultiLabelBinarizer):
                joblib.dump(transformer,
                            join(export_folder, "label.transformer.bin"),
                            protocol=2)
            if isinstance(transformer, TfidfVectorizer):
                joblib.dump(transformer,
                            join(export_folder, "tfidf.transformer.bin"),
                            protocol=2)
            if isinstance(transformer, CountVectorizer):
                joblib.dump(transformer,
                            join(export_folder, "count.transformer.bin"),
                            protocol=2)
            if isinstance(transformer, NumberRemover):
                joblib.dump(transformer,
                            join(export_folder, "number.transformer.bin"),
                            protocol=2)
        model = [model for model in self.models if model.name == model_name][0]
        e = Experiment(self.X, self.y, model.estimator, None)
        model_filename = join(export_folder, "model.bin")
        e.export(model_filename)

def export ( self , model_name , export_folder ) : for transformer in self . transformers : if isinstance ( transformer , MultiLabelBinarizer ) : joblib . dump ( transformer , join ( export_folder , "label.transformer.bin" ) , protocol = 2 ) if isinstance ( transformer , TfidfVectorizer ) : joblib . dump ( transformer , join ( export_folder , "tfidf.transformer.bin" ) , protocol = 2 ) if isinstance ( transformer , CountVectorizer ) : joblib . dump ( transformer , join ( export_folder , "count.transformer.bin" ) , protocol = 2 ) if isinstance ( transformer , NumberRemover ) : joblib . dump ( transformer , join ( export_folder , "number.transformer.bin" ) , protocol = 2 ) model = [ model for model in self . models if model . name == model_name ] [ 0 ] e = Experiment ( self . X , self . y , model . estimator , None ) model_filename = join ( export_folder , "model.bin" ) e . export ( model_filename )


def fit(self, X, y, coef_init=None, intercept_init=None,
            sample_weight=None):
        """Fit linear model with Stochastic Gradient Descent.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data

        y : numpy array, shape (n_samples,)
            Target values

        coef_init : array, shape (n_classes, n_features)
            The initial coefficients to warm-start the optimization.

        intercept_init : array, shape (n_classes,)
            The initial intercept to warm-start the optimization.

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples.
            If not provided, uniform weights are assumed. These weights will
            be multiplied with class_weight (passed through the
            constructor) if class_weight is specified

        Returns
        -------
        self : returns an instance of self.
        """
        super(SGDClassifier, self).fit(X, y, coef_init, intercept_init,
                                       sample_weight)

def fit ( self , X , y , coef_init = None , intercept_init = None , sample_weight = None ) : super ( SGDClassifier , self ) . fit ( X , y , coef_init , intercept_init , sample_weight )

Fit linear model with Stochastic Gradient Descent .


def print_cm(cm, labels, hide_zeroes=False, hide_diagonal=False, hide_threshold=None):
    """pretty print for confusion matrixes"""
    columnwidth = max([len(x) for x in labels] + [5])  # 5 is value length
    empty_cell = " " * columnwidth
    # Print header
    print("    " + empty_cell, end=" ")
    for label in labels:
        print("%{0}s".format(columnwidth) % label, end=" ")
    print()
    # Print rows
    for i, label1 in enumerate(labels):
        print("    %{0}s".format(columnwidth) % label1, end=" ")
        for j in range(len(labels)):
            cell = "%{0}.1f".format(columnwidth) % cm[i, j]
            if hide_zeroes:
                cell = cell if float(cm[i, j]) != 0 else empty_cell
            if hide_diagonal:
                cell = cell if i != j else empty_cell
            if hide_threshold:
                cell = cell if cm[i, j] > hide_threshold else empty_cell
            print(cell, end=" ")
        print()

def
print_cm
(
cm
,
labels
,
hide_zeroes
=
False
,
hide_diagonal
=
False
,
hide_threshold
=
)
:
columnwidth
=
max
(
[
len
(
x
)
for
x
in
labels
]
+
[
5
]
)
empty_cell
=
" "
*
columnwidth
print
(
"    "
+
empty_cell
,
end
=
" "
)
for
label
in
labels
:
print
(
"%{0}s"
.
format
(
columnwidth
)
%
label
,
end
=
" "
)
print
(
)
for
i
,
label1
in
enumerate
(
labels
)
:
print
(
"    %{0}s"
.
format
(
columnwidth
)
%
label1
,
end
=
" "
)
for
j
in
range
(
len
(
labels
)
)
:
cell
=
"%{0}.1f"
.
format
(
columnwidth
)
%
cm
[
i
,
j
]
if
hide_zeroes
:
cell
=
cell
if
float
(
cm
[
i
,
j
]
)
!=
0
else
empty_cell
if
hide_diagonal
:
cell
=
cell
if
i
!=
j
else
empty_cell
if
hide_threshold
:
cell
=
cell
if
cm
[
i
,
j
]
>
hide_threshold
else
empty_cell
print
(
cell
,
end
=
" "
)
print
(
)