def train_rf_classifier(
        collected_features,
        test_fraction=0.25,
        n_crossval_iterations=20,
        n_kfolds=5,
        crossval_scoring_metric='f1',
        classifier_to_pickle=None,
        nworkers=-1,
):

    '''This gets the best RF classifier after running cross-validation.

    - splits the training set into test/train samples
    - does `KFold` stratified cross-validation using `RandomizedSearchCV`
    - gets the `RandomForestClassifier` with the best performance after CV
    - gets the confusion matrix for the test set

    Runs on the output dict from functions that produce dicts similar to that
    produced by `collect_nonperiodic_features` above.

    Parameters
    ----------

    collected_features : dict or str
        This is either the dict produced by a `collect_*_features` function or
        the pickle produced by the same.

    test_fraction : float
        This sets the fraction of the input set that will be used as the
        test set after training.

    n_crossval_iterations : int
        This sets the number of iterations to use when running the
        cross-validation.

    n_kfolds : int
        This sets the number of K-folds to use on the data when doing a
        test-train split.

    crossval_scoring_metric : str
        This is a string that describes how the cross-validation score is
        calculated for each iteration. See the URL below for how to specify this
        parameter:

        http://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter
        By default, this is tuned for binary classification and uses the F1
        scoring metric. Change the `crossval_scoring_metric` to another metric
        (probably 'accuracy') for multi-class classification, e.g. for periodic
        variable classification.

    classifier_to_pickle : str
        If this is a string indicating the name of a pickle file to write, will
        write the trained classifier to the pickle that can be later loaded and
        used to classify data.

    nworkers : int
        This is the number of parallel workers to use in the
        RandomForestClassifier. Set to -1 to use all CPUs on your machine.

    Returns
    -------

    dict
        A dict containing the trained classifier, cross-validation results, the
        input data set, and all input kwargs used is returned, along with
        cross-validation score metrics.

    '''

    if (isinstance(collected_features,str) and
        os.path.exists(collected_features)):
        with open(collected_features,'rb') as infd:
            fdict = pickle.load(infd)
    elif isinstance(collected_features, dict):
        fdict = collected_features
    else:
        LOGERROR("can't figure out the input collected_features arg")
        return None

    tfeatures = fdict['features_array']
    tlabels = fdict['labels_array']
    tfeaturenames = fdict['availablefeatures']
    tmagcol = fdict['magcol']
    tobjectids = fdict['objectids']

    # split the training set into training/test samples using stratification
    # to keep the same fraction of variable/nonvariables in each
    training_features, testing_features, training_labels, testing_labels = (
        train_test_split(
            tfeatures,
            tlabels,
            test_size=test_fraction,
            random_state=RANDSEED,
            stratify=tlabels
        )
    )

    # get a random forest classifier
    clf = RandomForestClassifier(n_jobs=nworkers,
                                 random_state=RANDSEED)

    # this is the grid def for hyperparam optimization
    rf_hyperparams = {
        "max_depth": [3,4,5,None],
        "n_estimators":sp_randint(100,2000),
        "max_features": sp_randint(1, 5),
        "min_samples_split": sp_randint(2, 11),
        "min_samples_leaf": sp_randint(2, 11),
    }

    # run the stratified kfold cross-validation on training features using our
    # random forest classifier object
    cvsearch = RandomizedSearchCV(
        clf,
        param_distributions=rf_hyperparams,
        n_iter=n_crossval_iterations,
        scoring=crossval_scoring_metric,
        cv=StratifiedKFold(n_splits=n_kfolds,
                           shuffle=True,
                           random_state=RANDSEED),
        random_state=RANDSEED
    )

    LOGINFO('running grid-search CV to optimize RF hyperparameters...')
    cvsearch_classifiers = cvsearch.fit(training_features,
                                        training_labels)

    # report on the classifiers' performance
    _gridsearch_report(cvsearch_classifiers.cv_results_)

    # get the best classifier after CV is done
    bestclf = cvsearch_classifiers.best_estimator_
    bestclf_score = cvsearch_classifiers.best_score_
    bestclf_hyperparams = cvsearch_classifiers.best_params_

    # test this classifier on the testing set
    test_predicted_labels = bestclf.predict(testing_features)

    recscore = recall_score(testing_labels, test_predicted_labels)
    precscore = precision_score(testing_labels,test_predicted_labels)
    f1score = f1_score(testing_labels, test_predicted_labels)
    confmatrix = confusion_matrix(testing_labels, test_predicted_labels)

    # write the classifier, its training/testing set, and its stats to the
    # pickle if requested
    outdict = {'features':tfeatures,
               'labels':tlabels,
               'feature_names':tfeaturenames,
               'magcol':tmagcol,
               'objectids':tobjectids,
               'kwargs':{'test_fraction':test_fraction,
                         'n_crossval_iterations':n_crossval_iterations,
                         'n_kfolds':n_kfolds,
                         'crossval_scoring_metric':crossval_scoring_metric,
                         'nworkers':nworkers},
               'collect_kwargs':fdict['kwargs'],
               'testing_features':testing_features,
               'testing_labels':testing_labels,
               'training_features':training_features,
               'training_labels':training_labels,
               'best_classifier':bestclf,
               'best_score':bestclf_score,
               'best_hyperparams':bestclf_hyperparams,
               'best_recall':recscore,
               'best_precision':precscore,
               'best_f1':f1score,
               'best_confmatrix':confmatrix}

    if classifier_to_pickle:

        with open(classifier_to_pickle,'wb') as outfd:
            pickle.dump(outdict, outfd, pickle.HIGHEST_PROTOCOL)

    # return this classifier and accompanying info
    return outdict

This gets the best RF classifier after running cross - validation .


def apply_rf_classifier(classifier,
                        varfeaturesdir,
                        outpickle,
                        maxobjects=None):
    '''This applys an RF classifier trained using `train_rf_classifier`
    to varfeatures pickles in `varfeaturesdir`.

    Parameters
    ----------

    classifier : dict or str
        This is the output dict or pickle created by `get_rf_classifier`. This
        will contain a `features_name` key that will be used to collect the same
        features used to train the classifier from the varfeatures pickles in
        varfeaturesdir.

    varfeaturesdir : str
        The directory containing the varfeatures pickles for objects that will
        be classified by the trained `classifier`.

    outpickle : str
        This is a filename for the pickle that will be written containing the
        result dict from this function.

    maxobjects : int
        This sets the number of objects to process in `varfeaturesdir`.

    Returns
    -------

    dict
        The classification results after running the trained `classifier` as
        returned as a dict. This contains predicted labels and their prediction
        probabilities.

    '''

    if isinstance(classifier,str) and os.path.exists(classifier):
        with open(classifier,'rb') as infd:
            clfdict = pickle.load(infd)
    elif isinstance(classifier, dict):
        clfdict = classifier
    else:
        LOGERROR("can't figure out the input classifier arg")
        return None

    # get the features to extract from clfdict
    if 'feature_names' not in clfdict:
        LOGERROR("feature_names not present in classifier input, "
                 "can't figure out which ones to extract from "
                 "varfeature pickles in %s" % varfeaturesdir)
        return None

    # get the feature labeltype, pklglob, and maxobjects from classifier's
    # collect_kwargs elem.
    featurestouse = clfdict['feature_names']
    pklglob = clfdict['collect_kwargs']['pklglob']
    magcol = clfdict['magcol']

    # extract the features used by the classifier from the varfeatures pickles
    # in varfeaturesdir using the pklglob provided
    featfile = os.path.join(
        os.path.dirname(outpickle),
        'actual-collected-features.pkl'
    )

    features = collect_nonperiodic_features(
        varfeaturesdir,
        magcol,
        featfile,
        pklglob=pklglob,
        featurestouse=featurestouse,
        maxobjects=maxobjects
    )

    # now use the trained classifier on these features
    bestclf = clfdict['best_classifier']

    predicted_labels = bestclf.predict(features['features_array'])

    # FIXME: do we need to use the probability calibration curves to fix these
    # probabilities? probably. figure out how to do this.
    predicted_label_probs = bestclf.predict_proba(
        features['features_array']
    )

    outdict = {
        'features':features,
        'featfile':featfile,
        'classifier':clfdict,
        'predicted_labels':predicted_labels,
        'predicted_label_probs':predicted_label_probs,
    }

    with open(outpickle,'wb') as outfd:
        pickle.dump(outdict, outfd, pickle.HIGHEST_PROTOCOL)

    return outdict

This applys an RF classifier trained using train_rf_classifier to varfeatures pickles in varfeaturesdir .


def plot_training_results(classifier,
                          classlabels,
                          outfile):
    '''This plots the training results from the classifier run on the training
    set.

    - plots the confusion matrix

    - plots the feature importances

    - FIXME: plot the learning curves too, see:
      http://scikit-learn.org/stable/modules/learning_curve.html

    Parameters
    ----------

    classifier : dict or str
        This is the output dict or pickle created by `get_rf_classifier`
        containing the trained classifier.

    classlabels : list of str
        This contains all of the class labels for the current classification
        problem.

    outfile : str
        This is the filename where the plots will be written.

    Returns
    -------

    str
        The path to the generated plot file.

    '''

    if isinstance(classifier,str) and os.path.exists(classifier):
        with open(classifier,'rb') as infd:
            clfdict = pickle.load(infd)
    elif isinstance(classifier, dict):
        clfdict = classifier
    else:
        LOGERROR("can't figure out the input classifier arg")
        return None

    confmatrix = clfdict['best_confmatrix']
    overall_feature_importances = clfdict[
        'best_classifier'
    ].feature_importances_
    feature_importances_per_tree = np.array([
        tree.feature_importances_
        for tree in clfdict['best_classifier'].estimators_
    ])
    stdev_feature_importances = np.std(feature_importances_per_tree,axis=0)

    feature_names = np.array(clfdict['feature_names'])

    plt.figure(figsize=(6.4*3.0,4.8))

    # confusion matrix
    plt.subplot(121)
    classes = np.array(classlabels)
    plt.imshow(confmatrix, interpolation='nearest', cmap=plt.cm.Blues)
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes)
    plt.yticks(tick_marks, classes)
    plt.title('evaluation set confusion matrix')
    plt.ylabel('predicted class')
    plt.xlabel('actual class')

    thresh = confmatrix.max() / 2.
    for i, j in itertools.product(range(confmatrix.shape[0]),
                                  range(confmatrix.shape[1])):
        plt.text(j, i, confmatrix[i, j],
                 horizontalalignment="center",
                 color="white" if confmatrix[i, j] > thresh else "black")

    # feature importances
    plt.subplot(122)

    features = np.array(feature_names)
    sorted_ind = np.argsort(overall_feature_importances)[::-1]

    features = features[sorted_ind]
    feature_names = feature_names[sorted_ind]
    overall_feature_importances = overall_feature_importances[sorted_ind]
    stdev_feature_importances = stdev_feature_importances[sorted_ind]

    plt.bar(np.arange(0,features.size),
            overall_feature_importances,
            yerr=stdev_feature_importances,
            width=0.8,
            color='grey')
    plt.xticks(np.arange(0,features.size),
               features,
               rotation=90)
    plt.yticks([0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0])
    plt.xlim(-0.75, features.size - 1.0 + 0.75)
    plt.ylim(0.0,0.9)
    plt.ylabel('relative importance')
    plt.title('relative importance of features')

    plt.subplots_adjust(wspace=0.1)

    plt.savefig(outfile,
                bbox_inches='tight',
                dpi=100)
    plt.close('all')
    return outfile

This plots the training results from the classifier run on the training set .


def _fourier_func(fourierparams, phase, mags):
    '''This returns a summed Fourier cosine series.

    Parameters
    ----------

    fourierparams : list
        This MUST be a list of the following form like so::

            [period,
             epoch,
             [amplitude_1, amplitude_2, amplitude_3, ..., amplitude_X],
             [phase_1, phase_2, phase_3, ..., phase_X]]

        where X is the Fourier order.

    phase,mags : np.array
        The input phase and magnitude areas to use as the basis for the cosine
        series. The phases are used directly to generate the values of the
        function, while the mags array is used to generate the zeroth order
        amplitude coefficient.

    Returns
    -------

    np.array
        The Fourier cosine series function evaluated over `phase`.

    '''

    # figure out the order from the length of the Fourier param list
    order = int(len(fourierparams)/2)

    # get the amplitude and phase coefficients
    f_amp = fourierparams[:order]
    f_pha = fourierparams[order:]

    # calculate all the individual terms of the series
    f_orders = [f_amp[x]*npcos(2.0*pi_value*x*phase + f_pha[x])
                for x in range(order)]

    # this is the zeroth order coefficient - a constant equal to median mag
    total_f = npmedian(mags)

    # sum the series
    for fo in f_orders:
        total_f += fo

    return total_f

def _fourier_func ( fourierparams , phase , mags ) : order = int ( len ( fourierparams ) / 2 ) f_amp = fourierparams [ : order ] f_pha = fourierparams [ order : ] f_orders = [ f_amp [ x ] * npcos ( 2.0 * pi_value * x * phase + f_pha [ x ] ) for x in range ( order ) ] total_f = npmedian ( mags ) for fo in f_orders : total_f += fo return total_f

This returns a summed Fourier cosine series .


def _fourier_chisq(fourierparams,
                   phase,
                   mags,
                   errs):
    '''This is the chisq objective function to be minimized by `scipy.minimize`.

    The parameters are the same as `_fourier_func` above. `errs` is used to
    calculate the chisq value.

    '''

    f = _fourier_func(fourierparams, phase, mags)
    chisq = npsum(((mags - f)*(mags - f))/(errs*errs))

    return chisq

def _fourier_chisq ( fourierparams , phase , mags , errs ) : f = _fourier_func ( fourierparams , phase , mags ) chisq = npsum ( ( ( mags - f ) * ( mags - f ) ) / ( errs * errs ) ) return chisq

This is the chisq objective function to be minimized by scipy . minimize .


def _fourier_residual(fourierparams,
                      phase,
                      mags):
    '''
    This is the residual objective function to be minimized by `scipy.leastsq`.

    The parameters are the same as `_fourier_func` above.

    '''

    f = _fourier_func(fourierparams, phase, mags)
    residual = mags - f

    return residual

def _fourier_residual ( fourierparams , phase , mags ) : f = _fourier_func ( fourierparams , phase , mags ) residual = mags - f return residual

This is the residual objective function to be minimized by scipy . leastsq .


def skyview_stamp(ra, decl,
                  survey='DSS2 Red',
                  scaling='Linear',
                  flip=True,
                  convolvewith=None,
                  forcefetch=False,
                  cachedir='~/.astrobase/stamp-cache',
                  timeout=10.0,
                  retry_failed=False,
                  savewcsheader=True,
                  verbose=False):
    '''This downloads a DSS FITS stamp centered on the coordinates specified.

    This wraps the function :py:func:`astrobase.services.skyview.get_stamp`,
    which downloads Digitized Sky Survey stamps in FITS format from the NASA
    SkyView service:

    https://skyview.gsfc.nasa.gov/current/cgi/query.pl

    Also adds some useful operations on top of the FITS file returned.

    Parameters
    ----------

    ra,decl : float
        The center coordinates for the stamp in decimal degrees.

    survey : str
        The survey name to get the stamp from. This is one of the
        values in the 'SkyView Surveys' option boxes on the SkyView
        webpage. Currently, we've only tested using 'DSS2 Red' as the value for
        this kwarg, but the other ones should work in principle.

    scaling : str
        This is the pixel value scaling function to use.

    flip : bool
        Will flip the downloaded image top to bottom. This should usually be
        True because matplotlib and FITS have different image coord origin
        conventions. Alternatively, set this to False and use the
        `origin='lower'` in any call to `matplotlib.pyplot.imshow` when plotting
        this image.

    convolvewith : astropy.convolution Kernel object or None
        If `convolvewith` is an astropy.convolution Kernel object from:

        http://docs.astropy.org/en/stable/convolution/kernels.html

        then, this function will return the stamp convolved with that
        kernel. This can be useful to see effects of wide-field telescopes (like
        the HATNet and HATSouth lenses) degrading the nominal 1 arcsec/px of
        DSS, causing blending of targets and any variability.

    forcefetch : bool
        If True, will disregard any existing cached copies of the stamp already
        downloaded corresponding to the requested center coordinates and
        redownload the FITS from the SkyView service.

    cachedir : str
        This is the path to the astrobase cache directory. All downloaded FITS
        stamps are stored here as .fits.gz files so we can immediately respond
        with the cached copy when a request is made for a coordinate center
        that's already been downloaded.

    timeout : float
        Sets the timeout in seconds to wait for a response from the NASA SkyView
        service.

    retry_failed : bool
        If the initial request to SkyView fails, and this is True, will retry
        until it succeeds.

    savewcsheader : bool
        If this is True, also returns the WCS header of the downloaded FITS
        stamp in addition to the FITS image itself. Useful for projecting object
        coordinates onto image xy coordinates for visualization.

    verbose : bool
        If True, indicates progress.

    Returns
    -------

    tuple or array or None
        This returns based on the value of `savewcsheader`:

        - If `savewcsheader=True`, returns a tuple:
          (FITS stamp image as a numpy array, FITS header)
        - If `savewcsheader=False`, returns only the FITS stamp image as numpy
          array.
        - If the stamp retrieval fails, returns None.

    '''

    stampdict = get_stamp(ra, decl,
                          survey=survey,
                          scaling=scaling,
                          forcefetch=forcefetch,
                          cachedir=cachedir,
                          timeout=timeout,
                          retry_failed=retry_failed,
                          verbose=verbose)
    #
    # DONE WITH FETCHING STUFF
    #
    if stampdict:

        # open the frame
        stampfits = pyfits.open(stampdict['fitsfile'])
        header = stampfits[0].header
        frame = stampfits[0].data
        stampfits.close()

        # finally, we can process the frame
        if flip:
            frame = np.flipud(frame)

        if verbose:
            LOGINFO('fetched stamp successfully for (%.3f, %.3f)'
                    % (ra, decl))

        if convolvewith:

            convolved = aconv.convolve(frame, convolvewith)
            if savewcsheader:
                return convolved, header
            else:
                return convolved

        else:

            if savewcsheader:
                return frame, header
            else:
                return frame

    else:
        LOGERROR('could not fetch the requested stamp for '
                 'coords: (%.3f, %.3f) from survey: %s and scaling: %s'
                 % (ra, decl, survey, scaling))
        return None

This downloads a DSS FITS stamp centered on the coordinates specified .


def plot_periodbase_lsp(lspinfo, outfile=None, plotdpi=100):

    '''Makes a plot of periodograms obtained from `periodbase` functions.

    This takes the output dict produced by any `astrobase.periodbase`
    period-finder function or a pickle filename containing such a dict and makes
    a periodogram plot.

    Parameters
    ----------

    lspinfo : dict or str
        If lspinfo is a dict, it must be a dict produced by an
        `astrobase.periodbase` period-finder function or a dict from your own
        period-finder function or routine that is of the form below with at
        least these keys::

            {'periods': np.array of all periods searched by the period-finder,
             'lspvals': np.array of periodogram power value for each period,
             'bestperiod': a float value that is the period with the highest
                           peak in the periodogram, i.e. the most-likely actual
                           period,
             'method': a three-letter code naming the period-finder used; must
                       be one of the keys in the `METHODLABELS` dict above,
             'nbestperiods': a list of the periods corresponding to periodogram
                             peaks (`nbestlspvals` below) to annotate on the
                             periodogram plot so they can be called out
                             visually,
             'nbestlspvals': a list of the power values associated with
                             periodogram peaks to annotate on the periodogram
                             plot so they can be called out visually; should be
                             the same length as `nbestperiods` above}

        If lspinfo is a str, then it must be a path to a pickle file that
        contains a dict of the form described above.

    outfile : str or None
        If this is a str, will write the periodogram plot to the file specified
        by this string. If this is None, will write to a file called
        'lsp-plot.png' in the current working directory.

    plotdpi : int
        Sets the resolution in DPI of the output periodogram plot PNG file.

    Returns
    -------

    str
        Absolute path to the periodogram plot file created.

    '''

    # get the lspinfo from a pickle file transparently
    if isinstance(lspinfo,str) and os.path.exists(lspinfo):
        LOGINFO('loading LSP info from pickle %s' % lspinfo)
        with open(lspinfo,'rb') as infd:
            lspinfo = pickle.load(infd)

    try:

        # get the things to plot out of the data
        periods = lspinfo['periods']
        lspvals = lspinfo['lspvals']
        bestperiod = lspinfo['bestperiod']
        lspmethod = lspinfo['method']

        # make the LSP plot on the first subplot
        plt.plot(periods, lspvals)
        plt.xscale('log',basex=10)
        plt.xlabel('Period [days]')
        plt.ylabel(PLOTYLABELS[lspmethod])
        plottitle = '%s best period: %.6f d' % (METHODSHORTLABELS[lspmethod],
                                                bestperiod)
        plt.title(plottitle)

        # show the best five peaks on the plot
        for bestperiod, bestpeak in zip(lspinfo['nbestperiods'],
                                        lspinfo['nbestlspvals']):

            plt.annotate('%.6f' % bestperiod,
                         xy=(bestperiod, bestpeak), xycoords='data',
                         xytext=(0.0,25.0), textcoords='offset points',
                         arrowprops=dict(arrowstyle="->"),fontsize='x-small')

        # make a grid
        plt.grid(color='#a9a9a9',
                 alpha=0.9,
                 zorder=0,
                 linewidth=1.0,
                 linestyle=':')

        # make the figure
        if outfile and isinstance(outfile, str):

            if outfile.endswith('.png'):
                plt.savefig(outfile,bbox_inches='tight',dpi=plotdpi)
            else:
                plt.savefig(outfile,bbox_inches='tight')

            plt.close()
            return os.path.abspath(outfile)

        elif dispok:

            plt.show()
            plt.close()
            return

        else:

            LOGWARNING('no output file specified and no $DISPLAY set, '
                       'saving to lsp-plot.png in current directory')
            outfile = 'lsp-plot.png'
            plt.savefig(outfile,bbox_inches='tight',dpi=plotdpi)
            plt.close()
            return os.path.abspath(outfile)

    except Exception as e:

        LOGEXCEPTION('could not plot this LSP, appears to be empty')
        return

Makes a plot of periodograms obtained from periodbase functions .


def lcdict_to_pickle(lcdict, outfile=None):
    '''This just writes the lcdict to a pickle.

    If outfile is None, then will try to get the name from the
    lcdict['objectid'] and write to <objectid>-hptxtlc.pkl. If that fails, will
    write to a file named hptxtlc.pkl'.

    '''

    if not outfile and lcdict['objectid']:
        outfile = '%s-hplc.pkl' % lcdict['objectid']
    elif not outfile and not lcdict['objectid']:
        outfile = 'hplc.pkl'

    with open(outfile,'wb') as outfd:
        pickle.dump(lcdict, outfd, protocol=pickle.HIGHEST_PROTOCOL)

    if os.path.exists(outfile):
        LOGINFO('lcdict for object: %s -> %s OK' % (lcdict['objectid'],
                                                    outfile))
        return outfile
    else:
        LOGERROR('could not make a pickle for this lcdict!')
        return None

This just writes the lcdict to a pickle .


def read_hatpi_pklc(lcfile):
    '''
    This just reads a pickle LC. Returns an lcdict.

    '''

    try:

        if lcfile.endswith('.gz'):
            infd = gzip.open(lcfile,'rb')
        else:
            infd = open(lcfile,'rb')

        lcdict = pickle.load(infd)
        infd.close()

        return lcdict

    except UnicodeDecodeError:

        if lcfile.endswith('.gz'):
            infd = gzip.open(lcfile,'rb')
        else:
            infd = open(lcfile,'rb')

        LOGWARNING('pickle %s was probably from Python 2 '
                   'and failed to load without using "latin1" encoding. '
                   'This is probably a numpy issue: '
                   'http://stackoverflow.com/q/11305790' % lcfile)
        lcdict = pickle.load(infd, encoding='latin1')
        infd.close()

        return lcdict

This just reads a pickle LC . Returns an lcdict .


def concatenate_textlcs(lclist,
                        sortby='rjd',
                        normalize=True):
    '''This concatenates a list of light curves.

    Does not care about overlaps or duplicates. The light curves must all be
    from the same aperture.

    The intended use is to concatenate light curves across CCDs or instrument
    changes for a single object. These can then be normalized later using
    standard astrobase tools to search for variablity and/or periodicity.

    sortby is a column to sort the final concatenated light curve by in
    ascending order.

    If normalize is True, then each light curve's magnitude columns are
    normalized to zero.

    The returned lcdict has an extra column: 'lcn' that tracks which measurement
    belongs to which input light curve. This can be used with
    lcdict['concatenated'] which relates input light curve index to input light
    curve filepath. Finally, there is an 'nconcatenated' key in the lcdict that
    contains the total number of concatenated light curves.

    '''

    # read the first light curve
    lcdict = read_hatpi_textlc(lclist[0])

    # track which LC goes where
    # initial LC
    lccounter = 0
    lcdict['concatenated'] = {lccounter: os.path.abspath(lclist[0])}
    lcdict['lcn'] = np.full_like(lcdict['rjd'], lccounter)

    # normalize if needed
    if normalize:

        for col in MAGCOLS:

            if col in lcdict:
                thismedval = np.nanmedian(lcdict[col])

                # handle fluxes
                if col in ('ifl1','ifl2','ifl3'):
                    lcdict[col] = lcdict[col] / thismedval
                # handle mags
                else:
                    lcdict[col] = lcdict[col] - thismedval

    # now read the rest
    for lcf in lclist[1:]:

        thislcd = read_hatpi_textlc(lcf)

        # if the columns don't agree, skip this LC
        if thislcd['columns'] != lcdict['columns']:
            LOGERROR('file %s does not have the '
                     'same columns as first file %s, skipping...'
                     % (lcf, lclist[0]))
            continue

        # otherwise, go ahead and start concatenatin'
        else:

            LOGINFO('adding %s (ndet: %s) to %s (ndet: %s)'
                    % (lcf,
                       thislcd['objectinfo']['ndet'],
                       lclist[0],
                       lcdict[lcdict['columns'][0]].size))

            # update LC tracking
            lccounter = lccounter + 1
            lcdict['concatenated'][lccounter] = os.path.abspath(lcf)
            lcdict['lcn'] = np.concatenate((
                lcdict['lcn'],
                np.full_like(thislcd['rjd'],lccounter)
            ))

            # concatenate the columns
            for col in lcdict['columns']:

                # handle normalization for magnitude columns
                if normalize and col in MAGCOLS:

                    thismedval = np.nanmedian(thislcd[col])

                    # handle fluxes
                    if col in ('ifl1','ifl2','ifl3'):
                        thislcd[col] = thislcd[col] / thismedval
                    # handle mags
                    else:
                        thislcd[col] = thislcd[col] - thismedval

                # concatenate the values
                lcdict[col] = np.concatenate((lcdict[col], thislcd[col]))

    #
    # now we're all done concatenatin'
    #

    # make sure to add up the ndet
    lcdict['objectinfo']['ndet'] = lcdict[lcdict['columns'][0]].size

    # update the stations
    lcdict['objectinfo']['stations'] = [
        'HP%s' % x for x in np.unique(lcdict['stf']).tolist()
    ]

    # update the total LC count
    lcdict['nconcatenated'] = lccounter + 1

    # if we're supposed to sort by a column, do so
    if sortby and sortby in [x[0] for x in COLDEFS]:

        LOGINFO('sorting concatenated light curve by %s...' % sortby)
        sortind = np.argsort(lcdict[sortby])

        # sort all the measurement columns by this index
        for col in lcdict['columns']:
            lcdict[col] = lcdict[col][sortind]

        # make sure to sort the lcn index as well
        lcdict['lcn'] = lcdict['lcn'][sortind]

    LOGINFO('done. concatenated light curve has %s detections' %
            lcdict['objectinfo']['ndet'])
    return lcdict

This concatenates a list of light curves .


def concatenate_textlcs_for_objectid(lcbasedir,
                                     objectid,
                                     aperture='TF1',
                                     postfix='.gz',
                                     sortby='rjd',
                                     normalize=True,
                                     recursive=True):
    '''This concatenates all text LCs for an objectid with the given aperture.

    Does not care about overlaps or duplicates. The light curves must all be
    from the same aperture.

    The intended use is to concatenate light curves across CCDs or instrument
    changes for a single object. These can then be normalized later using
    standard astrobase tools to search for variablity and/or periodicity.

    lcbasedir is the directory to start searching in.

    objectid is the object to search for.

    aperture is the aperture postfix to use: (TF1 = aperture 1,
                                              TF2 = aperture 2,
                                              TF3 = aperture 3)

    sortby is a column to sort the final concatenated light curve by in
    ascending order.

    If normalize is True, then each light curve's magnitude columns are
    normalized to zero, and the whole light curve is then normalized to the
    global median magnitude for each magnitude column.

    If recursive is True, then the function will search recursively in lcbasedir
    for any light curves matching the specified criteria. This may take a while,
    especially on network filesystems.

    The returned lcdict has an extra column: 'lcn' that tracks which measurement
    belongs to which input light curve. This can be used with
    lcdict['concatenated'] which relates input light curve index to input light
    curve filepath. Finally, there is an 'nconcatenated' key in the lcdict that
    contains the total number of concatenated light curves.

    '''
    LOGINFO('looking for light curves for %s, aperture %s in directory: %s'
            % (objectid, aperture, lcbasedir))

    if recursive is False:

        matching = glob.glob(os.path.join(lcbasedir,
                                          '*%s*%s*%s' % (objectid,
                                                         aperture,
                                                         postfix)))
    else:
        # use recursive glob for Python 3.5+
        if sys.version_info[:2] > (3,4):

            matching = glob.glob(os.path.join(lcbasedir,
                                              '**',
                                              '*%s*%s*%s' % (objectid,
                                                             aperture,
                                                             postfix)),
                                 recursive=True)
            LOGINFO('found %s files: %s' % (len(matching), repr(matching)))

        # otherwise, use os.walk and glob
        else:

            # use os.walk to go through the directories
            walker = os.walk(lcbasedir)
            matching = []

            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              '*%s*%s*%s' % (objectid,
                                                             aperture,
                                                             postfix))
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)
                        LOGINFO(
                            'found %s in dir: %s' % (repr(foundfiles),
                                                     os.path.join(root,sdir))
                        )

    # now that we have all the files, concatenate them
    # a single file will be returned as normalized
    if matching and len(matching) > 0:
        clcdict = concatenate_textlcs(matching,
                                      sortby=sortby,
                                      normalize=normalize)
        return clcdict
    else:
        LOGERROR('did not find any light curves for %s and aperture %s' %
                 (objectid, aperture))
        return None

This concatenates all text LCs for an objectid with the given aperture .


def concat_write_pklc(lcbasedir,
                      objectid,
                      aperture='TF1',
                      postfix='.gz',
                      sortby='rjd',
                      normalize=True,
                      outdir=None,
                      recursive=True):
    '''This concatenates all text LCs for the given object and writes to a pklc.

    Basically a rollup for the concatenate_textlcs_for_objectid and
    lcdict_to_pickle functions.

    '''

    concatlcd = concatenate_textlcs_for_objectid(lcbasedir,
                                                 objectid,
                                                 aperture=aperture,
                                                 sortby=sortby,
                                                 normalize=normalize,
                                                 recursive=recursive)

    if not outdir:
        outdir = 'pklcs'

    if not os.path.exists(outdir):
        os.mkdir(outdir)

    outfpath = os.path.join(outdir, '%s-%s-pklc.pkl' % (concatlcd['objectid'],
                                                        aperture))
    pklc = lcdict_to_pickle(concatlcd, outfile=outfpath)
    return pklc

def concat_write_pklc ( lcbasedir , objectid , aperture = 'TF1' , postfix = '.gz' , sortby = 'rjd' , normalize = True , outdir = None , recursive = True ) : concatlcd = concatenate_textlcs_for_objectid ( lcbasedir , objectid , aperture = aperture , sortby = sortby , normalize = normalize , recursive = recursive ) if not outdir : outdir = 'pklcs' if not os . path . exists ( outdir ) : os . mkdir ( outdir ) outfpath = os . path . join ( outdir , '%s-%s-pklc.pkl' % ( concatlcd [ 'objectid' ] , aperture ) ) pklc = lcdict_to_pickle ( concatlcd , outfile = outfpath ) return pklc

This concatenates all text LCs for the given object and writes to a pklc .


def parallel_concat_worker(task):
    '''
    This is a worker for the function below.

    task[0] = lcbasedir
    task[1] = objectid
    task[2] = {'aperture','postfix','sortby','normalize','outdir','recursive'}

    '''

    lcbasedir, objectid, kwargs = task

    try:
        return concat_write_pklc(lcbasedir, objectid, **kwargs)
    except Exception as e:
        LOGEXCEPTION('failed LC concatenation for %s in %s'
                     % (objectid, lcbasedir))
        return None

def
parallel_concat_worker
(
task
)
:
lcbasedir
,
objectid
,
kwargs
=
task
try
:
return
concat_write_pklc
(
lcbasedir
,
objectid
,
**
kwargs
)
except
Exception
as
e
:
LOGEXCEPTION
(
'failed LC concatenation for %s in %s'
%
(
objectid
,
lcbasedir
)
)
return

This is a worker for the function below .


def parallel_concat_lcdir(lcbasedir,
                          objectidlist,
                          aperture='TF1',
                          postfix='.gz',
                          sortby='rjd',
                          normalize=True,
                          outdir=None,
                          recursive=True,
                          nworkers=32,
                          maxworkertasks=1000):
    '''This concatenates all text LCs for the given objectidlist.

    '''

    if not outdir:
        outdir = 'pklcs'

    if not os.path.exists(outdir):
        os.mkdir(outdir)

    tasks = [(lcbasedir, x, {'aperture':aperture,
                             'postfix':postfix,
                             'sortby':sortby,
                             'normalize':normalize,
                             'outdir':outdir,
                             'recursive':recursive}) for x in objectidlist]

    pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)
    results = pool.map(parallel_concat_worker, tasks)

    pool.close()
    pool.join()

    return {x:y for (x,y) in zip(objectidlist, results)}

def parallel_concat_lcdir ( lcbasedir , objectidlist , aperture = 'TF1' , postfix = '.gz' , sortby = 'rjd' , normalize = True , outdir = None , recursive = True , nworkers = 32 , maxworkertasks = 1000 ) : if not outdir : outdir = 'pklcs' if not os . path . exists ( outdir ) : os . mkdir ( outdir ) tasks = [ ( lcbasedir , x , { 'aperture' : aperture , 'postfix' : postfix , 'sortby' : sortby , 'normalize' : normalize , 'outdir' : outdir , 'recursive' : recursive } ) for x in objectidlist ] pool = mp . Pool ( nworkers , maxtasksperchild = maxworkertasks ) results = pool . map ( parallel_concat_worker , tasks ) pool . close ( ) pool . join ( ) return { x : y for ( x , y ) in zip ( objectidlist , results ) }

This concatenates all text LCs for the given objectidlist .


def merge_hatpi_textlc_apertures(lclist):
    '''This merges all TFA text LCs with separate apertures for a single object.

    The framekey column will be used as the join column across all light curves
    in lclist. Missing values will be filled in with nans. This function assumes
    all light curves are in the format specified in COLDEFS above and readable
    by read_hatpi_textlc above (i.e. have a single column for TFA mags for a
    specific aperture at the end).

    '''

    lcaps = {}
    framekeys = []

    for lc in lclist:

        lcd = read_hatpi_textlc(lc)

        # figure what aperture this is and put it into the lcdict. if two LCs
        # with the same aperture (i.e. TF1 and TF1) are provided, the later one
        # in the lclist will overwrite the previous one,
        for col in lcd['columns']:
            if col.startswith('itf'):
                lcaps[col] = lcd
        thisframekeys = lcd['frk'].tolist()
        framekeys.extend(thisframekeys)

    # uniqify the framekeys
    framekeys = sorted(list(set(framekeys)))

def merge_hatpi_textlc_apertures ( lclist ) : lcaps = { } framekeys = [ ] for lc in lclist : lcd = read_hatpi_textlc ( lc ) for col in lcd [ 'columns' ] : if col . startswith ( 'itf' ) : lcaps [ col ] = lcd thisframekeys = lcd [ 'frk' ] . tolist ( ) framekeys . extend ( thisframekeys ) framekeys = sorted ( list ( set ( framekeys ) ) )

This merges all TFA text LCs with separate apertures for a single object .


def generate_hatpi_binnedlc_pkl(binnedpklf, textlcf, timebinsec,
                                outfile=None):
    '''
    This reads the binned LC and writes it out to a pickle.

    '''

    binlcdict = read_hatpi_binnedlc(binnedpklf, textlcf, timebinsec)

    if binlcdict:
        if outfile is None:
            outfile = os.path.join(
                os.path.dirname(binnedpklf),
                '%s-hplc.pkl' % (
                    os.path.basename(binnedpklf).replace('sec-lc.pkl.gz','')
                )
            )

        return lcdict_to_pickle(binlcdict, outfile=outfile)
    else:
        LOGERROR('could not read binned HATPI LC: %s' % binnedpklf)
        return None

def
generate_hatpi_binnedlc_pkl
(
binnedpklf
,
textlcf
,
timebinsec
,
outfile
=
)
:
binlcdict
=
read_hatpi_binnedlc
(
binnedpklf
,
textlcf
,
timebinsec
)
if
binlcdict
:
if
outfile
is
:
outfile
=
os
.
path
.
join
(
os
.
path
.
dirname
(
binnedpklf
)
,
'%s-hplc.pkl'
%
(
os
.
path
.
basename
(
binnedpklf
)
.
replace
(
'sec-lc.pkl.gz'
,
''
)
)
)
return
lcdict_to_pickle
(
binlcdict
,
outfile
=
outfile
)
else
:
LOGERROR
(
'could not read binned HATPI LC: %s'
%
binnedpklf
)
return

This reads the binned LC and writes it out to a pickle .


def parallel_gen_binnedlc_pkls(binnedpkldir,
                               textlcdir,
                               timebinsec,
                               binnedpklglob='*binned*sec*.pkl',
                               textlcglob='*.tfalc.TF1*'):
    '''
    This generates the binnedlc pkls for a directory of such files.

    FIXME: finish this

    '''

    binnedpkls = sorted(glob.glob(os.path.join(binnedpkldir, binnedpklglob)))

    # find all the textlcs associated with these
    textlcs = []

    for bpkl in binnedpkls:

        objectid = HATIDREGEX.findall(bpkl)
        if objectid is not None:
            objectid = objectid[0]

        searchpath = os.path.join(textlcdir, '%s-%s' % (objectid, textlcglob))
        textlcf = glob.glob(searchpath)
        if textlcf:
            textlcs.append(textlcf)
        else:
            textlcs.append(None)

def parallel_gen_binnedlc_pkls ( binnedpkldir , textlcdir , timebinsec , binnedpklglob = '*binned*sec*.pkl' , textlcglob = '*.tfalc.TF1*' ) : binnedpkls = sorted ( glob . glob ( os . path . join ( binnedpkldir , binnedpklglob ) ) ) textlcs = [ ] for bpkl in binnedpkls : objectid = HATIDREGEX . findall ( bpkl ) if objectid is not None : objectid = objectid [ 0 ] searchpath = os . path . join ( textlcdir , '%s-%s' % ( objectid , textlcglob ) ) textlcf = glob . glob ( searchpath ) if textlcf : textlcs . append ( textlcf ) else : textlcs . append ( None )

This generates the binnedlc pkls for a directory of such files .


def pklc_fovcatalog_objectinfo(
        pklcdir,
        fovcatalog,
        fovcatalog_columns=[0,1,2,
                            6,7,
                            8,9,
                            10,11,
                            13,14,15,16,
                            17,18,19,
                            20,21],
        fovcatalog_colnames=['objectid','ra','decl',
                             'jmag','jmag_err',
                             'hmag','hmag_err',
                             'kmag','kmag_err',
                             'bmag','vmag','rmag','imag',
                             'sdssu','sdssg','sdssr',
                             'sdssi','sdssz'],
        fovcatalog_colformats=('U20,f8,f8,'
                               'f8,f8,'
                               'f8,f8,'
                               'f8,f8,'
                               'f8,f8,f8,f8,'
                               'f8,f8,f8,'
                               'f8,f8')
):
    '''Adds catalog info to objectinfo key of all pklcs in lcdir.

    If fovcatalog, fovcatalog_columns, fovcatalog_colnames are provided, uses
    them to find all the additional information listed in the fovcatalog_colname
    keys, and writes this info to the objectinfo key of each lcdict. This makes
    it easier for astrobase tools to work on these light curve.

    The default set up for fovcatalog is to use a text file generated by the
    HATPI pipeline before auto-calibrating a field. The format is specified as
    above in _columns,  _colnames, and _colformats.

    '''

    if fovcatalog.endswith('.gz'):
        catfd = gzip.open(fovcatalog)
    else:
        catfd = open(fovcatalog)

    # read the catalog using the colformats, etc.
    fovcat = np.genfromtxt(catfd,
                           usecols=fovcatalog_columns,
                           names=fovcatalog_colnames,
                           dtype=fovcatalog_colformats)
    catfd.close()

    pklclist = sorted(glob.glob(os.path.join(pklcdir, '*HAT*-pklc.pkl')))

    updatedpklcs, failedpklcs = [], []

    for pklc in pklclist:

        lcdict = read_hatpi_pklc(pklc)
        objectid = lcdict['objectid']

        catind = np.where(fovcat['objectid'] == objectid)

        # if we found catalog info for this object, put it into objectinfo
        if len(catind) > 0 and catind[0]:

            lcdict['objectinfo'].update(
                {x:y for x,y in zip(
                    fovcatalog_colnames,
                    [np.asscalar(fovcat[z][catind]) for
                     z in fovcatalog_colnames]
                )
                }
            )

            # write the LC back to the pickle (tempfile for atomicity)
            with open(pklc+'-tmp','wb') as outfd:
                pickle.dump(lcdict, outfd, pickle.HIGHEST_PROTOCOL)

            # overwrite previous once we know it exists
            if os.path.exists(pklc+'-tmp'):
                shutil.move(pklc+'-tmp',pklc)

                LOGINFO('updated %s with catalog info for %s at %.3f, %.3f OK' %
                        (pklc, objectid,
                         lcdict['objectinfo']['ra'],
                         lcdict['objectinfo']['decl']))

                updatedpklcs.append(pklc)

        # otherwise, do nothing
        else:
            failedpklcs.append(pklc)

    # end of pklclist processing
    return updatedpklcs, failedpklcs

Adds catalog info to objectinfo key of all pklcs in lcdir .


def _base64_to_file(b64str, outfpath, writetostrio=False):
    '''This converts the base64 encoded string to a file.

    Parameters
    ----------

    b64str : str
        A base64 encoded strin that is the output of `base64.b64encode`.

    outfpath : str
        The path to where the file will be written. This should include an
        appropriate extension for the file (e.g. a base64 encoded string that
        represents a PNG should have its `outfpath` end in a '.png') so the OS
        can open these files correctly.

    writetostrio : bool
        If this is True, will return a StringIO object with the binary stream
        decoded from the base64-encoded input string `b64str`. This can be
        useful to embed these into other files without having to write them to
        disk.

    Returns
    -------

    str or StringIO object
        If `writetostrio` is False, will return the output file's path as a
        str. If it is True, will return a StringIO object directly. If writing
        the file fails in either case, will return None.

    '''

    try:

        filebytes = base64.b64decode(b64str)

        # if we're writing back to a stringio object
        if writetostrio:

            outobj = StrIO(filebytes)
            return outobj

        # otherwise, we're writing to an actual file
        else:

            with open(outfpath,'wb') as outfd:
                outfd.write(filebytes)

            if os.path.exists(outfpath):
                return outfpath
            else:
                LOGERROR('could not write output file: %s' % outfpath)
                return None

    except Exception as e:

        LOGEXCEPTION('failed while trying to convert '
                     'b64 string to file %s' % outfpath)
        return None

def
_base64_to_file
(
b64str
,
outfpath
,
writetostrio
=
False
)
:
try
:
filebytes
=
base64
.
b64decode
(
b64str
)
if
writetostrio
:
outobj
=
StrIO
(
filebytes
)
return
outobj
else
:
with
open
(
outfpath
,
'wb'
)
as
outfd
:
outfd
.
write
(
filebytes
)
if
os
.
path
.
exists
(
outfpath
)
:
return
outfpath
else
:
LOGERROR
(
'could not write output file: %s'
%
outfpath
)
return
except
Exception
as
e
:
LOGEXCEPTION
(
'failed while trying to convert '
'b64 string to file %s'
%
outfpath
)
return

This converts the base64 encoded string to a file .


def _read_checkplot_picklefile(checkplotpickle):
    '''This reads a checkplot gzipped pickle file back into a dict.

    NOTE: the try-except is for Python 2 pickles that have numpy arrays in
    them. Apparently, these aren't compatible with Python 3. See here:

    http://stackoverflow.com/q/11305790

    The workaround is noted in this answer:

    http://stackoverflow.com/a/41366785

    Parameters
    ----------

    checkplotpickle : str
        The path to a checkplot pickle file. This can be a gzipped file (in
        which case the file extension should end in '.gz')

    Returns
    -------

    dict
        This returns a checkplotdict.

    '''

    if checkplotpickle.endswith('.gz'):

        try:
            with gzip.open(checkplotpickle,'rb') as infd:
                cpdict = pickle.load(infd)

        except UnicodeDecodeError:

            with gzip.open(checkplotpickle,'rb') as infd:
                cpdict = pickle.load(infd, encoding='latin1')

    else:

        try:
            with open(checkplotpickle,'rb') as infd:
                cpdict = pickle.load(infd)

        except UnicodeDecodeError:

            with open(checkplotpickle,'rb') as infd:
                cpdict = pickle.load(infd, encoding='latin1')

    return cpdict

def _read_checkplot_picklefile ( checkplotpickle ) : if checkplotpickle . endswith ( '.gz' ) : try : with gzip . open ( checkplotpickle , 'rb' ) as infd : cpdict = pickle . load ( infd ) except UnicodeDecodeError : with gzip . open ( checkplotpickle , 'rb' ) as infd : cpdict = pickle . load ( infd , encoding = 'latin1' ) else : try : with open ( checkplotpickle , 'rb' ) as infd : cpdict = pickle . load ( infd ) except UnicodeDecodeError : with open ( checkplotpickle , 'rb' ) as infd : cpdict = pickle . load ( infd , encoding = 'latin1' ) return cpdict

This reads a checkplot gzipped pickle file back into a dict .


def make_fit_plot(phase, pmags, perrs, fitmags,
                  period, mintime, magseriesepoch,
                  plotfit,
                  magsarefluxes=False,
                  wrap=False,
                  model_over_lc=False):
    '''This makes a plot of the LC model fit.

    Parameters
    ----------

    phase,pmags,perrs : np.array
        The actual mag/flux time-series.

    fitmags : np.array
        The model fit time-series.

    period : float
        The period at which the phased LC was generated.

    mintime : float
        The minimum time value.

    magseriesepoch : float
        The value of time around which the phased LC was folded.

    plotfit : str
        The name of a file to write the plot to.

    magsarefluxes : bool
        Set this to True if the values in `pmags` and `fitmags` are actually
        fluxes.

    wrap : bool
        If True, will wrap the phased LC around 0.0 to make some phased LCs
        easier to look at.

    model_over_lc : bool
        Usually, this function will plot the actual LC over the model LC. Set
        this to True to plot the model over the actual LC; this is most useful
        when you have a very dense light curve and want to be able to see how it
        follows the model.

    Returns
    -------

    Nothing.

    '''

    # set up the figure
    plt.close('all')
    plt.figure(figsize=(8,4.8))

    if model_over_lc:
        model_z = 100
        lc_z = 0
    else:
        model_z = 0
        lc_z = 100

    if not wrap:

        plt.plot(phase, fitmags, linewidth=3.0, color='red',zorder=model_z)
        plt.plot(phase,pmags,
                 marker='o',
                 markersize=1.0,
                 linestyle='none',
                 rasterized=True, color='k',zorder=lc_z)

        # set the x axis ticks and label
        plt.gca().set_xticks(
            [0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]
        )

    else:
        plt.plot(np.concatenate([phase-1.0,phase]),
                 np.concatenate([fitmags,fitmags]),
                 linewidth=3.0,
                 color='red',zorder=model_z)
        plt.plot(np.concatenate([phase-1.0,phase]),
                 np.concatenate([pmags,pmags]),
                 marker='o',
                 markersize=1.0,
                 linestyle='none',
                 rasterized=True, color='k',zorder=lc_z)

        plt.gca().set_xlim((-0.8,0.8))
        # set the x axis ticks and label
        plt.gca().set_xticks(
            [-0.8,-0.7,-0.6,-0.5,-0.4,-0.3,-0.2,-0.1,
             0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]
        )

    # set the y axis limit and label
    ymin, ymax = plt.ylim()
    if not magsarefluxes:
        plt.gca().invert_yaxis()
        plt.ylabel('magnitude')
    else:
        plt.ylabel('flux')

    plt.xlabel('phase')
    plt.title('period: %.6f, folded at %.6f, fit epoch: %.6f' %
              (period, mintime, magseriesepoch))
    plt.savefig(plotfit)
    plt.close()


def objectlist_conesearch(racenter,
                          declcenter,
                          searchradiusarcsec,
                          gaia_mirror=None,
                          columns=('source_id',
                                   'ra','dec',
                                   'phot_g_mean_mag',
                                   'l','b',
                                   'parallax', 'parallax_error',
                                   'pmra','pmra_error',
                                   'pmdec','pmdec_error'),
                          extra_filter=None,
                          returnformat='csv',
                          forcefetch=False,
                          cachedir='~/.astrobase/gaia-cache',
                          verbose=True,
                          timeout=15.0,
                          refresh=2.0,
                          maxtimeout=300.0,
                          maxtries=3,
                          complete_query_later=True):
    '''This queries the GAIA TAP service for a list of objects near the coords.

    Runs a conesearch around `(racenter, declcenter)` with radius in arcsec of
    `searchradiusarcsec`.

    Parameters
    ----------

    racenter,declcenter : float
        The center equatorial coordinates in decimal degrees.

    searchradiusarcsec : float
        The search radius of the cone-search in arcseconds.

    gaia_mirror : {'gaia','heidelberg','vizier'} or None
        This is the key used to select a GAIA catalog mirror from the
        `GAIA_URLS` dict above. If set, the specified mirror will be used. If
        None, a random mirror chosen from that dict will be used.

    columns : sequence of str
        This indicates which columns from the GAIA table to request for the
        objects found within the search radius.

    extra_filter: str or None
        If this is provided, must be a valid ADQL filter string that is used to
        further filter the cone-search results.

    returnformat : {'csv','votable','json'}
        The returned file format to request from the GAIA catalog service.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    completequerylater : bool
        If set to True, a submitted query that does not return a result before
        `maxtimeout` has passed will be cancelled but its input request
        parameters and the result URL provided by the service will be saved. If
        this function is then called later with these same input request
        parameters, it will check if the query finally finished and a result is
        available. If so, will download the results instead of submitting a new
        query. If it's not done yet, will start waiting for results again. To
        force launch a new query with the same request parameters, set the
        `forcefetch` kwarg to True.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    # this was generated using the awesome query generator at:
    # https://gea.esac.esa.int/archive/

    # NOTE: here we don't resolve the table name right away. this is because
    # some of the GAIA mirrors use different table names, so we leave the table
    # name to be resolved by the lower level tap_query function. this is done by
    # the {{table}} construct.
    query = (
        "select {columns}, "
        "(DISTANCE(POINT('ICRS', "
        "{{table}}.ra, {{table}}.dec), "
        "POINT('ICRS', {ra_center:.5f}, {decl_center:.5f})))*3600.0 "
        "AS dist_arcsec "
        "from {{table}} where "
        "CONTAINS(POINT('ICRS',{{table}}.ra, {{table}}.dec),"
        "CIRCLE('ICRS',{ra_center:.5f},{decl_center:.5f},"
        "{search_radius:.6f}))=1 "
        "{extra_filter_str}"
        "ORDER by dist_arcsec asc "
    )

    if extra_filter is not None:
        extra_filter_str = ' and %s ' % extra_filter
    else:
        extra_filter_str = ''

    formatted_query = query.format(ra_center=racenter,
                                   decl_center=declcenter,
                                   search_radius=searchradiusarcsec/3600.0,
                                   extra_filter_str=extra_filter_str,
                                   columns=', '.join(columns))

    return tap_query(formatted_query,
                     gaia_mirror=gaia_mirror,
                     returnformat=returnformat,
                     forcefetch=forcefetch,
                     cachedir=cachedir,
                     verbose=verbose,
                     timeout=timeout,
                     refresh=refresh,
                     maxtimeout=maxtimeout,
                     maxtries=maxtries,
                     complete_query_later=complete_query_later)

This queries the GAIA TAP service for a list of objects near the coords .


def objectlist_radeclbox(radeclbox,
                         gaia_mirror=None,
                         columns=('source_id',
                                  'ra','dec',
                                  'phot_g_mean_mag',
                                  'l','b',
                                  'parallax, parallax_error',
                                  'pmra','pmra_error',
                                  'pmdec','pmdec_error'),
                         extra_filter=None,
                         returnformat='csv',
                         forcefetch=False,
                         cachedir='~/.astrobase/gaia-cache',
                         verbose=True,
                         timeout=15.0,
                         refresh=2.0,
                         maxtimeout=300.0,
                         maxtries=3,
                         complete_query_later=True):

    '''This queries the GAIA TAP service for a list of objects in an equatorial
    coordinate box.

    Parameters
    ----------

    radeclbox : sequence of four floats
        This defines the box to search in::

            [ra_min, ra_max, decl_min, decl_max]

    gaia_mirror : {'gaia','heidelberg','vizier'} or None
        This is the key used to select a GAIA catalog mirror from the
        `GAIA_URLS` dict above. If set, the specified mirror will be used. If
        None, a random mirror chosen from that dict will be used.

    columns : sequence of str
        This indicates which columns from the GAIA table to request for the
        objects found within the search radius.

    extra_filter: str or None
        If this is provided, must be a valid ADQL filter string that is used to
        further filter the cone-search results.

    returnformat : {'csv','votable','json'}
        The returned file format to request from the GAIA catalog service.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    completequerylater : bool
        If set to True, a submitted query that does not return a result before
        `maxtimeout` has passed will be cancelled but its input request
        parameters and the result URL provided by the service will be saved. If
        this function is then called later with these same input request
        parameters, it will check if the query finally finished and a result is
        available. If so, will download the results instead of submitting a new
        query. If it's not done yet, will start waiting for results again. To
        force launch a new query with the same request parameters, set the
        `forcefetch` kwarg to True.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    # this was generated using the awesome query generator at:
    # https://gea.esac.esa.int/archive/

    # NOTE: here we don't resolve the table name right away. this is because
    # some of the GAIA mirrors use different table names, so we leave the table
    # name to be resolved by the lower level tap_query function. this is done by
    # the {{table}} construct.
    query = (
        "select {columns} from {{table}} where "
        "CONTAINS(POINT('ICRS',{{table}}.ra, {{table}}.dec),"
        "BOX('ICRS',{ra_center:.5f},{decl_center:.5f},"
        "{ra_width:.5f},{decl_height:.5f}))=1"
        "{extra_filter_str}"
    )

    ra_min, ra_max, decl_min, decl_max = radeclbox
    ra_center = (ra_max + ra_min)/2.0
    decl_center = (decl_max + decl_min)/2.0
    ra_width = ra_max - ra_min
    decl_height = decl_max - decl_min

    if extra_filter is not None:
        extra_filter_str = ' and %s ' % extra_filter
    else:
        extra_filter_str = ''

    formatted_query = query.format(columns=', '.join(columns),
                                   extra_filter_str=extra_filter_str,
                                   ra_center=ra_center,
                                   decl_center=decl_center,
                                   ra_width=ra_width,
                                   decl_height=decl_height)

    return tap_query(formatted_query,
                     gaia_mirror=gaia_mirror,
                     returnformat=returnformat,
                     forcefetch=forcefetch,
                     cachedir=cachedir,
                     verbose=verbose,
                     timeout=timeout,
                     refresh=refresh,
                     maxtimeout=maxtimeout,
                     maxtries=maxtries,
                     complete_query_later=complete_query_later)

This queries the GAIA TAP service for a list of objects in an equatorial coordinate box .


def objectid_search(gaiaid,
                    gaia_mirror=None,
                    columns=('source_id',
                             'ra','dec',
                             'phot_g_mean_mag',
                             'phot_bp_mean_mag',
                             'phot_rp_mean_mag',
                             'l','b',
                             'parallax, parallax_error',
                             'pmra','pmra_error',
                             'pmdec','pmdec_error'),
                    returnformat='csv',
                    forcefetch=False,
                    cachedir='~/.astrobase/gaia-cache',
                    verbose=True,
                    timeout=15.0,
                    refresh=2.0,
                    maxtimeout=300.0,
                    maxtries=3,
                    complete_query_later=True):

    '''This queries the GAIA TAP service for a single GAIA source ID.

    Parameters
    ----------

    gaiaid : str
        The source ID of the object whose info will be collected.

    gaia_mirror : {'gaia','heidelberg','vizier'} or None
        This is the key used to select a GAIA catalog mirror from the
        `GAIA_URLS` dict above. If set, the specified mirror will be used. If
        None, a random mirror chosen from that dict will be used.

    columns : sequence of str
        This indicates which columns from the GAIA table to request for the
        objects found within the search radius.

    returnformat : {'csv','votable','json'}
        The returned file format to request from the GAIA catalog service.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    completequerylater : bool
        If set to True, a submitted query that does not return a result before
        `maxtimeout` has passed will be cancelled but its input request
        parameters and the result URL provided by the service will be saved. If
        this function is then called later with these same input request
        parameters, it will check if the query finally finished and a result is
        available. If so, will download the results instead of submitting a new
        query. If it's not done yet, will start waiting for results again. To
        force launch a new query with the same request parameters, set the
        `forcefetch` kwarg to True.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    # NOTE: here we don't resolve the table name right away. this is because
    # some of the GAIA mirrors use different table names, so we leave the table
    # name to be resolved by the lower level tap_query function. this is done by
    # the {{table}} construct.
    query = (
        "select {columns} from {{table}} where "
        "source_id = {gaiaid}"
    )

    formatted_query = query.format(columns=', '.join(columns),
                                   gaiaid=gaiaid)

    return tap_query(formatted_query,
                     gaia_mirror=gaia_mirror,
                     returnformat=returnformat,
                     forcefetch=forcefetch,
                     cachedir=cachedir,
                     verbose=verbose,
                     timeout=timeout,
                     refresh=refresh,
                     maxtimeout=maxtimeout,
                     maxtries=maxtries,
                     complete_query_later=complete_query_later)

This queries the GAIA TAP service for a single GAIA source ID .


def generalized_lsp_value_notau(times, mags, errs, omega):
    '''
    This is the simplified version not using tau.

    The relations used are::

        W = sum (1.0/(errs*errs) )
        w_i = (1/W)*(1/(errs*errs))

        Y = sum( w_i*y_i )
        C = sum( w_i*cos(wt_i) )
        S = sum( w_i*sin(wt_i) )

        YY = sum( w_i*y_i*y_i ) - Y*Y
        YC = sum( w_i*y_i*cos(wt_i) ) - Y*C
        YS = sum( w_i*y_i*sin(wt_i) ) - Y*S

        CpC = sum( w_i*cos(w_t_i)*cos(w_t_i) )
        CC = CpC - C*C
        SS = (1 - CpC) - S*S
        CS = sum( w_i*cos(w_t_i)*sin(w_t_i) ) - C*S

        D(omega) = CC*SS - CS*CS
        P(omega) = (SS*YC*YC + CC*YS*YS - 2.0*CS*YC*YS)/(YY*D)

    Parameters
    ----------

    times,mags,errs : np.array
        The time-series to calculate the periodogram value for.

    omega : float
        The frequency to calculate the periodogram value at.

    Returns
    -------

    periodogramvalue : float
        The normalized periodogram at the specified test frequency `omega`.

    '''

    one_over_errs2 = 1.0/(errs*errs)

    W = npsum(one_over_errs2)
    wi = one_over_errs2/W

    sin_omegat = npsin(omega*times)
    cos_omegat = npcos(omega*times)

    sin2_omegat = sin_omegat*sin_omegat
    cos2_omegat = cos_omegat*cos_omegat
    sincos_omegat = sin_omegat*cos_omegat

    # calculate some more sums and terms
    Y = npsum( wi*mags )
    C = npsum( wi*cos_omegat )
    S = npsum( wi*sin_omegat )

    YpY = npsum( wi*mags*mags)

    YpC = npsum( wi*mags*cos_omegat )
    YpS = npsum( wi*mags*sin_omegat )

    CpC = npsum( wi*cos2_omegat )
    # SpS = npsum( wi*sin2_omegat )

    CpS = npsum( wi*sincos_omegat )

    # the final terms
    YY = YpY - Y*Y
    YC = YpC - Y*C
    YS = YpS - Y*S
    CC = CpC - C*C
    SS = 1 - CpC - S*S  # use SpS = 1 - CpC
    CS = CpS - C*S

    # P(omega) = (SS*YC*YC + CC*YS*YS - 2.0*CS*YC*YS)/(YY*D)
    # D(omega) = CC*SS - CS*CS
    Domega = CC*SS - CS*CS
    lspval = (SS*YC*YC + CC*YS*YS - 2.0*CS*YC*YS)/(YY*Domega)

    return lspval

def generalized_lsp_value_notau ( times , mags , errs , omega ) : one_over_errs2 = 1.0 / ( errs * errs ) W = npsum ( one_over_errs2 ) wi = one_over_errs2 / W sin_omegat = npsin ( omega * times ) cos_omegat = npcos ( omega * times ) sin2_omegat = sin_omegat * sin_omegat cos2_omegat = cos_omegat * cos_omegat sincos_omegat = sin_omegat * cos_omegat Y = npsum ( wi * mags ) C = npsum ( wi * cos_omegat ) S = npsum ( wi * sin_omegat ) YpY = npsum ( wi * mags * mags ) YpC = npsum ( wi * mags * cos_omegat ) YpS = npsum ( wi * mags * sin_omegat ) CpC = npsum ( wi * cos2_omegat ) CpS = npsum ( wi * sincos_omegat ) YY = YpY - Y * Y YC = YpC - Y * C YS = YpS - Y * S CC = CpC - C * C SS = 1 - CpC - S * S CS = CpS - C * S Domega = CC * SS - CS * CS lspval = ( SS * YC * YC + CC * YS * YS - 2.0 * CS * YC * YS ) / ( YY * Domega ) return lspval

This is the simplified version not using tau .


def specwindow_lsp_value(times, mags, errs, omega):
    '''This calculates the peak associated with the spectral window function
    for times and at the specified omega.

    NOTE: this is classical Lomb-Scargle, not the Generalized
    Lomb-Scargle. `mags` and `errs` are silently ignored since we're calculating
    the periodogram of the observing window function. These are kept to present
    a consistent external API so the `pgen_lsp` function below can call this
    transparently.

    Parameters
    ----------

    times,mags,errs : np.array
        The time-series to calculate the periodogram value for.

    omega : float
        The frequency to calculate the periodogram value at.

    Returns
    -------

    periodogramvalue : float
        The normalized periodogram at the specified test frequency `omega`.

    '''

    norm_times = times - times.min()

    tau = (
        (1.0/(2.0*omega)) *
        nparctan( npsum(npsin(2.0*omega*norm_times)) /
                  npsum(npcos(2.0*omega*norm_times)) )
    )

    lspval_top_cos = (npsum(1.0 * npcos(omega*(norm_times-tau))) *
                      npsum(1.0 * npcos(omega*(norm_times-tau))))
    lspval_bot_cos = npsum( (npcos(omega*(norm_times-tau))) *
                            (npcos(omega*(norm_times-tau))) )

    lspval_top_sin = (npsum(1.0 * npsin(omega*(norm_times-tau))) *
                      npsum(1.0 * npsin(omega*(norm_times-tau))))
    lspval_bot_sin = npsum( (npsin(omega*(norm_times-tau))) *
                            (npsin(omega*(norm_times-tau))) )

    lspval = 0.5 * ( (lspval_top_cos/lspval_bot_cos) +
                     (lspval_top_sin/lspval_bot_sin) )

    return lspval

def specwindow_lsp_value ( times , mags , errs , omega ) : norm_times = times - times . min ( ) tau = ( ( 1.0 / ( 2.0 * omega ) ) * nparctan ( npsum ( npsin ( 2.0 * omega * norm_times ) ) / npsum ( npcos ( 2.0 * omega * norm_times ) ) ) ) lspval_top_cos = ( npsum ( 1.0 * npcos ( omega * ( norm_times - tau ) ) ) * npsum ( 1.0 * npcos ( omega * ( norm_times - tau ) ) ) ) lspval_bot_cos = npsum ( ( npcos ( omega * ( norm_times - tau ) ) ) * ( npcos ( omega * ( norm_times - tau ) ) ) ) lspval_top_sin = ( npsum ( 1.0 * npsin ( omega * ( norm_times - tau ) ) ) * npsum ( 1.0 * npsin ( omega * ( norm_times - tau ) ) ) ) lspval_bot_sin = npsum ( ( npsin ( omega * ( norm_times - tau ) ) ) * ( npsin ( omega * ( norm_times - tau ) ) ) ) lspval = 0.5 * ( ( lspval_top_cos / lspval_bot_cos ) + ( lspval_top_sin / lspval_bot_sin ) ) return lspval

This calculates the peak associated with the spectral window function for times and at the specified omega .


def specwindow_lsp(
        times,
        mags,
        errs,
        magsarefluxes=False,
        startp=None,
        endp=None,
        stepsize=1.0e-4,
        autofreq=True,
        nbestpeaks=5,
        periodepsilon=0.1,
        sigclip=10.0,
        nworkers=None,
        glspfunc=_glsp_worker_specwindow,
        verbose=True
):
    '''This calculates the spectral window function.

    Wraps the `pgen_lsp` function above to use the specific worker for
    calculating the window-function.

    Parameters
    ----------

    times,mags,errs : np.array
        The mag/flux time-series with associated measurement errors to run the
        period-finding on.

    magsarefluxes : bool
        If the input measurement values in `mags` and `errs` are in fluxes, set
        this to True.

    startp,endp : float or None
        The minimum and maximum periods to consider for the transit search.

    stepsize : float
        The step-size in frequency to use when constructing a frequency grid for
        the period search.

    autofreq : bool
        If this is True, the value of `stepsize` will be ignored and the
        :py:func:`astrobase.periodbase.get_frequency_grid` function will be used
        to generate a frequency grid based on `startp`, and `endp`. If these are
        None as well, `startp` will be set to 0.1 and `endp` will be set to
        `times.max() - times.min()`.

    nbestpeaks : int
        The number of 'best' peaks to return from the periodogram results,
        starting from the global maximum of the periodogram peak values.

    periodepsilon : float
        The fractional difference between successive values of 'best' periods
        when sorting by periodogram power to consider them as separate periods
        (as opposed to part of the same periodogram peak). This is used to avoid
        broad peaks in the periodogram and make sure the 'best' periods returned
        are all actually independent.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    nworkers : int
        The number of parallel workers to use when calculating the periodogram.

    glspfunc : Python function
        The worker function to use to calculate the periodogram. This is used to
        used to make the `pgen_lsp` function calculate the time-series sampling
        window function instead of the time-series measurements' GLS periodogram
        by passing in `_glsp_worker_specwindow` instead of the default
        `_glsp_worker` function.

    verbose : bool
        If this is True, will indicate progress and details about the frequency
        grid used for the period search.

    Returns
    -------

    dict
        This function returns a dict, referred to as an `lspinfo` dict in other
        astrobase functions that operate on periodogram results. This is a
        standardized format across all astrobase period-finders, and is of the
        form below::

            {'bestperiod': the best period value in the periodogram,
             'bestlspval': the periodogram peak associated with the best period,
             'nbestpeaks': the input value of nbestpeaks,
             'nbestlspvals': nbestpeaks-size list of best period peak values,
             'nbestperiods': nbestpeaks-size list of best periods,
             'lspvals': the full array of periodogram powers,
             'periods': the full array of periods considered,
             'method':'win' -> the name of the period-finder method,
             'kwargs':{ dict of all of the input kwargs for record-keeping}}

    '''

    # run the LSP using glsp_worker_specwindow as the worker
    lspres = pgen_lsp(
        times,
        mags,
        errs,
        magsarefluxes=magsarefluxes,
        startp=startp,
        endp=endp,
        autofreq=autofreq,
        nbestpeaks=nbestpeaks,
        periodepsilon=periodepsilon,
        stepsize=stepsize,
        nworkers=nworkers,
        sigclip=sigclip,
        glspfunc=glspfunc,
        verbose=verbose
    )

    # update the resultdict to indicate we're a spectral window function
    lspres['method'] = 'win'

    if lspres['lspvals'] is not None:

        # renormalize the periodogram to between 0 and 1 like the usual GLS.
        lspmax = npnanmax(lspres['lspvals'])

        if npisfinite(lspmax):

            lspres['lspvals'] = lspres['lspvals']/lspmax
            lspres['nbestlspvals'] = [
                x/lspmax for x in lspres['nbestlspvals']
            ]
            lspres['bestlspval'] = lspres['bestlspval']/lspmax

    return lspres

def specwindow_lsp ( times , mags , errs , magsarefluxes = False , startp = None , endp = None , stepsize = 1.0e-4 , autofreq = True , nbestpeaks = 5 , periodepsilon = 0.1 , sigclip = 10.0 , nworkers = None , glspfunc = _glsp_worker_specwindow , verbose = True ) : lspres = pgen_lsp ( times , mags , errs , magsarefluxes = magsarefluxes , startp = startp , endp = endp , autofreq = autofreq , nbestpeaks = nbestpeaks , periodepsilon = periodepsilon , stepsize = stepsize , nworkers = nworkers , sigclip = sigclip , glspfunc = glspfunc , verbose = verbose ) lspres [ 'method' ] = 'win' if lspres [ 'lspvals' ] is not None : lspmax = npnanmax ( lspres [ 'lspvals' ] ) if npisfinite ( lspmax ) : lspres [ 'lspvals' ] = lspres [ 'lspvals' ] / lspmax lspres [ 'nbestlspvals' ] = [ x / lspmax for x in lspres [ 'nbestlspvals' ] ] lspres [ 'bestlspval' ] = lspres [ 'bestlspval' ] / lspmax return lspres

This calculates the spectral window function .


def check_existing_apikey(lcc_server):
    '''This validates if an API key for the specified LCC-Server is available.

    API keys are stored using the following file scheme::

        ~/.astrobase/lccs/apikey-domain.of.lccserver.org

    e.g. for the HAT LCC-Server at https://data.hatsurveys.org::

        ~/.astrobase/lccs/apikey-https-data.hatsurveys.org

    Parameters
    ----------

    lcc_server : str
        The base URL of the LCC-Server for which the existence of API keys will
        be checked.

    Returns
    -------

    (apikey_ok, apikey_str, expiry) : tuple
        The returned tuple contains the status of the API key, the API key
        itself if present, and its expiry date if present.

    '''

    USERHOME = os.path.expanduser('~')
    APIKEYFILE = os.path.join(USERHOME,
                              '.astrobase',
                              'lccs',
                              'apikey-%s' % lcc_server.replace(
                                  'https://',
                                  'https-'
                              ).replace(
                                  'http://',
                                  'http-'
                              ))

    if os.path.exists(APIKEYFILE):

        # check if this file is readable/writeable by user only
        fileperm = oct(os.stat(APIKEYFILE)[stat.ST_MODE])

        if fileperm == '0100600' or fileperm == '0o100600':

            with open(APIKEYFILE) as infd:
                apikey, expires = infd.read().strip('\n').split()

            # get today's datetime
            now = datetime.now(utc)

            if sys.version_info[:2] < (3,7):
                # this hideous incantation is required for lesser Pythons
                expdt = datetime.strptime(
                    expires.replace('Z',''),
                    '%Y-%m-%dT%H:%M:%S.%f'
                ).replace(tzinfo=utc)
            else:
                expdt = datetime.fromisoformat(expires.replace('Z','+00:00'))

            if now > expdt:
                LOGERROR('API key has expired. expiry was on: %s' % expires)
                return False, apikey, expires
            else:
                return True, apikey, expires

        else:
            LOGWARNING('The API key file %s has bad permissions '
                       'and is insecure, not reading it.\n'
                       '(you need to chmod 600 this file)'
                       % APIKEYFILE)

            return False, None, None
    else:
        LOGWARNING('No LCC-Server API key '
                   'found in: {apikeyfile}'.format(apikeyfile=APIKEYFILE))

        return False, None, None

This validates if an API key for the specified LCC - Server is available .


def get_new_apikey(lcc_server):
    '''This gets a new API key from the specified LCC-Server.

    NOTE: this only gets an anonymous API key. To get an API key tied to a user
    account (and associated privilege level), see the `import_apikey` function
    below.

    Parameters
    ----------

    lcc_server : str
        The base URL of the LCC-Server from where the API key will be fetched.

    Returns
    -------

    (apikey, expiry) : tuple
        This returns a tuple with the API key and its expiry date.

    '''

    USERHOME = os.path.expanduser('~')
    APIKEYFILE = os.path.join(USERHOME,
                              '.astrobase',
                              'lccs',
                              'apikey-%s' % lcc_server.replace(
                                  'https://',
                                  'https-'
                              ).replace(
                                  'http://',
                                  'http-'
                              ))

    # url for getting an API key
    url = '%s/api/key' % lcc_server

    # get the API key
    resp = urlopen(url)

    if resp.code == 200:

        respdict = json.loads(resp.read())

    else:

        LOGERROR('could not fetch the API key from LCC-Server at: %s' %
                 lcc_server)
        LOGERROR('the HTTP status code was: %s' % resp.status_code)
        return None

    #
    # now that we have an API key dict, get the API key out of it and write it
    # to the APIKEYFILE
    #
    apikey = respdict['result']['apikey']
    expires = respdict['result']['expires']

    # write this to the apikey file

    if not os.path.exists(os.path.dirname(APIKEYFILE)):
        os.makedirs(os.path.dirname(APIKEYFILE))

    with open(APIKEYFILE,'w') as outfd:
        outfd.write('%s %s\n' % (apikey, expires))

    # chmod it to the correct value
    os.chmod(APIKEYFILE, 0o100600)

    LOGINFO('key fetched successfully from: %s. expires on: %s' % (lcc_server,
                                                                   expires))
    LOGINFO('written to: %s' % APIKEYFILE)

    return apikey, expires

This gets a new API key from the specified LCC - Server .


def import_apikey(lcc_server, apikey_text_json):
    '''This imports an API key from text and writes it to the cache dir.

    Use this with the JSON text copied from the API key text box on your
    LCC-Server user home page. The API key will thus be tied to the privileges
    of that user account and can then access objects, datasets, and collections
    marked as private for the user only or shared with that user.

    Parameters
    ----------

    lcc_server : str
        The base URL of the LCC-Server to get the API key for.

    apikey_text_json : str
        The JSON string from the API key text box on the user's LCC-Server home
        page at `lcc_server/users/home`.

    Returns
    -------

    (apikey, expiry) : tuple
        This returns a tuple with the API key and its expiry date.

    '''
    USERHOME = os.path.expanduser('~')
    APIKEYFILE = os.path.join(USERHOME,
                              '.astrobase',
                              'lccs',
                              'apikey-%s' % lcc_server.replace(
                                  'https://',
                                  'https-'
                              ).replace(
                                  'http://',
                                  'http-'
                              ))

    respdict = json.loads(apikey_text_json)

    #
    # now that we have an API key dict, get the API key out of it and write it
    # to the APIKEYFILE
    #
    apikey = respdict['apikey']
    expires = respdict['expires']

    # write this to the apikey file

    if not os.path.exists(os.path.dirname(APIKEYFILE)):
        os.makedirs(os.path.dirname(APIKEYFILE))

    with open(APIKEYFILE,'w') as outfd:
        outfd.write('%s %s\n' % (apikey, expires))

    # chmod it to the correct value
    os.chmod(APIKEYFILE, 0o100600)

    LOGINFO('key fetched successfully from: %s. expires on: %s' % (lcc_server,
                                                                   expires))
    LOGINFO('written to: %s' % APIKEYFILE)

    return apikey, expires

This imports an API key from text and writes it to the cache dir .


def submit_post_searchquery(url, data, apikey):
    '''This submits a POST query to an LCC-Server search API endpoint.

    Handles streaming of the results, and returns the final JSON stream. Also
    handles results that time out.

    Parameters
    ----------

    url : str
        The URL of the search API endpoint to hit. This is something like
        `https://data.hatsurveys.org/api/conesearch`

    data : dict
        A dict of the search query parameters to pass to the search service.

    apikey : str
        The API key to use to access the search service. API keys are required
        for all POST request made to an LCC-Server's API endpoints.

    Returns
    -------

    (status_flag, data_dict, dataset_id) : tuple
        This returns a tuple containing the status of the request: ('complete',
        'failed', 'background', etc.), a dict parsed from the JSON result of the
        request, and a dataset ID, which can be used to reconstruct the URL on
        the LCC-Server where the results can be browsed.

    '''

    # first, we need to convert any columns and collections items to broken out
    # params
    postdata = {}

    for key in data:

        if key == 'columns':
            postdata['columns[]'] = data[key]
        elif key == 'collections':
            postdata['collections[]'] = data[key]
        else:
            postdata[key] = data[key]

    # do the urlencode with doseq=True
    # we also need to encode to bytes
    encoded_postdata = urlencode(postdata, doseq=True).encode()

    # if apikey is not None, add it in as an Authorization: Bearer [apikey]
    # header
    if apikey:
        headers = {'Authorization':'Bearer: %s' % apikey}
    else:
        headers = {}

    LOGINFO('submitting search query to LCC-Server API URL: %s' % url)

    try:

        # hit the server with a POST request
        req = Request(url, data=encoded_postdata, headers=headers)
        resp = urlopen(req)

        if resp.code == 200:

            # we'll iterate over the lines in the response
            # this works super-well for ND-JSON!
            for line in resp:

                data = json.loads(line)
                msg = data['message']
                status = data['status']

                if status != 'failed':
                    LOGINFO('status: %s, %s' % (status, msg))
                else:
                    LOGERROR('status: %s, %s' % (status, msg))

                # here, we'll decide what to do about the query

                # completed query or query sent to background...
                if status in ('ok','background'):

                    setid = data['result']['setid']
                    # save the data pickle to astrobase lccs directory
                    outpickle = os.path.join(os.path.expanduser('~'),
                                             '.astrobase',
                                             'lccs',
                                             'query-%s.pkl' % setid)
                    if not os.path.exists(os.path.dirname(outpickle)):
                        os.makedirs(os.path.dirname(outpickle))

                    with open(outpickle,'wb') as outfd:
                        pickle.dump(data, outfd, pickle.HIGHEST_PROTOCOL)
                        LOGINFO('saved query info to %s, use this to '
                                'download results later with '
                                'retrieve_dataset_files' % outpickle)

                    # we're done at this point, return
                    return status, data, data['result']['setid']

                # the query probably failed...
                elif status == 'failed':

                    # we're done at this point, return
                    return status, data, None

        # if the response was not OK, then we probably failed
        else:

            try:
                data = json.load(resp)
                msg = data['message']

                LOGERROR(msg)
                return 'failed', None, None

            except Exception as e:

                LOGEXCEPTION('failed to submit query to %s' % url)
                return 'failed', None, None

    except HTTPError as e:

        LOGERROR('could not submit query to LCC API at: %s' % url)
        LOGERROR('HTTP status code was %s, reason: %s' % (e.code, e.reason))
        return 'failed', None, None

This submits a POST query to an LCC - Server search API endpoint .