def parallel_timebin(lclist,
                     binsizesec,
                     maxobjects=None,
                     outdir=None,
                     lcformat='hat-sql',
                     lcformatdir=None,
                     timecols=None,
                     magcols=None,
                     errcols=None,
                     minbinelems=7,
                     nworkers=NCPUS,
                     maxworkertasks=1000):
    '''This time-bins all the LCs in the list using the specified bin size.

    Parameters
    ----------

    lclist : list of str
        The input LCs to process.

    binsizesec : float
        The time bin size to use in seconds.

    maxobjects : int or None
        If provided, LC processing will stop at `lclist[maxobjects]`.

    outdir : str or None
        The directory where output LCs will be written. If None, will write to
        the same directory as the input LCs.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curve file.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols,magcols,errcols : lists of str
        The keys in the lcdict produced by your light curve reader function that
        correspond to the times, mags/fluxes, and associated measurement errors
        that will be used as inputs to the binning process. If these are None,
        the default values for `timecols`, `magcols`, and `errcols` for your
        light curve format will be used here.

    minbinelems : int
        The minimum number of time-bin elements required to accept a time-bin as
        valid for the output binned light curve.

    nworkers : int
        Number of parallel workers to launch.

    maxworkertasks : int
        The maximum number of tasks a parallel worker will complete before being
        replaced to guard against memory leaks.

    Returns
    -------

    dict
        The returned dict contains keys = input LCs, vals = output LCs.

    '''

    if outdir and not os.path.exists(outdir):
        os.mkdir(outdir)

    if maxobjects is not None:
        lclist = lclist[:maxobjects]

    tasks = [(x, binsizesec, {'outdir':outdir,
                              'lcformat':lcformat,
                              'lcformatdir':lcformatdir,
                              'timecols':timecols,
                              'magcols':magcols,
                              'errcols':errcols,
                              'minbinelems':minbinelems}) for x in lclist]

    pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)
    results = pool.map(timebinlc_worker, tasks)
    pool.close()
    pool.join()

    resdict = {os.path.basename(x):y for (x,y) in zip(lclist, results)}

    return resdict

def parallel_timebin ( lclist , binsizesec , maxobjects = None , outdir = None , lcformat = 'hat-sql' , lcformatdir = None , timecols = None , magcols = None , errcols = None , minbinelems = 7 , nworkers = NCPUS , maxworkertasks = 1000 ) : if outdir and not os . path . exists ( outdir ) : os . mkdir ( outdir ) if maxobjects is not None : lclist = lclist [ : maxobjects ] tasks = [ ( x , binsizesec , { 'outdir' : outdir , 'lcformat' : lcformat , 'lcformatdir' : lcformatdir , 'timecols' : timecols , 'magcols' : magcols , 'errcols' : errcols , 'minbinelems' : minbinelems } ) for x in lclist ] pool = mp . Pool ( nworkers , maxtasksperchild = maxworkertasks ) results = pool . map ( timebinlc_worker , tasks ) pool . close ( ) pool . join ( ) resdict = { os . path . basename ( x ) : y for ( x , y ) in zip ( lclist , results ) } return resdict

This time - bins all the LCs in the list using the specified bin size .


def parallel_timebin_lcdir(lcdir,
                           binsizesec,
                           maxobjects=None,
                           outdir=None,
                           lcformat='hat-sql',
                           lcformatdir=None,
                           timecols=None,
                           magcols=None,
                           errcols=None,
                           minbinelems=7,
                           nworkers=NCPUS,
                           maxworkertasks=1000):
    '''
    This time bins all the light curves in the specified directory.

    Parameters
    ----------

    lcdir : list of str
        Directory containing the input LCs to process.

    binsizesec : float
        The time bin size to use in seconds.

    maxobjects : int or None
        If provided, LC processing will stop at `lclist[maxobjects]`.

    outdir : str or None
        The directory where output LCs will be written. If None, will write to
        the same directory as the input LCs.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curve file.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols,magcols,errcols : lists of str
        The keys in the lcdict produced by your light curve reader function that
        correspond to the times, mags/fluxes, and associated measurement errors
        that will be used as inputs to the binning process. If these are None,
        the default values for `timecols`, `magcols`, and `errcols` for your
        light curve format will be used here.

    minbinelems : int
        The minimum number of time-bin elements required to accept a time-bin as
        valid for the output binned light curve.

    nworkers : int
        Number of parallel workers to launch.

    maxworkertasks : int
        The maximum number of tasks a parallel worker will complete before being
        replaced to guard against memory leaks.

    Returns
    -------

    dict
        The returned dict contains keys = input LCs, vals = output LCs.

    '''
    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (fileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    lclist = sorted(glob.glob(os.path.join(lcdir, fileglob)))

    return parallel_timebin(lclist,
                            binsizesec,
                            maxobjects=maxobjects,
                            outdir=outdir,
                            lcformat=lcformat,
                            timecols=timecols,
                            magcols=magcols,
                            errcols=errcols,
                            minbinelems=minbinelems,
                            nworkers=nworkers,
                            maxworkertasks=maxworkertasks)

This time bins all the light curves in the specified directory .


def _varfeatures_worker(task):
    '''
    This wraps varfeatures.

    '''

    try:
        (lcfile, outdir, timecols, magcols, errcols,
         mindet, lcformat, lcformatdir) = task
        return get_varfeatures(lcfile, outdir,
                               timecols=timecols,
                               magcols=magcols,
                               errcols=errcols,
                               mindet=mindet,
                               lcformat=lcformat,
                               lcformatdir=lcformatdir)

    except Exception as e:
        return None

def _varfeatures_worker ( task ) : try : ( lcfile , outdir , timecols , magcols , errcols , mindet , lcformat , lcformatdir ) = task return get_varfeatures ( lcfile , outdir , timecols = timecols , magcols = magcols , errcols = errcols , mindet = mindet , lcformat = lcformat , lcformatdir = lcformatdir ) except Exception as e : return None


def serial_varfeatures(lclist,
                       outdir,
                       maxobjects=None,
                       timecols=None,
                       magcols=None,
                       errcols=None,
                       mindet=1000,
                       lcformat='hat-sql',
                       lcformatdir=None):
    '''This runs variability feature extraction for a list of LCs.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    Returns
    -------

    list of str
        List of the generated variability features pickles for the input LCs,
        with results for each magcol in the input `magcol` or light curve
        format's default `magcol` list.

    '''

    if maxobjects:
        lclist = lclist[:maxobjects]

    tasks = [(x, outdir, timecols, magcols, errcols,
              mindet, lcformat, lcformatdir)
             for x in lclist]

    for task in tqdm(tasks):
        result = _varfeatures_worker(task)

    return result

def serial_varfeatures ( lclist , outdir , maxobjects = None , timecols = None , magcols = None , errcols = None , mindet = 1000 , lcformat = 'hat-sql' , lcformatdir = None ) : if maxobjects : lclist = lclist [ : maxobjects ] tasks = [ ( x , outdir , timecols , magcols , errcols , mindet , lcformat , lcformatdir ) for x in lclist ] for task in tqdm ( tasks ) : result = _varfeatures_worker ( task ) return result

This runs variability feature extraction for a list of LCs .


def parallel_varfeatures(lclist,
                         outdir,
                         maxobjects=None,
                         timecols=None,
                         magcols=None,
                         errcols=None,
                         mindet=1000,
                         lcformat='hat-sql',
                         lcformatdir=None,
                         nworkers=NCPUS):
    '''This runs variable feature extraction in parallel for all LCs in `lclist`.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    nworkers : int
        The number of parallel workers to launch.

    Returns
    -------

    dict
        A dict with key:val pairs of input LC file name : the generated
        variability features pickles for each of the input LCs, with results for
        each magcol in the input `magcol` or light curve format's default
        `magcol` list.

    '''
    # make sure to make the output directory if it doesn't exist
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    if maxobjects:
        lclist = lclist[:maxobjects]

    tasks = [(x, outdir, timecols, magcols, errcols, mindet,
              lcformat, lcformatdir) for x in lclist]

    with ProcessPoolExecutor(max_workers=nworkers) as executor:
        resultfutures = executor.map(varfeatures_worker, tasks)

    results = [x for x in resultfutures]
    resdict = {os.path.basename(x):y for (x,y) in zip(lclist, results)}

    return resdict

def parallel_varfeatures ( lclist , outdir , maxobjects = None , timecols = None , magcols = None , errcols = None , mindet = 1000 , lcformat = 'hat-sql' , lcformatdir = None , nworkers = NCPUS ) : if not os . path . exists ( outdir ) : os . makedirs ( outdir ) if maxobjects : lclist = lclist [ : maxobjects ] tasks = [ ( x , outdir , timecols , magcols , errcols , mindet , lcformat , lcformatdir ) for x in lclist ] with ProcessPoolExecutor ( max_workers = nworkers ) as executor : resultfutures = executor . map ( varfeatures_worker , tasks ) results = [ x for x in resultfutures ] resdict = { os . path . basename ( x ) : y for ( x , y ) in zip ( lclist , results ) } return resdict

This runs variable feature extraction in parallel for all LCs in lclist .


def parallel_varfeatures_lcdir(lcdir,
                               outdir,
                               fileglob=None,
                               maxobjects=None,
                               timecols=None,
                               magcols=None,
                               errcols=None,
                               recursive=True,
                               mindet=1000,
                               lcformat='hat-sql',
                               lcformatdir=None,
                               nworkers=NCPUS):
    '''This runs parallel variable feature extraction for a directory of LCs.

    Parameters
    ----------

    lcdir : str
        The directory of light curve files to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    fileglob : str or None
        The file glob to use when looking for light curve files in `lcdir`. If
        None, the default file glob associated for this LC format will be used.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    nworkers : int
        The number of parallel workers to launch.

    Returns
    -------

    dict
        A dict with key:val pairs of input LC file name : the generated
        variability features pickles for each of the input LCs, with results for
        each magcol in the input `magcol` or light curve format's default
        `magcol` list.

    '''

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    if not fileglob:
        fileglob = dfileglob

    # now find the files
    LOGINFO('searching for %s light curves in %s ...' % (lcformat, lcdir))

    if recursive is False:
        matching = glob.glob(os.path.join(lcdir, fileglob))

    else:
        # use recursive glob for Python 3.5+
        if sys.version_info[:2] > (3,4):

            matching = glob.glob(os.path.join(lcdir,
                                              '**',
                                              fileglob),
                                 recursive=True)

        # otherwise, use os.walk and glob
        else:

            # use os.walk to go through the directories
            walker = os.walk(lcdir)
            matching = []

            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              fileglob)
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)

    # now that we have all the files, process them
    if matching and len(matching) > 0:

        LOGINFO('found %s light curves, getting varfeatures...' %
                len(matching))

        return parallel_varfeatures(matching,
                                    outdir,
                                    maxobjects=maxobjects,
                                    timecols=timecols,
                                    magcols=magcols,
                                    errcols=errcols,
                                    mindet=mindet,
                                    lcformat=lcformat,
                                    lcformatdir=lcformatdir,
                                    nworkers=nworkers)

    else:

        LOGERROR('no light curve files in %s format found in %s' % (lcformat,
                                                                    lcdir))
        return None

This runs parallel variable feature extraction for a directory of LCs .


def cp2png(checkplotin, extrarows=None):
    '''This is just a shortened form of the function above for convenience.

    This only handles pickle files as input.

    Parameters
    ----------

    checkplotin : str
        File name of a checkplot pickle file to convert to a PNG.

    extrarows : list of tuples
        This is a list of 4-element tuples containing paths to PNG files that
        will be added to the end of the rows generated from the checkplotin
        pickle/dict. Each tuple represents a row in the final output PNG
        file. If there are less than 4 elements per tuple, the missing elements
        will be filled in with white-space. If there are more than 4 elements
        per tuple, only the first four will be used.

        The purpose of this kwarg is to incorporate periodograms and phased LC
        plots (in the form of PNGs) generated from an external period-finding
        function or program (like VARTOOLS) to allow for comparison with
        astrobase results.

        NOTE: the PNG files specified in `extrarows` here will be added to those
        already present in the input `checkplotdict['externalplots']` if that is
        None because you passed in a similar list of external plots to the
        :py:func:`astrobase.checkplot.pkl.checkplot_pickle` function earlier. In
        this case, `extrarows` can be used to add even more external plots if
        desired.

        Each external plot PNG will be resized to 750 x 480 pixels to fit into
        an output image cell.

        By convention, each 4-element tuple should contain:

        - a periodiogram PNG
        - phased LC PNG with 1st best peak period from periodogram
        - phased LC PNG with 2nd best peak period from periodogram
        - phased LC PNG with 3rd best peak period from periodogram

        Example of extrarows::

            [('/path/to/external/bls-periodogram.png',
              '/path/to/external/bls-phasedlc-plot-bestpeak.png',
              '/path/to/external/bls-phasedlc-plot-peak2.png',
              '/path/to/external/bls-phasedlc-plot-peak3.png'),
             ('/path/to/external/pdm-periodogram.png',
              '/path/to/external/pdm-phasedlc-plot-bestpeak.png',
              '/path/to/external/pdm-phasedlc-plot-peak2.png',
              '/path/to/external/pdm-phasedlc-plot-peak3.png'),
            ...]

    Returns
    -------

    str
        The absolute path to the generated checkplot PNG.

    '''

    if checkplotin.endswith('.gz'):
        outfile = checkplotin.replace('.pkl.gz','.png')
    else:
        outfile = checkplotin.replace('.pkl','.png')

    return checkplot_pickle_to_png(checkplotin, outfile, extrarows=extrarows)

def cp2png ( checkplotin , extrarows = None ) : if checkplotin . endswith ( '.gz' ) : outfile = checkplotin . replace ( '.pkl.gz' , '.png' ) else : outfile = checkplotin . replace ( '.pkl' , '.png' ) return checkplot_pickle_to_png ( checkplotin , outfile , extrarows = extrarows )

This is just a shortened form of the function above for convenience .


def flare_model(flareparams, times, mags, errs):
    '''This is a flare model function, similar to Kowalski+ 2011.

    From the paper by Pitkin+ 2014:
    http://adsabs.harvard.edu/abs/2014MNRAS.445.2268P

    Parameters
    ----------

    flareparams : list of float
        This defines the flare model::

            [amplitude,
             flare_peak_time,
             rise_gaussian_stdev,
             decay_time_constant]

        where:

        `amplitude`: the maximum flare amplitude in mags or flux. If flux, then
        amplitude should be positive. If mags, amplitude should be negative.

        `flare_peak_time`: time at which the flare maximum happens.

        `rise_gaussian_stdev`: the stdev of the gaussian describing the rise of
        the flare.

        `decay_time_constant`: the time constant of the exponential fall of the
        flare.

    times,mags,errs : np.array
        The input time-series of measurements and associated errors for which
        the model will be generated. The times will be used to generate
        model mags.

    Returns
    -------

    (modelmags, times, mags, errs) : tuple
        Returns the model mags evaluated at the input time values. Also returns
        the input `times`, `mags`, and `errs`.

    '''

    (amplitude, flare_peak_time,
     rise_gaussian_stdev, decay_time_constant) = flareparams

    zerolevel = np.median(mags)
    modelmags = np.full_like(times, zerolevel)

    # before peak gaussian rise...
    modelmags[times < flare_peak_time] = (
        mags[times < flare_peak_time] +
        amplitude * np.exp(
            -((times[times < flare_peak_time] -
               flare_peak_time) *
              (times[times < flare_peak_time] -
               flare_peak_time)) /
            (2.0*rise_gaussian_stdev*rise_gaussian_stdev)
        )
    )

    # after peak exponential decay...
    modelmags[times > flare_peak_time] = (
        mags[times > flare_peak_time] +
        amplitude * np.exp(
            -((times[times > flare_peak_time] -
               flare_peak_time)) /
            (decay_time_constant)
        )
    )

    return modelmags, times, mags, errs

def flare_model ( flareparams , times , mags , errs ) : ( amplitude , flare_peak_time , rise_gaussian_stdev , decay_time_constant ) = flareparams zerolevel = np . median ( mags ) modelmags = np . full_like ( times , zerolevel ) modelmags [ times < flare_peak_time ] = ( mags [ times < flare_peak_time ] + amplitude * np . exp ( - ( ( times [ times < flare_peak_time ] - flare_peak_time ) * ( times [ times < flare_peak_time ] - flare_peak_time ) ) / ( 2.0 * rise_gaussian_stdev * rise_gaussian_stdev ) ) ) modelmags [ times > flare_peak_time ] = ( mags [ times > flare_peak_time ] + amplitude * np . exp ( - ( ( times [ times > flare_peak_time ] - flare_peak_time ) ) / ( decay_time_constant ) ) ) return modelmags , times , mags , errs

This is a flare model function similar to Kowalski + 2011 .


def flare_model_residual(flareparams, times, mags, errs):
    '''
    This returns the residual between model mags and the actual mags.

    Parameters
    ----------

    flareparams : list of float
        This defines the flare model::

            [amplitude,
             flare_peak_time,
             rise_gaussian_stdev,
             decay_time_constant]

        where:

        `amplitude`: the maximum flare amplitude in mags or flux. If flux, then
        amplitude should be positive. If mags, amplitude should be negative.

        `flare_peak_time`: time at which the flare maximum happens.

        `rise_gaussian_stdev`: the stdev of the gaussian describing the rise of
        the flare.

        `decay_time_constant`: the time constant of the exponential fall of the
        flare.

    times,mags,errs : np.array
        The input time-series of measurements and associated errors for which
        the model will be generated. The times will be used to generate
        model mags.

    Returns
    -------

    np.array
        The residuals between the input `mags` and generated `modelmags`,
        weighted by the measurement errors in `errs`.

    '''

    modelmags, _, _, _ = flare_model(flareparams, times, mags, errs)

    return (mags - modelmags)/errs

def flare_model_residual ( flareparams , times , mags , errs ) : modelmags , _ , _ , _ = flare_model ( flareparams , times , mags , errs ) return ( mags - modelmags ) / errs

This returns the residual between model mags and the actual mags .


def shutdown_check_handler():
    """This checks the AWS instance data URL to see if there's a pending
    shutdown for the instance.

    This is useful for AWS spot instances. If there is a pending shutdown posted
    to the instance data URL, we'll use the result of this function break out of
    the processing loop and shut everything down ASAP before the instance dies.

    Returns
    -------

    bool
        - True if the instance is going to die soon.
        - False if the instance is still safe.

    """

    url = 'http://169.254.169.254/latest/meta-data/spot/instance-action'

    try:
        resp = requests.get(url, timeout=1.0)
        resp.raise_for_status()

        stopinfo = resp.json()
        if 'action' in stopinfo and stopinfo['action'] in ('stop',
                                                           'terminate',
                                                           'hibernate'):
            stoptime = stopinfo['time']
            LOGWARNING('instance is going to %s at %s' % (stopinfo['action'],
                                                          stoptime))

            resp.close()
            return True
        else:
            resp.close()
            return False

    except HTTPError as e:
        resp.close()
        return False

    except Exception as e:
        resp.close()
        return False

This checks the AWS instance data URL to see if there s a pending shutdown for the instance .


def runcp_producer_loop_savedstate(
        use_saved_state=None,
        lightcurve_list=None,
        input_queue=None,
        input_bucket=None,
        result_queue=None,
        result_bucket=None,
        pfresult_list=None,
        runcp_kwargs=None,
        process_list_slice=None,
        download_when_done=True,
        purge_queues_when_done=True,
        save_state_when_done=True,
        delete_queues_when_done=False,
        s3_client=None,
        sqs_client=None
):
    """This wraps the function above to allow for loading previous state from a
    file.

    Parameters
    ----------

    use_saved_state : str or None
        This is the path to the saved state pickle file produced by a previous
        run of `runcp_producer_loop`. Will get all of the arguments to run
        another instance of the loop from that pickle file. If this is None, you
        MUST provide all of the appropriate arguments to that function.

    lightcurve_list : str or list of str or None
        This is either a string pointing to a file containing a list of light
        curves filenames to process or the list itself. The names must
        correspond to the full filenames of files stored on S3, including all
        prefixes, but not include the 's3://<bucket name>/' bit (these will be
        added automatically).

    input_queue : str or None
        This is the name of the SQS queue which will receive processing tasks
        generated by this function. The queue URL will automatically be obtained
        from AWS.

    input_bucket : str or None
        The name of the S3 bucket containing the light curve files to process.

    result_queue : str or None
        This is the name of the SQS queue that this function will listen to for
        messages from the workers as they complete processing on their input
        elements. This function will attempt to match input sent to the
        `input_queue` with results coming into the `result_queue` so it knows
        how many objects have been successfully processed. If this function
        receives task results that aren't in its own input queue, it will
        acknowledge them so they complete successfully, but not download them
        automatically. This handles leftover tasks completing from a previous
        run of this function.

    result_bucket : str or None
        The name of the S3 bucket which will receive the results from the
        workers.

    pfresult_list : list of str or None
        This is a list of periodfinder result pickle S3 URLs associated with
        each light curve. If provided, this will be used to add in phased light
        curve plots to each checkplot pickle. If this is None, the worker loop
        will produce checkplot pickles that only contain object information,
        neighbor information, and unphased light curves.

    runcp_kwargs : dict or None
        This is a dict used to pass any extra keyword arguments to the
        `lcproc.checkplotgen.runcp` function that will be run by the worker
        loop.

    process_list_slice : list or None
        This is used to index into the input light curve list so a subset of the
        full list can be processed in this specific run of this function.

        Use None for a slice index elem to emulate single slice spec behavior:

        process_list_slice = [10, None]  -> lightcurve_list[10:]
        process_list_slice = [None, 500] -> lightcurve_list[:500]

    purge_queues_when_done : bool or None
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C), all outstanding elements in the
        input/output queues that have not yet been acknowledged by workers or by
        this function will be purged. This effectively cancels all outstanding
        work.

    delete_queues_when_done : bool or None
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C'), all outstanding work items will be
        purged from the input/queues and the queues themselves will be deleted.

    download_when_done : bool or None
        If this is True, the generated checkplot pickle for each input work item
        will be downloaded immediately to the current working directory when the
        worker functions report they're done with it.

    save_state_when_done : bool or None
        If this is True, will save the current state of the work item queue and
        the work items acknowledged as completed to a pickle in the current
        working directory. Call the `runcp_producer_loop_savedstate` function
        below to resume processing from this saved state later.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 download operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    dict or str
        Returns the current work state as a dict or str path to the generated
        work state pickle depending on if `save_state_when_done` is True.

    """

    if use_saved_state is not None and os.path.exists(use_saved_state):

        with open(use_saved_state,'rb') as infd:
            saved_state = pickle.load(infd)

        # run the producer loop using the saved state's todo list
        return runcp_producer_loop(
            saved_state['in_progress'],
            saved_state['args'][1],
            saved_state['args'][2],
            saved_state['args'][3],
            saved_state['args'][4],
            **saved_state['kwargs']
        )

    else:

        return runcp_producer_loop(
            lightcurve_list,
            input_queue,
            input_bucket,
            result_queue,
            result_bucket,
            pfresult_list=pfresult_list,
            runcp_kwargs=runcp_kwargs,
            process_list_slice=process_list_slice,
            download_when_done=download_when_done,
            purge_queues_when_done=purge_queues_when_done,
            save_state_when_done=save_state_when_done,
            delete_queues_when_done=delete_queues_when_done,
            s3_client=s3_client,
            sqs_client=sqs_client
        )

def runcp_producer_loop_savedstate ( use_saved_state = None , lightcurve_list = None , input_queue = None , input_bucket = None , result_queue = None , result_bucket = None , pfresult_list = None , runcp_kwargs = None , process_list_slice = None , download_when_done = True , purge_queues_when_done = True , save_state_when_done = True , delete_queues_when_done = False , s3_client = None , sqs_client = None ) : if use_saved_state is not None and os . path . exists ( use_saved_state ) : with open ( use_saved_state , 'rb' ) as infd : saved_state = pickle . load ( infd ) return runcp_producer_loop ( saved_state [ 'in_progress' ] , saved_state [ 'args' ] [ 1 ] , saved_state [ 'args' ] [ 2 ] , saved_state [ 'args' ] [ 3 ] , saved_state [ 'args' ] [ 4 ] , ** saved_state [ 'kwargs' ] ) else : return runcp_producer_loop ( lightcurve_list , input_queue , input_bucket , result_queue , result_bucket , pfresult_list = pfresult_list , runcp_kwargs = runcp_kwargs , process_list_slice = process_list_slice , download_when_done = download_when_done , purge_queues_when_done = purge_queues_when_done , save_state_when_done = save_state_when_done , delete_queues_when_done = delete_queues_when_done , s3_client = s3_client , sqs_client = sqs_client )

This wraps the function above to allow for loading previous state from a file .


def spline_fit_magseries(times, mags, errs, period,
                         knotfraction=0.01,
                         maxknots=30,
                         sigclip=30.0,
                         plotfit=False,
                         ignoreinitfail=False,
                         magsarefluxes=False,
                         verbose=True):

    '''This fits a univariate cubic spline to the phased light curve.

    This fit may be better than the Fourier fit for sharply variable objects,
    like EBs, so can be used to distinguish them from other types of variables.

    Parameters
    ----------

    times,mags,errs : np.array
        The input mag/flux time-series to fit a spline to.

    period : float
        The period to use for the spline fit.

    knotfraction : float
        The knot fraction is the number of internal knots to use for the
        spline. A value of 0.01 (or 1%) of the total number of non-nan
        observations appears to work quite well, without over-fitting. maxknots
        controls the maximum number of knots that will be allowed.

    maxknots : int
        The maximum number of knots that will be used even if `knotfraction`
        gives a value to use larger than `maxknots`. This helps dealing with
        over-fitting to short time-scale variations.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    magsarefluxes : bool
        If True, will treat the input values of `mags` as fluxes for purposes of
        plotting the fit and sig-clipping.

    plotfit : str or False
        If this is a string, this function will make a plot for the fit to the
        mag/flux time-series and writes the plot to the path specified here.

    ignoreinitfail : bool
        If this is True, ignores the initial failure to find a set of optimized
        Fourier parameters using the global optimization function and proceeds
        to do a least-squares fit anyway.

    verbose : bool
        If True, will indicate progress and warn of any problems.

    Returns
    -------

    dict
        This function returns a dict containing the model fit parameters, the
        minimized chi-sq value and the reduced chi-sq value. The form of this
        dict is mostly standardized across all functions in this module::

            {
                'fittype':'spline',
                'fitinfo':{
                    'nknots': the number of knots used for the fit
                    'fitmags': the model fit mags,
                    'fitepoch': the epoch of minimum light for the fit,
                },
                'fitchisq': the minimized value of the fit's chi-sq,
                'fitredchisq':the reduced chi-sq value,
                'fitplotfile': the output fit plot if fitplot is not None,
                'magseries':{
                    'times':input times in phase order of the model,
                    'phase':the phases of the model mags,
                    'mags':input mags/fluxes in the phase order of the model,
                    'errs':errs in the phase order of the model,
                    'magsarefluxes':input value of magsarefluxes kwarg
                }
            }

    '''

    # this is required to fit the spline correctly
    if errs is None:
        errs = npfull_like(mags, 0.005)

    # sigclip the magnitude time series
    stimes, smags, serrs = sigclip_magseries(times, mags, errs,
                                             sigclip=sigclip,
                                             magsarefluxes=magsarefluxes)
    # get rid of zero errs
    nzind = npnonzero(serrs)
    stimes, smags, serrs = stimes[nzind], smags[nzind], serrs[nzind]

    # phase the mag series
    phase, pmags, perrs, ptimes, mintime = (
        get_phased_quantities(stimes, smags, serrs, period)
    )

    # now figure out the number of knots up to max knots (=100)
    nobs = len(phase)
    nknots = int(npfloor(knotfraction*nobs))
    nknots = maxknots if nknots > maxknots else nknots
    splineknots = nplinspace(phase[0] + 0.01,
                             phase[-1] - 0.01,
                             num=nknots)

    # NOTE: newer scipy needs x to be strictly increasing. this means we should
    # filter out anything that doesn't have np.diff(phase) > 0.0
    # FIXME: this needs to be tested
    phase_diffs_ind = npdiff(phase) > 0.0
    incphase_ind = npconcatenate((nparray([True]), phase_diffs_ind))
    phase, pmags, perrs = (phase[incphase_ind],
                           pmags[incphase_ind],
                           perrs[incphase_ind])

    # generate and fit the spline
    spl = LSQUnivariateSpline(phase, pmags, t=splineknots, w=1.0/perrs)

    # calculate the spline fit to the actual phases, the chisq and red-chisq
    fitmags = spl(phase)

    fitchisq = npsum(
        ((fitmags - pmags)*(fitmags - pmags)) / (perrs*perrs)
    )

    fitredchisq = fitchisq/(len(pmags) - nknots - 1)

    if verbose:
        LOGINFO(
            'spline fit done. nknots = %s,  '
            'chisq = %.5f, reduced chisq = %.5f' %
            (nknots, fitchisq, fitredchisq)
        )

    # figure out the time of light curve minimum (i.e. the fit epoch)
    # this is when the fit mag is maximum (i.e. the faintest)
    # or if magsarefluxes = True, then this is when fit flux is minimum
    if not magsarefluxes:
        fitmagminind = npwhere(fitmags == npmax(fitmags))
    else:
        fitmagminind = npwhere(fitmags == npmin(fitmags))
    if len(fitmagminind[0]) > 1:
        fitmagminind = (fitmagminind[0][0],)
    magseriesepoch = ptimes[fitmagminind]

    # assemble the returndict
    returndict = {
        'fittype':'spline',
        'fitinfo':{
            'nknots':nknots,
            'fitmags':fitmags,
            'fitepoch':magseriesepoch
        },
        'fitchisq':fitchisq,
        'fitredchisq':fitredchisq,
        'fitplotfile':None,
        'magseries':{
            'times':ptimes,
            'phase':phase,
            'mags':pmags,
            'errs':perrs,
            'magsarefluxes':magsarefluxes
        },
    }

    # make the fit plot if required
    if plotfit and isinstance(plotfit, str):

        make_fit_plot(phase, pmags, perrs, fitmags,
                      period, mintime, magseriesepoch,
                      plotfit,
                      magsarefluxes=magsarefluxes)

        returndict['fitplotfile'] = plotfit

    return returndict

def
spline_fit_magseries
(
times
,
mags
,
errs
,
period
,
knotfraction
=
0.01
,
maxknots
=
30
,
sigclip
=
30.0
,
plotfit
=
False
,
ignoreinitfail
=
False
,
magsarefluxes
=
False
,
verbose
=
True
)
:
if
errs
is
:
errs
=
npfull_like
(
mags
,
0.005
)
stimes
,
smags
,
serrs
=
sigclip_magseries
(
times
,
mags
,
errs
,
sigclip
=
sigclip
,
magsarefluxes
=
magsarefluxes
)
nzind
=
npnonzero
(
serrs
)
stimes
,
smags
,
serrs
=
stimes
[
nzind
]
,
smags
[
nzind
]
,
serrs
[
nzind
]
phase
,
pmags
,
perrs
,
ptimes
,
mintime
=
(
get_phased_quantities
(
stimes
,
smags
,
serrs
,
period
)
)
nobs
=
len
(
phase
)
nknots
=
int
(
npfloor
(
knotfraction
*
nobs
)
)
nknots
=
maxknots
if
nknots
>
maxknots
else
nknots
splineknots
=
nplinspace
(
phase
[
0
]
+
0.01
,
phase
[
1
]
0.01
,
num
=
nknots
)
phase_diffs_ind
=
npdiff
(
phase
)
>
0.0
incphase_ind
=
npconcatenate
(
(
nparray
(
[
True
]
)
,
phase_diffs_ind
)
)
phase
,
pmags
,
perrs
=
(
phase
[
incphase_ind
]
,
pmags
[
incphase_ind
]
,
perrs
[
incphase_ind
]
)
spl
=
LSQUnivariateSpline
(
phase
,
pmags
,
t
=
splineknots
,
w
=
1.0
/
perrs
)
fitmags
=
spl
(
phase
)
fitchisq
=
npsum
(
(
(
fitmags
pmags
)
*
(
fitmags
pmags
)
)
/
(
perrs
*
perrs
)
)
fitredchisq
=
fitchisq
/
(
len
(
pmags
)
nknots
1
)
if
verbose
:
LOGINFO
(
'spline fit done. nknots = %s,  '
'chisq = %.5f, reduced chisq = %.5f'
%
(
nknots
,
fitchisq
,
fitredchisq
)
)
if
not
magsarefluxes
:
fitmagminind
=
npwhere
(
fitmags
==
npmax
(
fitmags
)
)
else
:
fitmagminind
=
npwhere
(
fitmags
==
npmin
(
fitmags
)
)
if
len
(
fitmagminind
[
0
]
)
>
1
:
fitmagminind
=
(
fitmagminind
[
0
]
[
0
]
,
)
magseriesepoch
=
ptimes
[
fitmagminind
]
returndict
=
{
'fittype'
:
'spline'
,
'fitinfo'
:
{
'nknots'
:
nknots
,
'fitmags'
:
fitmags
,
'fitepoch'
:
magseriesepoch
}
,
'fitchisq'
:
fitchisq
,
'fitredchisq'
:
fitredchisq
,
'fitplotfile'
:
,
'magseries'
:
{
'times'
:
ptimes
,
'phase'
:
phase
,
'mags'
:
pmags
,
'errs'
:
perrs
,
'magsarefluxes'
:
magsarefluxes
}
,
}
if
plotfit
and
isinstance
(
plotfit
,
str
)
:
make_fit_plot
(
phase
,
pmags
,
perrs
,
fitmags
,
period
,
mintime
,
magseriesepoch
,
plotfit
,
magsarefluxes
=
magsarefluxes
)
returndict
[
'fitplotfile'
]
=
plotfit
return
returndict

This fits a univariate cubic spline to the phased light curve .


def runcp_worker(task):
    '''
    This is the worker for running checkplots.

    Parameters
    ----------

    task : tuple
        This is of the form: (pfpickle, outdir, lcbasedir, kwargs).

    Returns
    -------

    list of str
        The list of checkplot pickles returned by the `runcp` function.

    '''

    pfpickle, outdir, lcbasedir, kwargs = task

    try:

        return runcp(pfpickle, outdir, lcbasedir, **kwargs)

    except Exception as e:

        LOGEXCEPTION(' could not make checkplots for %s: %s' % (pfpickle, e))
        return None

def
runcp_worker
(
task
)
:
pfpickle
,
outdir
,
lcbasedir
,
kwargs
=
task
try
:
return
runcp
(
pfpickle
,
outdir
,
lcbasedir
,
**
kwargs
)
except
Exception
as
e
:
LOGEXCEPTION
(
' could not make checkplots for %s: %s'
%
(
pfpickle
,
e
)
)
return

This is the worker for running checkplots .


def parallel_cp(
        pfpicklelist,
        outdir,
        lcbasedir,
        fast_mode=False,
        lcfnamelist=None,
        cprenorm=False,
        lclistpkl=None,
        gaia_max_timeout=60.0,
        gaia_mirror=None,
        nbrradiusarcsec=60.0,
        maxnumneighbors=5,
        makeneighborlcs=True,
        xmatchinfo=None,
        xmatchradiusarcsec=3.0,
        sigclip=10.0,
        minobservations=99,
        lcformat='hat-sql',
        lcformatdir=None,
        timecols=None,
        magcols=None,
        errcols=None,
        skipdone=False,
        done_callback=None,
        done_callback_args=None,
        done_callback_kwargs=None,
        liststartindex=None,
        maxobjects=None,
        nworkers=NCPUS,
):
    '''This drives the parallel execution of `runcp` for a list of periodfinding
    result pickles.

    Parameters
    ----------

    pfpicklelist : list of str or list of Nones
        This is the list of the filenames of the period-finding result pickles
        to process. To make checkplots using the light curves directly, set this
        to a list of Nones with the same length as the list of light curve files
        that you provide in `lcfnamelist`.

    outdir : str
        The directory the checkplot pickles will be written to.

    lcbasedir : str
        The base directory that this function will look in to find the light
        curves pointed to by the period-finding result files. If you're using
        `lcfnamelist` to provide a list of light curve filenames directly, this
        arg is ignored.

    lcfnamelist : list of str or None
        If this is provided, it must be a list of the input light curve
        filenames to process. These can either be associated with each input
        period-finder result pickle, or can be provided standalone to make
        checkplots without phased LC plots in them. In the second case, you must
        set `pfpicklelist` to a list of Nones that matches the length of
        `lcfnamelist`.

    cprenorm : bool
        Set this to True if the light curves should be renormalized by
        `checkplot.checkplot_pickle`. This is set to False by default because we
        do our own normalization in this function using the light curve's
        registered normalization function and pass the normalized times, mags,
        errs to the `checkplot.checkplot_pickle` function.

    lclistpkl : str or dict
        This is either the filename of a pickle or the actual dict produced by
        lcproc.make_lclist. This is used to gather neighbor information.

    nbrradiusarcsec : float
        The radius in arcseconds to use for a search conducted around the
        coordinates of this object to look for any potential confusion and
        blending of variability amplitude caused by their proximity.

    maxnumneighbors : int
        The maximum number of neighbors that will have their light curves and
        magnitudes noted in this checkplot as potential blends with the target
        object.

    makeneighborlcs : bool
        If True, will make light curve and phased light curve plots for all
        neighbors found in the object collection for each input object.

    fast_mode : bool or float
        This runs the external catalog operations in a "fast" mode, with short
        timeouts and not trying to hit external catalogs that take a long time
        to respond.

        If this is set to True, the default settings for the external requests
        will then become::

                skyview_lookup = False
                skyview_timeout = 10.0
                skyview_retry_failed = False
                dust_timeout = 10.0
                gaia_submit_timeout = 7.0
                gaia_max_timeout = 10.0
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

        If this is a float, will run in "fast" mode with the provided timeout
        value in seconds and the following settings::

                skyview_lookup = True
                skyview_timeout = fast_mode
                skyview_retry_failed = False
                dust_timeout = fast_mode
                gaia_submit_timeout = 0.66*fast_mode
                gaia_max_timeout = fast_mode
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

    gaia_max_timeout : float
        Sets the timeout in seconds to use when waiting for the GAIA service to
        respond to our request for the object's information. Note that if
        `fast_mode` is set, this is ignored.

    gaia_mirror : str or None
        This sets the GAIA mirror to use. This is a key in the
        `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
        mirror.

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        `load_xmatch_external_catalogs` above, or the path to the xmatch info
        pickle file produced by that function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    minobservations : int
        The minimum of observations the input object's mag/flux time-series must
        have for this function to plot its light curve and phased light
        curve. If the object has less than this number, no light curves will be
        plotted, but the checkplotdict will still contain all of the other
        information.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the lcdict in generating this checkplot.

    magcols : list of str or None
        The magcol keys to use from the lcdict in generating this checkplot.

    errcols : list of str or None
        The errcol keys to use from the lcdict in generating this checkplot.

    skipdone : bool
        This indicates if this function will skip creating checkplots that
        already exist corresponding to the current `objectid` and `magcol`. If
        `skipdone` is set to True, this will be done.

    done_callback : Python function or None
        This is used to provide a function to execute after the checkplot
        pickles are generated. This is useful if you want to stream the results
        of checkplot making to some other process, e.g. directly running an
        ingestion into an LCC-Server collection. The function will always get
        the list of the generated checkplot pickles as its first arg, and all of
        the kwargs for runcp in the kwargs dict. Additional args and kwargs can
        be provided by giving a list in the `done_callbacks_args` kwarg and a
        dict in the `done_callbacks_kwargs` kwarg.

        NOTE: the function you pass in here should be pickleable by normal
        Python if you want to use it with the parallel_cp and parallel_cp_lcdir
        functions below.

    done_callback_args : tuple or None
        If not None, contains any args to pass into the `done_callback`
        function.

    done_callback_kwargs : dict or None
        If not None, contains any kwargs to pass into the `done_callback`
        function.

    liststartindex : int
        The index of the `pfpicklelist` (and `lcfnamelist` if provided) to start
        working at.

    maxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input period-finding result pickles (and light curves if `lcfnamelist`
        is also provided) over several sessions or machines.

    nworkers : int
        The number of parallel workers that will work on the checkplot
        generation process.

    Returns
    -------

    dict
        This returns a dict with keys = input period-finding pickles and vals =
        list of the corresponding checkplot pickles produced.

    '''

    # work around the Darwin segfault after fork if no network activity in
    # main thread bug: https://bugs.python.org/issue30385#msg293958
    if sys.platform == 'darwin':
        import requests
        requests.get('http://captive.apple.com/hotspot-detect.html')

    if not os.path.exists(outdir):
        os.mkdir(outdir)

    # handle the start and end indices
    if (liststartindex is not None) and (maxobjects is None):
        pfpicklelist = pfpicklelist[liststartindex:]
        if lcfnamelist is not None:
            lcfnamelist = lcfnamelist[liststartindex:]

    elif (liststartindex is None) and (maxobjects is not None):
        pfpicklelist = pfpicklelist[:maxobjects]
        if lcfnamelist is not None:
            lcfnamelist = lcfnamelist[:maxobjects]

    elif (liststartindex is not None) and (maxobjects is not None):
        pfpicklelist = (
            pfpicklelist[liststartindex:liststartindex+maxobjects]
        )
        if lcfnamelist is not None:
            lcfnamelist = lcfnamelist[liststartindex:liststartindex+maxobjects]

    # if the lcfnamelist is not provided, create a dummy
    if lcfnamelist is None:
        lcfnamelist = [None]*len(pfpicklelist)

    tasklist = [(x, outdir, lcbasedir,
                 {'lcformat':lcformat,
                  'lcformatdir':lcformatdir,
                  'lcfname':y,
                  'timecols':timecols,
                  'magcols':magcols,
                  'errcols':errcols,
                  'lclistpkl':lclistpkl,
                  'gaia_max_timeout':gaia_max_timeout,
                  'gaia_mirror':gaia_mirror,
                  'nbrradiusarcsec':nbrradiusarcsec,
                  'maxnumneighbors':maxnumneighbors,
                  'makeneighborlcs':makeneighborlcs,
                  'xmatchinfo':xmatchinfo,
                  'xmatchradiusarcsec':xmatchradiusarcsec,
                  'sigclip':sigclip,
                  'minobservations':minobservations,
                  'skipdone':skipdone,
                  'cprenorm':cprenorm,
                  'fast_mode':fast_mode,
                  'done_callback':done_callback,
                  'done_callback_args':done_callback_args,
                  'done_callback_kwargs':done_callback_kwargs}) for
                x,y in zip(pfpicklelist, lcfnamelist)]

    resultfutures = []
    results = []

    with ProcessPoolExecutor(max_workers=nworkers) as executor:
        resultfutures = executor.map(runcp_worker, tasklist)

    results = [x for x in resultfutures]

    executor.shutdown()
    return results

This drives the parallel execution of runcp for a list of periodfinding result pickles .


def parallel_cp_pfdir(pfpickledir,
                      outdir,
                      lcbasedir,
                      pfpickleglob='periodfinding-*.pkl*',
                      lclistpkl=None,
                      cprenorm=False,
                      nbrradiusarcsec=60.0,
                      maxnumneighbors=5,
                      makeneighborlcs=True,
                      fast_mode=False,
                      gaia_max_timeout=60.0,
                      gaia_mirror=None,
                      xmatchinfo=None,
                      xmatchradiusarcsec=3.0,
                      minobservations=99,
                      sigclip=10.0,
                      lcformat='hat-sql',
                      lcformatdir=None,
                      timecols=None,
                      magcols=None,
                      errcols=None,
                      skipdone=False,
                      done_callback=None,
                      done_callback_args=None,
                      done_callback_kwargs=None,
                      maxobjects=None,
                      nworkers=32):

    '''This drives the parallel execution of `runcp` for a directory of
    periodfinding pickles.

    Parameters
    ----------

    pfpickledir : str
        This is the directory containing all of the period-finding pickles to
        process.

    outdir : str
        The directory the checkplot pickles will be written to.

    lcbasedir : str
        The base directory that this function will look in to find the light
        curves pointed to by the period-finding result files. If you're using
        `lcfnamelist` to provide a list of light curve filenames directly, this
        arg is ignored.

    pkpickleglob : str
        This is a UNIX file glob to select period-finding result pickles in the
        specified `pfpickledir`.

    lclistpkl : str or dict
        This is either the filename of a pickle or the actual dict produced by
        lcproc.make_lclist. This is used to gather neighbor information.

    cprenorm : bool
        Set this to True if the light curves should be renormalized by
        `checkplot.checkplot_pickle`. This is set to False by default because we
        do our own normalization in this function using the light curve's
        registered normalization function and pass the normalized times, mags,
        errs to the `checkplot.checkplot_pickle` function.

    nbrradiusarcsec : float
        The radius in arcseconds to use for a search conducted around the
        coordinates of this object to look for any potential confusion and
        blending of variability amplitude caused by their proximity.

    maxnumneighbors : int
        The maximum number of neighbors that will have their light curves and
        magnitudes noted in this checkplot as potential blends with the target
        object.

    makeneighborlcs : bool
        If True, will make light curve and phased light curve plots for all
        neighbors found in the object collection for each input object.

    fast_mode : bool or float
        This runs the external catalog operations in a "fast" mode, with short
        timeouts and not trying to hit external catalogs that take a long time
        to respond.

        If this is set to True, the default settings for the external requests
        will then become::

                skyview_lookup = False
                skyview_timeout = 10.0
                skyview_retry_failed = False
                dust_timeout = 10.0
                gaia_submit_timeout = 7.0
                gaia_max_timeout = 10.0
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

        If this is a float, will run in "fast" mode with the provided timeout
        value in seconds and the following settings::

                skyview_lookup = True
                skyview_timeout = fast_mode
                skyview_retry_failed = False
                dust_timeout = fast_mode
                gaia_submit_timeout = 0.66*fast_mode
                gaia_max_timeout = fast_mode
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

    gaia_max_timeout : float
        Sets the timeout in seconds to use when waiting for the GAIA service to
        respond to our request for the object's information. Note that if
        `fast_mode` is set, this is ignored.

    gaia_mirror : str or None
        This sets the GAIA mirror to use. This is a key in the
        `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
        mirror.

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        `load_xmatch_external_catalogs` above, or the path to the xmatch info
        pickle file produced by that function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    minobservations : int
        The minimum of observations the input object's mag/flux time-series must
        have for this function to plot its light curve and phased light
        curve. If the object has less than this number, no light curves will be
        plotted, but the checkplotdict will still contain all of the other
        information.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the lcdict in generating this checkplot.

    magcols : list of str or None
        The magcol keys to use from the lcdict in generating this checkplot.

    errcols : list of str or None
        The errcol keys to use from the lcdict in generating this checkplot.

    skipdone : bool
        This indicates if this function will skip creating checkplots that
        already exist corresponding to the current `objectid` and `magcol`. If
        `skipdone` is set to True, this will be done.

    done_callback : Python function or None
        This is used to provide a function to execute after the checkplot
        pickles are generated. This is useful if you want to stream the results
        of checkplot making to some other process, e.g. directly running an
        ingestion into an LCC-Server collection. The function will always get
        the list of the generated checkplot pickles as its first arg, and all of
        the kwargs for runcp in the kwargs dict. Additional args and kwargs can
        be provided by giving a list in the `done_callbacks_args` kwarg and a
        dict in the `done_callbacks_kwargs` kwarg.

        NOTE: the function you pass in here should be pickleable by normal
        Python if you want to use it with the parallel_cp and parallel_cp_lcdir
        functions below.

    done_callback_args : tuple or None
        If not None, contains any args to pass into the `done_callback`
        function.

    done_callback_kwargs : dict or None
        If not None, contains any kwargs to pass into the `done_callback`
        function.

    maxobjects : int
        The maximum number of objects to process in this run.

    nworkers : int
        The number of parallel workers that will work on the checkplot
        generation process.

    Returns
    -------

    dict
        This returns a dict with keys = input period-finding pickles and vals =
        list of the corresponding checkplot pickles produced.

    '''

    pfpicklelist = sorted(glob.glob(os.path.join(pfpickledir, pfpickleglob)))

    LOGINFO('found %s period-finding pickles, running cp...' %
            len(pfpicklelist))

    return parallel_cp(pfpicklelist,
                       outdir,
                       lcbasedir,
                       fast_mode=fast_mode,
                       lclistpkl=lclistpkl,
                       nbrradiusarcsec=nbrradiusarcsec,
                       gaia_max_timeout=gaia_max_timeout,
                       gaia_mirror=gaia_mirror,
                       maxnumneighbors=maxnumneighbors,
                       makeneighborlcs=makeneighborlcs,
                       xmatchinfo=xmatchinfo,
                       xmatchradiusarcsec=xmatchradiusarcsec,
                       sigclip=sigclip,
                       minobservations=minobservations,
                       cprenorm=cprenorm,
                       maxobjects=maxobjects,
                       lcformat=lcformat,
                       lcformatdir=lcformatdir,
                       timecols=timecols,
                       magcols=magcols,
                       errcols=errcols,
                       skipdone=skipdone,
                       nworkers=nworkers,
                       done_callback=done_callback,
                       done_callback_args=done_callback_args,
                       done_callback_kwargs=done_callback_kwargs)

This drives the parallel execution of runcp for a directory of periodfinding pickles .


def _runpf_worker(task):
    '''
    This runs the runpf function.

    '''

    (lcfile, outdir, timecols, magcols, errcols, lcformat, lcformatdir,
     pfmethods, pfkwargs, getblssnr, sigclip, nworkers, minobservations,
     excludeprocessed) = task

    if os.path.exists(lcfile):
        pfresult = runpf(lcfile,
                         outdir,
                         timecols=timecols,
                         magcols=magcols,
                         errcols=errcols,
                         lcformat=lcformat,
                         lcformatdir=lcformatdir,
                         pfmethods=pfmethods,
                         pfkwargs=pfkwargs,
                         getblssnr=getblssnr,
                         sigclip=sigclip,
                         nworkers=nworkers,
                         minobservations=minobservations,
                         excludeprocessed=excludeprocessed)
        return pfresult
    else:
        LOGERROR('LC does not exist for requested file %s' % lcfile)
        return None


def parallel_pf(lclist,
                outdir,
                timecols=None,
                magcols=None,
                errcols=None,
                lcformat='hat-sql',
                lcformatdir=None,
                pfmethods=('gls','pdm','mav','win'),
                pfkwargs=({},{},{},{}),
                sigclip=10.0,
                getblssnr=False,
                nperiodworkers=NCPUS,
                ncontrolworkers=1,
                liststartindex=None,
                listmaxobjects=None,
                minobservations=500,
                excludeprocessed=True):
    '''This drives the overall parallel period processing for a list of LCs.

    As a rough benchmark, 25000 HATNet light curves with up to 50000 points per
    LC take about 26 days in total for an invocation of this function using
    GLS+PDM+BLS, 10 periodworkers, and 4 controlworkers (so all 40 'cores') on a
    2 x Xeon E5-2660v3 machine.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file to process.

    outdir : str
        The output directory where the period-finding result pickles will go.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    pfmethods : list of str
        This is a list of period finding methods to run. Each element is a
        string matching the keys of the `PFMETHODS` dict above. By default, this
        runs GLS, PDM, AoVMH, and the spectral window Lomb-Scargle periodogram.

    pfkwargs : list of dicts
        This is used to provide any special kwargs as dicts to each
        period-finding method function specified in `pfmethods`.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    getblssnr : bool
        If this is True and BLS is one of the methods specified in `pfmethods`,
        will also calculate the stats for each best period in the BLS results:
        transit depth, duration, ingress duration, refit period and epoch, and
        the SNR of the transit.

    nperiodworkers : int
        The number of parallel period-finding workers to launch per object task.

    ncontrolworkers : int
        The number of controlling processes to launch. This effectively sets how
        many objects from `lclist` will be processed in parallel.

    liststartindex : int or None
        This sets the index from where to start in `lclist`.

    listmaxobjects : int or None
        This sets the maximum number of objects in `lclist` to run
        period-finding for in this invocation. Together with `liststartindex`,
        `listmaxobjects` can be used to distribute processing over several
        independent machines if the number of light curves is very large.

    minobservations : int
        The minimum number of finite LC points required to process a light
        curve.

    excludeprocessed : bool
        If this is True, light curves that have existing period-finding result
        pickles in `outdir` will not be processed.

        FIXME: currently, this uses a dumb method of excluding already-processed
        files. A smarter way to do this is to (i) generate a SHA512 cachekey
        based on a repr of `{'lcfile', 'timecols', 'magcols', 'errcols',
        'lcformat', 'pfmethods', 'sigclip', 'getblssnr', 'pfkwargs'}`, (ii) make
        sure all list kwargs in the dict are sorted, (iii) check if the output
        file has the same cachekey in its filename (last 8 chars of cachekey
        should work), so the result was processed in exactly the same way as
        specifed in the input to this function, and can therefore be
        ignored. Will implement this later.

    Returns
    -------

    list of str
        A list of the period-finding pickles created for all of input LCs
        processed.

    '''

    # make the output directory if it doesn't exist
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    if (liststartindex is not None) and (listmaxobjects is None):
        lclist = lclist[liststartindex:]

    elif (liststartindex is None) and (listmaxobjects is not None):
        lclist = lclist[:listmaxobjects]

    elif (liststartindex is not None) and (listmaxobjects is not None):
        lclist = lclist[liststartindex:liststartindex+listmaxobjects]

    tasklist = [(x, outdir, timecols, magcols, errcols, lcformat, lcformatdir,
                 pfmethods, pfkwargs, getblssnr, sigclip, nperiodworkers,
                 minobservations,
                 excludeprocessed)
                for x in lclist]

    with ProcessPoolExecutor(max_workers=ncontrolworkers) as executor:
        resultfutures = executor.map(_runpf_worker, tasklist)

    results = [x for x in resultfutures]
    return results

def parallel_pf ( lclist , outdir , timecols = None , magcols = None , errcols = None , lcformat = 'hat-sql' , lcformatdir = None , pfmethods = ( 'gls' , 'pdm' , 'mav' , 'win' ) , pfkwargs = ( { } , { } , { } , { } ) , sigclip = 10.0 , getblssnr = False , nperiodworkers = NCPUS , ncontrolworkers = 1 , liststartindex = None , listmaxobjects = None , minobservations = 500 , excludeprocessed = True ) : if not os . path . exists ( outdir ) : os . makedirs ( outdir ) if ( liststartindex is not None ) and ( listmaxobjects is None ) : lclist = lclist [ liststartindex : ] elif ( liststartindex is None ) and ( listmaxobjects is not None ) : lclist = lclist [ : listmaxobjects ] elif ( liststartindex is not None ) and ( listmaxobjects is not None ) : lclist = lclist [ liststartindex : liststartindex + listmaxobjects ] tasklist = [ ( x , outdir , timecols , magcols , errcols , lcformat , lcformatdir , pfmethods , pfkwargs , getblssnr , sigclip , nperiodworkers , minobservations , excludeprocessed ) for x in lclist ] with ProcessPoolExecutor ( max_workers = ncontrolworkers ) as executor : resultfutures = executor . map ( _runpf_worker , tasklist ) results = [ x for x in resultfutures ] return results

This drives the overall parallel period processing for a list of LCs .


def parallel_pf_lcdir(lcdir,
                      outdir,
                      fileglob=None,
                      recursive=True,
                      timecols=None,
                      magcols=None,
                      errcols=None,
                      lcformat='hat-sql',
                      lcformatdir=None,
                      pfmethods=('gls','pdm','mav','win'),
                      pfkwargs=({},{},{},{}),
                      sigclip=10.0,
                      getblssnr=False,
                      nperiodworkers=NCPUS,
                      ncontrolworkers=1,
                      liststartindex=None,
                      listmaxobjects=None,
                      minobservations=500,
                      excludeprocessed=True):
    '''This runs parallel light curve period finding for directory of LCs.

    Parameters
    ----------

    lcdir : str
        The directory containing the LCs to process.

    outdir : str
        The directory where the resulting period-finding pickles will go.

    fileglob : str or None
        The UNIX file glob to use to search for LCs in `lcdir`. If None, the
        default file glob associated with the registered LC format will be used
        instead.

    recursive : bool
        If True, will search recursively in `lcdir` for light curves to process.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    pfmethods : list of str
        This is a list of period finding methods to run. Each element is a
        string matching the keys of the `PFMETHODS` dict above. By default, this
        runs GLS, PDM, AoVMH, and the spectral window Lomb-Scargle periodogram.

    pfkwargs : list of dicts
        This is used to provide any special kwargs as dicts to each
        period-finding method function specified in `pfmethods`.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    getblssnr : bool
        If this is True and BLS is one of the methods specified in `pfmethods`,
        will also calculate the stats for each best period in the BLS results:
        transit depth, duration, ingress duration, refit period and epoch, and
        the SNR of the transit.

    nperiodworkers : int
        The number of parallel period-finding workers to launch per object task.

    ncontrolworkers : int
        The number of controlling processes to launch. This effectively sets how
        many objects from `lclist` will be processed in parallel.

    liststartindex : int or None
        This sets the index from where to start in `lclist`.

    listmaxobjects : int or None
        This sets the maximum number of objects in `lclist` to run
        period-finding for in this invocation. Together with `liststartindex`,
        `listmaxobjects` can be used to distribute processing over several
        independent machines if the number of light curves is very large.

    minobservations : int
        The minimum number of finite LC points required to process a light
        curve.

    excludeprocessed : bool
        If this is True, light curves that have existing period-finding result
        pickles in `outdir` will not be processed.

        FIXME: currently, this uses a dumb method of excluding already-processed
        files. A smarter way to do this is to (i) generate a SHA512 cachekey
        based on a repr of `{'lcfile', 'timecols', 'magcols', 'errcols',
        'lcformat', 'pfmethods', 'sigclip', 'getblssnr', 'pfkwargs'}`, (ii) make
        sure all list kwargs in the dict are sorted, (iii) check if the output
        file has the same cachekey in its filename (last 8 chars of cachekey
        should work), so the result was processed in exactly the same way as
        specifed in the input to this function, and can therefore be
        ignored. Will implement this later.

    Returns
    -------

    list of str
        A list of the period-finding pickles created for all of input LCs
        processed.

    '''

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    if not fileglob:
        fileglob = dfileglob

    # now find the files
    LOGINFO('searching for %s light curves in %s ...' % (lcformat, lcdir))

    if recursive is False:
        matching = glob.glob(os.path.join(lcdir, fileglob))

    else:
        # use recursive glob for Python 3.5+
        if sys.version_info[:2] > (3,4):

            matching = glob.glob(os.path.join(lcdir,
                                              '**',
                                              fileglob),recursive=True)

        # otherwise, use os.walk and glob
        else:

            # use os.walk to go through the directories
            walker = os.walk(lcdir)
            matching = []

            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              fileglob)
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)

    # now that we have all the files, process them
    if matching and len(matching) > 0:

        # this helps us process things in deterministic order when we distribute
        # processing over several machines
        matching = sorted(matching)

        LOGINFO('found %s light curves, running pf...' % len(matching))

        return parallel_pf(matching,
                           outdir,
                           timecols=timecols,
                           magcols=magcols,
                           errcols=errcols,
                           lcformat=lcformat,
                           lcformatdir=lcformatdir,
                           pfmethods=pfmethods,
                           pfkwargs=pfkwargs,
                           getblssnr=getblssnr,
                           sigclip=sigclip,
                           nperiodworkers=nperiodworkers,
                           ncontrolworkers=ncontrolworkers,
                           liststartindex=liststartindex,
                           listmaxobjects=listmaxobjects,
                           minobservations=minobservations,
                           excludeprocessed=excludeprocessed)

    else:

        LOGERROR('no light curve files in %s format found in %s' % (lcformat,
                                                                    lcdir))
        return None

This runs parallel light curve period finding for directory of LCs .


def collect_nonperiodic_features(
        featuresdir,
        magcol,
        outfile,
        pklglob='varfeatures-*.pkl',
        featurestouse=NONPERIODIC_FEATURES_TO_COLLECT,
        maxobjects=None,
        labeldict=None,
        labeltype='binary',
):
    '''This collects variability features into arrays for use with the classifer.

    Parameters
    ----------

    featuresdir : str
        This is the directory where all the varfeatures pickles are. Use
        `pklglob` to specify the glob to search for. The `varfeatures` pickles
        contain objectids, a light curve magcol, and features as dict
        key-vals. The :py:mod:`astrobase.lcproc.lcvfeatures` module can be used
        to produce these.

    magcol : str
        This is the key in each varfeatures pickle corresponding to the magcol
        of the light curve the variability features were extracted from.

    outfile : str
        This is the filename of the output pickle that will be written
        containing a dict of all the features extracted into np.arrays.

    pklglob : str
        This is the UNIX file glob to use to search for varfeatures pickle files
        in `featuresdir`.

    featurestouse : list of str
        Each varfeatures pickle can contain any combination of non-periodic,
        stellar, and periodic features; these must have the same names as
        elements in the list of strings provided in `featurestouse`.  This tries
        to get all the features listed in NONPERIODIC_FEATURES_TO_COLLECT by
        default. If `featurestouse` is provided as a list, gets only the
        features listed in this kwarg instead.

    maxobjects : int or None
        The controls how many pickles from the featuresdir to process. If None,
        will process all varfeatures pickles.

    labeldict : dict or None
        If this is provided, it must be a dict with the following key:val list::

            '<objectid>':<label value>

        for each objectid collected from the varfeatures pickles. This will turn
        the collected information into a training set for classifiers.

        Example: to carry out non-periodic variable feature collection of fake
        LCS prepared by :py:mod:`astrobase.fakelcs.generation`, use the value
        of the 'isvariable' dict elem from the `fakelcs-info.pkl` here, like
        so::

            labeldict={x:y for x,y in zip(fakelcinfo['objectid'],
                                          fakelcinfo['isvariable'])}

    labeltype : {'binary', 'classes'}
        This is either 'binary' or 'classes' for binary/multi-class
        classification respectively.

    Returns
    -------

    dict
        This returns a dict with all of the features collected into np.arrays,
        ready to use as input to a scikit-learn classifier.

    '''

    # list of input pickles generated by varfeatures in lcproc.py
    pklist = glob.glob(os.path.join(featuresdir, pklglob))

    if maxobjects:
        pklist = pklist[:maxobjects]

    # fancy progress bar with tqdm if present
    if TQDM:
        listiterator = tqdm(pklist)
    else:
        listiterator = pklist

    # go through all the varfeatures arrays

    feature_dict = {'objectids':[],'magcol':magcol, 'availablefeatures':[]}

    LOGINFO('collecting features for magcol: %s' % magcol)

    for pkl in listiterator:

        with open(pkl,'rb') as infd:
            varf = pickle.load(infd)

        # update the objectid list
        objectid = varf['objectid']
        if objectid not in feature_dict['objectids']:
            feature_dict['objectids'].append(objectid)

        thisfeatures = varf[magcol]

        if featurestouse and len(featurestouse) > 0:
            featurestoget = featurestouse
        else:
            featurestoget = NONPERIODIC_FEATURES_TO_COLLECT

        # collect all the features for this magcol/objectid combination
        for feature in featurestoget:

            # update the global feature list if necessary
            if ((feature not in feature_dict['availablefeatures']) and
                (feature in thisfeatures)):

                feature_dict['availablefeatures'].append(feature)
                feature_dict[feature] = []

            if feature in thisfeatures:

                feature_dict[feature].append(
                    thisfeatures[feature]
                )

    # now that we've collected all the objects and their features, turn the list
    # into arrays, and then concatenate them
    for feat in feature_dict['availablefeatures']:
        feature_dict[feat] = np.array(feature_dict[feat])

    feature_dict['objectids'] = np.array(feature_dict['objectids'])

    feature_array = np.column_stack([feature_dict[feat] for feat in
                                     feature_dict['availablefeatures']])
    feature_dict['features_array'] = feature_array

    # if there's a labeldict available, use it to generate a label array. this
    # feature collection is now a training set.
    if isinstance(labeldict, dict):

        labelarray = np.zeros(feature_dict['objectids'].size, dtype=np.int64)

        # populate the labels for each object in the training set
        for ind, objectid in enumerate(feature_dict['objectids']):

            if objectid in labeldict:

                # if this is a binary classifier training set, convert bools to
                # ones and zeros
                if labeltype == 'binary':

                    if labeldict[objectid]:
                        labelarray[ind] = 1

                # otherwise, use the actual class label integer
                elif labeltype == 'classes':
                    labelarray[ind] = labeldict[objectid]

        feature_dict['labels_array'] = labelarray

    feature_dict['kwargs'] = {'pklglob':pklglob,
                              'featurestouse':featurestouse,
                              'maxobjects':maxobjects,
                              'labeltype':labeltype}

    # write the info to the output pickle
    with open(outfile,'wb') as outfd:
        pickle.dump(feature_dict, outfd, pickle.HIGHEST_PROTOCOL)

    # return the feature_dict
    return feature_dict

def
collect_nonperiodic_features
(
featuresdir
,
magcol
,
outfile
,
pklglob
=
'varfeatures-*.pkl'
,
featurestouse
=
NONPERIODIC_FEATURES_TO_COLLECT
,
maxobjects
=
,
labeldict
=
,
labeltype
=
'binary'
,
)
:
pklist
=
glob
.
glob
(
os
.
path
.
join
(
featuresdir
,
pklglob
)
)
if
maxobjects
:
pklist
=
pklist
[
:
maxobjects
]
if
TQDM
:
listiterator
=
tqdm
(
pklist
)
else
:
listiterator
=
pklist
feature_dict
=
{
'objectids'
:
[
]
,
'magcol'
:
magcol
,
'availablefeatures'
:
[
]
}
LOGINFO
(
'collecting features for magcol: %s'
%
magcol
)
for
pkl
in
listiterator
:
with
open
(
pkl
,
'rb'
)
as
infd
:
varf
=
pickle
.
load
(
infd
)
objectid
=
varf
[
'objectid'
]
if
objectid
not
in
feature_dict
[
'objectids'
]
:
feature_dict
[
'objectids'
]
.
append
(
objectid
)
thisfeatures
=
varf
[
magcol
]
if
featurestouse
and
len
(
featurestouse
)
>
0
:
featurestoget
=
featurestouse
else
:
featurestoget
=
NONPERIODIC_FEATURES_TO_COLLECT
for
feature
in
featurestoget
:
if
(
(
feature
not
in
feature_dict
[
'availablefeatures'
]
)
and
(
feature
in
thisfeatures
)
)
:
feature_dict
[
'availablefeatures'
]
.
append
(
feature
)
feature_dict
[
feature
]
=
[
]
if
feature
in
thisfeatures
:
feature_dict
[
feature
]
.
append
(
thisfeatures
[
feature
]
)
for
feat
in
feature_dict
[
'availablefeatures'
]
:
feature_dict
[
feat
]
=
np
.
array
(
feature_dict
[
feat
]
)
feature_dict
[
'objectids'
]
=
np
.
array
(
feature_dict
[
'objectids'
]
)
feature_array
=
np
.
column_stack
(
[
feature_dict
[
feat
]
for
feat
in
feature_dict
[
'availablefeatures'
]
]
)
feature_dict
[
'features_array'
]
=
feature_array
if
isinstance
(
labeldict
,
dict
)
:
labelarray
=
np
.
zeros
(
feature_dict
[
'objectids'
]
.
size
,
dtype
=
np
.
int64
)
for
ind
,
objectid
in
enumerate
(
feature_dict
[
'objectids'
]
)
:
if
objectid
in
labeldict
:
if
labeltype
==
'binary'
:
if
labeldict
[
objectid
]
:
labelarray
[
ind
]
=
1
elif
labeltype
==
'classes'
:
labelarray
[
ind
]
=
labeldict
[
objectid
]
feature_dict
[
'labels_array'
]
=
labelarray
feature_dict
[
'kwargs'
]
=
{
'pklglob'
:
pklglob
,
'featurestouse'
:
featurestouse
,
'maxobjects'
:
maxobjects
,
'labeltype'
:
labeltype
}
with
open
(
outfile
,
'wb'
)
as
outfd
:
pickle
.
dump
(
feature_dict
,
outfd
,
pickle
.
HIGHEST_PROTOCOL
)
return
feature_dict

This collects variability features into arrays for use with the classifer .