def fit_comp(new_comp, old_comp, **kwargs):
    """
    Fits a new component to an old component

    Calls do_levmarq to match the .get() fields of the two objects. The
    parameters of new_comp are modified in place.

    Parameters
    ----------
    new_comp : :class:`peri.comps.comp`
        The new object, whose parameters to update to fit the field of
        `old_comp`. Must have a .get() attribute which returns an ndarray
    old_comp : peri.comp
        The old ilm to match to.

    Other Parameters
    ----------------
        Any keyword arguments to be passed to the optimizer LMGlobals
        through do_levmarq.

    See Also
    --------
    do_levmarq : Levenberg-Marquardt minimization using a random subset
        of the image pixels.
    """
    #resetting the category to ilm:
    new_cat = new_comp.category
    new_comp.category = 'ilm'
    fake_s = states.ImageState(Image(old_comp.get().copy()), [new_comp], pad=0,
            mdl=mdl.SmoothFieldModel())
    do_levmarq(fake_s, new_comp.params, **kwargs)
    new_comp.category = new_cat

def fit_comp ( new_comp , old_comp , ** kwargs ) : new_cat = new_comp . category new_comp . category = 'ilm' fake_s = states . ImageState ( Image ( old_comp . get ( ) . copy ( ) ) , [ new_comp ] , pad = 0 , mdl = mdl . SmoothFieldModel ( ) ) do_levmarq ( fake_s , new_comp . params , ** kwargs ) new_comp . category = new_cat


def reset(self, new_damping=None):
        """
        Keeps all user supplied options the same, but resets counters etc.
        """
        self._num_iter = 0
        self._inner_run_counter = 0
        self._J_update_counter = self.update_J_frequency
        self._fresh_JTJ = False
        self._has_run = False
        if new_damping is not None:
            self.damping = np.array(new_damping).astype('float')
        self._set_err_paramvals()

def reset ( self , new_damping = None ) : self . _num_iter = 0 self . _inner_run_counter = 0 self . _J_update_counter = self . update_J_frequency self . _fresh_JTJ = False self . _has_run = False if new_damping is not None : self . damping = np . array ( new_damping ) . astype ( 'float' ) self . _set_err_paramvals ( )

Keeps all user supplied options the same but resets counters etc .


def do_run_1(self):
        """
        LM run, evaluating 1 step at a time.

        Broyden or eigendirection updates replace full-J updates until
        a full-J update occurs. Does not run with the calculated J (no
        internal run).
        """
        while not self.check_terminate():
            self._has_run = True
            self._run1()
            self._num_iter += 1; self._inner_run_counter += 1

def do_run_1 ( self ) : while not self . check_terminate ( ) : self . _has_run = True self . _run1 ( ) self . _num_iter += 1 self . _inner_run_counter += 1


def _run1(self):
        """workhorse for do_run_1"""
        if self.check_update_J():
            self.update_J()
        else:
            if self.check_Broyden_J():
                self.update_Broyden_J()
            if self.check_update_eig_J():
                self.update_eig_J()

        #1. Assuming that J starts updated:
        delta_vals = self.find_LM_updates(self.calc_grad())

        #2. Increase damping until we get a good step:
        er1 = self.update_function(self.param_vals + delta_vals)
        good_step = (find_best_step([self.error, er1]) == 1)
        if not good_step:
            er0 = self.update_function(self.param_vals)
            if np.abs(er0 -self.error)/er0 > 1e-7:
                raise RuntimeError('Function updates are not exact.')
            CLOG.debug('Bad step, increasing damping')
            CLOG.debug('\t\t%f\t%f' % (self.error, er1))
            grad = self.calc_grad()
            for _try in range(self._max_inner_loop):
                self.increase_damping()
                delta_vals = self.find_LM_updates(grad)
                er1 = self.update_function(self.param_vals + delta_vals)
                good_step = (find_best_step([self.error, er1]) == 1)
                if good_step:
                    break
            else:
                er0 = self.update_function(self.param_vals)
                CLOG.warn('Stuck!')
                if np.abs(er0 -self.error)/er0 > 1e-7:
                    raise RuntimeError('Function updates are not exact.')

        #state is updated, now params:
        if good_step:
            self._last_error = self.error
            self.error = er1
            CLOG.debug('Good step\t%f\t%f' % (self._last_error, self.error))
            self.update_param_vals(delta_vals, incremental=True)
            self.decrease_damping()

def
_run1
(
self
)
:
if
self
.
check_update_J
(
)
:
self
.
update_J
(
)
else
:
if
self
.
check_Broyden_J
(
)
:
self
.
update_Broyden_J
(
)
if
self
.
check_update_eig_J
(
)
:
self
.
update_eig_J
(
)
delta_vals
=
self
.
find_LM_updates
(
self
.
calc_grad
(
)
)
er1
=
self
.
update_function
(
self
.
param_vals
+
delta_vals
)
good_step
=
(
find_best_step
(
[
self
.
error
,
er1
]
)
==
1
)
if
not
good_step
:
er0
=
self
.
update_function
(
self
.
param_vals
)
if
np
.
abs
(
er0
self
.
error
)
/
er0
>
1e-7
:
raise
RuntimeError
(
'Function updates are not exact.'
)
CLOG
.
debug
(
'Bad step, increasing damping'
)
CLOG
.
debug
(
'\t\t%f\t%f'
%
(
self
.
error
,
er1
)
)
grad
=
self
.
calc_grad
(
)
for
_try
in
range
(
self
.
_max_inner_loop
)
:
self
.
increase_damping
(
)
delta_vals
=
self
.
find_LM_updates
(
grad
)
er1
=
self
.
update_function
(
self
.
param_vals
+
delta_vals
)
good_step
=
(
find_best_step
(
[
self
.
error
,
er1
]
)
==
1
)
if
good_step
:
break
else
:
er0
=
self
.
update_function
(
self
.
param_vals
)
CLOG
.
warn
(
'Stuck!'
)
if
np
.
abs
(
er0
self
.
error
)
/
er0
>
1e-7
:
raise
RuntimeError
(
'Function updates are not exact.'
)
if
good_step
:
self
.
_last_error
=
self
.
error
self
.
error
=
er1
CLOG
.
debug
(
'Good step\t%f\t%f'
%
(
self
.
_last_error
,
self
.
error
)
)
self
.
update_param_vals
(
delta_vals
,
incremental
=
True
)
self
.
decrease_damping
(
)


def _run2(self):
        """Workhorse for do_run_2"""
        if self.check_update_J():
            self.update_J()
        else:
            if self.check_Broyden_J():
                self.update_Broyden_J()
            if self.check_update_eig_J():
                self.update_eig_J()

        #0. Find _last_residuals, _last_error, etc:
        _last_residuals = self.calc_residuals().copy()
        _last_error = 1*self.error
        _last_vals = self.param_vals.copy()

        #1. Calculate 2 possible steps
        delta_params_1 = self.find_LM_updates(self.calc_grad(),
                do_correct_damping=False)
        self.decrease_damping()
        delta_params_2 = self.find_LM_updates(self.calc_grad(),
                do_correct_damping=False)
        self.decrease_damping(undo_decrease=True)

        #2. Check which step is best:
        er1 = self.update_function(self.param_vals + delta_params_1)
        er2 = self.update_function(self.param_vals + delta_params_2)

        triplet = (self.error, er1, er2)
        best_step = find_best_step(triplet)
        if best_step == 0:
            #Both bad steps, put back & increase damping:
            _ = self.update_function(self.param_vals.copy())
            grad = self.calc_grad()
            CLOG.debug('Bad step, increasing damping')
            CLOG.debug('%f\t%f\t%f' % triplet)
            for _try in range(self._max_inner_loop):
                self.increase_damping()
                delta_vals = self.find_LM_updates(grad)
                er_new = self.update_function(self.param_vals + delta_vals)
                good_step = er_new < self.error
                if good_step:
                    #Update params, error, break:
                    self.update_param_vals(delta_vals, incremental=True)
                    self.error = er_new
                    CLOG.debug('Sufficiently increased damping')
                    CLOG.debug('%f\t%f' % (triplet[0], self.error))
                    break
            else: #for-break-else
                #Throw a warning, put back the parameters
                CLOG.warn('Stuck!')
                self.error = self.update_function(self.param_vals.copy())

        elif best_step == 1:
            #er1 <= er2:
            good_step = True
            CLOG.debug('Good step, same damping')
            CLOG.debug('%f\t%f\t%f' % triplet)
            #Update to er1 params:
            er1_1 = self.update_function(self.param_vals + delta_params_1)
            if np.abs(er1_1 - er1) > 1e-6:
                raise RuntimeError('Function updates are not exact.')
            self.update_param_vals(delta_params_1, incremental=True)
            self.error = er1

        elif best_step == 2:
            #er2 < er1:
            good_step = True
            self.error = er2
            CLOG.debug('Good step, decreasing damping')
            CLOG.debug('%f\t%f\t%f' % triplet)
            #-we're already at the correct parameters
            self.update_param_vals(delta_params_2, incremental=True)
            self.decrease_damping()

        #3. Run with current J, damping; update what we need to::
        if good_step:
            self._last_residuals = _last_residuals
            self._last_error = _last_error
            self._last_vals = _last_vals
            self.error
            self.do_internal_run(initial_count=1)

def
_run2
(
self
)
:
if
self
.
check_update_J
(
)
:
self
.
update_J
(
)
else
:
if
self
.
check_Broyden_J
(
)
:
self
.
update_Broyden_J
(
)
if
self
.
check_update_eig_J
(
)
:
self
.
update_eig_J
(
)
_last_residuals
=
self
.
calc_residuals
(
)
.
copy
(
)
_last_error
=
1
*
self
.
error
_last_vals
=
self
.
param_vals
.
copy
(
)
delta_params_1
=
self
.
find_LM_updates
(
self
.
calc_grad
(
)
,
do_correct_damping
=
False
)
self
.
decrease_damping
(
)
delta_params_2
=
self
.
find_LM_updates
(
self
.
calc_grad
(
)
,
do_correct_damping
=
False
)
self
.
decrease_damping
(
undo_decrease
=
True
)
er1
=
self
.
update_function
(
self
.
param_vals
+
delta_params_1
)
er2
=
self
.
update_function
(
self
.
param_vals
+
delta_params_2
)
triplet
=
(
self
.
error
,
er1
,
er2
)
best_step
=
find_best_step
(
triplet
)
if
best_step
==
0
:
_
=
self
.
update_function
(
self
.
param_vals
.
copy
(
)
)
grad
=
self
.
calc_grad
(
)
CLOG
.
debug
(
'Bad step, increasing damping'
)
CLOG
.
debug
(
'%f\t%f\t%f'
%
triplet
)
for
_try
in
range
(
self
.
_max_inner_loop
)
:
self
.
increase_damping
(
)
delta_vals
=
self
.
find_LM_updates
(
grad
)
er_new
=
self
.
update_function
(
self
.
param_vals
+
delta_vals
)
good_step
=
er_new
<
self
.
error
if
good_step
:
self
.
update_param_vals
(
delta_vals
,
incremental
=
True
)
self
.
error
=
er_new
CLOG
.
debug
(
'Sufficiently increased damping'
)
CLOG
.
debug
(
'%f\t%f'
%
(
triplet
[
0
]
,
self
.
error
)
)
break
else
:
CLOG
.
warn
(
'Stuck!'
)
self
.
error
=
self
.
update_function
(
self
.
param_vals
.
copy
(
)
)
elif
best_step
==
1
:
good_step
=
True
CLOG
.
debug
(
'Good step, same damping'
)
CLOG
.
debug
(
'%f\t%f\t%f'
%
triplet
)
er1_1
=
self
.
update_function
(
self
.
param_vals
+
delta_params_1
)
if
np
.
abs
(
er1_1
er1
)
>
1e-6
:
raise
RuntimeError
(
'Function updates are not exact.'
)
self
.
update_param_vals
(
delta_params_1
,
incremental
=
True
)
self
.
error
=
er1
elif
best_step
==
2
:
good_step
=
True
self
.
error
=
er2
CLOG
.
debug
(
'Good step, decreasing damping'
)
CLOG
.
debug
(
'%f\t%f\t%f'
%
triplet
)
self
.
update_param_vals
(
delta_params_2
,
incremental
=
True
)
self
.
decrease_damping
(
)
if
good_step
:
self
.
_last_residuals
=
_last_residuals
self
.
_last_error
=
_last_error
self
.
_last_vals
=
_last_vals
self
.
error
self
.
do_internal_run
(
initial_count
=
1
)


def do_internal_run(self, initial_count=0, subblock=None, update_derr=True):
        """
        Takes more steps without calculating J again.

        Given a fixed damping, J, JTJ, iterates calculating steps, with
        optional Broyden or eigendirection updates. Iterates either until
        a bad step is taken or for self.run_length times.
        Called internally by do_run_2() but is also useful on its own.

        Parameters
        ----------
            initial_count : Int, optional
                The initial count of the run. Default is 0. Increasing from
                0 effectively temporarily decreases run_length.
            subblock : None or np.ndarray of bools, optional
                If not None, a boolean mask which determines which sub-
                block of parameters to run over. Default is None, i.e.
                all the parameters.
            update_derr : Bool, optional
                Set to False to not update the variable that determines
                delta_err, preventing premature termination through errtol.

        Notes
        -----
        It might be good to do something similar to update_derr with the
        parameter values, but this is trickier because of Broyden updates
        and _fresh_J.
        """
        self._inner_run_counter = initial_count; good_step = True
        n_good_steps = 0
        CLOG.debug('Running...')

        _last_residuals = self.calc_residuals().copy()
        while ((self._inner_run_counter < self.run_length) & good_step &
                (not self.check_terminate())):
            #1. Checking if we update J
            if self.check_Broyden_J() and self._inner_run_counter != 0:
                self.update_Broyden_J()
            if self.check_update_eig_J() and self._inner_run_counter != 0:
                self.update_eig_J()

            #2. Getting parameters, error
            er0 = 1*self.error
            delta_vals = self.find_LM_updates(self.calc_grad(),
                    do_correct_damping=False, subblock=subblock)
            er1 = self.update_function(self.param_vals + delta_vals)
            good_step = er1 < er0

            if good_step:
                n_good_steps += 1
                CLOG.debug('%f\t%f' % (er0, er1))
                #Updating:
                self.update_param_vals(delta_vals, incremental=True)
                self._last_residuals = _last_residuals.copy()
                if update_derr:
                    self._last_error = er0
                self.error = er1

                _last_residuals = self.calc_residuals().copy()
            else:
                er0_0 = self.update_function(self.param_vals)
                CLOG.debug('Bad step!')
                if np.abs(er0 - er0_0) > 1e-6:
                    raise RuntimeError('Function updates are not exact.')

            self._inner_run_counter += 1
        return n_good_steps

def
do_internal_run
(
self
,
initial_count
=
0
,
subblock
=
,
update_derr
=
True
)
:
self
.
_inner_run_counter
=
initial_count
good_step
=
True
n_good_steps
=
0
CLOG
.
debug
(
'Running...'
)
_last_residuals
=
self
.
calc_residuals
(
)
.
copy
(
)
while
(
(
self
.
_inner_run_counter
<
self
.
run_length
)
&
good_step
&
(
not
self
.
check_terminate
(
)
)
)
:
if
self
.
check_Broyden_J
(
)
and
self
.
_inner_run_counter
!=
0
:
self
.
update_Broyden_J
(
)
if
self
.
check_update_eig_J
(
)
and
self
.
_inner_run_counter
!=
0
:
self
.
update_eig_J
(
)
er0
=
1
*
self
.
error
delta_vals
=
self
.
find_LM_updates
(
self
.
calc_grad
(
)
,
do_correct_damping
=
False
,
subblock
=
subblock
)
er1
=
self
.
update_function
(
self
.
param_vals
+
delta_vals
)
good_step
=
er1
<
er0
if
good_step
:
n_good_steps
+=
1
CLOG
.
debug
(
'%f\t%f'
%
(
er0
,
er1
)
)
self
.
update_param_vals
(
delta_vals
,
incremental
=
True
)
self
.
_last_residuals
=
_last_residuals
.
copy
(
)
if
update_derr
:
self
.
_last_error
=
er0
self
.
error
=
er1
_last_residuals
=
self
.
calc_residuals
(
)
.
copy
(
)
else
:
er0_0
=
self
.
update_function
(
self
.
param_vals
)
CLOG
.
debug
(
'Bad step!'
)
if
np
.
abs
(
er0
er0_0
)
>
1e-6
:
raise
RuntimeError
(
'Function updates are not exact.'
)
self
.
_inner_run_counter
+=
1
return
n_good_steps

Takes more steps without calculating J again .


def find_LM_updates(self, grad, do_correct_damping=True, subblock=None):
        """
        Calculates LM updates, with or without the acceleration correction.

        Parameters
        ----------
            grad : numpy.ndarray
                The gradient of the model cost.
            do_correct_damping : Bool, optional
                If `self.use_accel`, then set to True to correct damping
                if the acceleration correction is too big. Default is True
                Does nothing is `self.use_accel` is False
            subblock : slice, numpy.ndarray, or None, optional
                Set to a slice or a valide numpy.ndarray to use only a
                certain subset of the parameters. Default is None, i.e.
                use all the parameters.

        Returns
        -------
            delta : numpy.ndarray
                The Levenberg-Marquadt step, relative to the old
                parameters. Size is always self.param_vals.size.
        """
        if subblock is not None:
            if (subblock.sum() == 0) or (subblock.size == 0):
                CLOG.fatal('Empty subblock in find_LM_updates')
                raise ValueError('Empty sub-block')
            j = self.J[subblock]
            JTJ = np.dot(j, j.T)
            damped_JTJ = self._calc_damped_jtj(JTJ, subblock=subblock)
            grad = grad[subblock]  #select the subblock of the grad
        else:
            damped_JTJ = self._calc_damped_jtj(self.JTJ, subblock=subblock)

        delta = self._calc_lm_step(damped_JTJ, grad, subblock=subblock)

        if self.use_accel:
            accel_correction = self.calc_accel_correction(damped_JTJ, delta)
            nrm_d0 = np.sqrt(np.sum(delta**2))
            nrm_corr = np.sqrt(np.sum(accel_correction**2))
            CLOG.debug('|correction| / |LM step|\t%e' % (nrm_corr/nrm_d0))
            if nrm_corr/nrm_d0 < self.max_accel_correction:
                delta += accel_correction
            elif do_correct_damping:
                CLOG.debug('Untrustworthy step! Increasing damping...')
                self.increase_damping()
                damped_JTJ = self._calc_damped_jtj(self.JTJ, subblock=subblock)
                delta = self._calc_lm_step(damped_JTJ, grad, subblock=subblock)

        if np.any(np.isnan(delta)):
            CLOG.fatal('Calculated steps have nans!?')
            raise FloatingPointError('Calculated steps have nans!?')
        return delta

Calculates LM updates with or without the acceleration correction .


def update_param_vals(self, new_vals, incremental=False):
        """
        Updates the current set of parameter values and previous values,
        sets a flag to re-calculate J.

        Parameters
        ----------
            new_vals : numpy.ndarray
                The new values to update to
            incremental : Bool, optional
                Set to True to make it an incremental update relative
                to the old parameters. Default is False
        """
        self._last_vals = self.param_vals.copy()
        if incremental:
            self.param_vals += new_vals
        else:
            self.param_vals = new_vals.copy()
        #And we've updated, so JTJ is no longer valid:
        self._fresh_JTJ = False

def update_param_vals ( self , new_vals , incremental = False ) : self . _last_vals = self . param_vals . copy ( ) if incremental : self . param_vals += new_vals else : self . param_vals = new_vals . copy ( ) self . _fresh_JTJ = False

Updates the current set of parameter values and previous values sets a flag to re - calculate J .


def get_termination_stats(self, get_cos=True):
        """
        Returns a dict of termination statistics

        Parameters
        ----------
            get_cos : Bool, optional
                Whether or not to calcualte the cosine of the residuals
                with the tangent plane of the model using the current J.
                The calculation may take some time. Default is True

        Returns
        -------
            dict
                Has keys
                    delta_vals  : The last change in parameter values.
                    delta_err   : The last change in the error.
                    exp_err     : The expected (last) change in the error.
                    frac_err    : The fractional change in the error.
                    num_iter    : The number of iterations completed.
                    error       : The current error.
        """
        delta_vals = self._last_vals - self.param_vals
        delta_err = self._last_error - self.error
        frac_err = delta_err / self.error
        to_return = {'delta_vals':delta_vals, 'delta_err':delta_err,
                'num_iter':1*self._num_iter, 'frac_err':frac_err,
                'error':self.error, 'exp_err':self._exp_err}
        if get_cos:
            model_cosine = self.calc_model_cosine()
            to_return.update({'model_cosine':model_cosine})
        return to_return

def get_termination_stats ( self , get_cos = True ) : delta_vals = self . _last_vals - self . param_vals delta_err = self . _last_error - self . error frac_err = delta_err / self . error to_return = { 'delta_vals' : delta_vals , 'delta_err' : delta_err , 'num_iter' : 1 * self . _num_iter , 'frac_err' : frac_err , 'error' : self . error , 'exp_err' : self . _exp_err } if get_cos : model_cosine = self . calc_model_cosine ( ) to_return . update ( { 'model_cosine' : model_cosine } ) return to_return


def check_completion(self):
        """
        Returns a Bool of whether the algorithm has found a satisfactory minimum
        """
        terminate = False
        term_dict = self.get_termination_stats(get_cos=self.costol is not None)
        terminate |= np.all(np.abs(term_dict['delta_vals']) < self.paramtol)
        terminate |= (term_dict['delta_err'] < self.errtol)
        terminate |= (term_dict['exp_err'] < self.exptol)
        terminate |= (term_dict['frac_err'] < self.fractol)
        if self.costol is not None:
            terminate |= (curcos < term_dict['model_cosine'])

        return terminate

def check_completion ( self ) : terminate = False term_dict = self . get_termination_stats ( get_cos = self . costol is not None ) terminate |= np . all ( np . abs ( term_dict [ 'delta_vals' ] ) < self . paramtol ) terminate |= ( term_dict [ 'delta_err' ] < self . errtol ) terminate |= ( term_dict [ 'exp_err' ] < self . exptol ) terminate |= ( term_dict [ 'frac_err' ] < self . fractol ) if self . costol is not None : terminate |= ( curcos < term_dict [ 'model_cosine' ] ) return terminate


def check_terminate(self):
        """
        Returns a Bool of whether to terminate.

        Checks whether a satisfactory minimum has been found or whether
        too many iterations have occurred.
        """
        if not self._has_run:
            return False
        else:
            #1-3. errtol, paramtol, model cosine low enough?
            terminate = self.check_completion()

            #4. too many iterations??
            terminate |= (self._num_iter >= self.max_iter)
            return terminate

def check_terminate ( self ) : if not self . _has_run : return False else : terminate = self . check_completion ( ) terminate |= ( self . _num_iter >= self . max_iter ) return terminate

Returns a Bool of whether to terminate .


def check_update_J(self):
        """
        Checks if the full J should be updated.

        Right now, just updates after update_J_frequency loops
        """
        self._J_update_counter += 1
        update = self._J_update_counter >= self.update_J_frequency
        return update & (not self._fresh_JTJ)

def check_update_J ( self ) : self . _J_update_counter += 1 update = self . _J_update_counter >= self . update_J_frequency return update & ( not self . _fresh_JTJ )

Checks if the full J should be updated .


def update_J(self):
        """Updates J, JTJ, and internal counters."""
        self.calc_J()
        # np.dot(j, j.T) is slightly faster but 2x as much mem
        step = np.ceil(1e-2 * self.J.shape[1]).astype('int')  # 1% more mem...
        self.JTJ = low_mem_sq(self.J, step=step)
        #copies still, since J is not C -ordered but a slice of j_e...
        #doing self.J.copy() works but takes 2x as much ram..
        self._fresh_JTJ = True
        self._J_update_counter = 0
        if np.any(np.isnan(self.JTJ)):
            raise FloatingPointError('J, JTJ have nans.')
        #Update self._exp_err
        self._exp_err = self.error - self.find_expected_error(delta_params='perfect')

def
update_J
(
self
)
:
self
.
calc_J
(
)
step
=
np
.
ceil
(
1e-2
*
self
.
J
.
shape
[
1
]
)
.
astype
(
'int'
)
self
.
JTJ
=
low_mem_sq
(
self
.
J
,
step
=
step
)
self
.
_fresh_JTJ
=
True
self
.
_J_update_counter
=
0
if
np
.
any
(
np
.
isnan
(
self
.
JTJ
)
)
:
raise
FloatingPointError
(
'J, JTJ have nans.'
)
self
.
_exp_err
=
self
.
error
self
.
find_expected_error
(
delta_params
=
'perfect'
)


def update_Broyden_J(self):
        """Execute a Broyden update of J"""
        CLOG.debug('Broyden update.')
        delta_vals = self.param_vals - self._last_vals
        delta_residuals = self.calc_residuals() - self._last_residuals
        nrm = np.sqrt(np.dot(delta_vals, delta_vals))
        direction = delta_vals / nrm
        vals = delta_residuals / nrm
        self._rank_1_J_update(direction, vals)
        self.JTJ = np.dot(self.J, self.J.T)

def
update_Broyden_J
(
self
)
:
CLOG
.
debug
(
'Broyden update.'
)
delta_vals
=
self
.
param_vals
self
.
_last_vals
delta_residuals
=
self
.
calc_residuals
(
)
self
.
_last_residuals
nrm
=
np
.
sqrt
(
np
.
dot
(
delta_vals
,
delta_vals
)
)
direction
=
delta_vals
/
nrm
vals
=
delta_residuals
/
nrm
self
.
_rank_1_J_update
(
direction
,
vals
)
self
.
JTJ
=
np
.
dot
(
self
.
J
,
self
.
J
.
T
)


def update_eig_J(self):
        """Execute an eigen update of J"""
        CLOG.debug('Eigen update.')
        vls, vcs = np.linalg.eigh(self.JTJ)
        res0 = self.calc_residuals()
        for a in range(min([self.num_eig_dirs, vls.size])):
            #1. Finding stiff directions
            stif_dir = vcs[-(a+1)] #already normalized

            #2. Evaluating derivative along that direction, we'll use dl=5e-4:
            dl = self.eig_dl #1e-5
            _ = self.update_function(self.param_vals + dl*stif_dir)
            res1 = self.calc_residuals()

            #3. Updating
            grad_stif = (res1-res0)/dl
            self._rank_1_J_update(stif_dir, grad_stif)

        self.JTJ = np.dot(self.J, self.J.T)
        #Putting the parameters back:
        _ = self.update_function(self.param_vals)

def
update_eig_J
(
self
)
:
CLOG
.
debug
(
'Eigen update.'
)
vls
,
vcs
=
np
.
linalg
.
eigh
(
self
.
JTJ
)
res0
=
self
.
calc_residuals
(
)
for
a
in
range
(
min
(
[
self
.
num_eig_dirs
,
vls
.
size
]
)
)
:
stif_dir
=
vcs
[
(
a
+
1
)
]
dl
=
self
.
eig_dl
_
=
self
.
update_function
(
self
.
param_vals
+
dl
*
stif_dir
)
res1
=
self
.
calc_residuals
(
)
grad_stif
=
(
res1
res0
)
/
dl
self
.
_rank_1_J_update
(
stif_dir
,
grad_stif
)
self
.
JTJ
=
np
.
dot
(
self
.
J
,
self
.
J
.
T
)
_
=
self
.
update_function
(
self
.
param_vals
)


def calc_accel_correction(self, damped_JTJ, delta0):
        """
        Geodesic acceleration correction to the LM step.

        Parameters
        ----------
            damped_JTJ : numpy.ndarray
                The damped JTJ used to calculate the initial step.
            delta0 : numpy.ndarray
                The initial LM step.

        Returns
        -------
            corr : numpy.ndarray
                The correction to the original LM step.
        """
        #Get the derivative:
        _ = self.update_function(self.param_vals)
        rm0 = self.calc_residuals().copy()
        _ = self.update_function(self.param_vals + delta0)
        rm1 = self.calc_residuals().copy()
        _ = self.update_function(self.param_vals - delta0)
        rm2 = self.calc_residuals().copy()
        der2 = (rm2 + rm1 - 2*rm0)

        corr, res, rank, s = np.linalg.lstsq(damped_JTJ, np.dot(self.J, der2),
                rcond=self.min_eigval)
        corr *= -0.5
        return corr

def calc_accel_correction ( self , damped_JTJ , delta0 ) : _ = self . update_function ( self . param_vals ) rm0 = self . calc_residuals ( ) . copy ( ) _ = self . update_function ( self . param_vals + delta0 ) rm1 = self . calc_residuals ( ) . copy ( ) _ = self . update_function ( self . param_vals - delta0 ) rm2 = self . calc_residuals ( ) . copy ( ) der2 = ( rm2 + rm1 - 2 * rm0 ) corr , res , rank , s = np . linalg . lstsq ( damped_JTJ , np . dot ( self . J , der2 ) , rcond = self . min_eigval ) corr *= - 0.5 return corr

Geodesic acceleration correction to the LM step .


def calc_J(self):
        """Updates self.J, returns nothing"""
        del self.J
        self.J = np.zeros([self.param_vals.size, self.data.size])
        dp = np.zeros_like(self.param_vals)
        f0 = self.model.copy()
        for a in range(self.param_vals.size):
            dp *= 0
            dp[a] = self.dl[a]
            f1 = self.func(self.param_vals + dp, *self.func_args, **self.func_kwargs)
            grad_func = (f1 - f0) / dp[a]
            #J = grad(residuals) = -grad(model)
            self.J[a] = -grad_func

def calc_J ( self ) : del self . J self . J = np . zeros ( [ self . param_vals . size , self . data . size ] ) dp = np . zeros_like ( self . param_vals ) f0 = self . model . copy ( ) for a in range ( self . param_vals . size ) : dp *= 0 dp [ a ] = self . dl [ a ] f1 = self . func ( self . param_vals + dp , * self . func_args , ** self . func_kwargs ) grad_func = ( f1 - f0 ) / dp [ a ] self . J [ a ] = - grad_func


def update_function(self, param_vals):
        """Takes an array param_vals, updates function, returns the new error"""
        self.model = self.func(param_vals, *self.func_args, **self.func_kwargs)
        d = self.calc_residuals()
        return np.dot(d.flat, d.flat)

def update_function ( self , param_vals ) : self . model = self . func ( param_vals , * self . func_args , ** self . func_kwargs ) d = self . calc_residuals ( ) return np . dot ( d . flat , d . flat )

Takes an array param_vals updates function returns the new error


def update_function(self, param_vals):
        """Updates the opt_obj, returns new error."""
        self.opt_obj.update_function(param_vals)
        return self.opt_obj.get_error()

def update_function ( self , param_vals ) : self . opt_obj . update_function ( param_vals ) return self . opt_obj . get_error ( )


def calc_J(self):
        """Calculates J along the direction."""
        r0 = self.state.residuals.copy().ravel()
        dl = np.zeros(self.param_vals.size)
        p0 = self.param_vals.copy()
        J = []
        for a in range(self.param_vals.size):
            dl *= 0
            dl[a] += self.dl
            self.update_function(p0 + dl)
            r1 = self.state.residuals.copy().ravel()
            J.append( (r1-r0)/self.dl)
        self.update_function(p0)
        return np.array(J)

def calc_J ( self ) : r0 = self . state . residuals . copy ( ) . ravel ( ) dl = np . zeros ( self . param_vals . size ) p0 = self . param_vals . copy ( ) J = [ ] for a in range ( self . param_vals . size ) : dl *= 0 dl [ a ] += self . dl self . update_function ( p0 + dl ) r1 = self . state . residuals . copy ( ) . ravel ( ) J . append ( ( r1 - r0 ) / self . dl ) self . update_function ( p0 ) return np . array ( J )


def reset(self, new_region_size=None, do_calc_size=True, new_damping=None,
            new_max_mem=None):
        """
        Resets the particle groups and optionally the region size and damping.

        Parameters
        ----------
            new_region_size : : Int or 3-element list-like of ints, optional
                The region size for sub-blocking particles. Default is 40
            do_calc_size : Bool, optional
                If True, calculates the region size internally based on
                the maximum allowed memory. Default is True
            new_damping : Float or None, optional
                The new damping of the optimizer. Set to None to leave
                as the default for LMParticles. Default is None.
            new_max_mem : Numeric, optional
                The maximum allowed memory for J to occupy. Default is 1e9
        """
        if new_region_size is not None:
            self.region_size = new_region_size
        if new_max_mem != None:
            self.max_mem = new_max_mem
        if do_calc_size:
            self.region_size = calc_particle_group_region_size(self.state,
                    region_size=self.region_size, max_mem=self.max_mem)
        self.stats = []
        self.particle_groups = separate_particles_into_groups(self.state,
                self.region_size, doshift='rand')
        if new_damping is not None:
            self._kwargs.update({'damping':new_damping})
        if self.save_J:
            if len(self.particle_groups) > 90:
                CLOG.warn('Attempting to create many open files. Consider increasing max_mem and/or region_size to avoid crashes.')
            self._tempfiles = []
            self._has_saved_J = []
            for a in range(len(self.particle_groups)):
                #TemporaryFile is automatically deleted
                for _ in ['j','tile']:
                    self._tempfiles.append(tempfile.TemporaryFile(dir=os.getcwd()))
                self._has_saved_J.append(False)

Resets the particle groups and optionally the region size and damping .


def _do_run(self, mode='1'):
        """workhorse for the self.do_run_xx methods."""
        for a in range(len(self.particle_groups)):
            group = self.particle_groups[a]
            lp = LMParticles(self.state, group, **self._kwargs)
            if mode == 'internal':
                lp.J, lp.JTJ, lp._dif_tile = self._load_j_diftile(a)

            if mode == '1':
                lp.do_run_1()
            if mode == '2':
                lp.do_run_2()
            if mode == 'internal':
                lp.do_internal_run()

            self.stats.append(lp.get_termination_stats(get_cos=self.get_cos))
            if self.save_J and (mode != 'internal'):
                self._dump_j_diftile(a, lp.J, lp._dif_tile)
                self._has_saved_J[a] = True

def _do_run ( self , mode = '1' ) : for a in range ( len ( self . particle_groups ) ) : group = self . particle_groups [ a ] lp = LMParticles ( self . state , group , ** self . _kwargs ) if mode == 'internal' : lp . J , lp . JTJ , lp . _dif_tile = self . _load_j_diftile ( a ) if mode == '1' : lp . do_run_1 ( ) if mode == '2' : lp . do_run_2 ( ) if mode == 'internal' : lp . do_internal_run ( ) self . stats . append ( lp . get_termination_stats ( get_cos = self . get_cos ) ) if self . save_J and ( mode != 'internal' ) : self . _dump_j_diftile ( a , lp . J , lp . _dif_tile ) self . _has_saved_J [ a ] = True

workhorse for the self . do_run_xx methods .


def do_internal_run(self):
        """Calls LMParticles.do_internal_run for each group of particles."""
        if not self.save_J:
            raise RuntimeError('self.save_J=True required for do_internal_run()')
        if not np.all(self._has_saved_J):
            raise RuntimeError('J, JTJ have not been pre-computed. Call do_run_1 or do_run_2')
        self._do_run(mode='internal')

Calls LMParticles . do_internal_run for each group of particles .


def reset(self):
        """
        Resets the initial radii used for updating the particles. Call
        if any of the particle radii or positions have been changed
        external to the augmented state.
        """
        inds = list(range(self.state.obj_get_positions().shape[0]))
        self._rad_nms = self.state.param_particle_rad(inds)
        self._pos_nms = self.state.param_particle_pos(inds)
        self._initial_rad = np.copy(self.state.state[self._rad_nms])
        self._initial_pos = np.copy(self.state.state[self._pos_nms]).reshape((-1,3))
        self.param_vals[self.rscale_mask] = 0

def reset ( self ) : inds = list ( range ( self . state . obj_get_positions ( ) . shape [ 0 ] ) ) self . _rad_nms = self . state . param_particle_rad ( inds ) self . _pos_nms = self . state . param_particle_pos ( inds ) self . _initial_rad = np . copy ( self . state . state [ self . _rad_nms ] ) self . _initial_pos = np . copy ( self . state . state [ self . _pos_nms ] ) . reshape ( ( - 1 , 3 ) ) self . param_vals [ self . rscale_mask ] = 0

Resets the initial radii used for updating the particles . Call if any of the particle radii or positions have been changed external to the augmented state .


def reset(self, **kwargs):
        """Resets the aug_state and the LMEngine"""
        self.aug_state.reset()
        super(LMAugmentedState, self).reset(**kwargs)

def reset ( self , ** kwargs ) : self . aug_state . reset ( ) super ( LMAugmentedState , self ) . reset ( ** kwargs )


def get_shares(self):
    '''
      Returns an object with a the numbers of shares a link has had using
      Buffer.

      www will be stripped, but other subdomains will not.
    '''

    self.shares = self.api.get(url=PATHS['GET_SHARES'] % self.url)['shares']

    return self.shares

def get_shares ( self ) : self . shares = self . api . get ( url = PATHS [ 'GET_SHARES' ] % self . url ) [ 'shares' ] return self . shares

Returns an object with a the numbers of shares a link has had using Buffer .


def sample(field, inds=None, slicer=None, flat=True):
    """
    Take a sample from a field given flat indices or a shaped slice

    Parameters
    -----------
    inds : list of indices
        One dimensional (raveled) indices to return from the field

    slicer : slice object
        A shaped (3D) slicer that returns a section of image

    flat : boolean
        Whether to flatten the sampled item before returning
    """
    if inds is not None:
        out = field.ravel()[inds]
    elif slicer is not None:
        out = field[slicer].ravel()
    else:
        out = field

    if flat:
        return out.ravel()
    return out

def sample ( field , inds = None , slicer = None , flat = True ) : if inds is not None : out = field . ravel ( ) [ inds ] elif slicer is not None : out = field [ slicer ] . ravel ( ) else : out = field if flat : return out . ravel ( ) return out


def update(self, params, values):
        """
        Update a single parameter or group of parameters ``params``
        with ``values``.

        Parameters
        ----------
        params : string or list of strings
            Parameter names which to update

        value : number or list of numbers
            Values of those parameters which to update
        """
        return super(State, self).update(params, values)

def update ( self , params , values ) : return super ( State , self ) . update ( params , values )

Update a single parameter or group of parameters params with values .


def build_funcs(self):
        """
        Here, we build gradient and hessian functions based on the properties
        of a state that are generally wanted. For each one, we fill in _grad or
        _hess with a function that takes care of various options such as
        slicing and flattening. For example, `m` below takes the model, selects
        different indices from it, maybe flattens it and copies it. This is
        then used in the fisherinformation, gradmodel, and hessmodel functions.
        """
        # create essentially lambda functions, but with a nice signature
        def m(inds=None, slicer=None, flat=True):
            return sample(self.model, inds=inds, slicer=slicer, flat=flat).copy()

        def r(inds=None, slicer=None, flat=True):
            return sample(self.residuals, inds=inds, slicer=slicer, flat=flat).copy()

        def l():
            return self.loglikelihood

        def r_e(**kwargs):
            """sliced etc residuals, with state.error appended on"""
            return r(**kwargs), np.copy(self.error)

        def m_e(**kwargs):
            """sliced etc residuals, with state.error appended on"""
            return m(**kwargs), np.copy(self.error)

        # set the member functions using partial
        self.fisherinformation = partial(self._jtj, funct=m)
        self.gradloglikelihood = partial(self._grad, funct=l)
        self.hessloglikelihood = partial(self._hess, funct=l)
        self.gradmodel = partial(self._grad, funct=m)
        self.hessmodel = partial(self._hess, funct=m)
        self.JTJ = partial(self._jtj, funct=r)
        self.J = partial(self._grad, funct=r)
        self.J_e = partial(self._grad, funct=r_e, nout=2)
        self.gradmodel_e = partial(self._grad, funct=m_e, nout=2)

        # add the appropriate documentation to the following functions
        self.fisherinformation.__doc__ = _graddoc + _sampledoc
        self.gradloglikelihood.__doc__ = _graddoc
        self.hessloglikelihood.__doc__ = _graddoc
        self.gradmodel.__doc__ = _graddoc + _sampledoc
        self.hessmodel.__doc__ = _graddoc + _sampledoc
        self.JTJ.__doc__ = _graddoc + _sampledoc
        self.J.__doc__ = _graddoc + _sampledoc

        # add documentation to the private functions as well. this is done
        # slightly differently, hence the function call
        self._dograddoc(self._grad_one_param)
        self._dograddoc(self._hess_two_param)
        self._dograddoc(self._grad)
        self._dograddoc(self._hess)

        # the state object is a workaround so that other interfaces still
        # work. this should probably be removed in the long run
        class _Statewrap(object):
            def __init__(self, obj):
                self.obj = obj
            def __getitem__(self, d=None):
                if d is None:
                    d = self.obj.params
                return util.delistify(self.obj.get_values(d), d)

        self.state = _Statewrap(self)

def build_funcs ( self ) : def m ( inds = None , slicer = None , flat = True ) : return sample ( self . model , inds = inds , slicer = slicer , flat = flat ) . copy ( ) def r ( inds = None , slicer = None , flat = True ) : return sample ( self . residuals , inds = inds , slicer = slicer , flat = flat ) . copy ( ) def l ( ) : return self . loglikelihood def r_e ( ** kwargs ) : return r ( ** kwargs ) , np . copy ( self . error ) def m_e ( ** kwargs ) : return m ( ** kwargs ) , np . copy ( self . error ) self . fisherinformation = partial ( self . _jtj , funct = m ) self . gradloglikelihood = partial ( self . _grad , funct = l ) self . hessloglikelihood = partial ( self . _hess , funct = l ) self . gradmodel = partial ( self . _grad , funct = m ) self . hessmodel = partial ( self . _hess , funct = m ) self . JTJ = partial ( self . _jtj , funct = r ) self . J = partial ( self . _grad , funct = r ) self . J_e = partial ( self . _grad , funct = r_e , nout = 2 ) self . gradmodel_e = partial ( self . _grad , funct = m_e , nout = 2 ) self . fisherinformation . __doc__ = _graddoc + _sampledoc self . gradloglikelihood . __doc__ = _graddoc self . hessloglikelihood . __doc__ = _graddoc self . gradmodel . __doc__ = _graddoc + _sampledoc self . hessmodel . __doc__ = _graddoc + _sampledoc self . JTJ . __doc__ = _graddoc + _sampledoc self . J . __doc__ = _graddoc + _sampledoc self . _dograddoc ( self . _grad_one_param ) self . _dograddoc ( self . _hess_two_param ) self . _dograddoc ( self . _grad ) self . _dograddoc ( self . _hess ) class _Statewrap ( object ) : def __init__ ( self , obj ) : self . obj = obj def __getitem__ ( self , d = None ) : if d is None : d = self . obj . params return util . delistify ( self . obj . get_values ( d ) , d ) self . state = _Statewrap ( self )

Here we build gradient and hessian functions based on the properties of a state that are generally wanted . For each one we fill in _grad or _hess with a function that takes care of various options such as slicing and flattening . For example m below takes the model selects different indices from it maybe flattens it and copies it . This is then used in the fisherinformation gradmodel and hessmodel functions .


def set_model(self, mdl):
        """
        Setup the image model formation equation and corresponding objects into
        their various objects. `mdl` is a `peri.models.Model` object
        """
        self.mdl = mdl
        self.mdl.check_inputs(self.comps)

        for c in self.comps:
            setattr(self, '_comp_'+c.category, c)

def set_model ( self , mdl ) : self . mdl = mdl self . mdl . check_inputs ( self . comps ) for c in self . comps : setattr ( self , '_comp_' + c . category , c )

Setup the image model formation equation and corresponding objects into their various objects . mdl is a peri . models . Model object


def model_to_data(self, sigma=0.0):
        """ Switch out the data for the model's recreation of the data. """
        im = self.model.copy()
        im += sigma*np.random.randn(*im.shape)
        self.set_image(util.NullImage(image=im))

def model_to_data ( self , sigma = 0.0 ) : im = self . model . copy ( ) im += sigma * np . random . randn ( * im . shape ) self . set_image ( util . NullImage ( image = im ) )

Switch out the data for the model s recreation of the data .


def get_update_io_tiles(self, params, values):
        """
        Get the tiles corresponding to a particular section of image needed to
        be updated. Inputs are the parameters and values. Returned is the
        padded tile, inner tile, and slicer to go between, but accounting for
        wrap with the edge of the image as necessary.
        """
        # get the affected area of the model image
        otile = self.get_update_tile(params, values)
        if otile is None:
            return [None]*3
        ptile = self.get_padding_size(otile) or util.Tile(0, dim=otile.dim)

        otile = util.Tile.intersection(otile, self.oshape)

        if (otile.shape <= 0).any():
            raise UpdateError("update triggered invalid tile size")

        if (ptile.shape < 0).any() or (ptile.shape > self.oshape.shape).any():
            raise UpdateError("update triggered invalid padding tile size")

        # now remove the part of the tile that is outside the image and pad the
        # interior part with that overhang. reflect the necessary padding back
        # into the image itself for the outer slice which we will call outer
        outer = otile.pad((ptile.shape+1)//2)
        inner, outer = outer.reflect_overhang(self.oshape)
        iotile = inner.translate(-outer.l)

        outer = util.Tile.intersection(outer, self.oshape)
        inner = util.Tile.intersection(inner, self.oshape)
        return outer, inner, iotile

Get the tiles corresponding to a particular section of image needed to be updated . Inputs are the parameters and values . Returned is the padded tile inner tile and slicer to go between but accounting for wrap with the edge of the image as necessary .


def get(self, name):
        """ Return component by category name """
        for c in self.comps:
            if c.category == name:
                return c
        return None

def get ( self , name ) : for c in self . comps : if c . category == name : return c return None


def _calc_loglikelihood(self, model=None, tile=None):
        """Allows for fast local updates of log-likelihood"""
        if model is None:
            res = self.residuals
        else:
            res = model - self._data[tile.slicer]

        sig, isig = self.sigma, 1.0/self.sigma
        nlogs = -np.log(np.sqrt(2*np.pi)*sig)*res.size
        return -0.5*isig*isig*np.dot(res.flat, res.flat) + nlogs

def _calc_loglikelihood ( self , model = None , tile = None ) : if model is None : res = self . residuals else : res = model - self . _data [ tile . slicer ] sig , isig = self . sigma , 1.0 / self . sigma nlogs = - np . log ( np . sqrt ( 2 * np . pi ) * sig ) * res . size return - 0.5 * isig * isig * np . dot ( res . flat , res . flat ) + nlogs

Allows for fast local updates of log - likelihood


def update_from_model_change(self, oldmodel, newmodel, tile):
        """
        Update various internal variables from a model update from oldmodel to
        newmodel for the tile `tile`
        """
        self._loglikelihood -= self._calc_loglikelihood(oldmodel, tile=tile)
        self._loglikelihood += self._calc_loglikelihood(newmodel, tile=tile)
        self._residuals[tile.slicer] = self._data[tile.slicer] - newmodel

def update_from_model_change ( self , oldmodel , newmodel , tile ) : self . _loglikelihood -= self . _calc_loglikelihood ( oldmodel , tile = tile ) self . _loglikelihood += self . _calc_loglikelihood ( newmodel , tile = tile ) self . _residuals [ tile . slicer ] = self . _data [ tile . slicer ] - newmodel

Update various internal variables from a model update from oldmodel to newmodel for the tile tile


def set_mem_level(self, mem_level='hi'):
        """
        Sets the memory usage level of the state.

        Parameters
        ----------
        mem_level : string
            Can be set to one of:
                * hi      : all mem's are np.float64
                * med-hi  : image, platonic are float32, rest are float64
                * med     : all mem's are float32
                * med-lo  : image, platonic are float16, rest float32
                * lo      : all are float16, which is bad for accuracy.

        Notes
        -----
        Right now the PSF is not affected by the mem-level changes, which is
        OK for mem but it means that self._model, self._residuals are always
        float64, which can be a chunk of mem.
        """
        #A little thing to parse strings for convenience:
        key = ''.join([c if c in 'mlh' else '' for c in mem_level])
        if key not in ['h','mh','m','ml','m', 'l']:
            raise ValueError('mem_level must be one of hi, med-hi, med, med-lo, lo.')
        mem_levels = {  'h':     [np.float64, np.float64],
                        'mh': [np.float64, np.float32],
                        'm':   [np.float32, np.float32],
                        'ml':  [np.float32, np.float16],
                        'l':      [np.float16, np.float16]
                    }
        hi_lvl, lo_lvl = mem_levels[key]
        cat_lvls = {'obj':lo_lvl,
                    'ilm':hi_lvl,
                    'bkg':lo_lvl
                    }  #no psf...

        self.image.float_precision = hi_lvl
        self.image.image = self.image.image.astype(lo_lvl)
        self.set_image(self.image)

        for cat in cat_lvls.keys():
            obj = self.get(cat)
            #check if it's a component collection
            if hasattr(obj, 'comps'):
                for c in obj.comps:
                    c.float_precision = lo_lvl
            else:
                obj.float_precision = lo_lvl
        self._model = self._model.astype(hi_lvl)
        self._residuals = self._model.astype(hi_lvl)
        self.reset()

Sets the memory usage level of the state .


def scramble_positions(p, delete_frac=0.1):
    """randomly deletes particles and adds 1-px noise for a realistic
    initial featuring guess"""
    probs = [1-delete_frac, delete_frac]
    m = np.random.choice([True, False], p.shape[0], p=probs)
    jumble = np.random.randn(m.sum(), 3)
    return p[m] + jumble

def scramble_positions ( p , delete_frac = 0.1 ) : probs = [ 1 - delete_frac , delete_frac ] m = np . random . choice ( [ True , False ] , p . shape [ 0 ] , p = probs ) jumble = np . random . randn ( m . sum ( ) , 3 ) return p [ m ] + jumble

randomly deletes particles and adds 1 - px noise for a realistic initial featuring guess


def create_img():
    """Creates an image, as a `peri.util.Image`, which is similar
    to the image in the tutorial"""
    # 1. particles + coverslip
    rad = 0.5 * np.random.randn(POS.shape[0]) + 4.5  # 4.5 +- 0.5 px particles
    part = objs.PlatonicSpheresCollection(POS, rad, zscale=0.89)
    slab = objs.Slab(zpos=4.92, angles=(-4.7e-3, -7.3e-4))
    objects = comp.ComponentCollection([part, slab], category='obj')

    # 2. psf, ilm
    p = exactpsf.FixedSSChebLinePSF(kfki=1.07, zslab=-29.3, alpha=1.17,
            n2n1=0.98, sigkf=-0.33, zscale=0.89, laser_wavelength=0.45)
    i = ilms.BarnesStreakLegPoly2P1D(npts=(16,10,8,4), zorder=8)
    b = ilms.LegendrePoly2P1D(order=(7,2,2), category='bkg')
    off = comp.GlobalScalar(name='offset', value=-2.11)
    mdl = models.ConfocalImageModel()
    st = states.ImageState(util.NullImage(shape=[48,64,64]),
            [objects, p, i, b, off], mdl=mdl, model_as_data=True)
    b.update(b.params, BKGVALS)
    i.update(i.params, ILMVALS)
    im = st.model + np.random.randn(*st.model.shape) * 0.03
    return util.Image(im)

def create_img ( ) : rad = 0.5 * np . random . randn ( POS . shape [ 0 ] ) + 4.5 part = objs . PlatonicSpheresCollection ( POS , rad , zscale = 0.89 ) slab = objs . Slab ( zpos = 4.92 , angles = ( - 4.7e-3 , - 7.3e-4 ) ) objects = comp . ComponentCollection ( [ part , slab ] , category = 'obj' ) p = exactpsf . FixedSSChebLinePSF ( kfki = 1.07 , zslab = - 29.3 , alpha = 1.17 , n2n1 = 0.98 , sigkf = - 0.33 , zscale = 0.89 , laser_wavelength = 0.45 ) i = ilms . BarnesStreakLegPoly2P1D ( npts = ( 16 , 10 , 8 , 4 ) , zorder = 8 ) b = ilms . LegendrePoly2P1D ( order = ( 7 , 2 , 2 ) , category = 'bkg' ) off = comp . GlobalScalar ( name = 'offset' , value = - 2.11 ) mdl = models . ConfocalImageModel ( ) st = states . ImageState ( util . NullImage ( shape = [ 48 , 64 , 64 ] ) , [ objects , p , i , b , off ] , mdl = mdl , model_as_data = True ) b . update ( b . params , BKGVALS ) i . update ( i . params , ILMVALS ) im = st . model + np . random . randn ( * st . model . shape ) * 0.03 return util . Image ( im )

Creates an image as a peri . util . Image which is similar to the image in the tutorial


def get_values(self, params):
        """
        Get the value of a list or single parameter.

        Parameters
        ----------
        params : string, list of string
            name of parameters which to retrieve
        """
        return util.delistify(
            [self.param_dict[p] for p in util.listify(params)], params
        )

def get_values ( self , params ) : return util . delistify ( [ self . param_dict [ p ] for p in util . listify ( params ) ] , params )

Get the value of a list or single parameter .


def set_shape(self, shape, inner):
        """
        Set the overall shape of the calculation area. The total shape of that
        the calculation can possibly occupy, in pixels. The second, inner, is
        the region of interest within the image.
        """
        if self.shape != shape or self.inner != inner:
            self.shape = shape
            self.inner = inner
            self.initialize()

def set_shape ( self , shape , inner ) : if self . shape != shape or self . inner != inner : self . shape = shape self . inner = inner self . initialize ( )

Set the overall shape of the calculation area . The total shape of that the calculation can possibly occupy in pixels . The second inner is the region of interest within the image .


def trigger_update(self, params, values):
        """ Notify parent of a parameter change """
        if self._parent:
            self._parent.trigger_update(params, values)
        else:
            self.update(params, values)

def trigger_update ( self , params , values ) : if self . _parent : self . _parent . trigger_update ( params , values ) else : self . update ( params , values )


def get(self):
        """ Combine the fields from all components """
        fields = [c.get() for c in self.comps]
        return self.field_reduce_func(fields)

def get ( self ) : fields = [ c . get ( ) for c in self . comps ] return self . field_reduce_func ( fields )


def set_shape(self, shape, inner):
        """ Set the shape for all components """
        for c in self.comps:
            c.set_shape(shape, inner)

def set_shape ( self , shape , inner ) : for c in self . comps : c . set_shape ( shape , inner )


def sync_params(self):
        """ Ensure that shared parameters are the same value everywhere """
        def _normalize(comps, param):
            vals = [c.get_values(param) for c in comps]
            diff = any([vals[i] != vals[i+1] for i in range(len(vals)-1)])

            if diff:
                for c in comps:
                    c.set_values(param, vals[0])

        for param, comps in iteritems(self.lmap):
            if isinstance(comps, list) and len(comps) > 1:
                _normalize(comps, param)

def sync_params ( self ) : def _normalize ( comps , param ) : vals = [ c . get_values ( param ) for c in comps ] diff = any ( [ vals [ i ] != vals [ i + 1 ] for i in range ( len ( vals ) - 1 ) ] ) if diff : for c in comps : c . set_values ( param , vals [ 0 ] ) for param , comps in iteritems ( self . lmap ) : if isinstance ( comps , list ) and len ( comps ) > 1 : _normalize ( comps , param )


def setup_passthroughs(self):
        """
        Inherit some functions from the components that we own. In particular,
        let's grab all functions that begin with `param_` so the super class
        knows how to get parameter groups. Also, take anything that is listed
        under Component.exports and rename with the category type, i.e.,
        SphereCollection.add_particle -> Component.obj_add_particle
        """
        self._nopickle = []

        for c in self.comps:
            # take all member functions that start with 'param_'
            funcs = inspect.getmembers(c, predicate=inspect.ismethod)
            for func in funcs:
                if func[0].startswith('param_'):
                    setattr(self, func[0], func[1])
                    self._nopickle.append(func[0])

            # add everything from exports
            funcs = c.exports()
            for func in funcs:
                newname = c.category + '_' + func.__func__.__name__
                setattr(self, newname, func)
                self._nopickle.append(newname)

def setup_passthroughs ( self ) : self . _nopickle = [ ] for c in self . comps : funcs = inspect . getmembers ( c , predicate = inspect . ismethod ) for func in funcs : if func [ 0 ] . startswith ( 'param_' ) : setattr ( self , func [ 0 ] , func [ 1 ] ) self . _nopickle . append ( func [ 0 ] ) funcs = c . exports ( ) for func in funcs : newname = c . category + '_' + func . __func__ . __name__ setattr ( self , newname , func ) self . _nopickle . append ( newname )

Inherit some functions from the components that we own . In particular let s grab all functions that begin with param_ so the super class knows how to get parameter groups . Also take anything that is listed under Component . exports and rename with the category type i . e . SphereCollection . add_particle - > Component . obj_add_particle


def read_environment():
    """ Read all environment variables to see if they contain PERI """
    out = {}
    for k,v in iteritems(os.environ):
        if transform(k) in default_conf:
            out[transform(k)] = v
    return out

def read_environment ( ) : out = { } for k , v in iteritems ( os . environ ) : if transform ( k ) in default_conf : out [ transform ( k ) ] = v return out


def get_group_name(id_group):
    """Used for breadcrumb dynamic_list_constructor."""
    group = Group.query.get(id_group)
    if group is not None:
        return group.name

def get_group_name ( id_group ) : group = Group . query . get ( id_group ) if group is not None : return group . name

Used for breadcrumb dynamic_list_constructor .


def index():
    """List all user memberships."""
    page = request.args.get('page', 1, type=int)
    per_page = request.args.get('per_page', 5, type=int)
    q = request.args.get('q', '')

    groups = Group.query_by_user(current_user, eager=True)
    if q:
        groups = Group.search(groups, q)
    groups = groups.paginate(page, per_page=per_page)

    requests = Membership.query_requests(current_user).count()
    invitations = Membership.query_invitations(current_user).count()

    return render_template(
        'invenio_groups/index.html',
        groups=groups,
        requests=requests,
        invitations=invitations,
        page=page,
        per_page=per_page,
        q=q
    )

def index ( ) : page = request . args . get ( 'page' , 1 , type = int ) per_page = request . args . get ( 'per_page' , 5 , type = int ) q = request . args . get ( 'q' , '' ) groups = Group . query_by_user ( current_user , eager = True ) if q : groups = Group . search ( groups , q ) groups = groups . paginate ( page , per_page = per_page ) requests = Membership . query_requests ( current_user ) . count ( ) invitations = Membership . query_invitations ( current_user ) . count ( ) return render_template ( 'invenio_groups/index.html' , groups = groups , requests = requests , invitations = invitations , page = page , per_page = per_page , q = q )


def requests():
    """List all pending memberships, listed only for group admins."""
    page = request.args.get('page', 1, type=int)
    per_page = request.args.get('per_page', 5, type=int)
    memberships = Membership.query_requests(current_user, eager=True).all()

    return render_template(
        'invenio_groups/pending.html',
        memberships=memberships,
        requests=True,
        page=page,
        per_page=per_page,
    )

def requests ( ) : page = request . args . get ( 'page' , 1 , type = int ) per_page = request . args . get ( 'per_page' , 5 , type = int ) memberships = Membership . query_requests ( current_user , eager = True ) . all ( ) return render_template ( 'invenio_groups/pending.html' , memberships = memberships , requests = True , page = page , per_page = per_page , )

List all pending memberships listed only for group admins .


def invitations():
    """List all user pending memberships."""
    page = request.args.get('page', 1, type=int)
    per_page = request.args.get('per_page', 5, type=int)
    memberships = Membership.query_invitations(current_user, eager=True).all()

    return render_template(
        'invenio_groups/pending.html',
        memberships=memberships,
        page=page,
        per_page=per_page,
    )

def invitations ( ) : page = request . args . get ( 'page' , 1 , type = int ) per_page = request . args . get ( 'per_page' , 5 , type = int ) memberships = Membership . query_invitations ( current_user , eager = True ) . all ( ) return render_template ( 'invenio_groups/pending.html' , memberships = memberships , page = page , per_page = per_page , )


def new():
    """Create new group."""
    form = GroupForm(request.form)

    if form.validate_on_submit():
        try:
            group = Group.create(admins=[current_user], **form.data)

            flash(_('Group "%(name)s" created', name=group.name), 'success')
            return redirect(url_for(".index"))
        except IntegrityError:
            flash(_('Group creation failure'), 'error')

    return render_template(
        "invenio_groups/new.html",
        form=form,
    )

def
new
(
)
:
form
=
GroupForm
(
request
.
form
)
if
form
.
validate_on_submit
(
)
:
try
:
group
=
Group
.
create
(
admins
=
[
current_user
]
,
**
form
.
data
)
flash
(
_
(
'Group "%(name)s" created'
,
name
=
group
.
name
)
,
'success'
)
return
redirect
(
url_for
(
".index"
)
)
except
IntegrityError
:
flash
(
_
(
'Group creation failure'
)
,
'error'
)
return
render_template
(
"invenio_groups/new.html"
,
form
=
form
,
)


def manage(group_id):
    """Manage your group."""
    group = Group.query.get_or_404(group_id)
    form = GroupForm(request.form, obj=group)

    if form.validate_on_submit():
        if group.can_edit(current_user):
            try:
                group.update(**form.data)
                flash(_('Group "%(name)s" was updated', name=group.name),
                      'success')
            except Exception as e:
                flash(str(e), 'error')
                return render_template(
                    "invenio_groups/new.html",
                    form=form,
                    group=group,
                )
        else:
            flash(
                _(
                    'You cannot edit group %(group_name)s',
                    group_name=group.name
                ),
                'error'
            )

    return render_template(
        "invenio_groups/new.html",
        form=form,
        group=group,
    )

def
manage
(
group_id
)
:
group
=
Group
.
query
.
get_or_404
(
group_id
)
form
=
GroupForm
(
request
.
form
,
obj
=
group
)
if
form
.
validate_on_submit
(
)
:
if
group
.
can_edit
(
current_user
)
:
try
:
group
.
update
(
**
form
.
data
)
flash
(
_
(
'Group "%(name)s" was updated'
,
name
=
group
.
name
)
,
'success'
)
except
Exception
as
e
:
flash
(
str
(
e
)
,
'error'
)
return
render_template
(
"invenio_groups/new.html"
,
form
=
form
,
group
=
group
,
)
else
:
flash
(
_
(
'You cannot edit group %(group_name)s'
,
group_name
=
group
.
name
)
,
'error'
)
return
render_template
(
"invenio_groups/new.html"
,
form
=
form
,
group
=
group
,
)


def delete(group_id):
    """Delete group."""
    group = Group.query.get_or_404(group_id)

    if group.can_edit(current_user):
        try:
            group.delete()
        except Exception as e:
            flash(str(e), "error")
            return redirect(url_for(".index"))

        flash(_('Successfully removed group "%(group_name)s"',
                group_name=group.name), 'success')
        return redirect(url_for(".index"))

    flash(
        _(
            'You cannot delete the group %(group_name)s',
            group_name=group.name
        ),
        'error'
    )
    return redirect(url_for(".index"))


def members(group_id):
    """List user group members."""
    page = request.args.get('page', 1, type=int)
    per_page = request.args.get('per_page', 5, type=int)
    q = request.args.get('q', '')
    s = request.args.get('s', '')

    group = Group.query.get_or_404(group_id)
    if group.can_see_members(current_user):
        members = Membership.query_by_group(group_id, with_invitations=True)
        if q:
            members = Membership.search(members, q)
        if s:
            members = Membership.order(members, Membership.state, s)
        members = members.paginate(page, per_page=per_page)

        return render_template(
            "invenio_groups/members.html",
            group=group,
            members=members,
            page=page,
            per_page=per_page,
            q=q,
            s=s,
        )

    flash(
        _(
            'You are not allowed to see members of this group %(group_name)s.',
            group_name=group.name
        ),
        'error'
    )
    return redirect(url_for('.index'))


def leave(group_id):
    """Leave group."""
    group = Group.query.get_or_404(group_id)

    if group.can_leave(current_user):
        try:
            group.remove_member(current_user)
        except Exception as e:
            flash(str(e), "error")
            return redirect(url_for('.index'))

        flash(
            _(
                'You have successfully left %(group_name)s group.',
                group_name=group.name
            ),
            'success'
        )
        return redirect(url_for('.index'))

    flash(
        _(
            'You cannot leave the group %(group_name)s',
            group_name=group.name
        ),
        'error'
    )
    return redirect(url_for('.index'))


def approve(group_id, user_id):
    """Approve a user."""
    membership = Membership.query.get_or_404((user_id, group_id))
    group = membership.group

    if group.can_edit(current_user):
        try:
            membership.accept()
        except Exception as e:
            flash(str(e), 'error')
            return redirect(url_for('.requests', group_id=membership.group.id))

        flash(_('%(user)s accepted to %(name)s group.',
                user=membership.user.email,
                name=membership.group.name), 'success')
        return redirect(url_for('.requests', group_id=membership.group.id))

    flash(
        _(
            'You cannot approve memberships for the group %(group_name)s',
            group_name=group.name
        ),
        'error'
    )
    return redirect(url_for('.index'))


def remove(group_id, user_id):
    """Remove user from a group."""
    group = Group.query.get_or_404(group_id)
    user = User.query.get_or_404(user_id)

    if group.can_edit(current_user):
        try:
            group.remove_member(user)
        except Exception as e:
            flash(str(e), "error")
            return redirect(urlparse(request.referrer).path)

        flash(_('User %(user_email)s was removed from %(group_name)s group.',
                user_email=user.email, group_name=group.name), 'success')
        return redirect(urlparse(request.referrer).path)

    flash(
        _(
            'You cannot delete users of the group %(group_name)s',
            group_name=group.name
        ),
        'error'
    )
    return redirect(url_for('.index'))


def accept(group_id):
    """Accpet pending invitation."""
    membership = Membership.query.get_or_404((current_user.get_id(), group_id))

    # no permission check, because they are checked during Memberships creating

    try:
        membership.accept()
    except Exception as e:
        flash(str(e), 'error')
        return redirect(url_for('.invitations', group_id=membership.group.id))

    flash(_('You are now part of %(name)s group.',
            user=membership.user.email,
            name=membership.group.name), 'success')
    return redirect(url_for('.invitations', group_id=membership.group.id))

def
accept
(
group_id
)
:
membership
=
Membership
.
query
.
get_or_404
(
(
current_user
.
get_id
(
)
,
group_id
)
)
try
:
membership
.
accept
(
)
except
Exception
as
e
:
flash
(
str
(
e
)
,
'error'
)
return
redirect
(
url_for
(
'.invitations'
,
group_id
=
membership
.
group
.
id
)
)
flash
(
_
(
'You are now part of %(name)s group.'
,
user
=
membership
.
user
.
email
,
name
=
membership
.
group
.
name
)
,
'success'
)
return
redirect
(
url_for
(
'.invitations'
,
group_id
=
membership
.
group
.
id
)
)


def locate_spheres(image, feature_rad, dofilter=False, order=(3 ,3, 3),
                    trim_edge=True, **kwargs):
    """
    Get an initial featuring of sphere positions in an image.

    Parameters
    -----------
    image : :class:`peri.util.Image` object
        Image object which defines the image file as well as the region.

    feature_rad : float
        Radius of objects to find, in pixels. This is a featuring radius
        and not a real radius, so a better value is frequently smaller
        than the real radius (half the actual radius is good). If ``use_tp``
        is True, then the twice ``feature_rad`` is passed as trackpy's
        ``diameter`` keyword.

    dofilter : boolean, optional
        Whether to remove the background before featuring. Doing so can
        often greatly increase the success of initial featuring and
        decrease later optimization time. Filtering functions by fitting
        the image to a low-order polynomial and featuring the residuals.
        In doing so, this will change the mean intensity of the featured
        image and hence the good value of ``minmass`` will change when
        ``dofilter`` is True. Default is False.

    order : 3-element tuple, optional
        If `dofilter`, the 2+1D Leg Poly approximation to the background
        illumination field. Default is (3,3,3).

    Other Parameters
    ----------------
    invert : boolean, optional
        Whether to invert the image for featuring. Set to True if the
        image is dark particles on a bright background. Default is True
    minmass : Float or None, optional
        The minimum mass/masscut of a particle. Default is None, which
        calculates internally.
    use_tp : Bool, optional
        Whether or not to use trackpy. Default is False, since trackpy
        cuts out particles at the edge.

    Returns
    --------
    positions : np.ndarray [N,3]
        Positions of the particles in order (z,y,x) in image pixel units.

    Notes
    -----
    Optionally filters the image by fitting the image I(x,y,z) to a
    polynomial, then subtracts this fitted intensity variation and uses
    centroid methods to find the particles.
    """
    # We just want a smoothed field model of the image so that the residuals
    # are simply the particles without other complications
    m = models.SmoothFieldModel()
    I = ilms.LegendrePoly2P1D(order=order, constval=image.get_image().mean())
    s = states.ImageState(image, [I], pad=0, mdl=m)
    if dofilter:
        opt.do_levmarq(s, s.params)
    pos = addsub.feature_guess(s, feature_rad, trim_edge=trim_edge, **kwargs)[0]
    return pos

def locate_spheres ( image , feature_rad , dofilter = False , order = ( 3 , 3 , 3 ) , trim_edge = True , ** kwargs ) : m = models . SmoothFieldModel ( ) I = ilms . LegendrePoly2P1D ( order = order , constval = image . get_image ( ) . mean ( ) ) s = states . ImageState ( image , [ I ] , pad = 0 , mdl = m ) if dofilter : opt . do_levmarq ( s , s . params ) pos = addsub . feature_guess ( s , feature_rad , trim_edge = trim_edge , ** kwargs ) [ 0 ] return pos

Get an initial featuring of sphere positions in an image .


def get_initial_featuring(statemaker, feature_rad, actual_rad=None,
        im_name=None, tile=None, invert=True, desc='', use_full_path=False,
        featuring_params={}, statemaker_kwargs={}, **kwargs):
    """
    Completely optimizes a state from an image of roughly monodisperse
    particles.

    The user can interactively select the image. The state is periodically
    saved during optimization, with different filename for different stages
    of the optimization.

    Parameters
    ----------
        statemaker : Function
            A statemaker function. Given arguments `im` (a
            :class:`~peri.util.Image`), `pos` (numpy.ndarray), `rad` (ndarray),
            and any additional `statemaker_kwargs`, must return a
            :class:`~peri.states.ImageState`.  There is an example function in
            scripts/statemaker_example.py
        feature_rad : Int, odd
            The particle radius for featuring, as passed to locate_spheres.
        actual_rad : Float, optional
            The actual radius of the particles. Default is feature_rad
        im_name : string, optional
            The file name of the image to load. If not set, it is selected
            interactively through Tk.
        tile : :class:`peri.util.Tile`, optional
            The tile of the raw image to be analyzed. Default is None, the
            entire image.
        invert : Bool, optional
            Whether to invert the image for featuring, as passed to trackpy.
            Default is True.
        desc : String, optional
            A description to be inserted in saved state. The save name will
            be, e.g., '0.tif-peri-' + desc + 'initial-burn.pkl'. Default is ''
        use_full_path : Bool, optional
            Set to True to use the full path name for the image. Default
            is False.
        featuring_params : Dict, optional
            kwargs-like dict of any additional keyword arguments to pass to
            ``get_initial_featuring``, such as ``'use_tp'`` or ``'minmass'``.
            Default is ``{}``.
        statemaker_kwargs : Dict, optional
            kwargs-like dict of any additional keyword arguments to pass to
            the statemaker function. Default is ``{}``.

    Other Parameters
    ----------------
        max_mem : Numeric
            The maximum additional memory to use for the optimizers, as
            passed to optimize.burn. Default is 1e9.
        min_rad : Float, optional
            The minimum particle radius, as passed to addsubtract.add_subtract.
            Particles with a fitted radius smaller than this are identified
            as fake and removed. Default is 0.5 * actual_rad.
        max_rad : Float, optional
            The maximum particle radius, as passed to addsubtract.add_subtract.
            Particles with a fitted radius larger than this are identified
            as fake and removed. Default is 1.5 * actual_rad, however you
            may find better results if you make this more stringent.
        rz_order : int, optional
            If nonzero, the order of an additional augmented rscl(z)
            parameter for optimization. Default is 0; i.e. no rscl(z)
            optimization.
        zscale : Float, optional
            The zscale of the image. Default is 1.0

    Returns
    -------
        s : :class:`peri.states.ImageState`
            The optimized state.

    See Also
    --------
        feature_from_pos_rad    : Using a previous state's globals and
            user-provided positions and radii as an initial guess,
            completely optimizes a state.

        get_particle_featuring  : Using a previous state's globals and
            positions as an initial guess, completely optimizes a state.

        translate_featuring     : Use a previous state's globals and
            centroids methods for an initial particle guess, completely
            optimizes a state.

    Notes
    -----
    Proceeds by centroid-featuring the image for an initial guess of
    particle positions, then optimizing the globals + positions until
    termination as called in _optimize_from_centroid.
    The ``Other Parameters`` are passed to _optimize_from_centroid.
    """
    if actual_rad is None:
        actual_rad = feature_rad

    _,  im_name = _pick_state_im_name('', im_name, use_full_path=use_full_path)
    im = util.RawImage(im_name, tile=tile)

    pos = locate_spheres(im, feature_rad, invert=invert, **featuring_params)
    if np.size(pos) == 0:
        msg = 'No particles found. Try using a smaller `feature_rad`.'
        raise ValueError(msg)

    rad = np.ones(pos.shape[0], dtype='float') * actual_rad
    s = statemaker(im, pos, rad, **statemaker_kwargs)
    RLOG.info('State Created.')
    if desc is not None:
        states.save(s, desc=desc+'initial')
    optimize_from_initial(s, invert=invert, desc=desc, **kwargs)
    return s

Completely optimizes a state from an image of roughly monodisperse particles .


def feature_from_pos_rad(statemaker, pos, rad, im_name=None, tile=None,
        desc='', use_full_path=False, statemaker_kwargs={}, **kwargs):
    """
    Gets a completely-optimized state from an image and an initial guess of
    particle positions and radii.

    The state is periodically saved during optimization, with different
    filename for different stages of the optimization. The user can select
    the image.

    Parameters
    ----------
        statemaker : Function
            A statemaker function. Given arguments `im` (a
            :class:`~peri.util.Image`), `pos` (numpy.ndarray), `rad` (ndarray),
            and any additional `statemaker_kwargs`, must return a
            :class:`~peri.states.ImageState`.  There is an example function in
            scripts/statemaker_example.py
        pos : [N,3] element numpy.ndarray.
            The initial guess for the N particle positions.
        rad : N element numpy.ndarray.
            The initial guess for the N particle radii.
        im_name : string or None, optional
            The filename of the image to feature. Default is None, in which
            the user selects the image.
        tile : :class:`peri.util.Tile`, optional
            A tile of the sub-region of the image to feature. Default is
            None, i.e. entire image.
        desc : String, optional
            A description to be inserted in saved state. The save name will
            be, e.g., '0.tif-peri-' + desc + 'initial-burn.pkl'. Default is ''
        use_full_path : Bool, optional
            Set to True to use the full path name for the image. Default
            is False.
        statemaker_kwargs : Dict, optional
            kwargs-like dict of any additional keyword arguments to pass to
            the statemaker function. Default is ``{}``.

    Other Parameters
    ----------------
        max_mem : Numeric
            The maximum additional memory to use for the optimizers, as
            passed to optimize.burn. Default is 1e9.
        min_rad : Float, optional
            The minimum particle radius, as passed to addsubtract.add_subtract.
            Particles with a fitted radius smaller than this are identified
            as fake and removed. Default is 0.5 * actual_rad.
        max_rad : Float, optional
            The maximum particle radius, as passed to addsubtract.add_subtract.
            Particles with a fitted radius larger than this are identified
            as fake and removed. Default is 1.5 * actual_rad, however you
            may find better results if you make this more stringent.
        invert : {'guess', True, False}
            Whether to invert the image for featuring, as passed to
            addsubtract.add_subtract. Default is to guess from the
            current state's particle positions.
        rz_order : int, optional
            If nonzero, the order of an additional augmented rscl(z)
            parameter for optimization. Default is 0; i.e. no rscl(z)
            optimization.
        zscale : Float, optional
            The zscale of the image. Default is 1.0

    Returns
    -------
        s : :class:`peri.states.ImageState`
            The optimized state.

    See Also
    --------
        get_initial_featuring   : Features an image from scratch, using
            centroid methods as initial particle locations.

        get_particle_featuring  : Using a previous state's globals and
            positions as an initial guess, completely optimizes a state.

        translate_featuring     : Use a previous state's globals and
            centroids methods for an initial particle guess, completely
            optimizes a state.

    Notes
    -----
    The ``Other Parameters`` are passed to _optimize_from_centroid.
    Proceeds by centroid-featuring the image for an initial guess of
    particle positions, then optimizing the globals + positions until
    termination as called in _optimize_from_centroid.
    """
    if np.size(pos) == 0:
        raise ValueError('`pos` is an empty array.')
    elif np.shape(pos)[1] != 3:
        raise ValueError('`pos` must be an [N,3] element numpy.ndarray.')
    _,  im_name = _pick_state_im_name('', im_name, use_full_path=use_full_path)
    im = util.RawImage(im_name, tile=tile)
    s = statemaker(im, pos, rad, **statemaker_kwargs)
    RLOG.info('State Created.')
    if desc is not None:
        states.save(s, desc=desc+'initial')
    optimize_from_initial(s, desc=desc, **kwargs)
    return s

Gets a completely - optimized state from an image and an initial guess of particle positions and radii .


def optimize_from_initial(s, max_mem=1e9, invert='guess', desc='', rz_order=3,
        min_rad=None, max_rad=None):
    """
    Optimizes a state from an initial set of positions and radii, without
    any known microscope parameters.

    Parameters
    ----------
        s : :class:`peri.states.ImageState`
            The state to optimize. It is modified internally and returned.
        max_mem : Numeric, optional
            The maximum memory for the optimizer to use. Default is 1e9 (bytes)
        invert : Bool or `'guess'`, optional
            Set to True if the image is dark particles on a bright
            background, False otherwise. Used for add-subtract. The
            default is to guess from the state's current particles.
        desc : String, optional
            An additional description to infix for periodic saving along the
            way. Default is the null string ``''``.
        rz_order : int, optional
            ``rz_order`` as passed to opt.burn. Default is 3
        min_rad : Float or None, optional
            The minimum radius to identify a particles as bad, as passed to
            add-subtract. Default is None, which picks half the median radii.
            If your sample is not monodisperse you should pick a different
            value.
        max_rad : Float or None, optional
            The maximum radius to identify a particles as bad, as passed to
            add-subtract. Default is None, which picks 1.5x the median radii.
            If your sample is not monodisperse you should pick a different
            value.

    Returns
    -------
        s : :class:`peri.states.ImageState`
            The optimized state, which is the same as the input ``s`` but
            modified in-place.
    """
    RLOG.info('Initial burn:')
    if desc is not None:
        desc_burn = desc + 'initial-burn'
        desc_polish = desc + 'addsub-polish'
    else:
        desc_burn, desc_polish = [None] * 2
    opt.burn(s, mode='burn', n_loop=3, fractol=0.1, desc=desc_burn,
            max_mem=max_mem, include_rad=False, dowarn=False)
    opt.burn(s, mode='burn', n_loop=3, fractol=0.1, desc=desc_burn,
            max_mem=max_mem, include_rad=True, dowarn=False)

    RLOG.info('Start add-subtract')
    rad = s.obj_get_radii()
    if min_rad is None:
        min_rad = 0.5 * np.median(rad)
    if max_rad is None:
        max_rad = 1.5 * np.median(rad)
    addsub.add_subtract(s, tries=30, min_rad=min_rad, max_rad=max_rad,
            invert=invert)
    if desc is not None:
        states.save(s, desc=desc + 'initial-addsub')

    RLOG.info('Final polish:')
    d = opt.burn(s, mode='polish', n_loop=8, fractol=3e-4, desc=desc_polish,
            max_mem=max_mem, rz_order=rz_order, dowarn=False)
    if not d['converged']:
        RLOG.warn('Optimization did not converge; consider re-running')
    return s

Optimizes a state from an initial set of positions and radii without any known microscope parameters .


def get_particles_featuring(feature_rad, state_name=None, im_name=None,
        use_full_path=False, actual_rad=None, invert=True, featuring_params={},
        **kwargs):
    """
    Combines centroid featuring with the globals from a previous state.

    Runs trackpy.locate on an image, sets the globals from a previous state,
    calls _translate_particles

    Parameters
    ----------
        feature_rad : Int, odd
            The particle radius for featuring, as passed to locate_spheres.

        state_name : String or None, optional
            The name of the initially-optimized state. Default is None,
            which prompts the user to select the name interactively
            through a Tk window.
        im_name : String or None, optional
            The name of the new image to optimize. Default is None,
            which prompts the user to select the name interactively
            through a Tk window.
        use_full_path : Bool, optional
            Set to True to use the full path of the state instead of
            partial path names (e.g. /full/path/name/state.pkl vs
            state.pkl). Default is False
        actual_rad : Float or None, optional
            The initial guess for the particle radii. Default is the median
            of the previous state.
        invert : Bool
            Whether to invert the image for featuring, as passed to
            addsubtract.add_subtract and locate_spheres. Set to False
            if the image is bright particles on a dark background.
            Default is True (dark particles on bright background).
        featuring_params : Dict, optional
            kwargs-like dict of any additional keyword arguments to pass to
            ``get_initial_featuring``, such as ``'use_tp'`` or ``'minmass'``.
            Default is ``{}``.

    Other Parameters
    ----------------
        max_mem : Numeric
            The maximum additional memory to use for the optimizers, as
            passed to optimize.burn. Default is 1e9.
        desc : String, optional
            A description to be inserted in saved state. The save name will
            be, e.g., '0.tif-peri-' + desc + 'initial-burn.pkl'. Default is ''
        min_rad : Float, optional
            The minimum particle radius, as passed to addsubtract.add_subtract.
            Particles with a fitted radius smaller than this are identified
            as fake and removed. Default is 0.5 * actual_rad.
        max_rad : Float, optional
            The maximum particle radius, as passed to addsubtract.add_subtract.
            Particles with a fitted radius larger than this are identified
            as fake and removed. Default is 1.5 * actual_rad, however you
            may find better results if you make this more stringent.
        rz_order : int, optional
            If nonzero, the order of an additional augmented rscl(z)
            parameter for optimization. Default is 0; i.e. no rscl(z)
            optimization.
        do_polish : Bool, optional
            Set to False to only optimize the particles and add-subtract.
            Default is True, which then runs a polish afterwards.

    Returns
    -------
        s : :class:`peri.states.ImageState`
            The optimized state.

    See Also
    --------
        get_initial_featuring   : Features an image from scratch, using
            centroid methods as initial particle locations.

        feature_from_pos_rad    : Using a previous state's globals and
            user-provided positions and radii as an initial guess,
            completely optimizes a state.

        translate_featuring     : Use a previous state's globals and
            centroids methods for an initial particle guess, completely
            optimizes a state.

    Notes
    -----
        The ``Other Parameters`` are passed to _translate_particles.
    Proceeds by:
        1. Find a guess of the particle positions through centroid methods.
        2. Optimize particle positions only.
        3. Optimize particle positions and radii only.
        4. Add-subtract missing and bad particles.
        5. If polish, optimize the illumination, background, and particles.
        6. If polish, optimize everything.
    """
    state_name, im_name = _pick_state_im_name(
            state_name, im_name, use_full_path=use_full_path)
    s = states.load(state_name)

    if actual_rad is None:
        actual_rad = np.median(s.obj_get_radii())
    im = util.RawImage(im_name, tile=s.image.tile)
    pos = locate_spheres(im, feature_rad, invert=invert, **featuring_params)
    _ = s.obj_remove_particle(np.arange(s.obj_get_radii().size))
    s.obj_add_particle(pos, np.ones(pos.shape[0])*actual_rad)

    s.set_image(im)
    _translate_particles(s, invert=invert, **kwargs)
    return s

def get_particles_featuring ( feature_rad , state_name = None , im_name = None , use_full_path = False , actual_rad = None , invert = True , featuring_params = { } , ** kwargs ) : state_name , im_name = _pick_state_im_name ( state_name , im_name , use_full_path = use_full_path ) s = states . load ( state_name ) if actual_rad is None : actual_rad = np . median ( s . obj_get_radii ( ) ) im = util . RawImage ( im_name , tile = s . image . tile ) pos = locate_spheres ( im , feature_rad , invert = invert , ** featuring_params ) _ = s . obj_remove_particle ( np . arange ( s . obj_get_radii ( ) . size ) ) s . obj_add_particle ( pos , np . ones ( pos . shape [ 0 ] ) * actual_rad ) s . set_image ( im ) _translate_particles ( s , invert = invert , ** kwargs ) return s

Combines centroid featuring with the globals from a previous state .


def _pick_state_im_name(state_name, im_name, use_full_path=False):
    """
    If state_name or im_name is None, picks them interactively through Tk,
    and then sets with or without the full path.

    Parameters
    ----------
        state_name : {string, None}
            The name of the state. If None, selected through Tk.
        im_name : {string, None}
            The name of the image. If None, selected through Tk.
        use_full_path : Bool, optional
            Set to True to return the names as full paths rather than
            relative paths. Default is False (relative path).
    """
    initial_dir = os.getcwd()
    if (state_name is None) or (im_name is None):
        wid = tk.Tk()
        wid.withdraw()
    if state_name is None:
        state_name = tkfd.askopenfilename(
                initialdir=initial_dir, title='Select pre-featured state')
        os.chdir(os.path.dirname(state_name))

    if im_name is None:
        im_name = tkfd.askopenfilename(
                initialdir=initial_dir, title='Select new image')

    if (not use_full_path) and (os.path.dirname(im_name) != ''):
        im_path = os.path.dirname(im_name)
        os.chdir(im_path)
        im_name = os.path.basename(im_name)
    else:
        os.chdir(initial_dir)
    return state_name, im_name

def
_pick_state_im_name
(
state_name
,
im_name
,
use_full_path
=
False
)
:
initial_dir
=
os
.
getcwd
(
)
if
(
state_name
is
)
or
(
im_name
is
)
:
wid
=
tk
.
Tk
(
)
wid
.
withdraw
(
)
if
state_name
is
:
state_name
=
tkfd
.
askopenfilename
(
initialdir
=
initial_dir
,
title
=
'Select pre-featured state'
)
os
.
chdir
(
os
.
path
.
dirname
(
state_name
)
)
if
im_name
is
:
im_name
=
tkfd
.
askopenfilename
(
initialdir
=
initial_dir
,
title
=
'Select new image'
)
if
(
not
use_full_path
)
and
(
os
.
path
.
dirname
(
im_name
)
!=
''
)
:
im_path
=
os
.
path
.
dirname
(
im_name
)
os
.
chdir
(
im_path
)
im_name
=
os
.
path
.
basename
(
im_name
)
else
:
os
.
chdir
(
initial_dir
)
return
state_name
,
im_name

If state_name or im_name is None picks them interactively through Tk and then sets with or without the full path .


def _translate_particles(s, max_mem=1e9, desc='', min_rad='calc',
        max_rad='calc', invert='guess', rz_order=0, do_polish=True):
    """
    Workhorse for translating particles. See get_particles_featuring for docs.
    """
    if desc is not None:
        desc_trans = desc + 'translate-particles'
        desc_burn = desc + 'addsub_burn'
        desc_polish = desc + 'addsub_polish'
    else:
        desc_trans, desc_burn, desc_polish = [None]*3
    RLOG.info('Translate Particles:')
    opt.burn(s, mode='do-particles', n_loop=4, fractol=0.1, desc=desc_trans,
            max_mem=max_mem, include_rad=False, dowarn=False)
    opt.burn(s, mode='do-particles', n_loop=4, fractol=0.05, desc=desc_trans,
            max_mem=max_mem, include_rad=True, dowarn=False)

    RLOG.info('Start add-subtract')
    addsub.add_subtract(s, tries=30, min_rad=min_rad, max_rad=max_rad,
        invert=invert)
    if desc is not None:
        states.save(s, desc=desc + 'translate-addsub')

    if do_polish:
        RLOG.info('Final Burn:')
        opt.burn(s, mode='burn', n_loop=3, fractol=3e-4, desc=desc_burn,
                max_mem=max_mem, rz_order=rz_order,dowarn=False)
        RLOG.info('Final Polish:')
        d = opt.burn(s, mode='polish', n_loop=4, fractol=3e-4, desc=desc_polish,
                max_mem=max_mem, rz_order=rz_order, dowarn=False)
        if not d['converged']:
            RLOG.warn('Optimization did not converge; consider re-running')

Workhorse for translating particles . See get_particles_featuring for docs .


def link_zscale(st):
    """Links the state ``st`` psf zscale with the global zscale"""
    # FIXME should be made more generic to other parameters and categories
    psf = st.get('psf')
    psf.param_dict['zscale'] = psf.param_dict['psf-zscale']
    psf.params[psf.params.index('psf-zscale')] = 'zscale'
    psf.global_zscale = True
    psf.param_dict.pop('psf-zscale')
    st.trigger_parameter_change()
    st.reset()

def link_zscale ( st ) : psf = st . get ( 'psf' ) psf . param_dict [ 'zscale' ] = psf . param_dict [ 'psf-zscale' ] psf . params [ psf . params . index ( 'psf-zscale' ) ] = 'zscale' psf . global_zscale = True psf . param_dict . pop ( 'psf-zscale' ) st . trigger_parameter_change ( ) st . reset ( )

Links the state st psf zscale with the global zscale


def finish_state(st, desc='finish-state', invert='guess'):
    """
    Final optimization for the best-possible state.

    Runs a local add-subtract to capture any difficult-to-feature particles,
    then does another set of optimization designed to get to the best
    possible fit.

    Parameters
    ----------
        st : :class:`peri.states.ImageState`
            The state to finish
        desc : String, optional
            Description to intermittently save the state as, as passed to
            state.save. Default is `'finish-state'`.
        invert : {'guess', True, False}
            Whether to invert the image for featuring, as passed to
            addsubtract.add_subtract. Default is to guess from the
            state's current particles.

    See Also
    --------
        `peri.opt.addsubtract.add_subtract_locally`
        `peri.opt.optimize.finish`
    """
    for minmass in [None, 0]:
        for _ in range(3):
            npart, poses = addsub.add_subtract_locally(st, region_depth=7,
                    minmass=minmass, invert=invert)
            if npart == 0:
                break
    opt.finish(st, n_loop=1, separate_psf=True, desc=desc, dowarn=False)
    opt.burn(st, mode='polish', desc=desc, n_loop=2, dowarn=False)
    d = opt.finish(st, desc=desc, n_loop=4, dowarn=False)
    if not d['converged']:
        RLOG.warn('Optimization did not converge; consider re-running')

Final optimization for the best - possible state .