def makestate(im, pos, rad, slab=None, mem_level='hi'):
    """
    Workhorse for creating & optimizing states with an initial centroid
    guess.

    This is an example function that works for a particular microscope. For
    your own microscope, you'll need to change particulars such as the psf
    type and the orders of the background and illumination.

    Parameters
    ----------
        im : :class:`~peri.util.RawImage`
            A RawImage of the data.
        pos : [N,3] element numpy.ndarray.
            The initial guess for the N particle positions.
        rad : N element numpy.ndarray.
            The initial guess for the N particle radii.

        slab : :class:`peri.comp.objs.Slab` or None, optional
            If not None, a slab corresponding to that in the image. Default
            is None.
        mem_level : {'lo', 'med-lo', 'med', 'med-hi', 'hi'}, optional
            A valid memory level for the state to control the memory overhead
            at the expense of accuracy. Default is `'hi'`

    Returns
    -------
        :class:`~peri.states.ImageState`
            An ImageState with a linked z-scale, a ConfocalImageModel, and
            all the necessary components with orders at which are useful for
            my particular test case.
    """
    if slab is not None:
        o = comp.ComponentCollection(
                [
                    objs.PlatonicSpheresCollection(pos, rad, zscale=zscale),
                    slab
                ],
                category='obj'
            )
    else:
        o = objs.PlatonicSpheresCollection(pos, rad, zscale=zscale)

    p = exactpsf.FixedSSChebLinePSF()
    npts, iorder = _calc_ilm_order(im.get_image().shape)
    i = ilms.BarnesStreakLegPoly2P1D(npts=npts, zorder=iorder)
    b = ilms.LegendrePoly2P1D(order=(9 ,3, 5), category='bkg')
    c = comp.GlobalScalar('offset', 0.0)
    s = states.ImageState(im, [o, i, b, c, p])
    runner.link_zscale(s)
    if mem_level != 'hi':
        s.set_mem_level(mem_level)

    opt.do_levmarq(s, ['ilm-scale'], max_iter=1, run_length=6, max_mem=1e4)
    return s

def makestate ( im , pos , rad , slab = None , mem_level = 'hi' ) : if slab is not None : o = comp . ComponentCollection ( [ objs . PlatonicSpheresCollection ( pos , rad , zscale = zscale ) , slab ] , category = 'obj' ) else : o = objs . PlatonicSpheresCollection ( pos , rad , zscale = zscale ) p = exactpsf . FixedSSChebLinePSF ( ) npts , iorder = _calc_ilm_order ( im . get_image ( ) . shape ) i = ilms . BarnesStreakLegPoly2P1D ( npts = npts , zorder = iorder ) b = ilms . LegendrePoly2P1D ( order = ( 9 , 3 , 5 ) , category = 'bkg' ) c = comp . GlobalScalar ( 'offset' , 0.0 ) s = states . ImageState ( im , [ o , i , b , c , p ] ) runner . link_zscale ( s ) if mem_level != 'hi' : s . set_mem_level ( mem_level ) opt . do_levmarq ( s , [ 'ilm-scale' ] , max_iter = 1 , run_length = 6 , max_mem = 1e4 ) return s

Workhorse for creating & optimizing states with an initial centroid guess .


def _calc_ilm_order(imshape):
    """
    Calculates an ilm order based on the shape of an image. This is based on
    something that works for our particular images. Your mileage will vary.

    Parameters
    ----------
        imshape : 3-element list-like
            The shape of the image.

    Returns
    -------
        npts : tuple
            The number of points to use for the ilm.
        zorder : int
            The order of the z-polynomial.
    """
    zorder = int(imshape[0] / 6.25) + 1
    l_npts = int(imshape[1] / 42.5)+1
    npts = ()
    for a in range(l_npts):
        if a < 5:
            npts += (int(imshape[2] * [59, 39, 29, 19, 14][a]/512.) + 1,)
        else:
            npts += (int(imshape[2] * 11/512.) + 1,)
    return npts, zorder

def _calc_ilm_order ( imshape ) : zorder = int ( imshape [ 0 ] / 6.25 ) + 1 l_npts = int ( imshape [ 1 ] / 42.5 ) + 1 npts = ( ) for a in range ( l_npts ) : if a < 5 : npts += ( int ( imshape [ 2 ] * [ 59 , 39 , 29 , 19 , 14 ] [ a ] / 512. ) + 1 , ) else : npts += ( int ( imshape [ 2 ] * 11 / 512. ) + 1 , ) return npts , zorder

Calculates an ilm order based on the shape of an image . This is based on something that works for our particular images . Your mileage will vary .


def _check_for_inception(self, root_dict):
    '''
      Used to check if there is a dict in a dict
    '''

    for key in root_dict:
      if isinstance(root_dict[key], dict):
          root_dict[key] = ResponseObject(root_dict[key])

    return root_dict

def _check_for_inception ( self , root_dict ) : for key in root_dict : if isinstance ( root_dict [ key ] , dict ) : root_dict [ key ] = ResponseObject ( root_dict [ key ] ) return root_dict


def randomize_parameters(self, ptp=0.2, fourier=False, vmin=None, vmax=None):
        """
        Create random parameters for this ILM that mimic experiments
        as closely as possible without real assumptions.
        """
        if vmin is not None and vmax is not None:
            ptp = vmax - vmin
        elif vmax is not None and vmin is None:
            vmin = vmax - ptp
        elif vmin is not None and vmax is None:
            vmax = vmin + ptp
        else:
            vmax = 1.0
            vmin = vmax - ptp

        self.set_values(self.category+'-scale', 1.0)
        self.set_values(self.category+'-off', 0.0)

        for k, v in iteritems(self.poly_params):
            norm = (self.zorder + 1.0)*2
            self.set_values(k, ptp*(np.random.rand() - 0.5) / norm)

        for i, p in enumerate(self.barnes_params):
            N = len(p)
            if fourier:
                t = ((np.random.rand(N)-0.5) + 1.j*(np.random.rand(N)-0.5))/(np.arange(N)+1)
                q = np.real(np.fft.ifftn(t)) / (i+1)
            else:
                t = ptp*np.sqrt(N)*(np.random.rand(N)-0.5)
                q = np.cumsum(t) / (i+1)

            q = ptp * q / q.ptp() / len(self.barnes_params)
            q -= q.mean()
            self.set_values(p, q)

        self._norm_stat = [ptp, vmin]

        if self.shape:
            self.initialize()

        if self._parent:
            param = self.category+'-scale'
            self.trigger_update(param, self.get_values(param))

def randomize_parameters ( self , ptp = 0.2 , fourier = False , vmin = None , vmax = None ) : if vmin is not None and vmax is not None : ptp = vmax - vmin elif vmax is not None and vmin is None : vmin = vmax - ptp elif vmin is not None and vmax is None : vmax = vmin + ptp else : vmax = 1.0 vmin = vmax - ptp self . set_values ( self . category + '-scale' , 1.0 ) self . set_values ( self . category + '-off' , 0.0 ) for k , v in iteritems ( self . poly_params ) : norm = ( self . zorder + 1.0 ) * 2 self . set_values ( k , ptp * ( np . random . rand ( ) - 0.5 ) / norm ) for i , p in enumerate ( self . barnes_params ) : N = len ( p ) if fourier : t = ( ( np . random . rand ( N ) - 0.5 ) + 1.j * ( np . random . rand ( N ) - 0.5 ) ) / ( np . arange ( N ) + 1 ) q = np . real ( np . fft . ifftn ( t ) ) / ( i + 1 ) else : t = ptp * np . sqrt ( N ) * ( np . random . rand ( N ) - 0.5 ) q = np . cumsum ( t ) / ( i + 1 ) q = ptp * q / q . ptp ( ) / len ( self . barnes_params ) q -= q . mean ( ) self . set_values ( p , q ) self . _norm_stat = [ ptp , vmin ] if self . shape : self . initialize ( ) if self . _parent : param = self . category + '-scale' self . trigger_update ( param , self . get_values ( param ) )

Create random parameters for this ILM that mimic experiments as closely as possible without real assumptions .


def _barnes(self, pos):
        """Creates a barnes interpolant & calculates its values"""
        b_in = self.b_in
        dist = lambda x: np.sqrt(np.dot(x,x))
        #we take a filter size as the max distance between the grids along
        #x or y:
        sz = self.npts[1]
        coeffs = self.get_values(self.barnes_params)

        b = BarnesInterpolationND(
            b_in, coeffs, filter_size=self.filtsize, damp=0.9, iterations=3,
            clip=self.local_updates, clipsize=self.barnes_clip_size,
            blocksize=100  # FIXME magic blocksize
        )
        return b(pos)

def _barnes ( self , pos ) : b_in = self . b_in dist = lambda x : np . sqrt ( np . dot ( x , x ) ) sz = self . npts [ 1 ] coeffs = self . get_values ( self . barnes_params ) b = BarnesInterpolationND ( b_in , coeffs , filter_size = self . filtsize , damp = 0.9 , iterations = 3 , clip = self . local_updates , clipsize = self . barnes_clip_size , blocksize = 100 ) return b ( pos )


def schedules(self):
    '''
      Returns details of the posting schedules associated with a social media
      profile.
    '''

    url = PATHS['GET_SCHEDULES'] % self.id

    self.__schedules = self.api.get(url=url)

    return self.__schedules

def schedules ( self ) : url = PATHS [ 'GET_SCHEDULES' ] % self . id self . __schedules = self . api . get ( url = url ) return self . __schedules

Returns details of the posting schedules associated with a social media profile .


def schedules(self, schedules):
    '''
      Set the posting schedules for the specified social media profile.
    '''

    url = PATHS['UPDATE_SCHEDULES'] % self.id

    data_format = "schedules[0][%s][]=%s&"
    post_data = ""

    for format_type, values in schedules.iteritems():
      for value in values:
        post_data += data_format % (format_type, value)

    self.api.post(url=url, data=post_data)

def schedules ( self , schedules ) : url = PATHS [ 'UPDATE_SCHEDULES' ] % self . id data_format = "schedules[0][%s][]=%s&" post_data = "" for format_type , values in schedules . iteritems ( ) : for value in values : post_data += data_format % ( format_type , value ) self . api . post ( url = url , data = post_data )

Set the posting schedules for the specified social media profile .


def moment(p, v, order=1):
    """ Calculates the moments of the probability distribution p with vector v """
    if order == 1:
        return (v*p).sum()
    elif order == 2:
        return np.sqrt( ((v**2)*p).sum() - (v*p).sum()**2 )

def moment ( p , v , order = 1 ) : if order == 1 : return ( v * p ) . sum ( ) elif order == 2 : return np . sqrt ( ( ( v ** 2 ) * p ) . sum ( ) - ( v * p ) . sum ( ) ** 2 )


def psf_slice(self, zint, size=11, zoffset=0., getextent=False):
        """
        Calculates the 3D psf at a particular z pixel height

        Parameters
        ----------
        zint : float
            z pixel height in image coordinates , converted to 1/k by the
            function using the slab position as well

        size : int, list, tuple
            The size over which to calculate the psf, can be 1 or 3 elements
            for the different axes in image pixel coordinates

        zoffset : float
            Offset in pixel units to use in the calculation of the psf

        cutval : float
            If not None, the psf will be cut along a curve corresponding to
            p(r) == 0 with exponential damping exp(-d^4)

        getextent : boolean
            If True, also return the extent of the psf in pixels for example
            to get the support size. Can only be used with cutval.
        """
        # calculate the current pixel value in 1/k, making sure we are above the slab
        zint = max(self._p2k(self._tz(zint)), 0)
        offset = np.array([zoffset*(zint>0), 0, 0])
        scale = [self.param_dict[self.zscale], 1.0, 1.0]

        # create the coordinate vectors for where to actually calculate the
        tile = util.Tile(left=0, size=size, centered=True)
        vecs = tile.coords(form='flat')
        vecs = [self._p2k(s*i+o) for i,s,o in zip(vecs, scale, offset)]

        psf = self.psffunc(*vecs[::-1], zint=zint, **self.pack_args()).T
        vec = tile.coords(form='meshed')

        # create a smoothly varying point spread function by cutting off the psf
        # at a certain value and smoothly taking it to zero
        if self.cutoffval is not None and not self.cutbyval:
            # find the edges of the PSF
            edge = psf > psf.max() * self.cutoffval
            dd = nd.morphology.distance_transform_edt(~edge)

            # calculate the new PSF and normalize it to the new support
            psf = psf * np.exp(-dd**4)
            psf /= psf.sum()

            if getextent:
                # the size is determined by the edge plus a 2 pad for the
                # exponential damping to zero at the edge
                size = np.array([
                    (vec*edge).min(axis=(1,2,3))-2,
                    (vec*edge).max(axis=(1,2,3))+2,
                ]).T
                return psf, vec, size
            return psf, vec

        # perform a cut by value instead
        if self.cutoffval is not None and self.cutbyval:
            cutval = self.cutoffval * psf.max()

            dd = (psf - cutval) / cutval
            dd[dd > 0] = 0.

            # calculate the new PSF and normalize it to the new support
            psf = psf * np.exp(-(dd / self.cutfallrate)**4)
            psf /= psf.sum()

            # let the small values determine the edges
            edge = psf > cutval * self.cutedgeval
            if getextent:
                # the size is determined by the edge plus a 2 pad for the
                # exponential damping to zero at the edge
                size = np.array([
                    (vec*edge).min(axis=(1,2,3))-2,
                    (vec*edge).max(axis=(1,2,3))+2,
                ]).T
                return psf, vec, size
            return psf, vec

        return psf, vec

def psf_slice ( self , zint , size = 11 , zoffset = 0. , getextent = False ) : zint = max ( self . _p2k ( self . _tz ( zint ) ) , 0 ) offset = np . array ( [ zoffset * ( zint > 0 ) , 0 , 0 ] ) scale = [ self . param_dict [ self . zscale ] , 1.0 , 1.0 ] tile = util . Tile ( left = 0 , size = size , centered = True ) vecs = tile . coords ( form = 'flat' ) vecs = [ self . _p2k ( s * i + o ) for i , s , o in zip ( vecs , scale , offset ) ] psf = self . psffunc ( * vecs [ : : - 1 ] , zint = zint , ** self . pack_args ( ) ) . T vec = tile . coords ( form = 'meshed' ) if self . cutoffval is not None and not self . cutbyval : edge = psf > psf . max ( ) * self . cutoffval dd = nd . morphology . distance_transform_edt ( ~ edge ) psf = psf * np . exp ( - dd ** 4 ) psf /= psf . sum ( ) if getextent : size = np . array ( [ ( vec * edge ) . min ( axis = ( 1 , 2 , 3 ) ) - 2 , ( vec * edge ) . max ( axis = ( 1 , 2 , 3 ) ) + 2 , ] ) . T return psf , vec , size return psf , vec if self . cutoffval is not None and self . cutbyval : cutval = self . cutoffval * psf . max ( ) dd = ( psf - cutval ) / cutval dd [ dd > 0 ] = 0. psf = psf * np . exp ( - ( dd / self . cutfallrate ) ** 4 ) psf /= psf . sum ( ) edge = psf > cutval * self . cutedgeval if getextent : size = np . array ( [ ( vec * edge ) . min ( axis = ( 1 , 2 , 3 ) ) - 2 , ( vec * edge ) . max ( axis = ( 1 , 2 , 3 ) ) + 2 , ] ) . T return psf , vec , size return psf , vec return psf , vec


def _tz(self, z):
        """ Transform z to real-space coordinates from tile coordinates """
        return (z-self.param_dict['psf-zslab'])*self.param_dict[self.zscale]

def _tz ( self , z ) : return ( z - self . param_dict [ 'psf-zslab' ] ) * self . param_dict [ self . zscale ]

Transform z to real - space coordinates from tile coordinates


def _kpad(self, field, finalshape, zpad=False, norm=True):
        """
        fftshift and pad the field with zeros until it has size finalshape.
        if zpad is off, then no padding is put on the z direction. returns
        the fourier transform of the field
        """
        currshape = np.array(field.shape)

        if any(finalshape < currshape):
            raise IndexError("PSF tile size is less than minimum support size")

        d = finalshape - currshape

        # fix off-by-one issues when going odd to even tile sizes
        o = d % 2
        d = np.floor_divide(d, 2)

        if not zpad:
            o[0] = 0

        axes = None
        pad = tuple((d[i]+o[i],d[i]) for i in [0,1,2])
        rpsf = np.pad(field, pad, mode='constant', constant_values=0)
        rpsf = np.fft.ifftshift(rpsf, axes=axes)
        kpsf = fft.rfftn(rpsf, **fftkwargs)

        if norm:
            kpsf /= kpsf[0,0,0]
        return kpsf

fftshift and pad the field with zeros until it has size finalshape . if zpad is off then no padding is put on the z direction . returns the fourier transform of the field


def pack_args(self):
        """
        Pack the parameters into the form necessary for the integration
        routines above.  For example, packs for calculate_linescan_psf
        """
        mapper = {
            'psf-kfki': 'kfki',
            'psf-alpha': 'alpha',
            'psf-n2n1': 'n2n1',
            'psf-sigkf': 'sigkf',
            'psf-sph6-ab': 'sph6_ab',
            'psf-laser-wavelength': 'laser_wavelength',
            'psf-pinhole-width': 'pinhole_width'
        }
        bads = [self.zscale, 'psf-zslab']

        d = {}
        for k,v in iteritems(mapper):
            if k in self.param_dict:
                d[v] = self.param_dict[k]

        d.update({
            'polar_angle': self.polar_angle,
            'normalize': self.normalize,
            'include_K3_det':self.use_J1
        })

        if self.polychromatic:
            d.update({'nkpts': self.nkpts})
            d.update({'k_dist': self.k_dist})

        if self.do_pinhole:
            d.update({'nlpts': self.num_line_pts})

        d.update({'use_laggauss': True})
        return d

def pack_args ( self ) : mapper = { 'psf-kfki' : 'kfki' , 'psf-alpha' : 'alpha' , 'psf-n2n1' : 'n2n1' , 'psf-sigkf' : 'sigkf' , 'psf-sph6-ab' : 'sph6_ab' , 'psf-laser-wavelength' : 'laser_wavelength' , 'psf-pinhole-width' : 'pinhole_width' } bads = [ self . zscale , 'psf-zslab' ] d = { } for k , v in iteritems ( mapper ) : if k in self . param_dict : d [ v ] = self . param_dict [ k ] d . update ( { 'polar_angle' : self . polar_angle , 'normalize' : self . normalize , 'include_K3_det' : self . use_J1 } ) if self . polychromatic : d . update ( { 'nkpts' : self . nkpts } ) d . update ( { 'k_dist' : self . k_dist } ) if self . do_pinhole : d . update ( { 'nlpts' : self . num_line_pts } ) d . update ( { 'use_laggauss' : True } ) return d

Pack the parameters into the form necessary for the integration routines above . For example packs for calculate_linescan_psf


def psffunc(self, *args, **kwargs):
        """Calculates a linescan psf"""
        if self.polychromatic:
            func = psfcalc.calculate_polychrome_linescan_psf
        else:
            func = psfcalc.calculate_linescan_psf
        return func(*args, **kwargs)

def psffunc ( self , * args , ** kwargs ) : if self . polychromatic : func = psfcalc . calculate_polychrome_linescan_psf else : func = psfcalc . calculate_linescan_psf return func ( * args , ** kwargs )


def psffunc(self, x, y, z, **kwargs):
        """Calculates a pinhole psf"""
        #do_pinhole?? FIXME
        if self.polychromatic:
            func = psfcalc.calculate_polychrome_pinhole_psf
        else:
            func = psfcalc.calculate_pinhole_psf
        x0, y0 = [psfcalc.vec_to_halfvec(v) for v in [x,y]]
        vls = psfcalc.wrap_and_calc_psf(x0, y0, z, func, **kwargs)
        return vls / vls.sum()

def psffunc ( self , x , y , z , ** kwargs ) : if self . polychromatic : func = psfcalc . calculate_polychrome_pinhole_psf else : func = psfcalc . calculate_pinhole_psf x0 , y0 = [ psfcalc . vec_to_halfvec ( v ) for v in [ x , y ] ] vls = psfcalc . wrap_and_calc_psf ( x0 , y0 , z , func , ** kwargs ) return vls / vls . sum ( )


def _req(self, url, method='GET', **kw):
        '''Make request and convert JSON response to python objects'''
        send = requests.post if method == 'POST' else requests.get
        try:
            r = send(
                url,
                headers=self._token_header(),
                timeout=self.settings['timeout'],
                **kw)
        except requests.exceptions.Timeout:
            raise ApiError('Request timed out (%s seconds)' % self.settings['timeout'])
        try:
            json = r.json()
        except ValueError:
            raise ApiError('Received not JSON response from API')
        if json.get('status') != 'ok':
            raise ApiError('API error: received unexpected json from API: %s' % json)
        return json


def get_active_bets(self, project_id=None):
        '''Returns all active bets'''
        url = urljoin(
            self.settings['bets_url'],
            'bets?state=fresh,active,accept_end&page=1&page_size=100')

        if project_id is not None:
            url += '&kava_project_id={}'.format(project_id)

        bets = []
        has_next_page = True
        while has_next_page:
            res = self._req(url)
            bets.extend(res['bets']['results'])
            url = res['bets'].get('next')
            has_next_page = bool(url)

        return bets


def get_bets(self, type=None, order_by=None, state=None, project_id=None,
                 page=None, page_size=None):
        """Return bets with given filters and ordering.

        :param type: return bets only with this type.
                     Use None to include all (default).
        :param order_by: '-last_stake' or 'last_stake' to sort by stake's
                         created date or None for default ordering.
        :param state: one of 'active', 'closed', 'all' (default 'active').
        :param project_id: return bets associated with given project id in kava
        :param page: default 1.
        :param page_site: page size (default 100).
        """
        if page is None:
            page = 1
        if page_size is None:
            page_size = 100
        if state == 'all':
            _states = []  # all states == no filter
        elif state == 'closed':
            _states = self.CLOSED_STATES
        else:
            _states = self.ACTIVE_STATES

        url = urljoin(
            self.settings['bets_url'],
            'bets?page={}&page_size={}'.format(page, page_size))
        url += '&state={}'.format(','.join(_states))
        if type is not None:
            url += '&type={}'.format(type)
        if order_by in ['-last_stake', 'last_stake']:
            url += '&order_by={}'.format(order_by)
        if project_id is not None:
            url += '&kava_project_id={}'.format(project_id)

        res = self._req(url)
        return res['bets']['results']

def get_bets ( self , type = None , order_by = None , state = None , project_id = None , page = None , page_size = None ) : if page is None : page = 1 if page_size is None : page_size = 100 if state == 'all' : _states = [ ] elif state == 'closed' : _states = self . CLOSED_STATES else : _states = self . ACTIVE_STATES url = urljoin ( self . settings [ 'bets_url' ] , 'bets?page={}&page_size={}' . format ( page , page_size ) ) url += '&state={}' . format ( ',' . join ( _states ) ) if type is not None : url += '&type={}' . format ( type ) if order_by in [ '-last_stake' , 'last_stake' ] : url += '&order_by={}' . format ( order_by ) if project_id is not None : url += '&kava_project_id={}' . format ( project_id ) res = self . _req ( url ) return res [ 'bets' ] [ 'results' ]

Return bets with given filters and ordering .


def get_project_slug(self, bet):
        '''Return slug of a project that given bet is associated with
        or None if bet is not associated with any project.
        '''
        if bet.get('form_params'):
            params = json.loads(bet['form_params'])
            return params.get('project')
        return None

def get_project_slug ( self , bet ) : if bet . get ( 'form_params' ) : params = json . loads ( bet [ 'form_params' ] ) return params . get ( 'project' ) return None

Return slug of a project that given bet is associated with or None if bet is not associated with any project .


def subscribe(self, event, bet_ids):
        '''Subscribe to event for given bet ids.'''
        if not self._subscriptions.get(event):
            self._subscriptions[event] = set()
        self._subscriptions[event] = self._subscriptions[event].union(bet_ids)

def subscribe ( self , event , bet_ids ) : if not self . _subscriptions . get ( event ) : self . _subscriptions [ event ] = set ( ) self . _subscriptions [ event ] = self . _subscriptions [ event ] . union ( bet_ids )


def preview(context):
    """Opens local preview of your blog website"""

    config = context.obj

    pelican(config, '--verbose', '--ignore-cache')

    server_proc = None
    os.chdir(config['OUTPUT_DIR'])
    try:
        try:
            command = 'python -m http.server ' + str(PORT)
            server_proc = run(command, bg=True)

            time.sleep(3)
            click.launch('http://localhost:8000')

            time.sleep(5)
            pelican(config, '--autoreload')
        except Exception:
            if server_proc is not None:
                server_proc.kill()
            raise
    except KeyboardInterrupt:
        abort(context)

def
preview
(
context
)
:
config
=
context
.
obj
pelican
(
config
,
'--verbose'
,
'--ignore-cache'
)
server_proc
=
os
.
chdir
(
config
[
'OUTPUT_DIR'
]
)
try
:
try
:
command
=
'python -m http.server '
+
str
(
PORT
)
server_proc
=
run
(
command
,
bg
=
True
)
time
.
sleep
(
3
)
click
.
launch
(
'http://localhost:8000'
)
time
.
sleep
(
5
)
pelican
(
config
,
'--autoreload'
)
except
Exception
:
if
server_proc
is
not
:
server_proc
.
kill
(
)
raise
except
KeyboardInterrupt
:
abort
(
context
)


def get_collection_endpoint(cls):
        """
        Get the relative path to the API resource collection

        If self.collection_endpoint is not set, it will default to the lowercase name of the resource class plus an "s" and the terminating "/"
        :param cls: Resource class
        :return: Relative path to the resource collection
        """
        return cls.Meta.collection_endpoint if cls.Meta.collection_endpoint is not None else cls.__name__.lower() + "s/"

def get_collection_endpoint ( cls ) : return cls . Meta . collection_endpoint if cls . Meta . collection_endpoint is not None else cls . __name__ . lower ( ) + "s/"


def write(context):
    """Starts a new article"""

    config = context.obj

    title = click.prompt('Title')
    author = click.prompt('Author', default=config.get('DEFAULT_AUTHOR'))

    slug = slugify(title)
    creation_date = datetime.now()
    basename = '{:%Y-%m-%d}_{}.md'.format(creation_date, slug)
    meta = (
        ('Title', title),
        ('Date', '{:%Y-%m-%d %H:%M}:00'.format(creation_date)),
        ('Modified', '{:%Y-%m-%d %H:%M}:00'.format(creation_date)),
        ('Author', author),
    )

    file_content = ''
    for key, value in meta:
        file_content += '{}: {}\n'.format(key, value)
    file_content += '\n\n'
    file_content += 'Text...\n\n'
    file_content += '![image description]({filename}/images/my-photo.jpg)\n\n'
    file_content += 'Text...\n\n'

    os.makedirs(config['CONTENT_DIR'], exist_ok=True)
    path = os.path.join(config['CONTENT_DIR'], basename)
    with click.open_file(path, 'w') as f:
        f.write(file_content)

    click.echo(path)
    click.launch(path)


def lint(context):
    """Looks for errors in source code of your blog"""

    config = context.obj
    try:
        run('flake8 {dir} --exclude={exclude}'.format(
            dir=config['CWD'],
            exclude=','.join(EXCLUDE),
        ))
    except SubprocessError:
        context.exit(1)

def
lint
(
context
)
:
config
=
context
.
obj
try
:
run
(
'flake8 {dir} --exclude={exclude}'
.
format
(
dir
=
config
[
'CWD'
]
,
exclude
=
','
.
join
(
EXCLUDE
)
,
)
)
except
SubprocessError
:
context
.
exit
(
1
)


def set_real_value_class(self):
        """
        value_class is initially a string with the import path to the resource class, but we need to get the actual class before doing any work

        We do not expect the actual clas to be in value_class since the beginning to avoid nasty import egg-before-chicken errors
        """
        if self.value_class is not None and isinstance(self.value_class, str):
            module_name, dot, class_name = self.value_class.rpartition(".")
            module = __import__(module_name, fromlist=[class_name])
            self.value_class = getattr(module, class_name)
            self._initialized = True

def set_real_value_class ( self ) : if self . value_class is not None and isinstance ( self . value_class , str ) : module_name , dot , class_name = self . value_class . rpartition ( "." ) module = __import__ ( module_name , fromlist = [ class_name ] ) self . value_class = getattr ( module , class_name ) self . _initialized = True

value_class is initially a string with the import path to the resource class but we need to get the actual class before doing any work


def publish(context):
    """Saves changes and sends them to GitHub"""

    header('Recording changes...')
    run('git add -A')

    header('Displaying changes...')
    run('git -c color.status=always status')

    if not click.confirm('\nContinue publishing'):
        run('git reset HEAD --')
        abort(context)

    header('Saving changes...')
    try:
        run('git commit -m "{message}"'.format(
            message='Publishing {}'.format(choose_commit_emoji())
        ), capture=True)
    except subprocess.CalledProcessError as e:
        if 'nothing to commit' not in e.stdout:
            raise
        else:
            click.echo('Nothing to commit.')

    header('Pushing to GitHub...')
    branch = get_branch()
    run('git push origin {branch}:{branch}'.format(branch=branch))

    pr_link = get_pr_link(branch)
    if pr_link:
        click.launch(pr_link)

def
publish
(
context
)
:
header
(
'Recording changes...'
)
run
(
'git add -A'
)
header
(
'Displaying changes...'
)
run
(
'git -c color.status=always status'
)
if
not
click
.
confirm
(
'\nContinue publishing'
)
:
run
(
'git reset HEAD --'
)
abort
(
context
)
header
(
'Saving changes...'
)
try
:
run
(
'git commit -m "{message}"'
.
format
(
message
=
'Publishing {}'
.
format
(
choose_commit_emoji
(
)
)
)
,
capture
=
True
)
except
subprocess
.
CalledProcessError
as
e
:
if
'nothing to commit'
not
in
e
.
stdout
:
raise
else
:
click
.
echo
(
'Nothing to commit.'
)
header
(
'Pushing to GitHub...'
)
branch
=
get_branch
(
)
run
(
'git push origin {branch}:{branch}'
.
format
(
branch
=
branch
)
)
pr_link
=
get_pr_link
(
branch
)
if
pr_link
:
click
.
launch
(
pr_link
)


def deploy(context):
    """Uploads new version of the blog website"""

    config = context.obj

    header('Generating HTML...')
    pelican(config, '--verbose', production=True)

    header('Removing unnecessary output...')
    unnecessary_paths = [
        'author', 'category', 'tag', 'feeds', 'tags.html',
        'authors.html', 'categories.html', 'archives.html',
    ]
    for path in unnecessary_paths:
        remove_path(os.path.join(config['OUTPUT_DIR'], path))

    if os.environ.get('TRAVIS'):  # Travis CI
        header('Setting up Git...')
        run(
            'git config user.name ' +
            run('git show --format="%cN" -s', capture=True)
        )
        run(
            'git config user.email ' +
            run('git show --format="%cE" -s', capture=True)
        )

        github_token = os.environ.get('GITHUB_TOKEN')
        repo_slug = os.environ.get('TRAVIS_REPO_SLUG')
        origin = 'https://{}@github.com/{}.git'.format(github_token, repo_slug)
        run('git remote set-url origin ' + origin)

    header('Rewriting gh-pages branch...')
    run('ghp-import -m "{message}" {dir}'.format(
        message='Deploying {}'.format(choose_commit_emoji()),
        dir=config['OUTPUT_DIR'],
    ))

    header('Pushing to GitHub...')
    run('git push origin gh-pages:gh-pages --force')


def signed_number(number, precision=2):
    """
    Return the given number as a string with a sign in front of it, ie. `+` if the number is positive, `-` otherwise.
    """
    prefix = '' if number <= 0 else '+'
    number_str = '{}{:.{precision}f}'.format(prefix, number, precision=precision)

    return number_str

def signed_number ( number , precision = 2 ) : prefix = '' if number <= 0 else '+' number_str = '{}{:.{precision}f}' . format ( prefix , number , precision = precision ) return number_str

Return the given number as a string with a sign in front of it ie . + if the number is positive - otherwise .


def balance(ctx):
    """
    Show Zebra balance.

    Like the hours balance, vacation left, etc.
    """
    backend = plugins_registry.get_backends_by_class(ZebraBackend)[0]

    timesheet_collection = get_timesheet_collection_for_context(ctx, None)
    hours_to_be_pushed = timesheet_collection.get_hours(pushed=False, ignored=False, unmapped=False)

    today = datetime.date.today()
    user_info = backend.get_user_info()
    timesheets = backend.get_timesheets(get_first_dow(today), get_last_dow(today))
    total_duration = sum([float(timesheet['time']) for timesheet in timesheets])

    vacation = hours_to_days(user_info['vacation']['difference'])
    vacation_balance = '{} days, {:.2f} hours'.format(*vacation)

    hours_balance = user_info['hours']['hours']['balance']

    click.echo("Hours balance: {}".format(signed_number(hours_balance)))
    click.echo("Hours balance after push: {}".format(signed_number(hours_balance + hours_to_be_pushed)))
    click.echo("Hours done this week: {:.2f}".format(total_duration))
    click.echo("Vacation left: {}".format(vacation_balance))

def
balance
(
ctx
)
:
backend
=
plugins_registry
.
get_backends_by_class
(
ZebraBackend
)
[
0
]
timesheet_collection
=
get_timesheet_collection_for_context
(
ctx
,
)
hours_to_be_pushed
=
timesheet_collection
.
get_hours
(
pushed
=
False
,
ignored
=
False
,
unmapped
=
False
)
today
=
datetime
.
date
.
today
(
)
user_info
=
backend
.
get_user_info
(
)
timesheets
=
backend
.
get_timesheets
(
get_first_dow
(
today
)
,
get_last_dow
(
today
)
)
total_duration
=
sum
(
[
float
(
timesheet
[
'time'
]
)
for
timesheet
in
timesheets
]
)
vacation
=
hours_to_days
(
user_info
[
'vacation'
]
[
'difference'
]
)
vacation_balance
=
'{} days, {:.2f} hours'
.
format
(
*
vacation
)
hours_balance
=
user_info
[
'hours'
]
[
'hours'
]
[
'balance'
]
click
.
echo
(
"Hours balance: {}"
.
format
(
signed_number
(
hours_balance
)
)
)
click
.
echo
(
"Hours balance after push: {}"
.
format
(
signed_number
(
hours_balance
+
hours_to_be_pushed
)
)
)
click
.
echo
(
"Hours done this week: {:.2f}"
.
format
(
total_duration
)
)
click
.
echo
(
"Vacation left: {}"
.
format
(
vacation_balance
)
)


def show_response_messages(response_json):
    """
    Show all messages in the `messages` key of the given dict.
    """
    message_type_kwargs = {
        'warning': {'fg': 'yellow'},
        'error': {'fg': 'red'},
    }
    for message in response_json.get('messages', []):
        click.secho(message['text'], **message_type_kwargs.get(message['type'], {}))

def show_response_messages ( response_json ) : message_type_kwargs = { 'warning' : { 'fg' : 'yellow' } , 'error' : { 'fg' : 'red' } , } for message in response_json . get ( 'messages' , [ ] ) : click . secho ( message [ 'text' ] , ** message_type_kwargs . get ( message [ 'type' ] , { } ) )

Show all messages in the messages key of the given dict .


def photos(context, path):
    """Adds images to the last article"""

    config = context.obj

    header('Looking for the latest article...')
    article_filename = find_last_article(config['CONTENT_DIR'])
    if not article_filename:
        return click.secho('No articles.', fg='red')
    click.echo(os.path.basename(article_filename))

    header('Looking for images...')
    images = list(sorted(find_images(path)))
    if not images:
        return click.secho('Found no images.', fg='red')

    for filename in images:
        click.secho(filename, fg='green')

    if not click.confirm('\nAdd these images to the latest article'):
        abort(config)

    url_prefix = os.path.join('{filename}', IMAGES_PATH)
    images_dir = os.path.join(config['CONTENT_DIR'], IMAGES_PATH)
    os.makedirs(images_dir, exist_ok=True)

    header('Processing images...')
    urls = []
    for filename in images:
        image_basename = os.path.basename(filename).replace(' ', '-').lower()
        urls.append(os.path.join(url_prefix, image_basename))
        image_filename = os.path.join(images_dir, image_basename)
        print(filename, image_filename)
        import_image(filename, image_filename)

    content = '\n'
    for url in urls:
        url = url.replace('\\', '/')
        content += '\n![image description]({})\n'.format(url)

    header('Adding to article: {}'.format(article_filename))
    with click.open_file(article_filename, 'a') as f:
        f.write(content)
    click.launch(article_filename)


def _generate_circle(self):
        """Generates the circle.
        """
        total_weight = 0
        for node in self.nodes:
            total_weight += self.weights.get(node, 1)

        for node in self.nodes:
            weight = 1

            if node in self.weights:
                weight = self.weights.get(node)

            factor = math.floor((40 * len(self.nodes) * weight) / total_weight)

            for j in range(0, int(factor)):
                b_key = bytearray(self._hash_digest('%s-%s' % (node, j)))

                for i in range(0, 3):
                    key = self._hash_val(b_key, lambda x: x + i * 4)
                    self.ring[key] = node
                    self._sorted_keys.append(key)

        self._sorted_keys.sort()

def _generate_circle ( self ) : total_weight = 0 for node in self . nodes : total_weight += self . weights . get ( node , 1 ) for node in self . nodes : weight = 1 if node in self . weights : weight = self . weights . get ( node ) factor = math . floor ( ( 40 * len ( self . nodes ) * weight ) / total_weight ) for j in range ( 0 , int ( factor ) ) : b_key = bytearray ( self . _hash_digest ( '%s-%s' % ( node , j ) ) ) for i in range ( 0 , 3 ) : key = self . _hash_val ( b_key , lambda x : x + i * 4 ) self . ring [ key ] = node self . _sorted_keys . append ( key ) self . _sorted_keys . sort ( )


def get_node(self, string_key):
        """Given a string key a corresponding node in the hash ring is returned.

        If the hash ring is empty, `None` is returned.
        """
        pos = self.get_node_pos(string_key)
        if pos is None:
            return None
        return self.ring[self._sorted_keys[pos]]

def get_node ( self , string_key ) : pos = self . get_node_pos ( string_key ) if pos is None : return None return self . ring [ self . _sorted_keys [ pos ] ]

Given a string key a corresponding node in the hash ring is returned .


def gen_key(self, key):
        """Given a string key it returns a long value,
        this long value represents a place on the hash ring.

        md5 is currently used because it mixes well.
        """
        b_key = self._hash_digest(key)
        return self._hash_val(b_key, lambda x: x)

def gen_key ( self , key ) : b_key = self . _hash_digest ( key ) return self . _hash_val ( b_key , lambda x : x )

Given a string key it returns a long value this long value represents a place on the hash ring .


def _get_networking_mode(app):
    """
    Get the Marathon networking mode for the app.
    """
    # Marathon 1.5+: there is a `networks` field
    networks = app.get('networks')
    if networks:
        # Modes cannot be mixed, so assigning the last mode is fine
        return networks[-1].get('mode', 'container')

    # Older Marathon: determine equivalent network mode
    container = app.get('container')
    if container is not None and 'docker' in container:
        docker_network = container['docker'].get('network')
        if docker_network == 'USER':
            return 'container'
        elif docker_network == 'BRIDGE':
            return 'container/bridge'

    return 'container' if _is_legacy_ip_per_task(app) else 'host'

def _get_networking_mode ( app ) : networks = app . get ( 'networks' ) if networks : return networks [ - 1 ] . get ( 'mode' , 'container' ) container = app . get ( 'container' ) if container is not None and 'docker' in container : docker_network = container [ 'docker' ] . get ( 'network' ) if docker_network == 'USER' : return 'container' elif docker_network == 'BRIDGE' : return 'container/bridge' return 'container' if _is_legacy_ip_per_task ( app ) else 'host'

Get the Marathon networking mode for the app .


def _get_container_port_mappings(app):
    """
    Get the ``portMappings`` field for the app container.
    """
    container = app['container']

    # Marathon 1.5+: container.portMappings field
    port_mappings = container.get('portMappings')

    # Older Marathon: container.docker.portMappings field
    if port_mappings is None and 'docker' in container:
        port_mappings = container['docker'].get('portMappings')

    return port_mappings

def _get_container_port_mappings ( app ) : container = app [ 'container' ] port_mappings = container . get ( 'portMappings' ) if port_mappings is None and 'docker' in container : port_mappings = container [ 'docker' ] . get ( 'portMappings' ) return port_mappings

Get the portMappings field for the app container .


def sort_pem_objects(pem_objects):
    """
    Given a list of pem objects, sort the objects into the private key, leaf
    certificate, and list of CA certificates in the trust chain. This function
    assumes that the list of pem objects will contain exactly one private key
    and exactly one leaf certificate and that only key and certificate type
    objects are provided.
    """
    keys, certs, ca_certs = [], [], []
    for pem_object in pem_objects:
        if isinstance(pem_object, pem.Key):
            keys.append(pem_object)
        else:
            # This assumes all pem objects provided are either of type pem.Key
            # or pem.Certificate. Technically, there are CSR and CRL types, but
            # we should never be passed those.
            if _is_ca(pem_object):
                ca_certs.append(pem_object)
            else:
                certs.append(pem_object)

    [key], [cert] = keys, certs
    return key, cert, ca_certs

def sort_pem_objects ( pem_objects ) : keys , certs , ca_certs = [ ] , [ ] , [ ] for pem_object in pem_objects : if isinstance ( pem_object , pem . Key ) : keys . append ( pem_object ) else : if _is_ca ( pem_object ) : ca_certs . append ( pem_object ) else : certs . append ( pem_object ) [ key ] , [ cert ] = keys , certs return key , cert , ca_certs

Given a list of pem objects sort the objects into the private key leaf certificate and list of CA certificates in the trust chain . This function assumes that the list of pem objects will contain exactly one private key and exactly one leaf certificate and that only key and certificate type objects are provided .


def raise_for_not_ok_status(response):
    """
    Raises a `requests.exceptions.HTTPError` if the response has a non-200
    status code.
    """
    if response.code != OK:
        raise HTTPError('Non-200 response code (%s) for url: %s' % (
            response.code, uridecode(response.request.absoluteURI)))

    return response

Raises a requests . exceptions . HTTPError if the response has a non - 200 status code .


def _sse_content_with_protocol(response, handler, **sse_kwargs):
    """
    Sometimes we need the protocol object so that we can manipulate the
    underlying transport in tests.
    """
    protocol = SseProtocol(handler, **sse_kwargs)
    finished = protocol.when_finished()

    response.deliverBody(protocol)

    return finished, protocol

def _sse_content_with_protocol ( response , handler , ** sse_kwargs ) : protocol = SseProtocol ( handler , ** sse_kwargs ) finished = protocol . when_finished ( ) response . deliverBody ( protocol ) return finished , protocol

Sometimes we need the protocol object so that we can manipulate the underlying transport in tests .


def sse_content(response, handler, **sse_kwargs):
    """
    Callback to collect the Server-Sent Events content of a response. Callbacks
    passed will receive event data.

    :param response:
        The response from the SSE request.
    :param handler:
        The handler for the SSE protocol.
    """
    # An SSE response must be 200/OK and have content-type 'text/event-stream'
    raise_for_not_ok_status(response)
    raise_for_header(response, 'Content-Type', 'text/event-stream')

    finished, _ = _sse_content_with_protocol(response, handler, **sse_kwargs)
    return finished

def sse_content ( response , handler , ** sse_kwargs ) : raise_for_not_ok_status ( response ) raise_for_header ( response , 'Content-Type' , 'text/event-stream' ) finished , _ = _sse_content_with_protocol ( response , handler , ** sse_kwargs ) return finished

Callback to collect the Server - Sent Events content of a response . Callbacks passed will receive event data .


def _request(self, failure, endpoints, *args, **kwargs):
        """
        Recursively make requests to each endpoint in ``endpoints``.
        """
        # We've run out of endpoints, fail
        if not endpoints:
            return failure

        endpoint = endpoints.pop(0)
        d = super(MarathonClient, self).request(*args, url=endpoint, **kwargs)

        # If something goes wrong, call ourselves again with the remaining
        # endpoints
        d.addErrback(self._request, endpoints, *args, **kwargs)
        return d

def _request ( self , failure , endpoints , * args , ** kwargs ) : if not endpoints : return failure endpoint = endpoints . pop ( 0 ) d = super ( MarathonClient , self ) . request ( * args , url = endpoint , ** kwargs ) d . addErrback ( self . _request , endpoints , * args , ** kwargs ) return d

Recursively make requests to each endpoint in endpoints .


def get_json_field(self, field, **kwargs):
        """
        Perform a GET request and get the contents of the JSON response.

        Marathon's JSON responses tend to contain an object with a single key
        which points to the actual data of the response. For example /v2/apps
        returns something like {"apps": [ {"app1"}, {"app2"} ]}. We're
        interested in the contents of "apps".

        This method will raise an error if:
        * There is an error response code
        * The field with the given name cannot be found
        """
        d = self.request(
            'GET', headers={'Accept': 'application/json'}, **kwargs)
        d.addCallback(raise_for_status)
        d.addCallback(raise_for_header, 'Content-Type', 'application/json')
        d.addCallback(json_content)
        d.addCallback(self._get_json_field, field)
        return d

def get_json_field ( self , field , ** kwargs ) : d = self . request ( 'GET' , headers = { 'Accept' : 'application/json' } , ** kwargs ) d . addCallback ( raise_for_status ) d . addCallback ( raise_for_header , 'Content-Type' , 'application/json' ) d . addCallback ( json_content ) d . addCallback ( self . _get_json_field , field ) return d

Perform a GET request and get the contents of the JSON response .


def _get_json_field(self, response_json, field_name):
        """
        Get a JSON field from the response JSON.

        :param: response_json:
            The parsed JSON content of the response.
        :param: field_name:
            The name of the field in the JSON to get.
        """
        if field_name not in response_json:
            raise KeyError('Unable to get value for "%s" from Marathon '
                           'response: "%s"' % (
                               field_name, json.dumps(response_json),))

        return response_json[field_name]

Get a JSON field from the response JSON .


def parse(
        self,
        value: str,
        type_: typing.Type[typing.Any] = str,
        subtype: typing.Type[typing.Any] = str,
    ) -> typing.Any:
        """
        Parse value from string.

        Convert :code:`value` to

        .. code-block:: python

           >>> parser = Config()
           >>> parser.parse('12345', type_=int)
           <<< 12345
           >>>
           >>> parser.parse('1,2,3,4', type_=list, subtype=int)
           <<< [1, 2, 3, 4]

        :param value: string
        :param type\\_: the type to return
        :param subtype: subtype for iterator types
        :return: the parsed config value

        """
        if type_ is bool:
            return type_(value.lower() in self.TRUE_STRINGS)

        try:
            if isinstance(type_, type) and issubclass(
                type_, (list, tuple, set, frozenset)
            ):
                return type_(
                    self.parse(v.strip(" "), subtype)
                    for v in value.split(",")
                    if value.strip(" ")
                )

            return type_(value)
        except ValueError as e:
            raise ConfigError(*e.args)

def
parse
(
self
,
value
:
str
,
type_
:
typing
.
Type
[
typing
.
Any
]
=
str
,
subtype
:
typing
.
Type
[
typing
.
Any
]
=
str
,
)
->
typing
.
Any
:
if
type_
is
bool
:
return
type_
(
value
.
lower
(
)
in
self
.
TRUE_STRINGS
)
try
:
if
isinstance
(
type_
,
type
)
and
issubclass
(
type_
,
(
list
,
tuple
,
set
,
frozenset
)
)
:
return
type_
(
self
.
parse
(
v
.
strip
(
" "
)
,
subtype
)
for
v
in
value
.
split
(
","
)
if
value
.
strip
(
" "
)
)
return
type_
(
value
)
except
ValueError
as
e
:
raise
ConfigError
(
*
e
.
args
)


def get(
        self,
        key: str,
        default: typing.Any = UNSET,
        type_: typing.Type[typing.Any] = str,
        subtype: typing.Type[typing.Any] = str,
        mapper: typing.Optional[typing.Callable[[object], object]] = None,
    ) -> typing.Any:
        """
        Parse a value from an environment variable.

        .. code-block:: python

           >>> os.environ['FOO']
           <<< '12345'
           >>>
           >>> os.environ['BAR']
           <<< '1,2,3,4'
           >>>
           >>> 'BAZ' in os.environ
           <<< False
           >>>
           >>> parser = Config()
           >>> parser.get('FOO', type_=int)
           <<< 12345
           >>>
           >>> parser.get('BAR', type_=list, subtype=int)
           <<< [1, 2, 3, 4]
           >>>
           >>> parser.get('BAZ', default='abc123')
           <<< 'abc123'
           >>>
           >>> parser.get('FOO', type_=int, mapper=lambda x: x*10)
           <<< 123450

        :param key: the key to look up the value under
        :param default: default value to return when when no value is present
        :param type\\_: the type to return
        :param subtype: subtype for iterator types
        :param mapper: a function to post-process the value with
        :return: the parsed config value

        """
        value = self.environ.get(key, UNSET)

        if value is UNSET and default is UNSET:
            raise ConfigError("Unknown environment variable: {0}".format(key))

        if value is UNSET:
            value = default
        else:
            value = self.parse(typing.cast(str, value), type_, subtype)

        if mapper:
            value = mapper(value)

        return value

def
get
(
self
,
key
:
str
,
default
:
typing
.
Any
=
UNSET
,
type_
:
typing
.
Type
[
typing
.
Any
]
=
str
,
subtype
:
typing
.
Type
[
typing
.
Any
]
=
str
,
mapper
:
typing
.
Optional
[
typing
.
Callable
[
[
object
]
,
object
]
]
=
,
)
->
typing
.
Any
:
value
=
self
.
environ
.
get
(
key
,
UNSET
)
if
value
is
UNSET
and
default
is
UNSET
:
raise
ConfigError
(
"Unknown environment variable: {0}"
.
format
(
key
)
)
if
value
is
UNSET
:
value
=
default
else
:
value
=
self
.
parse
(
typing
.
cast
(
str
,
value
)
,
type_
,
subtype
)
if
mapper
:
value
=
mapper
(
value
)
return
value

Parse a value from an environment variable .


def _request(self, endpoint, *args, **kwargs):
        """
        Perform a request to a specific endpoint. Raise an error if the status
        code indicates a client or server error.
        """
        kwargs['url'] = endpoint
        return (super(MarathonLbClient, self).request(*args, **kwargs)
                .addCallback(raise_for_status))

def _request ( self , endpoint , * args , ** kwargs ) : kwargs [ 'url' ] = endpoint return ( super ( MarathonLbClient , self ) . request ( * args , ** kwargs ) . addCallback ( raise_for_status ) )

Perform a request to a specific endpoint . Raise an error if the status code indicates a client or server error .


def _check_request_results(self, results):
        """
        Check the result of each request that we made. If a failure occurred,
        but some requests succeeded, log and count the failures. If all
        requests failed, raise an error.

        :return:
            The list of responses, with a None value for any requests that
            failed.
        """
        responses = []
        failed_endpoints = []
        for index, result_tuple in enumerate(results):
            success, result = result_tuple
            if success:
                responses.append(result)
            else:
                endpoint = self.endpoints[index]
                self.log.failure(
                    'Failed to make a request to a marathon-lb instance: '
                    '{endpoint}', result, LogLevel.error, endpoint=endpoint)
                responses.append(None)
                failed_endpoints.append(endpoint)

        if len(failed_endpoints) == len(self.endpoints):
            raise RuntimeError(
                'Failed to make a request to all marathon-lb instances')

        if failed_endpoints:
            self.log.error(
                'Failed to make a request to {x}/{y} marathon-lb instances: '
                '{endpoints}', x=len(failed_endpoints), y=len(self.endpoints),
                endpoints=failed_endpoints)

        return responses

Check the result of each request that we made . If a failure occurred but some requests succeeded log and count the failures . If all requests failed raise an error .


def maybe_key(pem_path):
    """
    Set up a client key if one does not exist already.

    https://gist.github.com/glyph/27867a478bb71d8b6046fbfb176e1a33#file-local-certs-py-L32-L50

    :type pem_path: twisted.python.filepath.FilePath
    :param pem_path:
        The path to the certificate directory to use.
    :rtype: twisted.internet.defer.Deferred
    """
    acme_key_file = pem_path.child(u'client.key')
    if acme_key_file.exists():
        key = _load_pem_private_key_bytes(acme_key_file.getContent())
    else:
        key = generate_private_key(u'rsa')
        acme_key_file.setContent(_dump_pem_private_key_bytes(key))
    return succeed(JWKRSA(key=key))

def maybe_key ( pem_path ) : acme_key_file = pem_path . child ( u'client.key' ) if acme_key_file . exists ( ) : key = _load_pem_private_key_bytes ( acme_key_file . getContent ( ) ) else : key = generate_private_key ( u'rsa' ) acme_key_file . setContent ( _dump_pem_private_key_bytes ( key ) ) return succeed ( JWKRSA ( key = key ) )

Set up a client key if one does not exist already .


def maybe_key_vault(client, mount_path):
    """
    Set up a client key in Vault if one does not exist already.

    :param client:
        The Vault API client to use.
    :param mount_path:
        The Vault key/value mount path to use.
    :rtype: twisted.internet.defer.Deferred
    """
    d = client.read_kv2('client_key', mount_path=mount_path)

    def get_or_create_key(client_key):
        if client_key is not None:
            key_data = client_key['data']['data']
            key = _load_pem_private_key_bytes(key_data['key'].encode('utf-8'))
            return JWKRSA(key=key)
        else:
            key = generate_private_key(u'rsa')
            key_data = {
                'key': _dump_pem_private_key_bytes(key).decode('utf-8')
            }
            d = client.create_or_update_kv2(
                'client_key', key_data, mount_path=mount_path)

            return d.addCallback(lambda _result: JWKRSA(key=key))

    return d.addCallback(get_or_create_key)

def maybe_key_vault ( client , mount_path ) : d = client . read_kv2 ( 'client_key' , mount_path = mount_path ) def get_or_create_key ( client_key ) : if client_key is not None : key_data = client_key [ 'data' ] [ 'data' ] key = _load_pem_private_key_bytes ( key_data [ 'key' ] . encode ( 'utf-8' ) ) return JWKRSA ( key = key ) else : key = generate_private_key ( u'rsa' ) key_data = { 'key' : _dump_pem_private_key_bytes ( key ) . decode ( 'utf-8' ) } d = client . create_or_update_kv2 ( 'client_key' , key_data , mount_path = mount_path ) return d . addCallback ( lambda _result : JWKRSA ( key = key ) ) return d . addCallback ( get_or_create_key )

Set up a client key in Vault if one does not exist already .


def read(self, path, **params):
        """
        Read data from Vault. Returns the JSON-decoded response.
        """
        d = self.request('GET', '/v1/' + path, params=params)
        return d.addCallback(self._handle_response)

def read ( self , path , ** params ) : d = self . request ( 'GET' , '/v1/' + path , params = params ) return d . addCallback ( self . _handle_response )

Read data from Vault . Returns the JSON - decoded response .


def write(self, path, **data):
        """
        Write data to Vault. Returns the JSON-decoded response.
        """
        d = self.request('PUT', '/v1/' + path, json=data)
        return d.addCallback(self._handle_response, check_cas=True)

def write ( self , path , ** data ) : d = self . request ( 'PUT' , '/v1/' + path , json = data ) return d . addCallback ( self . _handle_response , check_cas = True )

Write data to Vault . Returns the JSON - decoded response .


def get_single_header(headers, key):
    """
    Get a single value for the given key out of the given set of headers.

    :param twisted.web.http_headers.Headers headers:
        The set of headers in which to look for the header value
    :param str key:
        The header key
    """
    raw_headers = headers.getRawHeaders(key)
    if raw_headers is None:
        return None

    # Take the final header as the authorative
    header, _ = cgi.parse_header(raw_headers[-1])
    return header

def get_single_header ( headers , key ) : raw_headers = headers . getRawHeaders ( key ) if raw_headers is None : return None header , _ = cgi . parse_header ( raw_headers [ - 1 ] ) return header

Get a single value for the given key out of the given set of headers .


def request(self, method, url=None, **kwargs):
        """
        Perform a request.

        :param: method:
            The HTTP method to use (example is `GET`).
        :param: url:
            The URL to use. The default value is the URL this client was
            created with (`self.url`) (example is `http://localhost:8080`)
        :param: kwargs:
            Any other parameters that will be passed to `treq.request`, for
            example headers. Or any URL parameters to override, for example
            path, query or fragment.
        """
        url = self._compose_url(url, kwargs)

        kwargs.setdefault('timeout', self._timeout)

        d = self._client.request(method, url, reactor=self._reactor, **kwargs)

        d.addCallback(self._log_request_response, method, url, kwargs)
        d.addErrback(self._log_request_error, url)

        return d

def request ( self , method , url = None , ** kwargs ) : url = self . _compose_url ( url , kwargs ) kwargs . setdefault ( 'timeout' , self . _timeout ) d = self . _client . request ( method , url , reactor = self . _reactor , ** kwargs ) d . addCallback ( self . _log_request_response , method , url , kwargs ) d . addErrback ( self . _log_request_error , url ) return d


def listen(self, reactor, endpoint_description):
        """
        Run the server, i.e. start listening for requests on the given host and
        port.

        :param reactor: The ``IReactorTCP`` to use.
        :param endpoint_description:
            The Twisted description for the endpoint to listen on.
        :return:
            A deferred that returns an object that provides ``IListeningPort``.
        """
        endpoint = serverFromString(reactor, endpoint_description)
        return endpoint.listen(Site(self.app.resource()))

def listen ( self , reactor , endpoint_description ) : endpoint = serverFromString ( reactor , endpoint_description ) return endpoint . listen ( Site ( self . app . resource ( ) ) )

Run the server i . e . start listening for requests on the given host and port .


def create_marathon_acme(
    client_creator, cert_store, acme_email, allow_multiple_certs,
    marathon_addrs, marathon_timeout, sse_timeout, mlb_addrs, group,
        reactor):
    """
    Create a marathon-acme instance.

    :param client_creator:
        The txacme client creator function.
    :param cert_store:
        The txacme certificate store instance.
    :param acme_email:
        Email address to use when registering with the ACME service.
    :param allow_multiple_certs:
        Whether to allow multiple certificates per app port.
    :param marathon_addr:
        Address for the Marathon instance to find app domains that require
        certificates.
    :param marathon_timeout:
        Amount of time in seconds to wait for response headers to be received
        from Marathon.
    :param sse_timeout:
        Amount of time in seconds to wait for some event data to be received
        from Marathon.
    :param mlb_addrs:
        List of addresses for marathon-lb instances to reload when a new
        certificate is issued.
    :param group:
        The marathon-lb group (``HAPROXY_GROUP``) to consider when finding
        app domains.
    :param reactor: The reactor to use.
    """
    marathon_client = MarathonClient(marathon_addrs, timeout=marathon_timeout,
                                     sse_kwargs={'timeout': sse_timeout},
                                     reactor=reactor)
    marathon_lb_client = MarathonLbClient(mlb_addrs, reactor=reactor)

    return MarathonAcme(
        marathon_client,
        group,
        cert_store,
        marathon_lb_client,
        client_creator,
        reactor,
        acme_email,
        allow_multiple_certs
    )

def create_marathon_acme ( client_creator , cert_store , acme_email , allow_multiple_certs , marathon_addrs , marathon_timeout , sse_timeout , mlb_addrs , group , reactor ) : marathon_client = MarathonClient ( marathon_addrs , timeout = marathon_timeout , sse_kwargs = { 'timeout' : sse_timeout } , reactor = reactor ) marathon_lb_client = MarathonLbClient ( mlb_addrs , reactor = reactor ) return MarathonAcme ( marathon_client , group , cert_store , marathon_lb_client , client_creator , reactor , acme_email , allow_multiple_certs )


def init_storage_dir(storage_dir):
    """
    Initialise the storage directory with the certificates directory and a
    default wildcard self-signed certificate for HAProxy.

    :return: the storage path and certs path
    """
    storage_path = FilePath(storage_dir)

    # Create the default wildcard certificate if it doesn't already exist
    default_cert_path = storage_path.child('default.pem')
    if not default_cert_path.exists():
        default_cert_path.setContent(generate_wildcard_pem_bytes())

    # Create a directory for unmanaged certs. We don't touch this again, but it
    # needs to be there and it makes sense to create it at the same time as
    # everything else.
    unmanaged_certs_path = storage_path.child('unmanaged-certs')
    if not unmanaged_certs_path.exists():
        unmanaged_certs_path.createDirectory()

    # Store certificates in a directory inside the storage directory, so
    # HAProxy will read just the certificates there.
    certs_path = storage_path.child('certs')
    if not certs_path.exists():
        certs_path.createDirectory()

    return storage_path, certs_path

def init_storage_dir ( storage_dir ) : storage_path = FilePath ( storage_dir ) default_cert_path = storage_path . child ( 'default.pem' ) if not default_cert_path . exists ( ) : default_cert_path . setContent ( generate_wildcard_pem_bytes ( ) ) unmanaged_certs_path = storage_path . child ( 'unmanaged-certs' ) if not unmanaged_certs_path . exists ( ) : unmanaged_certs_path . createDirectory ( ) certs_path = storage_path . child ( 'certs' ) if not certs_path . exists ( ) : certs_path . createDirectory ( ) return storage_path , certs_path

Initialise the storage directory with the certificates directory and a default wildcard self - signed certificate for HAProxy .


def init_logging(log_level):
    """
    Initialise the logging by adding an observer to the global log publisher.

    :param str log_level: The minimum log level to log messages for.
    """
    log_level_filter = LogLevelFilterPredicate(
        LogLevel.levelWithName(log_level))
    log_level_filter.setLogLevelForNamespace(
        'twisted.web.client._HTTP11ClientFactory', LogLevel.warn)
    log_observer = FilteringLogObserver(
        textFileLogObserver(sys.stdout), [log_level_filter])
    globalLogPublisher.addObserver(log_observer)

Initialise the logging by adding an observer to the global log publisher .


def _parse_field_value(line):
    """ Parse the field and value from a line. """
    if line.startswith(':'):
        # Ignore the line
        return None, None

    if ':' not in line:
        # Treat the entire line as the field, use empty string as value
        return line, ''

    # Else field is before the ':' and value is after
    field, value = line.split(':', 1)

    # If value starts with a space, remove it.
    value = value[1:] if value.startswith(' ') else value

    return field, value

def
_parse_field_value
(
line
)
:
if
line
.
startswith
(
':'
)
:
return
,
if
':'
not
in
line
:
return
line
,
''
field
,
value
=
line
.
split
(
':'
,
1
)
value
=
value
[
1
:
]
if
value
.
startswith
(
' '
)
else
value
return
field
,
value


def dataReceived(self, data):
        """
        Translates bytes into lines, and calls lineReceived.

        Copied from ``twisted.protocols.basic.LineOnlyReceiver`` but using
        str.splitlines() to split on ``\r\n``, ``\n``, and ``\r``.
        """
        self.resetTimeout()
        lines = (self._buffer + data).splitlines()

        # str.splitlines() doesn't split the string after a trailing newline
        # character so we must check if there is a trailing newline and, if so,
        # clear the buffer as the line is "complete". Else, the line is
        # incomplete and we keep the last line in the buffer.
        if data.endswith(b'\n') or data.endswith(b'\r'):
            self._buffer = b''
        else:
            self._buffer = lines.pop(-1)

        for line in lines:
            if self.transport.disconnecting:
                # this is necessary because the transport may be told to lose
                # the connection by a line within a larger packet, and it is
                # important to disregard all the lines in that packet following
                # the one that told it to close.
                return
            if len(line) > self._max_length:
                self.lineLengthExceeded(line)
                return
            else:
                self.lineReceived(line)
        if len(self._buffer) > self._max_length:
            self.lineLengthExceeded(self._buffer)
            return

def dataReceived ( self , data ) : self . resetTimeout ( ) lines = ( self . _buffer + data ) . splitlines ( ) if data . endswith ( b'\n' ) or data . endswith ( b'\r' ) : self . _buffer = b'' else : self . _buffer = lines . pop ( - 1 ) for line in lines : if self . transport . disconnecting : return if len ( line ) > self . _max_length : self . lineLengthExceeded ( line ) return else : self . lineReceived ( line ) if len ( self . _buffer ) > self . _max_length : self . lineLengthExceeded ( self . _buffer ) return

Translates bytes into lines and calls lineReceived .


def _handle_field_value(self, field, value):
        """ Handle the field, value pair. """
        if field == 'event':
            self._event = value
        elif field == 'data':
            self._data_lines.append(value)
        elif field == 'id':
            # Not implemented
            pass
        elif field == 'retry':
            # Not implemented
            pass

def _handle_field_value ( self , field , value ) : if field == 'event' : self . _event = value elif field == 'data' : self . _data_lines . append ( value ) elif field == 'id' : pass elif field == 'retry' : pass


def _dispatch_event(self):
        """
        Dispatch the event to the handler.
        """
        data = self._prepare_data()
        if data is not None:
            self._handler(self._event, data)

        self._reset_event_data()

def _dispatch_event ( self ) : data = self . _prepare_data ( ) if data is not None : self . _handler ( self . _event , data ) self . _reset_event_data ( )


def listen_events(self, reconnects=0):
        """
        Start listening for events from Marathon, running a sync when we first
        successfully subscribe and triggering a sync on API request events.
        """
        self.log.info('Listening for events from Marathon...')
        self._attached = False

        def on_finished(result, reconnects):
            # If the callback fires then the HTTP request to the event stream
            # went fine, but the persistent connection for the SSE stream was
            # dropped. Just reconnect for now- if we can't actually connect
            # then the errback will fire rather.
            self.log.warn('Connection lost listening for events, '
                          'reconnecting... ({reconnects} so far)',
                          reconnects=reconnects)
            reconnects += 1
            return self.listen_events(reconnects)

        def log_failure(failure):
            self.log.failure('Failed to listen for events', failure)
            return failure

        return self.marathon_client.get_events({
            'event_stream_attached': self._sync_on_event_stream_attached,
            'api_post_event': self._sync_on_api_post_event
        }).addCallbacks(on_finished, log_failure, callbackArgs=[reconnects])

Start listening for events from Marathon running a sync when we first successfully subscribe and triggering a sync on API request events .


def sync(self):
        """
        Fetch the list of apps from Marathon, find the domains that require
        certificates, and issue certificates for any domains that don't already
        have a certificate.
        """
        self.log.info('Starting a sync...')

        def log_success(result):
            self.log.info('Sync completed successfully')
            return result

        def log_failure(failure):
            self.log.failure('Sync failed', failure, LogLevel.error)
            return failure

        return (self.marathon_client.get_apps()
                .addCallback(self._apps_acme_domains)
                .addCallback(self._filter_new_domains)
                .addCallback(self._issue_certs)
                .addCallbacks(log_success, log_failure))

def
sync
(
self
)
:
self
.
log
.
info
(
'Starting a sync...'
)
def
log_success
(
result
)
:
self
.
log
.
info
(
'Sync completed successfully'
)
return
result
def
log_failure
(
failure
)
:
self
.
log
.
failure
(
'Sync failed'
,
failure
,
LogLevel
.
error
)
return
failure
return
(
self
.
marathon_client
.
get_apps
(
)
.
addCallback
(
self
.
_apps_acme_domains
)
.
addCallback
(
self
.
_filter_new_domains
)
.
addCallback
(
self
.
_issue_certs
)
.
addCallbacks
(
log_success
,
log_failure
)
)

Fetch the list of apps from Marathon find the domains that require certificates and issue certificates for any domains that don t already have a certificate .


def _issue_cert(self, domain):
        """
        Issue a certificate for the given domain.
        """
        def errback(failure):
            # Don't fail on some of the errors we could get from the ACME
            # server, rather just log an error so that we can continue with
            # other domains.
            failure.trap(txacme_ServerError)
            acme_error = failure.value.message

            if acme_error.code in ['rateLimited', 'serverInternal',
                                   'connection', 'unknownHost']:
                # TODO: Fire off an error to Sentry or something?
                self.log.error(
                    'Error ({code}) issuing certificate for "{domain}": '
                    '{detail}', code=acme_error.code, domain=domain,
                    detail=acme_error.detail)
            else:
                # There are more error codes but if they happen then something
                # serious has gone wrong-- carry on error-ing.
                return failure

        d = self.txacme_service.issue_cert(domain)
        return d.addErrback(errback)

Issue a certificate for the given domain .


def rm_fwd_refs(obj):
    """When removing an object, other objects with references to the current
    object should remove those references. This function identifies objects
    with forward references to the current object, then removes those
    references.

    :param obj: Object to which forward references should be removed

    """
    for stack, key in obj._backrefs_flat:

        # Unpack stack
        backref_key, parent_schema_name, parent_field_name = stack

        # Get parent info
        parent_schema = obj._collections[parent_schema_name]
        parent_key_store = parent_schema._pk_to_storage(key)
        parent_object = parent_schema.load(parent_key_store)
        if parent_object is None:
            continue

        # Remove forward references
        if parent_object._fields[parent_field_name]._list:
            getattr(parent_object, parent_field_name).remove(obj)
        else:
            parent_field_object = parent_object._fields[parent_field_name]
            setattr(parent_object, parent_field_name, parent_field_object._gen_default())

        # Save
        parent_object.save()

def rm_fwd_refs ( obj ) : for stack , key in obj . _backrefs_flat : backref_key , parent_schema_name , parent_field_name = stack parent_schema = obj . _collections [ parent_schema_name ] parent_key_store = parent_schema . _pk_to_storage ( key ) parent_object = parent_schema . load ( parent_key_store ) if parent_object is None : continue if parent_object . _fields [ parent_field_name ] . _list : getattr ( parent_object , parent_field_name ) . remove ( obj ) else : parent_field_object = parent_object . _fields [ parent_field_name ] setattr ( parent_object , parent_field_name , parent_field_object . _gen_default ( ) ) parent_object . save ( )

When removing an object other objects with references to the current object should remove those references . This function identifies objects with forward references to the current object then removes those references .


def rm_back_refs(obj):
    """When removing an object with foreign fields, back-references from
    other objects to the current object should be deleted. This function
    identifies foreign fields of the specified object whose values are not
    None and which specify back-reference keys, then removes back-references
    from linked objects to the specified object.

    :param obj: Object for which back-references should be removed

    """
    for ref in _collect_refs(obj):
        ref['value']._remove_backref(
            ref['field_instance']._backref_field_name,
            obj,
            ref['field_name'],
            strict=False
        )

def rm_back_refs ( obj ) : for ref in _collect_refs ( obj ) : ref [ 'value' ] . _remove_backref ( ref [ 'field_instance' ] . _backref_field_name , obj , ref [ 'field_name' ] , strict = False )

When removing an object with foreign fields back - references from other objects to the current object should be deleted . This function identifies foreign fields of the specified object whose values are not None and which specify back - reference keys then removes back - references from linked objects to the specified object .


def ensure_backrefs(obj, fields=None):
    """Ensure that all forward references on the provided object have the
    appropriate backreferences.

    :param StoredObject obj: Database record
    :param list fields: Optional list of field names to check

    """
    for ref in _collect_refs(obj, fields):
        updated = ref['value']._update_backref(
            ref['field_instance']._backref_field_name,
            obj,
            ref['field_name'],
        )
        if updated:
            logging.debug('Updated reference {}:{}:{}:{}:{}'.format(
                obj._name, obj._primary_key, ref['field_name'],
                ref['value']._name, ref['value']._primary_key,
            ))

def
ensure_backrefs
(
obj
,
fields
=
)
:
for
ref
in
_collect_refs
(
obj
,
fields
)
:
updated
=
ref
[
'value'
]
.
_update_backref
(
ref
[
'field_instance'
]
.
_backref_field_name
,
obj
,
ref
[
'field_name'
]
,
)
if
updated
:
logging
.
debug
(
'Updated reference {}:{}:{}:{}:{}'
.
format
(
obj
.
_name
,
obj
.
_primary_key
,
ref
[
'field_name'
]
,
ref
[
'value'
]
.
_name
,
ref
[
'value'
]
.
_primary_key
,
)
)

Ensure that all forward references on the provided object have the appropriate backreferences .


def _remove_by_pk(self, key, flush=True):
        """Retrieve value from store.

        :param key: Key

        """
        try:
            del self.store[key]
        except Exception as error:
            pass
        if flush:
            self.flush()

def _remove_by_pk ( self , key , flush = True ) : try : del self . store [ key ] except Exception as error : pass if flush : self . flush ( )


def start(self):
        """Initializes the bot, plugins, and everything."""
        self.bot_start_time = datetime.now()
        self.webserver = Webserver(self.config['webserver']['host'], self.config['webserver']['port'])
        self.plugins.load()
        self.plugins.load_state()
        self._find_event_handlers()
        self.sc = ThreadedSlackClient(self.config['slack_token'])

        self.always_send_dm = ['_unauthorized_']
        if 'always_send_dm' in self.config:
            self.always_send_dm.extend(map(lambda x: '!' + x, self.config['always_send_dm']))

        # Rocket is very noisy at debug
        logging.getLogger('Rocket.Errors.ThreadPool').setLevel(logging.INFO)

        self.is_setup = True
        if self.test_mode:
            self.metrics['startup_time'] = (datetime.now() - self.bot_start_time).total_seconds() * 1000.0

def start ( self ) : self . bot_start_time = datetime . now ( ) self . webserver = Webserver ( self . config [ 'webserver' ] [ 'host' ] , self . config [ 'webserver' ] [ 'port' ] ) self . plugins . load ( ) self . plugins . load_state ( ) self . _find_event_handlers ( ) self . sc = ThreadedSlackClient ( self . config [ 'slack_token' ] ) self . always_send_dm = [ '_unauthorized_' ] if 'always_send_dm' in self . config : self . always_send_dm . extend ( map ( lambda x : '!' + x , self . config [ 'always_send_dm' ] ) ) logging . getLogger ( 'Rocket.Errors.ThreadPool' ) . setLevel ( logging . INFO ) self . is_setup = True if self . test_mode : self . metrics [ 'startup_time' ] = ( datetime . now ( ) - self . bot_start_time ) . total_seconds ( ) * 1000.0

Initializes the bot plugins and everything .


def run(self, start=True):
        """
        Connects to slack and enters the main loop.

        * start - If True, rtm.start API is used. Else rtm.connect API is used

        For more info, refer to
        https://python-slackclient.readthedocs.io/en/latest/real_time_messaging.html#rtm-start-vs-rtm-connect
        """
        # Fail out if setup wasn't run
        if not self.is_setup:
            raise NotSetupError

        # Start the web server
        self.webserver.start()

        first_connect = True

        try:
            while self.runnable:
                if self.reconnect_needed:
                    if not self.sc.rtm_connect(with_team_state=start):
                        return False
                    self.reconnect_needed = False
                    if first_connect:
                        first_connect = False
                        self.plugins.connect()

                # Get all waiting events - this always returns a list
                try:
                    events = self.sc.rtm_read()
                except AttributeError:
                    self.log.exception('Something has failed in the slack rtm library.  This is fatal.')
                    self.runnable = False
                    events = []
                except:
                    self.log.exception('Unhandled exception in rtm_read()')
                    self.reconnect_needed = True
                    events = []
                for e in events:
                    try:
                        self._handle_event(e)
                    except KeyboardInterrupt:
                        # Gracefully shutdown
                        self.runnable = False
                    except:
                        self.log.exception('Unhandled exception in event handler')
                sleep(0.1)
        except KeyboardInterrupt:
            # On ctrl-c, just exit
            pass
        except:
            self.log.exception('Unhandled exception')

Connects to slack and enters the main loop .


def stop(self):
        """Does cleanup of bot and plugins."""
        if self.webserver is not None:
            self.webserver.stop()
        if not self.test_mode:
            self.plugins.save_state()

def stop ( self ) : if self . webserver is not None : self . webserver . stop ( ) if not self . test_mode : self . plugins . save_state ( )


def send_message(self, channel, text, thread=None, reply_broadcast=None):
        """
        Sends a message to the specified channel

        * channel - The channel to send to.  This can be a SlackChannel object, a channel id, or a channel name
        (without the #)
        * text - String to send
        * thread - reply to the thread. See https://api.slack.com/docs/message-threading#threads_party
        * reply_broadcast - Set to true to indicate your reply is germane to all members of a channel
        """
        # This doesn't want the # in the channel name
        if isinstance(channel, SlackRoomIMBase):
            channel = channel.id
        self.log.debug("Trying to send to %s: %s", channel, text)
        self.sc.rtm_send_message(channel, text, thread=thread, reply_broadcast=reply_broadcast)

def
send_message
(
self
,
channel
,
text
,
thread
=
,
reply_broadcast
=
)
:
if
isinstance
(
channel
,
SlackRoomIMBase
)
:
channel
=
channel
.
id
self
.
log
.
debug
(
"Trying to send to %s: %s"
,
channel
,
text
)
self
.
sc
.
rtm_send_message
(
channel
,
text
,
thread
=
thread
,
reply_broadcast
=
reply_broadcast
)


def send_im(self, user, text):
        """
        Sends a message to a user as an IM

        * user - The user to send to.  This can be a SlackUser object, a user id, or the username (without the @)
        * text - String to send
        """
        if isinstance(user, SlackUser):
            user = user.id
            channelid = self._find_im_channel(user)
        else:
            channelid = user.id
        self.send_message(channelid, text)

def send_im ( self , user , text ) : if isinstance ( user , SlackUser ) : user = user . id channelid = self . _find_im_channel ( user ) else : channelid = user . id self . send_message ( channelid , text )


def push(self, message):
        """
        Takes a SlackEvent, parses it for a command, and runs against registered plugin
        """
        if self._ignore_event(message):
            return None, None
        args = self._parse_message(message)
        self.log.debug("Searching for command using chunks: %s", args)
        cmd, msg_args = self._find_longest_prefix_command(args)
        if cmd is not None:
            if message.user is None:
                self.log.debug("Discarded message with no originating user: %s", message)
                return None, None
            sender = message.user.username
            if message.channel is not None:
                sender = "#%s/%s" % (message.channel.name, sender)
            self.log.info("Received from %s: %s, args %s", sender, cmd, msg_args)
            f = self._get_command(cmd, message.user)
            if f:
                if self._is_channel_ignored(f, message.channel):
                    self.log.info("Channel %s is ignored, discarding command %s", message.channel, cmd)
                    return '_ignored_', ""
                return cmd, f.execute(message, msg_args)
            return '_unauthorized_', "Sorry, you are not authorized to run %s" % cmd
        return None, None

Takes a SlackEvent parses it for a command and runs against registered plugin


def _ignore_event(self, message):
        """
        message_replied event is not truly a message event and does not have a message.text
        don't process such events

        commands may not be idempotent, so ignore message_changed events.
        """
        if hasattr(message, 'subtype') and message.subtype in self.ignored_events:
            return True
        return False

def _ignore_event ( self , message ) : if hasattr ( message , 'subtype' ) and message . subtype in self . ignored_events : return True return False

message_replied event is not truly a message event and does not have a message . text don t process such events


def acl_show(self, msg, args):
        """Show current allow and deny blocks for the given acl."""
        name = args[0] if len(args) > 0 else None
        if name is None:
            return "%s: The following ACLs are defined: %s" % (msg.user, ', '.join(self._acl.keys()))

        if name not in self._acl:
            return "Sorry, couldn't find an acl named '%s'" % name

        return '\n'.join([
            "%s: ACL '%s' is defined as follows:" % (msg.user, name),
            "allow: %s" % ', '.join(self._acl[name]['allow']),
            "deny: %s" % ', '.join(self._acl[name]['deny'])
        ])

Show current allow and deny blocks for the given acl .


def add_user_to_allow(self, name, user):
        """Add a user to the given acl allow block."""

        # Clear user from both allow and deny before adding
        if not self.remove_user_from_acl(name, user):
            return False

        if name not in self._acl:
            return False

        self._acl[name]['allow'].append(user)
        return True

def add_user_to_allow ( self , name , user ) : if not self . remove_user_from_acl ( name , user ) : return False if name not in self . _acl : return False self . _acl [ name ] [ 'allow' ] . append ( user ) return True

Add a user to the given acl allow block .


def create_acl(self, name):
        """Create a new acl."""
        if name in self._acl:
            return False

        self._acl[name] = {
            'allow': [],
            'deny': []
        }
        return True

def create_acl ( self , name ) : if name in self . _acl : return False self . _acl [ name ] = { 'allow' : [ ] , 'deny' : [ ] } return True


def delete_acl(self, name):
        """Delete an acl."""
        if name not in self._acl:
            return False

        del self._acl[name]
        return True

def delete_acl ( self , name ) : if name not in self . _acl : return False del self . _acl [ name ] return True


def mongo(daemon=False, port=20771):
    '''Run the mongod process.
    '''
    cmd = "mongod --port {0}".format(port)
    if daemon:
        cmd += " --fork"
    run(cmd)

def
mongo
(
daemon
=
False
,
port
=
20771
)
:
cmd
=
"mongod --port {0}"
.
format
(
port
)
if
daemon
:
cmd
+=
" --fork"
run
(
cmd
)


def proxy_factory(BaseSchema, label, ProxiedClass, get_key):
    """Create a proxy to a class instance stored in ``proxies``.

    :param class BaseSchema: Base schema (e.g. ``StoredObject``)
    :param str label: Name of class variable to set
    :param class ProxiedClass: Class to get or create
    :param function get_key: Extension-specific key function; may return e.g.
        the current Flask request

    """
    def local():
        key = get_key()
        try:
            return proxies[BaseSchema][label][key]
        except KeyError:
            proxies[BaseSchema][label][key] = ProxiedClass()
            return proxies[BaseSchema][label][key]
    return LocalProxy(local)

def proxy_factory ( BaseSchema , label , ProxiedClass , get_key ) : def local ( ) : key = get_key ( ) try : return proxies [ BaseSchema ] [ label ] [ key ] except KeyError : proxies [ BaseSchema ] [ label ] [ key ] = ProxiedClass ( ) return proxies [ BaseSchema ] [ label ] [ key ] return LocalProxy ( local )

Create a proxy to a class instance stored in proxies .


def with_proxies(proxy_map, get_key):
    """Class decorator factory; adds proxy class variables to target class.

    :param dict proxy_map: Mapping between class variable labels and proxied
        classes
    :param function get_key: Extension-specific key function; may return e.g.
        the current Flask request

    """
    def wrapper(cls):
        for label, ProxiedClass in six.iteritems(proxy_map):
            proxy = proxy_factory(cls, label, ProxiedClass, get_key)
            setattr(cls, label, proxy)
        return cls
    return wrapper

def with_proxies ( proxy_map , get_key ) : def wrapper ( cls ) : for label , ProxiedClass in six . iteritems ( proxy_map ) : proxy = proxy_factory ( cls , label , ProxiedClass , get_key ) setattr ( cls , label , proxy ) return cls return wrapper

Class decorator factory ; adds proxy class variables to target class .


def _to_primary_key(self, value):
        """
        Return primary key; if value is StoredObject, verify
        that it is loaded.

        """
        if value is None:
            return None
        if isinstance(value, self.base_class):
            if not value._is_loaded:
                raise exceptions.DatabaseError('Record must be loaded.')
            return value._primary_key

        return self.base_class._to_primary_key(value)

def
_to_primary_key
(
self
,
value
)
:
if
value
is
:
return
if
isinstance
(
value
,
self
.
base_class
)
:
if
not
value
.
_is_loaded
:
raise
exceptions
.
DatabaseError
(
'Record must be loaded.'
)
return
value
.
_primary_key
return
self
.
base_class
.
_to_primary_key
(
value
)

Return primary key ; if value is StoredObject verify that it is loaded .


def set_nested(data, value, *keys):
    """Assign to a nested dictionary.

    :param dict data: Dictionary to mutate
    :param value: Value to set
    :param list *keys: List of nested keys

    >>> data = {}
    >>> set_nested(data, 'hi', 'k0', 'k1', 'k2')
    >>> data
    {'k0': {'k1': {'k2': 'hi'}}}

    """
    if len(keys) == 1:
        data[keys[0]] = value
    else:
        if keys[0] not in data:
            data[keys[0]] = {}
        set_nested(data[keys[0]], value, *keys[1:])

def set_nested ( data , value , * keys ) : if len ( keys ) == 1 : data [ keys [ 0 ] ] = value else : if keys [ 0 ] not in data : data [ keys [ 0 ] ] = { } set_nested ( data [ keys [ 0 ] ] , value , * keys [ 1 : ] )


def get_by_username(self, username):
        """Retrieve user by username"""
        res = filter(lambda x: x.username == username, self.users.values())
        if len(res) > 0:
            return res[0]
        return None

def get_by_username ( self , username ) : res = filter ( lambda x : x . username == username , self . users . values ( ) ) if len ( res ) > 0 : return res [ 0 ] return None


def set(self, user):
        """
        Adds a user object to the user manager

        user - a SlackUser object
        """

        self.log.info("Loading user information for %s/%s", user.id, user.username)
        self.load_user_info(user)
        self.log.info("Loading user rights for %s/%s", user.id, user.username)
        self.load_user_rights(user)
        self.log.info("Added user: %s/%s", user.id, user.username)
        self._add_user_to_cache(user)
        return user

def
set
(
self
,
user
)
:
self
.
log
.
info
(
"Loading user information for %s/%s"
,
user
.
id
,
user
.
username
)
self
.
load_user_info
(
user
)
self
.
log
.
info
(
"Loading user rights for %s/%s"
,
user
.
id
,
user
.
username
)
self
.
load_user_rights
(
user
)
self
.
log
.
info
(
"Added user: %s/%s"
,
user
.
id
,
user
.
username
)
self
.
_add_user_to_cache
(
user
)
return
user


def load_user_rights(self, user):
        """Sets permissions on user object"""
        if user.username in self.admins:
            user.is_admin = True
        elif not hasattr(user, 'is_admin'):
            user.is_admin = False

def load_user_rights ( self , user ) : if user . username in self . admins : user . is_admin = True elif not hasattr ( user , 'is_admin' ) : user . is_admin = False


def send_message(self, channel, text):
        """
        Used to send a message to the specified channel.

        * channel - can be a channel or user
        * text - message to send
        """
        if isinstance(channel, SlackIM) or isinstance(channel, SlackUser):
            self._bot.send_im(channel, text)
        elif isinstance(channel, SlackRoom):
            self._bot.send_message(channel, text)
        elif isinstance(channel, basestring):
            if channel[0] == '@':
                self._bot.send_im(channel[1:], text)
            elif channel[0] == '#':
                self._bot.send_message(channel[1:], text)
            else:
                self._bot.send_message(channel, text)
        else:
            self._bot.send_message(channel, text)

def send_message ( self , channel , text ) : if isinstance ( channel , SlackIM ) or isinstance ( channel , SlackUser ) : self . _bot . send_im ( channel , text ) elif isinstance ( channel , SlackRoom ) : self . _bot . send_message ( channel , text ) elif isinstance ( channel , basestring ) : if channel [ 0 ] == '@' : self . _bot . send_im ( channel [ 1 : ] , text ) elif channel [ 0 ] == '#' : self . _bot . send_message ( channel [ 1 : ] , text ) else : self . _bot . send_message ( channel , text ) else : self . _bot . send_message ( channel , text )

Used to send a message to the specified channel .


def start_timer(self, duration, func, *args):
        """
        Schedules a function to be called after some period of time.

        * duration - time in seconds to wait before firing
        * func - function to be called
        * args - arguments to pass to the function
        """
        t = threading.Timer(duration, self._timer_callback, (func, args))
        self._timer_callbacks[func] = t
        t.start()
        self.log.info("Scheduled call to %s in %ds", func.__name__, duration)

def
start_timer
(
self
,
duration
,
func
,
*
args
)
:
t
=
threading
.
Timer
(
duration
,
self
.
_timer_callback
,
(
func
,
args
)
)
self
.
_timer_callbacks
[
func
]
=
t
t
.
start
(
)
self
.
log
.
info
(
"Scheduled call to %s in %ds"
,
func
.
__name__
,
duration
)

Schedules a function to be called after some period of time .


def stop_timer(self, func):
        """
        Stops a timer if it hasn't fired yet

        * func - the function passed in start_timer
        """
        if func in self._timer_callbacks:
            t = self._timer_callbacks[func]
            t.cancel()
            del self._timer_callbacks[func]

def stop_timer ( self , func ) : if func in self . _timer_callbacks : t = self . _timer_callbacks [ func ] t . cancel ( ) del self . _timer_callbacks [ func ]


def get_user(self, username):
        """
        Utility function to query slack for a particular user

        :param username: The username of the user to lookup
        :return: SlackUser object or None
        """
        if hasattr(self._bot, 'user_manager'):
            user = self._bot.user_manager.get_by_username(username)
            if user:
                return user
            user = SlackUser.get_user(self._bot.sc, username)
            self._bot.user_manager.set(user)
            return user
        return SlackUser.get_user(self._bot.sc, username)

def get_user ( self , username ) : if hasattr ( self . _bot , 'user_manager' ) : user = self . _bot . user_manager . get_by_username ( username ) if user : return user user = SlackUser . get_user ( self . _bot . sc , username ) self . _bot . user_manager . set ( user ) return user return SlackUser . get_user ( self . _bot . sc , username )


def webhook(*args, **kwargs):
    """
    Decorator to mark plugin functions as entry points for web calls

    * route - web route to register, uses Flask syntax
    * method - GET/POST, defaults to POST
    """
    def wrapper(func):
        func.is_webhook = True
        func.route = args[0]
        func.form_params = kwargs.get('form_params', [])
        func.method = kwargs.get('method', 'POST')
        return func
    return wrapper

def webhook ( * args , ** kwargs ) : def wrapper ( func ) : func . is_webhook = True func . route = args [ 0 ] func . form_params = kwargs . get ( 'form_params' , [ ] ) func . method = kwargs . get ( 'method' , 'POST' ) return func return wrapper


def freeze(value):
    """ Cast value to its frozen counterpart. """
    if isinstance(value, list):
        return FrozenList(*value)
    if isinstance(value, dict):
        return FrozenDict(**value)
    return value

def freeze ( value ) : if isinstance ( value , list ) : return FrozenList ( * value ) if isinstance ( value , dict ) : return FrozenDict ( ** value ) return value


def help(self, msg, args):
        """Displays help for each command"""
        output = []
        if len(args) == 0:
            commands = sorted(self._bot.dispatcher.commands.items(), key=itemgetter(0))
            commands = filter(lambda x: x[1].is_subcmd is False, commands)
            # Filter commands if auth is enabled, hide_admin_commands is enabled, and user is not admin
            if self._should_filter_help_commands(msg.user):
                commands = filter(lambda x: x[1].admin_only is False, commands)
            for name, cmd in commands:
                output.append(self._get_short_help_for_command(name))
        else:
            name = '!' + args[0]
            output = [self._get_help_for_command(name)]
        return '\n'.join(output)

def help ( self , msg , args ) : output = [ ] if len ( args ) == 0 : commands = sorted ( self . _bot . dispatcher . commands . items ( ) , key = itemgetter ( 0 ) ) commands = filter ( lambda x : x [ 1 ] . is_subcmd is False , commands ) if self . _should_filter_help_commands ( msg . user ) : commands = filter ( lambda x : x [ 1 ] . admin_only is False , commands ) for name , cmd in commands : output . append ( self . _get_short_help_for_command ( name ) ) else : name = '!' + args [ 0 ] output = [ self . _get_help_for_command ( name ) ] return '\n' . join ( output )


def save(self, msg, args):
        """Causes the bot to write its current state to backend."""
        self.send_message(msg.channel, "Saving current state...")
        self._bot.plugins.save_state()
        self.send_message(msg.channel, "Done.")

def
save
(
self
,
msg
,
args
)
:
self
.
send_message
(
msg
.
channel
,
"Saving current state..."
)
self
.
_bot
.
plugins
.
save_state
(
)
self
.
send_message
(
msg
.
channel
,
"Done."
)

Causes the bot to write its current state to backend .


def shutdown(self, msg, args):
        """Causes the bot to gracefully shutdown."""
        self.log.info("Received shutdown from %s", msg.user.username)
        self._bot.runnable = False
        return "Shutting down..."

def
shutdown
(
self
,
msg
,
args
)
:
self
.
log
.
info
(
"Received shutdown from %s"
,
msg
.
user
.
username
)
self
.
_bot
.
runnable
=
False
return
"Shutting down..."