def from_image(cls, filename, start, stop, legend,
                   source="Image",
                   col_offset=0.1,
                   row_offset=2,
                   tolerance=0):
        """
        Read an image and generate Striplog.

        Args:
            filename (str): An image file, preferably high-res PNG.
            start (float or int): The depth at the top of the image.
            stop (float or int): The depth at the bottom of the image.
            legend (Legend): A legend to look up the components in.
            source (str): A source for the data. Default: 'Image'.
            col_offset (Number): The proportion of the way across the image
                from which to extract the pixel column. Default: 0.1 (ie 10%).
            row_offset (int): The number of pixels to skip at the top of
                each change in colour. Default: 2.
            tolerance (float): The Euclidean distance between hex colours,
                which has a maximum (black to white) of 441.67 in base 10.
                Default: 0.

        Returns:
            Striplog: The ``striplog`` object.
        """
        rgb = utils.loglike_from_image(filename, col_offset)
        loglike = np.array([utils.rgb_to_hex(t) for t in rgb])

        # Get the pixels and colour values at 'tops' (i.e. changes).
        tops, hexes = utils.tops_from_loglike(loglike, offset=row_offset)

        # If there are consecutive tops, we assume it's because there is a
        # single-pixel row that we don't want. So take the second one only.
        # We used to do this reduction in ``utils.tops_from_loglike()`` but
        # it was prventing us from making intervals only one sample thick.
        nonconsecutive = np.append(np.diff(tops), 2)
        tops = tops[nonconsecutive > 1]
        hexes = hexes[nonconsecutive > 1]

        # Get the set of unique colours.
        hexes_reduced = list(set(hexes))

        # Get the components corresponding to the colours.
        components = [legend.get_component(h, tolerance=tolerance)
                      for h in hexes_reduced]

        # Turn them into integers.
        values = [hexes_reduced.index(i) for i in hexes]

        basis = np.linspace(start, stop, loglike.size)

        list_of_Intervals = cls.__intervals_from_tops(tops,
                                                      values,
                                                      basis,
                                                      components)

        return cls(list_of_Intervals, source="Image")

def from_image ( cls , filename , start , stop , legend , source = "Image" , col_offset = 0.1 , row_offset = 2 , tolerance = 0 ) : rgb = utils . loglike_from_image ( filename , col_offset ) loglike = np . array ( [ utils . rgb_to_hex ( t ) for t in rgb ] ) tops , hexes = utils . tops_from_loglike ( loglike , offset = row_offset ) nonconsecutive = np . append ( np . diff ( tops ) , 2 ) tops = tops [ nonconsecutive > 1 ] hexes = hexes [ nonconsecutive > 1 ] hexes_reduced = list ( set ( hexes ) ) components = [ legend . get_component ( h , tolerance = tolerance ) for h in hexes_reduced ] values = [ hexes_reduced . index ( i ) for i in hexes ] basis = np . linspace ( start , stop , loglike . size ) list_of_Intervals = cls . __intervals_from_tops ( tops , values , basis , components ) return cls ( list_of_Intervals , source = "Image" )


def from_log(cls, log,
                 cutoff=None,
                 components=None,
                 legend=None,
                 legend_field=None,
                 field=None,
                 right=False,
                 basis=None,
                 source='Log'):
        """
        Turn a 1D array into a striplog, given a cutoff.

        Args:
            log (array-like): A 1D array or a list of integers.
            cutoff (number or array-like): The log value(s) at which to bin
                the log. Optional.
            components (array-like): A list of components. Use this or
                ``legend``.
            legend (``Legend``): A legend object. Use this or ``components``.
            legend_field ('str'): If you're not trying to match against
                components, then you can match the log values to this field in
                the Decors.
            field (str): The field in the Interval's ``data`` to store the log
                values as.
            right (bool): Which side of the cutoff to send things that are
                equal to, i.e. right on, the cutoff.
            basis (array-like): A depth basis for the log, so striplog knows
                where to put the boundaries.
            source (str): The source of the data. Default 'Log'.

        Returns:
            Striplog: The ``striplog`` object.
        """
        if (components is None) and (legend is None) and (field is None):
            m = 'You must provide a list of components, and legend, or a field.'
            raise StriplogError(m)

        if (legend is not None) and (legend_field is None):
            try:  # To treat it like a legend.
                components = [deepcopy(decor.component) for decor in legend]
            except AttributeError:  # It's just a list of components.
                pass

        if legend_field is not None:
            field_values = [getattr(d, legend_field, 0) for d in legend]
            components = [Component() for i in range(int(max(field_values)+1))]
            for i, decor in enumerate(legend):
                components[i] = deepcopy(decor.component)

        if cutoff is not None:

            # First make sure we have enough components.
            try:
                n = len(cutoff)
            except TypeError:
                n = 1
            if len(components) < n+1:
                m = 'For n cutoffs, you need to provide at least'
                m += 'n+1 components.'
                raise StriplogError(m)

            # Digitize.
            try:  # To use cutoff as a list.
                a = np.digitize(log, cutoff, right)
            except ValueError:  # It's just a number.
                a = np.digitize(log, [cutoff], right)

        else:
            a = np.copy(log)

        tops, values = utils.tops_from_loglike(a)

        if basis is None:
            m = 'You must provide a depth or elevation basis.'
            raise StriplogError(m)

        list_of_Intervals = cls.__intervals_from_tops(tops,
                                                      values,
                                                      basis,
                                                      components,
                                                      field=field
                                                      )

        return cls(list_of_Intervals, source=source)

Turn a 1D array into a striplog given a cutoff .


def from_las3(cls, string, lexicon=None,
                  source="LAS",
                  dlm=',',
                  abbreviations=False):
        """
        Turn LAS3 'lithology' section into a Striplog.

        Args:
            string (str): A section from an LAS3 file.
            lexicon (Lexicon): The language for conversion to components.
            source (str): A source for the data.
            dlm (str): The delimiter.
            abbreviations (bool): Whether to expand abbreviations.

        Returns:
            Striplog: The ``striplog`` object.

        Note:
            Handles multiple 'Data' sections. It would be smarter for it
            to handle one at a time, and to deal with parsing the multiple
            sections in the Well object.

            Does not read an actual LAS file. Use the Well object for that.
        """
        f = re.DOTALL | re.IGNORECASE
        regex = r'\~\w+?_Data.+?\n(.+?)(?:\n\n+|\n*\~|\n*$)'
        pattern = re.compile(regex, flags=f)
        text = pattern.search(string).group(1)

        s = re.search(r'\.(.+?)\: ?.+?source', string)
        if s:
            source = s.group(1).strip()

        return cls.from_descriptions(text, lexicon,
                                     source=source,
                                     dlm=dlm,
                                     abbreviations=abbreviations)

Turn LAS3 lithology section into a Striplog .


def from_canstrat(cls, filename, source='canstrat'):
        """
        Eat a Canstrat DAT file and make a striplog.
        """
        with open(filename) as f:
            dat = f.read()

        data = parse_canstrat(dat)

        list_of_Intervals = []
        for d in data[7]:  # 7 is the 'card type' for lithology info.
            if d.pop('skip'):
                continue
            top = d.pop('top')
            base = d.pop('base')
            comps = [Component({'lithology': d['rtc'],
                                'colour': d['colour_name']
                                })]
            iv = Interval(top=top, base=base, components=comps, data=d)
            list_of_Intervals.append(iv)

        return cls(list_of_Intervals, source=source)

def from_canstrat ( cls , filename , source = 'canstrat' ) : with open ( filename ) as f : dat = f . read ( ) data = parse_canstrat ( dat ) list_of_Intervals = [ ] for d in data [ 7 ] : if d . pop ( 'skip' ) : continue top = d . pop ( 'top' ) base = d . pop ( 'base' ) comps = [ Component ( { 'lithology' : d [ 'rtc' ] , 'colour' : d [ 'colour_name' ] } ) ] iv = Interval ( top = top , base = base , components = comps , data = d ) list_of_Intervals . append ( iv ) return cls ( list_of_Intervals , source = source )

Eat a Canstrat DAT file and make a striplog .


def copy(self):
        """Returns a shallow copy."""
        return Striplog([i.copy() for i in self],
                        order=self.order,
                        source=self.source)

def copy ( self ) : return Striplog ( [ i . copy ( ) for i in self ] , order = self . order , source = self . source )


def to_csv(self,
               filename=None,
               as_text=True,
               use_descriptions=False,
               dlm=",",
               header=True):
        """
        Returns a CSV string built from the summaries of the Intervals.

        Args:
            use_descriptions (bool): Whether to use descriptions instead
                of summaries, if available.
            dlm (str): The delimiter.
            header (bool): Whether to form a header row.

        Returns:
            str: A string of comma-separated values.
        """
        if (filename is None):
            if (not as_text):
                raise StriplogError("You must provide a filename or set as_text to True.")
        else:
            as_text = False

        if as_text:
            output = StringIO()
        else:
            output = open(filename, 'w')

        fieldnames = ['Top', 'Base', 'Component']
        writer = csv.DictWriter(output,
                                delimiter=dlm,
                                fieldnames=fieldnames,
                                quoting=csv.QUOTE_MINIMAL)

        if header:
            writer.writeheader()

        for i in self.__list:
            if use_descriptions and i.description:
                text = i.description
            elif i.primary:
                text = i.primary.summary()
            else:
                text = ''
            data = {j: k for j, k in zip(fieldnames, [i.top.z, i.base.z, text])}
            writer.writerow(data)

        if as_text:
            return output.getvalue()
            #return output
        else:
            output.close
            return None

Returns a CSV string built from the summaries of the Intervals .


def to_las3(self, use_descriptions=False, dlm=",", source="Striplog"):
        """
        Returns an LAS 3.0 section string.

        Args:
            use_descriptions (bool): Whether to use descriptions instead
                of summaries, if available.
            dlm (str): The delimiter.
            source (str): The sourse of the data.

        Returns:
            str: A string forming Lithology section of an LAS3 file.
        """
        data = self.to_csv(use_descriptions=use_descriptions,
                           dlm=dlm,
                           header=False)

        return templates.section.format(name='Lithology',
                                        short="LITH",
                                        source=source,
                                        data=data)

def to_las3 ( self , use_descriptions = False , dlm = "," , source = "Striplog" ) : data = self . to_csv ( use_descriptions = use_descriptions , dlm = dlm , header = False ) return templates . section . format ( name = 'Lithology' , short = "LITH" , source = source , data = data )


def plot_axis(self,
                  ax,
                  legend,
                  ladder=False,
                  default_width=1,
                  match_only=None,
                  colour=None,
                  colour_function=None,
                  cmap=None,
                  default=None,
                  width_field=None,
                  **kwargs
                  ):
        """
        Plotting, but only the Rectangles. You have to set up the figure.
        Returns a matplotlib axis object.

        Args:
            ax (axis): The matplotlib axis to plot into.
            legend (Legend): The Legend to use for colours, etc.
            ladder (bool): Whether to use widths or not. Default False.
            default_width (int): A width for the plot if not using widths.
                Default 1.
            match_only (list): A list of strings matching the attributes you
                want to compare when plotting.
            colour (str): Which data field to use for colours.
            cmap (cmap): Matplotlib colourmap. Default ``viridis``.
            default (float): The default (null) value.
            width_field (str): The field to use for the width of the patches.
            **kwargs are passed through to matplotlib's ``patches.Rectangle``.

        Returns:
            axis: The matplotlib.pyplot axis.
        """
        default_c = None
        patches = []
        for iv in self.__list:
            origin = (0, iv.top.z)
            d = legend.get_decor(iv.primary, match_only=match_only)
            thick = iv.base.z - iv.top.z

            if ladder:
                if width_field is not None:
                    w = iv.data.get(width_field, 1)
                    w = default_width * w/self.max_field(width_field)
                    default_c = 'gray'
                elif legend is not None:
                    w = d.width or default_width
                    try:
                        w = default_width * w/legend.max_width
                    except:
                        w = default_width
            else:
                w = default_width

            # Allow override of lw
            this_patch_kwargs = kwargs.copy()
            lw = this_patch_kwargs.pop('lw', 0)
            ec = this_patch_kwargs.pop('ec', 'k')
            fc = this_patch_kwargs.pop('fc', None) or default_c or d.colour

            if colour is None:
                rect = mpl.patches.Rectangle(origin,
                                             w,
                                             thick,
                                             fc=fc,
                                             lw=lw,
                                             hatch=d.hatch,
                                             ec=ec,  # edgecolour for hatching
                                             **this_patch_kwargs)
                ax.add_patch(rect)
            else:
                rect = mpl.patches.Rectangle(origin,
                                             w,
                                             thick,
                                             lw=lw,
                                             ec=ec,  # edgecolour for hatching
                                             **this_patch_kwargs)
                patches.append(rect)

        if colour is not None:
            cmap = cmap or 'viridis'
            p = mpl.collections.PatchCollection(patches, cmap=cmap, lw=lw)
            p.set_array(self.get_data(colour, colour_function, default=default))
            ax.add_collection(p)
            cb = plt.colorbar(p)  #  orientation='horizontal' only really works with ticks=[0, 0.1, 0.2] say
            cb.outline.set_linewidth(0)

        return ax

def plot_axis ( self , ax , legend , ladder = False , default_width = 1 , match_only = None , colour = None , colour_function = None , cmap = None , default = None , width_field = None , ** kwargs ) : default_c = None patches = [ ] for iv in self . __list : origin = ( 0 , iv . top . z ) d = legend . get_decor ( iv . primary , match_only = match_only ) thick = iv . base . z - iv . top . z if ladder : if width_field is not None : w = iv . data . get ( width_field , 1 ) w = default_width * w / self . max_field ( width_field ) default_c = 'gray' elif legend is not None : w = d . width or default_width try : w = default_width * w / legend . max_width except : w = default_width else : w = default_width this_patch_kwargs = kwargs . copy ( ) lw = this_patch_kwargs . pop ( 'lw' , 0 ) ec = this_patch_kwargs . pop ( 'ec' , 'k' ) fc = this_patch_kwargs . pop ( 'fc' , None ) or default_c or d . colour if colour is None : rect = mpl . patches . Rectangle ( origin , w , thick , fc = fc , lw = lw , hatch = d . hatch , ec = ec , ** this_patch_kwargs ) ax . add_patch ( rect ) else : rect = mpl . patches . Rectangle ( origin , w , thick , lw = lw , ec = ec , ** this_patch_kwargs ) patches . append ( rect ) if colour is not None : cmap = cmap or 'viridis' p = mpl . collections . PatchCollection ( patches , cmap = cmap , lw = lw ) p . set_array ( self . get_data ( colour , colour_function , default = default ) ) ax . add_collection ( p ) cb = plt . colorbar ( p ) cb . outline . set_linewidth ( 0 ) return ax

Plotting but only the Rectangles . You have to set up the figure . Returns a matplotlib axis object .


def get_data(self, field, function=None, default=None):
        """
        Get data from the striplog.
        """
        f = function or utils.null
        data = []
        for iv in self:
            d = iv.data.get(field)
            if d is None:
                if default is not None:
                    d = default
                else:
                    d = np.nan
            data.append(f(d))

        return np.array(data)

def get_data ( self , field , function = None , default = None ) : f = function or utils . null data = [ ] for iv in self : d = iv . data . get ( field ) if d is None : if default is not None : d = default else : d = np . nan data . append ( f ( d ) ) return np . array ( data )


def extract(self, log, basis, name, function=None):
        """
        'Extract' a log into the components of a striplog.

        Args:
            log (array_like). A log or other 1D data.
            basis (array_like). The depths or elevations of the log samples.
            name (str). The name of the attribute to store in the components.
            function (function). A function that takes an array as the only
                input, and returns whatever you want to store in the 'name'
                attribute of the primary component.
        Returns:
            None. The function works on the striplog in place.
        """
        # Build a dict of {index: [log values]} to keep track.
        intervals = {}
        previous_ix = -1
        for i, z in enumerate(basis):
            ix = self.read_at(z, index=True)
            if ix is None:
                continue
            if ix == previous_ix:
                intervals[ix].append(log[i])
            else:
                intervals[ix] = [log[i]]
            previous_ix = ix

        # Set the requested attribute in the primary comp of each interval.
        for ix, data in intervals.items():
            f = function or utils.null
            d = f(np.array(data))
            self[ix].data[name] = d

        return None

def extract ( self , log , basis , name , function = None ) : intervals = { } previous_ix = - 1 for i , z in enumerate ( basis ) : ix = self . read_at ( z , index = True ) if ix is None : continue if ix == previous_ix : intervals [ ix ] . append ( log [ i ] ) else : intervals [ ix ] = [ log [ i ] ] previous_ix = ix for ix , data in intervals . items ( ) : f = function or utils . null d = f ( np . array ( data ) ) self [ ix ] . data [ name ] = d return None

Extract a log into the components of a striplog .


def find(self, search_term, index=False):
        """
        Look for a regex expression in the descriptions of the striplog.
        If there's no description, it looks in the summaries.

        If you pass a Component, then it will search the components, not the
        descriptions or summaries.

        Case insensitive.

        Args:
            search_term (string or Component): The thing you want to search
                for. Strings are treated as regular expressions.
            index (bool): Whether to return the index instead of the interval.
        Returns:
            Striplog: A striplog that contains only the 'hit' Intervals.
                However, if ``index`` was ``True``, then that's what you get.
        """
        hits = []
        for i, iv in enumerate(self):
            try:
                search_text = iv.description or iv.primary.summary()
                pattern = re.compile(search_term, flags=re.IGNORECASE)
                if pattern.search(search_text):
                    hits.append(i)
            except TypeError:
                if search_term in iv.components:
                    hits.append(i)
        if hits and index:
            return hits
        elif hits:
            return self[hits]
        else:
            return

def find ( self , search_term , index = False ) : hits = [ ] for i , iv in enumerate ( self ) : try : search_text = iv . description or iv . primary . summary ( ) pattern = re . compile ( search_term , flags = re . IGNORECASE ) if pattern . search ( search_text ) : hits . append ( i ) except TypeError : if search_term in iv . components : hits . append ( i ) if hits and index : return hits elif hits : return self [ hits ] else : return

Look for a regex expression in the descriptions of the striplog . If there s no description it looks in the summaries .


def find_overlaps(self, index=False):
        """
        Find overlaps in a striplog.

        Args:
            index (bool): If True, returns indices of intervals with
            gaps after them.

        Returns:
            Striplog: A striplog of all the overlaps as intervals.
        """
        return self.__find_incongruities(op=operator.gt, index=index)

def find_overlaps ( self , index = False ) : return self . __find_incongruities ( op = operator . gt , index = index )


def find_gaps(self, index=False):
        """
        Finds gaps in a striplog.

        Args:
            index (bool): If True, returns indices of intervals with
            gaps after them.

        Returns:
            Striplog: A striplog of all the gaps. A sort of anti-striplog.
        """
        return self.__find_incongruities(op=operator.lt, index=index)

def find_gaps ( self , index = False ) : return self . __find_incongruities ( op = operator . lt , index = index )


def prune(self, limit=None, n=None, percentile=None, keep_ends=False):
        """
        Remove intervals below a certain limit thickness. In place.

        Args:
            limit (float): Anything thinner than this will be pruned.
            n (int): The n thinnest beds will be pruned.
            percentile (float): The thinnest specified percentile will be
                pruned.
            keep_ends (bool): Whether to keep the first and last, regardless
                of whether they meet the pruning criteria.
        """
        strip = self.copy()

        if not (limit or n or percentile):
            m = "You must provide a limit or n or percentile for pruning."
            raise StriplogError(m)
        if limit:
            prune = [i for i, iv in enumerate(strip) if iv.thickness < limit]
        if n:
            prune = strip.thinnest(n=n, index=True)
        if percentile:
            n = np.floor(len(strip)*percentile/100)
            prune = strip.thinnest(n=n, index=True)

        if keep_ends:
            first, last = 0, len(strip) - 1
            if first in prune:
                prune.remove(first)
            if last in prune:
                prune.remove(last)

        del strip[prune]

        return strip

Remove intervals below a certain limit thickness . In place .


def anneal(self):
        """
        Fill in empty intervals by growing from top and base.

        Note that this operation happens in-place and destroys any information
        about the ``Position`` (e.g. metadata associated with the top or base).
        See GitHub issue #54.
        """
        strip = self.copy()

        gaps = strip.find_gaps(index=True)

        if not gaps:
            return

        for gap in gaps:
            before = strip[gap]
            after = strip[gap + 1]

            if strip.order == 'depth':
                t = (after.top.z-before.base.z)/2
                before.base = before.base.z + t
                after.top = after.top.z - t
            else:
                t = (after.base-before.top)/2
                before.top = before.top.z + t
                after.base = after.base.z - t

        return strip

def anneal ( self ) : strip = self . copy ( ) gaps = strip . find_gaps ( index = True ) if not gaps : return for gap in gaps : before = strip [ gap ] after = strip [ gap + 1 ] if strip . order == 'depth' : t = ( after . top . z - before . base . z ) / 2 before . base = before . base . z + t after . top = after . top . z - t else : t = ( after . base - before . top ) / 2 before . top = before . top . z + t after . base = after . base . z - t return strip

Fill in empty intervals by growing from top and base .


def fill(self, component=None):
        """
        Fill gaps with the component provided.

        Example
            t = s.fill(Component({'lithology': 'cheese'}))
        """
        c = [component] if component is not None else []

        # Make the intervals to go in the gaps.
        gaps = self.find_gaps()
        if not gaps:
            return self
        for iv in gaps:
            iv.components = c

        return deepcopy(self) + gaps

def fill ( self , component = None ) : c = [ component ] if component is not None else [ ] gaps = self . find_gaps ( ) if not gaps : return self for iv in gaps : iv . components = c return deepcopy ( self ) + gaps


def union(self, other):
        """
        Makes a striplog of all unions.

        Args:
            Striplog. The striplog instance to union with.

        Returns:
            Striplog. The result of the union.
        """
        if not isinstance(other, self.__class__):
            m = "You can only union striplogs with each other."
            raise StriplogError(m)

        result = []
        for iv in deepcopy(self):
            for jv in other:
                if iv.any_overlaps(jv):
                    iv = iv.union(jv)
            result.append(iv)
        return Striplog(result)


def intersect(self, other):
        """
        Makes a striplog of all intersections.

        Args:
            Striplog. The striplog instance to intersect with.

        Returns:
            Striplog. The result of the intersection.
        """
        if not isinstance(other, self.__class__):
            m = "You can only intersect striplogs with each other."
            raise StriplogError(m)

        result = []
        for iv in self:
            for jv in other:
                try:
                    result.append(iv.intersect(jv))
                except IntervalError:
                    # The intervals don't overlap
                    pass
        return Striplog(result)


def merge_overlaps(self):
        """
        Merges overlaps by merging overlapping Intervals.

        The function takes no arguments and returns ``None``. It operates on
        the striplog 'in place'

        TODO: This function will not work if any interval overlaps more than
            one other intervals at either its base or top.
        """
        overlaps = np.array(self.find_overlaps(index=True))

        if not overlaps.any():
            return

        for overlap in overlaps:
            before = self[overlap].copy()
            after = self[overlap + 1].copy()

            # Get rid of the before and after pieces.
            del self[overlap]
            del self[overlap]

            # Make the new piece.
            new_segment = before.merge(after)

            # Insert it.
            self.__insert(overlap, new_segment)

            overlaps += 1

        return

def merge_overlaps ( self ) : overlaps = np . array ( self . find_overlaps ( index = True ) ) if not overlaps . any ( ) : return for overlap in overlaps : before = self [ overlap ] . copy ( ) after = self [ overlap + 1 ] . copy ( ) del self [ overlap ] del self [ overlap ] new_segment = before . merge ( after ) self . __insert ( overlap , new_segment ) overlaps += 1 return

Merges overlaps by merging overlapping Intervals .


def hist(self,
             lumping=None,
             summary=False,
             sort=True,
             plot=True,
             legend=None,
             ax=None
             ):
        """
        Plots a histogram and returns the data for it.

        Args:
            lumping (str): If given, the bins will be lumped based on this
                attribute of the primary components of the intervals
                encountered.
            summary (bool): If True, the summaries of the components are
                returned as the bins. Otherwise, the default behaviour is to
                return the Components themselves.
            sort (bool): If True (default), the histogram is sorted by value,
                starting with the largest.
            plot (bool): If True (default), produce a bar plot.
            legend (Legend): The legend with which to colour the bars.
            ax (axis): An axis object, which will be returned if provided.
                If you don't provide one, it will be created but not returned.

        Returns:
            Tuple: A tuple of tuples of entities and counts.

        TODO:
            Deal with numeric properties, so I can histogram 'Vp' values, say.
        """
        # This seems like overkill, but collecting all this stuff gives
        # the user some choice about what they get back.
        comps = []
        labels = []
        entries = defaultdict(int)
        for i in self:
            if lumping:
                k = i.primary[lumping]
            else:
                if summary:
                    k = i.primary.summary()
                else:
                    k = i.primary
            comps.append(i.primary)
            labels.append(i.primary.summary())
            entries[k] += i.thickness

        if sort:
            allitems = sorted(entries.items(), key=lambda i: i[1], reverse=True)
            ents, counts = zip(*allitems)
        else:
            ents, counts = tuple(entries.keys()), tuple(entries.values())

        # Make plot.
        if plot:
            if ax is None:
                fig, ax = plt.subplots()
                return_ax = False
            else:
                return_ax = True
            ind = np.arange(len(ents))
            bars = ax.bar(ind, counts, align='center')
            ax.set_xticks(ind)
            ax.set_xticklabels(labels)
            if legend:
                colours = [legend.get_colour(c) for c in comps]
                for b, c in zip(bars, colours):
                    b.set_color(c)
            ax.set_ylabel('Thickness [m]')
        else:
            bars = []

        if plot and return_ax:
            return counts, ents, ax

        return counts, ents, bars

def
hist
(
self
,
lumping
=
,
summary
=
False
,
sort
=
True
,
plot
=
True
,
legend
=
,
ax
=
)
:
comps
=
[
]
labels
=
[
]
entries
=
defaultdict
(
int
)
for
i
in
self
:
if
lumping
:
k
=
i
.
primary
[
lumping
]
else
:
if
summary
:
k
=
i
.
primary
.
summary
(
)
else
:
k
=
i
.
primary
comps
.
append
(
i
.
primary
)
labels
.
append
(
i
.
primary
.
summary
(
)
)
entries
[
k
]
+=
i
.
thickness
if
sort
:
allitems
=
sorted
(
entries
.
items
(
)
,
key
=
lambda
i
:
i
[
1
]
,
reverse
=
True
)
ents
,
counts
=
zip
(
*
allitems
)
else
:
ents
,
counts
=
tuple
(
entries
.
keys
(
)
)
,
tuple
(
entries
.
values
(
)
)
if
plot
:
if
ax
is
:
fig
,
ax
=
plt
.
subplots
(
)
return_ax
=
False
else
:
return_ax
=
True
ind
=
np
.
arange
(
len
(
ents
)
)
bars
=
ax
.
bar
(
ind
,
counts
,
align
=
'center'
)
ax
.
set_xticks
(
ind
)
ax
.
set_xticklabels
(
labels
)
if
legend
:
colours
=
[
legend
.
get_colour
(
c
)
for
c
in
comps
]
for
b
,
c
in
zip
(
bars
,
colours
)
:
b
.
set_color
(
c
)
ax
.
set_ylabel
(
'Thickness [m]'
)
else
:
bars
=
[
]
if
plot
and
return_ax
:
return
counts
,
ents
,
ax
return
counts
,
ents
,
bars

Plots a histogram and returns the data for it .


def invert(self, copy=False):
        """
        Inverts the striplog, changing its order and the order of its contents.

        Operates in place by default.

        Args:
            copy (bool): Whether to operate in place or make a copy.

        Returns:
            None if operating in-place, or an inverted copy of the striplog
                if not.
        """
        if copy:
            return Striplog([i.invert(copy=True) for i in self])
        else:
            for i in self:
                i.invert()
            self.__sort()
            o = self.order
            self.order = {'depth': 'elevation', 'elevation': 'depth'}[o]
            return

def invert ( self , copy = False ) : if copy : return Striplog ( [ i . invert ( copy = True ) for i in self ] ) else : for i in self : i . invert ( ) self . __sort ( ) o = self . order self . order = { 'depth' : 'elevation' , 'elevation' : 'depth' } [ o ] return

Inverts the striplog changing its order and the order of its contents .


def crop(self, extent, copy=False):
        """
        Crop to a new depth range.

        Args:
            extent (tuple): The new start and stop depth. Must be 'inside'
                existing striplog.
            copy (bool): Whether to operate in place or make a copy.

        Returns:
            Operates in place by deault; if copy is True, returns a striplog.
        """
        try:
            if extent[0] is None:
                extent = (self.start.z, extent[1])
            if extent[1] is None:
                extent = (extent[0], self.stop.z)
        except:
            m = "You must provide a 2-tuple for the new extents. Use None for"
            m += " the existing start or stop."
            raise StriplogError(m)

        first_ix = self.read_at(extent[0], index=True)
        last_ix = self.read_at(extent[1], index=True)

        first = self[first_ix].split_at(extent[0])[1]
        last = self[last_ix].split_at(extent[1])[0]

        new_list = self.__list[first_ix:last_ix+1].copy()
        new_list[0] = first
        new_list[-1] = last

        if copy:
            return Striplog(new_list)
        else:
            self.__list = new_list
            return


def quality(self, tests, alias=None):
        """
        Run a series of tests and return the corresponding results.

        Based on curve testing for ``welly``.

        Args:
            tests (list): a list of functions.

        Returns:
            list. The results. Stick to booleans (True = pass) or ints.
        """
        # This is hacky... striplog should probably merge with welly...

        # Ignore aliases
        alias = alias or {}
        alias = alias.get('striplog', alias.get('Striplog', []))

        # Gather the tests.
        # First, anything called 'all', 'All', or 'ALL'.
        # Second, anything with the name of the curve we're in now.
        # Third, anything that the alias list has for this curve.
        # (This requires a reverse look-up so it's a bit messy.)
        this_tests =\
            tests.get('all', [])+tests.get('All', [])+tests.get('ALL', [])\
            + tests.get('striplog', tests.get('Striplog', []))\
            + utils.flatten_list([tests.get(a) for a in alias])
        this_tests = filter(None, this_tests)

        # If we explicitly set zero tests for a particular key, then this
        # overrides the 'all' tests.
        if not tests.get('striplog', tests.get('Striplog', 1)):
            this_tests = []

        return {test.__name__: test(self) for test in this_tests}

def quality ( self , tests , alias = None ) : alias = alias or { } alias = alias . get ( 'striplog' , alias . get ( 'Striplog' , [ ] ) ) this_tests = tests . get ( 'all' , [ ] ) + tests . get ( 'All' , [ ] ) + tests . get ( 'ALL' , [ ] ) + tests . get ( 'striplog' , tests . get ( 'Striplog' , [ ] ) ) + utils . flatten_list ( [ tests . get ( a ) for a in alias ] ) this_tests = filter ( None , this_tests ) if not tests . get ( 'striplog' , tests . get ( 'Striplog' , 1 ) ) : this_tests = [ ] return { test . __name__ : test ( self ) for test in this_tests }

Run a series of tests and return the corresponding results .


def hex_to_name(hexx):
    """
    Convert hex to a color name, using matplotlib's colour names.

    Args:
        hexx (str): A hexadecimal colour, starting with '#'.

    Returns:
        str: The name of the colour, or None if not found.
    """
    for n, h in defaults.COLOURS.items():
        if (len(n) > 1) and (h == hexx.upper()):
            return n.lower()
    return None

def hex_to_name ( hexx ) : for n , h in defaults . COLOURS . items ( ) : if ( len ( n ) > 1 ) and ( h == hexx . upper ( ) ) : return n . lower ( ) return None

Convert hex to a color name using matplotlib s colour names .


def loglike_from_image(filename, offset):
    """
    Get a log-like stream of RGB values from an image.

    Args:
        filename (str): The filename of a PNG image.
        offset (Number): If < 1, interpreted as proportion of way across
            the image. If > 1, interpreted as pixels from left.

    Returns:
        ndarray: A 2d array (a column of RGB triples) at the specified
        offset.

    TODO:
        Generalize this to extract 'logs' from images in other ways, such
        as giving the mean of a range of pixel columns, or an array of
        columns. See also a similar routine in pythonanywhere/freqbot.
    """
    im = plt.imread(filename)
    if offset < 1:
        col = int(im.shape[1] * offset)
    else:
        col = offset
    return im[:, col, :3]

def loglike_from_image ( filename , offset ) : im = plt . imread ( filename ) if offset < 1 : col = int ( im . shape [ 1 ] * offset ) else : col = offset return im [ : , col , : 3 ]

Get a log - like stream of RGB values from an image .


def get_field(self, field_name, args, kwargs):
        """
        Return an underscore if the attribute is absent.
        Not all components have the same attributes.
        """
        try:
            s = super(CustomFormatter, self)
            return s.get_field(field_name, args, kwargs)
        except KeyError:    # Key is missing
            return ("_", field_name)
        except IndexError:  # Value is missing
            return ("_", field_name)

def get_field ( self , field_name , args , kwargs ) : try : s = super ( CustomFormatter , self ) return s . get_field ( field_name , args , kwargs ) except KeyError : return ( "_" , field_name ) except IndexError : return ( "_" , field_name )

Return an underscore if the attribute is absent . Not all components have the same attributes .


def get_jobs(self, prefix=None):
        """ Lists all the jobs registered with Nomad.

           https://www.nomadproject.io/docs/http/jobs.html
            arguments:
              - prefix :(str) optional, specifies a string to filter jobs on based on an prefix.
                        This is specified as a querystring parameter.
            returns: list
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        params = {"prefix": prefix}
        return self.request(method="get", params=params).json()

def get_jobs ( self , prefix = None ) : params = { "prefix" : prefix } return self . request ( method = "get" , params = params ) . json ( )

Lists all the jobs registered with Nomad .


def parse(self, hcl, canonicalize=False):
        """ Parse a HCL Job file. Returns a dict with the JSON formatted job.
            This API endpoint is only supported from Nomad version 0.8.3.

            https://www.nomadproject.io/api/jobs.html#parse-job

            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        return self.request("parse", json={"JobHCL": hcl, "Canonicalize": canonicalize}, method="post", allow_redirects=True).json()

def parse ( self , hcl , canonicalize = False ) : return self . request ( "parse" , json = { "JobHCL" : hcl , "Canonicalize" : canonicalize } , method = "post" , allow_redirects = True ) . json ( )

Parse a HCL Job file . Returns a dict with the JSON formatted job . This API endpoint is only supported from Nomad version 0 . 8 . 3 .


def update_token(self, id, token):
        """ Update token.

            https://www.nomadproject.io/api/acl-tokens.html

            arguments:
                - AccdesorID
                - token
            returns: dict

            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        return self.request("token", id, json=token, method="post").json()

def update_token ( self , id , token ) : return self . request ( "token" , id , json = token , method = "post" ) . json ( )


def get_allocations(self, prefix=None):
        """ Lists all the allocations.

           https://www.nomadproject.io/docs/http/allocs.html
            arguments:
              - prefix :(str) optional, specifies a string to filter allocations on based on an prefix.
                        This is specified as a querystring parameter.
            returns: list
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        params = {"prefix": prefix}
        return self.request(method="get", params=params).json()

def get_allocations ( self , prefix = None ) : params = { "prefix" : prefix } return self . request ( method = "get" , params = params ) . json ( )


def fail_deployment(self, id):
        """ This endpoint is used to mark a deployment as failed. This should be done to force the scheduler to stop
            creating allocations as part of the deployment or to cause a rollback to a previous job version.

           https://www.nomadproject.io/docs/http/deployments.html

            arguments:
              - id
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        fail_json = {"DeploymentID": id}
        return self.request("fail", id, json=fail_json, method="post").json()

def fail_deployment ( self , id ) : fail_json = { "DeploymentID" : id } return self . request ( "fail" , id , json = fail_json , method = "post" ) . json ( )

This endpoint is used to mark a deployment as failed . This should be done to force the scheduler to stop creating allocations as part of the deployment or to cause a rollback to a previous job version .


def pause_deployment(self, id, pause):
        """ This endpoint is used to pause or unpause a deployment.
            This is done to pause a rolling upgrade or resume it.

           https://www.nomadproject.io/docs/http/deployments.html

            arguments:
              - id
              - pause, Specifies whether to pause or resume the deployment.
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        pause_json = {"Pause": pause,
                      "DeploymentID": id}
        return self.request("pause", id, json=pause_json, method="post").json()

def pause_deployment ( self , id , pause ) : pause_json = { "Pause" : pause , "DeploymentID" : id } return self . request ( "pause" , id , json = pause_json , method = "post" ) . json ( )

This endpoint is used to pause or unpause a deployment . This is done to pause a rolling upgrade or resume it .


def deployment_allocation_health(self, id, healthy_allocations=list(), unhealthy_allocations=list()):
        """ This endpoint is used to set the health of an allocation that is in the deployment manually. In some use
            cases, automatic detection of allocation health may not be desired. As such those task groups can be marked
            with an upgrade policy that uses health_check = "manual". Those allocations must have their health marked
            manually using this endpoint. Marking an allocation as healthy will allow the rolling upgrade to proceed.
            Marking it as failed will cause the deployment to fail.

           https://www.nomadproject.io/docs/http/deployments.html

            arguments:
              - id
              - healthy_allocations, Specifies the set of allocation that should be marked as healthy.
              - unhealthy_allocations,  Specifies the set of allocation that should be marked as unhealthy.
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        allocations = {"HealthyAllocationIDs": healthy_allocations,
                       "UnHealthyAllocationIDs": unhealthy_allocations,
                       "DeploymentID": id}
        return self.request("allocation-health", id, json=allocations, method="post").json()

def deployment_allocation_health ( self , id , healthy_allocations = list ( ) , unhealthy_allocations = list ( ) ) : allocations = { "HealthyAllocationIDs" : healthy_allocations , "UnHealthyAllocationIDs" : unhealthy_allocations , "DeploymentID" : id } return self . request ( "allocation-health" , id , json = allocations , method = "post" ) . json ( )

This endpoint is used to set the health of an allocation that is in the deployment manually . In some use cases automatic detection of allocation health may not be desired . As such those task groups can be marked with an upgrade policy that uses health_check = manual . Those allocations must have their health marked manually using this endpoint . Marking an allocation as healthy will allow the rolling upgrade to proceed . Marking it as failed will cause the deployment to fail .


def drain_node(self, id, enable=False):
        """ Toggle the drain mode of the node.
            When enabled, no further allocations will be
            assigned and existing allocations will be migrated.

           https://www.nomadproject.io/docs/http/node.html

            arguments:
              - id (str uuid): node id
              - enable (bool): enable node drain or not to enable node drain
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """

        return self.request(id, "drain", params={"enable": enable}, method="post").json()

def drain_node ( self , id , enable = False ) : return self . request ( id , "drain" , params = { "enable" : enable } , method = "post" ) . json ( )

Toggle the drain mode of the node . When enabled no further allocations will be assigned and existing allocations will be migrated .


def drain_node_with_spec(self, id, drain_spec, mark_eligible=None):
        """ This endpoint toggles the drain mode of the node. When draining is enabled,
            no further allocations will be assigned to this node, and existing allocations
            will be migrated to new nodes.

            If an empty dictionary is given as drain_spec this will disable/toggle the drain.

            https://www.nomadproject.io/docs/http/node.html

            arguments:
              - id (str uuid): node id
              - drain_spec (dict): https://www.nomadproject.io/api/nodes.html#drainspec
              - mark_eligible (bool): https://www.nomadproject.io/api/nodes.html#markeligible
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        payload = {}

        if drain_spec and mark_eligible is not None:
            payload = {
                "NodeID": id,
                "DrainSpec": drain_spec,
                "MarkEligible": mark_eligible
            }
        elif drain_spec and mark_eligible is None:
            payload = {
                "NodeID": id,
                "DrainSpec": drain_spec
            }
        elif not drain_spec and mark_eligible is not None:
            payload = {
                "NodeID": id,
                "DrainSpec": None,
                "MarkEligible": mark_eligible
            }
        elif not drain_spec and mark_eligible is None:
            payload = {
                "NodeID": id,
                "DrainSpec": None,
            }

        return self.request(id, "drain", json=payload, method="post").json()

def drain_node_with_spec ( self , id , drain_spec , mark_eligible = None ) : payload = { } if drain_spec and mark_eligible is not None : payload = { "NodeID" : id , "DrainSpec" : drain_spec , "MarkEligible" : mark_eligible } elif drain_spec and mark_eligible is None : payload = { "NodeID" : id , "DrainSpec" : drain_spec } elif not drain_spec and mark_eligible is not None : payload = { "NodeID" : id , "DrainSpec" : None , "MarkEligible" : mark_eligible } elif not drain_spec and mark_eligible is None : payload = { "NodeID" : id , "DrainSpec" : None , } return self . request ( id , "drain" , json = payload , method = "post" ) . json ( )

This endpoint toggles the drain mode of the node . When draining is enabled no further allocations will be assigned to this node and existing allocations will be migrated to new nodes .


def eligible_node(self, id, eligible=None, ineligible=None):
        """ Toggle the eligibility of the node.

           https://www.nomadproject.io/docs/http/node.html

            arguments:
              - id (str uuid): node id
              - eligible (bool): Set to True to mark node eligible
              - ineligible (bool): Set to True to mark node ineligible
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        payload = {}

        if eligible is not None and ineligible is not None:
            raise nomad.api.exceptions.InvalidParameters
        if eligible is None and ineligible is None:
            raise nomad.api.exceptions.InvalidParameters

        if eligible is not None and eligible:
            payload = {"Eligibility": "eligible", "NodeID": id}
        elif eligible is not None and not eligible:
            payload = {"Eligibility": "ineligible", "NodeID": id}
        elif ineligible is not None:
            payload = {"Eligibility": "ineligible", "NodeID": id}
        elif ineligible is not None and not ineligible:
            payload = {"Eligibility": "eligible", "NodeID": id}

        return self.request(id, "eligibility", json=payload, method="post").json()

def eligible_node ( self , id , eligible = None , ineligible = None ) : payload = { } if eligible is not None and ineligible is not None : raise nomad . api . exceptions . InvalidParameters if eligible is None and ineligible is None : raise nomad . api . exceptions . InvalidParameters if eligible is not None and eligible : payload = { "Eligibility" : "eligible" , "NodeID" : id } elif eligible is not None and not eligible : payload = { "Eligibility" : "ineligible" , "NodeID" : id } elif ineligible is not None : payload = { "Eligibility" : "ineligible" , "NodeID" : id } elif ineligible is not None and not ineligible : payload = { "Eligibility" : "eligible" , "NodeID" : id } return self . request ( id , "eligibility" , json = payload , method = "post" ) . json ( )


def list_files(self, id=None, path="/"):
        """ List files in an allocation directory.

           https://www.nomadproject.io/docs/http/client-fs-ls.html

            arguments:
              - id
              - path
            returns: list
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        if id:
            return self.request(id, params={"path": path}, method="get").json()
        else:
            return self.request(params={"path": path}, method="get").json()

def list_files ( self , id = None , path = "/" ) : if id : return self . request ( id , params = { "path" : path } , method = "get" ) . json ( ) else : return self . request ( params = { "path" : path } , method = "get" ) . json ( )


def stream(self, id, offset, origin, path="/"):
        """ This endpoint streams the contents of a file in an allocation directory.

            https://www.nomadproject.io/api/client.html#stream-file

            arguments:
              - id: (str) allocation_id required
              - offset: (int) required
              - origin: (str) either start|end
              - path: (str) optional
            returns: (str) text
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.BadRequestNomadException
        """
        params = {
            "path": path,
            "offset": offset,
            "origin": origin
        }
        return self.request(id, params=params, method="get").text

def stream ( self , id , offset , origin , path = "/" ) : params = { "path" : path , "offset" : offset , "origin" : origin } return self . request ( id , params = params , method = "get" ) . text

This endpoint streams the contents of a file in an allocation directory .


def stat_file(self, id=None, path="/"):
        """ Stat a file in an allocation directory.

           https://www.nomadproject.io/docs/http/client-fs-stat.html

            arguments:
              - id
              - path
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        if id:
            return self.request(id, params={"path": path}, method="get").json()
        else:
            return self.request(params={"path": path}, method="get").json()

def stat_file ( self , id = None , path = "/" ) : if id : return self . request ( id , params = { "path" : path } , method = "get" ) . json ( ) else : return self . request ( params = { "path" : path } , method = "get" ) . json ( )

Stat a file in an allocation directory .


def join_agent(self, addresses):
        """Initiate a join between the agent and target peers.

           https://www.nomadproject.io/docs/http/agent-join.html

            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        params = {"address": addresses}
        return self.request("join", params=params, method="post").json()

def join_agent ( self , addresses ) : params = { "address" : addresses } return self . request ( "join" , params = params , method = "post" ) . json ( )

Initiate a join between the agent and target peers .


def update_servers(self, addresses):
        """Updates the list of known servers to the provided list.
           Replaces all previous server addresses with the new list.

           https://www.nomadproject.io/docs/http/agent-servers.html

            returns: 200 status code
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        params = {"address": addresses}
        return self.request("servers", params=params, method="post").status_code

def update_servers ( self , addresses ) : params = { "address" : addresses } return self . request ( "servers" , params = params , method = "post" ) . status_code

Updates the list of known servers to the provided list . Replaces all previous server addresses with the new list .


def force_leave(self, node):
        """Force a failed gossip member into the left state.

            https://www.nomadproject.io/docs/http/agent-force-leave.html

            returns: 200 status code
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        params = {"node": node}
        return self.request("force-leave", params=params, method="post").status_code

def force_leave ( self , node ) : params = { "node" : node } return self . request ( "force-leave" , params = params , method = "post" ) . status_code

Force a failed gossip member into the left state .


def get_nodes(self, prefix=None):
        """ Lists all the client nodes registered with Nomad.

           https://www.nomadproject.io/docs/http/nodes.html
            arguments:
              - prefix :(str) optional, specifies a string to filter nodes on based on an prefix.
                        This is specified as a querystring parameter.
            returns: list
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        params = {"prefix": prefix}
        return self.request(method="get", params=params).json()

def get_nodes ( self , prefix = None ) : params = { "prefix" : prefix } return self . request ( method = "get" , params = params ) . json ( )

Lists all the client nodes registered with Nomad .


def get_evaluations(self, prefix=None):
        """ Lists all the evaluations.

           https://www.nomadproject.io/docs/http/evals.html
            arguments:
              - prefix :(str) optional, specifies a string to filter evaluations on based on an prefix.
                        This is specified as a querystring parameter.
            returns: list
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        params = {"prefix": prefix}
        return self.request(method="get", params=params).json()

def get_evaluations ( self , prefix = None ) : params = { "prefix" : prefix } return self . request ( method = "get" , params = params ) . json ( )


def get_namespaces(self, prefix=None):
        """ Lists all the namespaces registered with Nomad.

           https://www.nomadproject.io/docs/enterprise/namespaces/index.html
            arguments:
              - prefix :(str) optional, specifies a string to filter namespaces on based on an prefix.
                        This is specified as a querystring parameter.
            returns: list
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        params = {"prefix": prefix}
        return self.request(method="get", params=params).json()

def get_namespaces ( self , prefix = None ) : params = { "prefix" : prefix } return self . request ( method = "get" , params = params ) . json ( )

Lists all the namespaces registered with Nomad .


def register_job(self, id, job):
        """ Registers a new job or updates an existing job

           https://www.nomadproject.io/docs/http/job.html

            arguments:
              - id
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        return self.request(id, json=job, method="post").json()

def register_job ( self , id , job ) : return self . request ( id , json = job , method = "post" ) . json ( )


def plan_job(self, id, job, diff=False, policy_override=False):
        """ Invoke a dry-run of the scheduler for the job.

           https://www.nomadproject.io/docs/http/job.html

            arguments:
              - id
              - job, dict
              - diff, boolean
              - policy_override, boolean
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        json_dict = {}
        json_dict.update(job)
        json_dict.setdefault('Diff', diff)
        json_dict.setdefault('PolicyOverride', policy_override)
        return self.request(id, "plan", json=json_dict, method="post").json()

def plan_job ( self , id , job , diff = False , policy_override = False ) : json_dict = { } json_dict . update ( job ) json_dict . setdefault ( 'Diff' , diff ) json_dict . setdefault ( 'PolicyOverride' , policy_override ) return self . request ( id , "plan" , json = json_dict , method = "post" ) . json ( )

Invoke a dry - run of the scheduler for the job .


def dispatch_job(self, id, payload=None, meta=None):
        """ Dispatches a new instance of a parameterized job.

           https://www.nomadproject.io/docs/http/job.html

            arguments:
              - id
              - payload
              - meta
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        dispatch_json = {"Meta": meta, "Payload": payload}
        return self.request(id, "dispatch", json=dispatch_json, method="post").json()

def dispatch_job ( self , id , payload = None , meta = None ) : dispatch_json = { "Meta" : meta , "Payload" : payload } return self . request ( id , "dispatch" , json = dispatch_json , method = "post" ) . json ( )

Dispatches a new instance of a parameterized job .


def revert_job(self, id, version, enforce_prior_version=None):
        """ This endpoint reverts the job to an older version.

           https://www.nomadproject.io/docs/http/job.html

            arguments:
              - id
              - version, Specifies the job version to revert to.
            optional_arguments:
              - enforce_prior_version, Optional value specifying the current job's version.
                                       This is checked and acts as a check-and-set value before reverting to the
                                       specified job.
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        revert_json = {"JobID": id,
                       "JobVersion": version,
                       "EnforcePriorVersion": enforce_prior_version}
        return self.request(id, "revert", json=revert_json, method="post").json()

def revert_job ( self , id , version , enforce_prior_version = None ) : revert_json = { "JobID" : id , "JobVersion" : version , "EnforcePriorVersion" : enforce_prior_version } return self . request ( id , "revert" , json = revert_json , method = "post" ) . json ( )

This endpoint reverts the job to an older version .


def stable_job(self, id, version, stable):
        """ This endpoint sets the job's stability.

           https://www.nomadproject.io/docs/http/job.html

            arguments:
              - id
              - version, Specifies the job version to revert to.
              - stable, Specifies whether the job should be marked as stable or not.
            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        revert_json = {"JobID": id,
                       "JobVersion": version,
                       "Stable": stable}
        return self.request(id, "stable", json=revert_json, method="post").json()

def stable_job ( self , id , version , stable ) : revert_json = { "JobID" : id , "JobVersion" : version , "Stable" : stable } return self . request ( id , "stable" , json = revert_json , method = "post" ) . json ( )

This endpoint sets the job s stability .


def deregister_job(self, id, purge=None):
        """ Deregisters a job, and stops all allocations part of it.

           https://www.nomadproject.io/docs/http/job.html

            arguments:
              - id
              - purge (bool), optionally specifies whether the job should be
                stopped and purged immediately (`purge=True`) or deferred to the
                Nomad garbage collector (`purge=False`).

            returns: dict
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
              - nomad.api.exceptions.InvalidParameters
        """
        params = None
        if purge is not None:
            if not isinstance(purge, bool):
                raise nomad.api.exceptions.InvalidParameters("purge is invalid "
                        "(expected type %s but got %s)"%(type(bool()), type(purge)))
            params = {"purge": purge}
        return self.request(id, params=params, method="delete").json()

def
deregister_job
(
self
,
id
,
purge
=
)
:
params
=
if
purge
is
not
:
if
not
isinstance
(
purge
,
bool
)
:
raise
nomad
.
api
.
exceptions
.
InvalidParameters
(
"purge is invalid "
"(expected type %s but got %s)"
%
(
type
(
bool
(
)
)
,
type
(
purge
)
)
)
params
=
{
"purge"
:
purge
}
return
self
.
request
(
id
,
params
=
params
,
method
=
"delete"
)
.
json
(
)

Deregisters a job and stops all allocations part of it .


def get_configuration(self, stale=False):
        """ Query the status of a client node registered with Nomad.

            https://www.nomadproject.io/docs/http/operator.html

            returns: dict
            optional arguments:
              - stale, (defaults to False), Specifies if the cluster should respond without an active leader.
                                            This is specified as a querystring parameter.
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """

        params = {"stale": stale}
        return self.request("raft", "configuration", params=params, method="get").json()

def get_configuration ( self , stale = False ) : params = { "stale" : stale } return self . request ( "raft" , "configuration" , params = params , method = "get" ) . json ( )

Query the status of a client node registered with Nomad .


def delete_peer(self, peer_address, stale=False):
        """ Remove the Nomad server with given address from the Raft configuration.
            The return code signifies success or failure.

            https://www.nomadproject.io/docs/http/operator.html

            arguments:
              - peer_address, The address specifies the server to remove and is given as an IP:port
            optional arguments:
              - stale, (defaults to False), Specifies if the cluster should respond without an active leader.
                                            This is specified as a querystring parameter.
            returns: Boolean
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """

        params = {"address": peer_address, "stale": stale}
        return self.request("raft", "peer", params=params, method="delete").ok

def delete_peer ( self , peer_address , stale = False ) : params = { "address" : peer_address , "stale" : stale } return self . request ( "raft" , "peer" , params = params , method = "delete" ) . ok

Remove the Nomad server with given address from the Raft configuration . The return code signifies success or failure .


def get_deployments(self, prefix=""):
        """ This endpoint lists all deployments.

           https://www.nomadproject.io/docs/http/deployments.html

            optional_arguments:
              - prefix, (default "") Specifies a string to filter deployments on based on an index prefix.
                        This is specified as a querystring parameter.

            returns: list of dicts
            raises:
              - nomad.api.exceptions.BaseNomadException
              - nomad.api.exceptions.URLNotFoundNomadException
        """
        params = {"prefix": prefix}
        return self.request(params=params, method="get").json()

def get_deployments ( self , prefix = "" ) : params = { "prefix" : prefix } return self . request ( params = params , method = "get" ) . json ( )


def _get_random(self, obj_type):
        """
        Get a random mutator from a list of mutators
        """
        return self.mutator[obj_type][random.randint(0, self.config.level)]

def _get_random ( self , obj_type ) : return self . mutator [ obj_type ] [ random . randint ( 0 , self . config . level ) ]


def get_mutator(self, obj, obj_type):
        """
        Get a random mutator for the given type
        """
        if obj_type == unicode:
            obj_type = str
            obj = str(obj)
        return self._get_random(obj_type)(obj)

def get_mutator ( self , obj , obj_type ) : if obj_type == unicode : obj_type = str obj = str ( obj ) return self . _get_random ( obj_type ) ( obj )


def get_string_polyglot_attack(self, obj):
        """
        Return a polyglot attack containing the original object
        """
        return self.polyglot_attacks[random.choice(self.config.techniques)] % obj

def get_string_polyglot_attack ( self , obj ) : return self . polyglot_attacks [ random . choice ( self . config . techniques ) ] % obj


def fuzz(self, obj):
        """
        Perform the fuzzing
        """
        buf = list(obj)
        FuzzFactor = random.randrange(1, len(buf))
        numwrites=random.randrange(math.ceil((float(len(buf)) / FuzzFactor)))+1
        for j in range(numwrites):
            self.random_action(buf)
        return self.safe_unicode(buf)

def fuzz ( self , obj ) : buf = list ( obj ) FuzzFactor = random . randrange ( 1 , len ( buf ) ) numwrites = random . randrange ( math . ceil ( ( float ( len ( buf ) ) / FuzzFactor ) ) ) + 1 for j in range ( numwrites ) : self . random_action ( buf ) return self . safe_unicode ( buf )


def safe_unicode(self, buf):
        """
        Safely return an unicode encoded string
        """
        tmp = ""
        buf = "".join(b for b in buf)
        for character in buf:
            tmp += character
        return tmp

def safe_unicode ( self , buf ) : tmp = "" buf = "" . join ( b for b in buf ) for character in buf : tmp += character return tmp


def run(self):
        """
        Start the servers
        """
        route("/")(self.serve)
        if self.config.html:
            route("/<filepath:path>")(self.custom_html)
        if self.config.fuzz_web:
            self.request_checker.start()
        self.httpd.start()
        self.httpsd.start()

def run ( self ) : route ( "/" ) ( self . serve ) if self . config . html : route ( "/<filepath:path>" ) ( self . custom_html ) if self . config . fuzz_web : self . request_checker . start ( ) self . httpd . start ( ) self . httpsd . start ( )


def stop(self):
        """
        Kill the servers
        """
        os.kill(self.httpd.pid, signal.SIGKILL)
        os.kill(self.httpsd.pid, signal.SIGKILL)
        self.client_queue.put((0,0))
        if self.config.fuzz_web:
            self.request_checker.join()
        self.logger.debug("[{0}] - PJFServer successfully completed".format(time.strftime("%H:%M:%S")))


def custom_html(self, filepath):
        """
        Serve custom HTML page
        """
        try:
            response.headers.append("Access-Control-Allow-Origin", "*")
            response.headers.append("Accept-Encoding", "identity")
            response.headers.append("Content-Type", "text/html")
            return static_file(filepath, root=self.config.html)
        except Exception as e:
            raise PJFBaseException(e.message if hasattr(e, "message") else str(e))

def custom_html ( self , filepath ) : try : response . headers . append ( "Access-Control-Allow-Origin" , "*" ) response . headers . append ( "Accept-Encoding" , "identity" ) response . headers . append ( "Content-Type" , "text/html" ) return static_file ( filepath , root = self . config . html ) except Exception as e : raise PJFBaseException ( e . message if hasattr ( e , "message" ) else str ( e ) )


def serve(self):
        """
        Serve fuzzed JSON object
        """
        try:
            fuzzed = self.json.fuzzed
            if self.config.fuzz_web:
                self.client_queue.put((request.environ.get('REMOTE_ADDR'), fuzzed))
            response.headers.append("Access-Control-Allow-Origin", "*")
            response.headers.append("Accept-Encoding", "identity")
            response.headers.append("Content-Type", self.config.content_type)
            if self.config.notify:
                PJFTestcaseServer.send_testcase(fuzzed, '127.0.0.1', self.config.ports["servers"]["TCASE_PORT"])
            yield fuzzed
        except Exception as e:
            raise PJFBaseException(e.message if hasattr(e, "message") else str(e))

def serve ( self ) : try : fuzzed = self . json . fuzzed if self . config . fuzz_web : self . client_queue . put ( ( request . environ . get ( 'REMOTE_ADDR' ) , fuzzed ) ) response . headers . append ( "Access-Control-Allow-Origin" , "*" ) response . headers . append ( "Accept-Encoding" , "identity" ) response . headers . append ( "Content-Type" , self . config . content_type ) if self . config . notify : PJFTestcaseServer . send_testcase ( fuzzed , '127.0.0.1' , self . config . ports [ "servers" ] [ "TCASE_PORT" ] ) yield fuzzed except Exception as e : raise PJFBaseException ( e . message if hasattr ( e , "message" ) else str ( e ) )


def fuzz(self, obj):
        """
        Generic fuzz mutator, use a decorator for the given type
        """
        decorators = self.decorators

        @decorators.mutate_object_decorate
        def mutate():
            return obj
        return mutate()

def fuzz ( self , obj ) : decorators = self . decorators @ decorators . mutate_object_decorate def mutate ( ) : return obj return mutate ( )

Generic fuzz mutator use a decorator for the given type


def spawn(self, cmd, stdin_content="", stdin=False, shell=False, timeout=2):
        """
        Spawn a new process using subprocess
        """
        try:
            if type(cmd) != list:
                raise PJFInvalidType(type(cmd), list)
            if type(stdin_content) != str:
                raise PJFInvalidType(type(stdin_content), str)
            if type(stdin) != bool:
                raise PJFInvalidType(type(stdin), bool)
            self._in = stdin_content
            try:
                self.process = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=shell)
                self.finish_read(timeout, stdin_content, stdin)
                if self.process.poll() is not None:
                    self.close()
            except KeyboardInterrupt:
                return
        except OSError:
            raise PJFProcessExecutionError("Binary <%s> does not exist" % cmd[0])
        except Exception as e:
            raise PJFBaseException(e.message if hasattr(e, "message") else str(e))

def
spawn
(
self
,
cmd
,
stdin_content
=
""
,
stdin
=
False
,
shell
=
False
,
timeout
=
2
)
:
try
:
if
type
(
cmd
)
!=
list
:
raise
PJFInvalidType
(
type
(
cmd
)
,
list
)
if
type
(
stdin_content
)
!=
str
:
raise
PJFInvalidType
(
type
(
stdin_content
)
,
str
)
if
type
(
stdin
)
!=
bool
:
raise
PJFInvalidType
(
type
(
stdin
)
,
bool
)
self
.
_in
=
stdin_content
try
:
self
.
process
=
subprocess
.
Popen
(
cmd
,
stdout
=
PIPE
,
stderr
=
PIPE
,
stdin
=
PIPE
,
shell
=
shell
)
self
.
finish_read
(
timeout
,
stdin_content
,
stdin
)
if
self
.
process
.
poll
(
)
is
not
:
self
.
close
(
)
except
KeyboardInterrupt
:
return
except
OSError
:
raise
PJFProcessExecutionError
(
"Binary <%s> does not exist"
%
cmd
[
0
]
)
except
Exception
as
e
:
raise
PJFBaseException
(
e
.
message
if
hasattr
(
e
,
"message"
)
else
str
(
e
)
)


def get_output(self, stdin_content, stdin):
        """
        Try to get output in a separate thread
        """
        try:
            if stdin:
                if sys.version_info >= (3, 0):
                    self.process.stdin.write(bytes(stdin_content, "utf-8"))
                else:
                    self.process.stdin.write(stdin_content)
            self._out = self.process.communicate()[0]
        except (error, IOError):
            self._out = self._in
            pass

def get_output ( self , stdin_content , stdin ) : try : if stdin : if sys . version_info >= ( 3 , 0 ) : self . process . stdin . write ( bytes ( stdin_content , "utf-8" ) ) else : self . process . stdin . write ( stdin_content ) self . _out = self . process . communicate ( ) [ 0 ] except ( error , IOError ) : self . _out = self . _in pass


def finish_read(self, timeout=2, stdin_content="", stdin=False):
        """
        Wait until we got output or until timeout is over
        """
        process = Thread(target=self.get_output, args=(stdin_content, stdin))
        process.start()
        if timeout > 0:
            process.join(timeout)
        else:
            process.join()
        if process.is_alive():
            self.close()
            self.return_code = -signal.SIGHUP
        else:
            self.return_code = self.process.returncode

def finish_read ( self , timeout = 2 , stdin_content = "" , stdin = False ) : process = Thread ( target = self . get_output , args = ( stdin_content , stdin ) ) process . start ( ) if timeout > 0 : process . join ( timeout ) else : process . join ( ) if process . is_alive ( ) : self . close ( ) self . return_code = - signal . SIGHUP else : self . return_code = self . process . returncode


def close(self):
        """
        Terminate the newly created process
        """
        try:
            self.process.terminate()
            self.return_code = self.process.returncode
        except OSError:
            pass
        self.process.stdin.close()
        self.process.stdout.close()
        self.process.stderr.close()
        self.logger.debug("[{0}] - PJFExecutor successfully completed".format(time.strftime("%H:%M:%S")))


def start(self):
        """
        Parse the command line and start PyJFuzz
        """
        from .pjf_worker import PJFWorker
        worker = PJFWorker(self)
        if self.update_pjf:
            worker.update_library()
        elif self.browser_auto:
            worker.browser_autopwn()
        elif self.fuzz_web:
            worker.web_fuzzer()
        elif self.json:
            if not self.web_server and not self.ext_fuzz and not self.cmd_fuzz:
                worker.fuzz()
            elif self.ext_fuzz:
                if self.stdin:
                    worker.fuzz_stdin()
                else:
                    worker.fuzz_command_line()
            elif self.cmd_fuzz:
                if self.stdin:
                    worker.fuzz_external(True)
                else:
                    worker.fuzz_external()
            else:
                worker.start_http_server()
        elif self.json_file:
            worker.start_file_fuzz()
        elif self.process_to_monitor:
            worker.start_process_monitor()

def start ( self ) : from . pjf_worker import PJFWorker worker = PJFWorker ( self ) if self . update_pjf : worker . update_library ( ) elif self . browser_auto : worker . browser_autopwn ( ) elif self . fuzz_web : worker . web_fuzzer ( ) elif self . json : if not self . web_server and not self . ext_fuzz and not self . cmd_fuzz : worker . fuzz ( ) elif self . ext_fuzz : if self . stdin : worker . fuzz_stdin ( ) else : worker . fuzz_command_line ( ) elif self . cmd_fuzz : if self . stdin : worker . fuzz_external ( True ) else : worker . fuzz_external ( ) else : worker . start_http_server ( ) elif self . json_file : worker . start_file_fuzz ( ) elif self . process_to_monitor : worker . start_process_monitor ( )


def execute(self, obj):
        """
        Perform the actual external fuzzing, you may replace this method in order to increase performance
        """
        try:
            if self.config.stdin:
                    self.spawn(self.config.command, stdin_content=obj, stdin=True, timeout=1)
            else:
                if "@@" not in self.config.command:
                    raise PJFMissingArgument("Missing @@ filename indicator while using non-stdin fuzzing method")
                for x in self.config.command:
                    if "@@" in x:
                        self.config.command[self.config.command.index(x)] = x.replace("@@", obj)
                self.spawn(self.config.command, timeout=2)
            self.logger.debug("[{0}] - PJFExternalFuzzer successfully completed".format(time.strftime("%H:%M:%S")))
            return self._out
        except KeyboardInterrupt:
            return ""
        except Exception as e:
            raise PJFBaseException(e.message if hasattr(e, "message") else str(e))

Perform the actual external fuzzing you may replace this method in order to increase performance


def json_encode(func):
        """
        Decorator used to change the return value from PJFFactory.fuzzed, it makes the structure printable
        """
        def func_wrapper(self, indent, utf8):
            if utf8:
                encoding = "\\x%02x"
            else:
                encoding = "\\u%04x"
            hex_regex = re.compile(r"(\\\\x[a-fA-F0-9]{2})")
            unicode_regex = re.compile(r"(\\u[a-fA-F0-9]{4})")

            def encode_decode_all(d, _decode=True):
                if type(d) == dict:
                    for k in d:
                        if type(d[k]) in [dict, list]:
                            if _decode:
                                d[k] = encode_decode_all(d[k])
                            else:
                                d[k] = encode_decode_all(d[k], _decode=False)
                        elif type(d[k]) == str:
                            if _decode:
                                d[k] = decode(d[k])
                            else:
                                d[k] = encode(d[k])
                elif type(d) == list:
                    arr = []
                    for e in d:
                        if type(e) == str:
                            if _decode:
                                arr.append(decode(e))
                            else:
                                arr.append(encode(e))
                        elif type(e) in [dict, list]:
                            if _decode:
                                arr.append(encode_decode_all(e))
                            else:
                                arr.append(encode_decode_all(e, _decode=False))
                        else:
                            arr.append(e)
                    return arr
                else:
                    if _decode:
                        return decode(d)
                    else:
                        return encode(d)
                return d

            def decode(x):
                tmp = "".join(encoding % ord(c) if c not in p else c for c in x)
                if sys.version_info >= (3, 0):
                    return str(tmp)
                else:
                    for encoded in unicode_regex.findall(tmp):
                        tmp = tmp.replace(encoded, encoded.decode("unicode_escape"))
                    return unicode(tmp)

            def encode(x):
                for encoded in hex_regex.findall(x):
                    if sys.version_info >= (3, 0):
                        x = x.replace(encoded, bytes(str(encoded).replace("\\\\x", "\\x"),"utf-8").decode("unicode_escape"))
                    else:
                        x = x.replace(encoded, str(encoded).replace("\\\\x", "\\x").decode("string_escape"))
                return x

            if indent:
                return encode_decode_all("{0}".format(json.dumps(encode_decode_all(func(self)), indent=5)),
                                         _decode=False)
            else:
                return encode_decode_all("{0}".format(json.dumps(encode_decode_all(func(self)))), _decode=False)

        return func_wrapper

def json_encode ( func ) : def func_wrapper ( self , indent , utf8 ) : if utf8 : encoding = "\\x%02x" else : encoding = "\\u%04x" hex_regex = re . compile ( r"(\\\\x[a-fA-F0-9]{2})" ) unicode_regex = re . compile ( r"(\\u[a-fA-F0-9]{4})" ) def encode_decode_all ( d , _decode = True ) : if type ( d ) == dict : for k in d : if type ( d [ k ] ) in [ dict , list ] : if _decode : d [ k ] = encode_decode_all ( d [ k ] ) else : d [ k ] = encode_decode_all ( d [ k ] , _decode = False ) elif type ( d [ k ] ) == str : if _decode : d [ k ] = decode ( d [ k ] ) else : d [ k ] = encode ( d [ k ] ) elif type ( d ) == list : arr = [ ] for e in d : if type ( e ) == str : if _decode : arr . append ( decode ( e ) ) else : arr . append ( encode ( e ) ) elif type ( e ) in [ dict , list ] : if _decode : arr . append ( encode_decode_all ( e ) ) else : arr . append ( encode_decode_all ( e , _decode = False ) ) else : arr . append ( e ) return arr else : if _decode : return decode ( d ) else : return encode ( d ) return d def decode ( x ) : tmp = "" . join ( encoding % ord ( c ) if c not in p else c for c in x ) if sys . version_info >= ( 3 , 0 ) : return str ( tmp ) else : for encoded in unicode_regex . findall ( tmp ) : tmp = tmp . replace ( encoded , encoded . decode ( "unicode_escape" ) ) return unicode ( tmp ) def encode ( x ) : for encoded in hex_regex . findall ( x ) : if sys . version_info >= ( 3 , 0 ) : x = x . replace ( encoded , bytes ( str ( encoded ) . replace ( "\\\\x" , "\\x" ) , "utf-8" ) . decode ( "unicode_escape" ) ) else : x = x . replace ( encoded , str ( encoded ) . replace ( "\\\\x" , "\\x" ) . decode ( "string_escape" ) ) return x if indent : return encode_decode_all ( "{0}" . format ( json . dumps ( encode_decode_all ( func ( self ) ) , indent = 5 ) ) , _decode = False ) else : return encode_decode_all ( "{0}" . format ( json . dumps ( encode_decode_all ( func ( self ) ) ) ) , _decode = False ) return func_wrapper

Decorator used to change the return value from PJFFactory . fuzzed it makes the structure printable


def build(self, pre=None, shortest=False):
        """Build the String instance

        :param list pre: The prerequisites list (optional, default=None)
        :param bool shortest: Whether or not the shortest reference-chain (most minimal) version of the field should be generated.
        """
        if pre is None:
            pre = []

        if self.value is not None and rand.maybe():
            return utils.val(self.value, pre, shortest=shortest)

        length = super(String, self).build(pre, shortest=shortest)
        res = rand.data(length, self.charset)
        return res

def build ( self , pre = None , shortest = False ) : if pre is None : pre = [ ] if self . value is not None and rand . maybe ( ) : return utils . val ( self . value , pre , shortest = shortest ) length = super ( String , self ) . build ( pre , shortest = shortest ) res = rand . data ( length , self . charset ) return res


def build(self, pre=None, shortest=False):
        """Build the ``And`` instance

        :param list pre: The prerequisites list
        :param bool shortest: Whether or not the shortest reference-chain (most minimal) version of the field should be generated.
        """
        if pre is None:
            pre = []

        res = deque()
        for x in self.values:
            try:
                res.append(utils.val(x, pre, shortest=shortest))
            except errors.OptGram as e:
                continue
            except errors.FlushGrams as e:
                prev = "".join(res)
                res.clear()
                # this is assuming a scope was pushed!
                if len(self.fuzzer._scope_stack) == 1:
                    pre.append(prev)
                else:
                    stmts = self.fuzzer._curr_scope.setdefault("prev_append", deque())
                    stmts.extend(pre)
                    stmts.append(prev)
                    pre.clear()
                continue

        return self.sep.join(res)

def build ( self , pre = None , shortest = False ) : if pre is None : pre = [ ] res = deque ( ) for x in self . values : try : res . append ( utils . val ( x , pre , shortest = shortest ) ) except errors . OptGram as e : continue except errors . FlushGrams as e : prev = "" . join ( res ) res . clear ( ) if len ( self . fuzzer . _scope_stack ) == 1 : pre . append ( prev ) else : stmts = self . fuzzer . _curr_scope . setdefault ( "prev_append" , deque ( ) ) stmts . extend ( pre ) stmts . append ( prev ) pre . clear ( ) continue return self . sep . join ( res )


def build(self, pre=None, shortest=False):
        """Build the ``Quote`` instance

        :param list pre: The prerequisites list
        :param bool shortest: Whether or not the shortest reference-chain (most minimal) version of the field should be generated.
        """
        res = super(Q, self).build(pre, shortest=shortest)

        if self.escape:
            return repr(res)
        elif self.html_js_escape:
            return ("'" + res.encode("string_escape").replace("<", "\\x3c").replace(">", "\\x3e") + "'")
        else:
            return "{q}{r}{q}".format(q=self.quote, r=res)

def build ( self , pre = None , shortest = False ) : res = super ( Q , self ) . build ( pre , shortest = shortest ) if self . escape : return repr ( res ) elif self . html_js_escape : return ( "'" + res . encode ( "string_escape" ) . replace ( "<" , "\\x3c" ) . replace ( ">" , "\\x3e" ) + "'" ) else : return "{q}{r}{q}" . format ( q = self . quote , r = res )


def build(self, pre=None, shortest=False):
        """Build the ``Or`` instance

        :param list pre: The prerequisites list
        :param bool shortest: Whether or not the shortest reference-chain (most minimal) version of the field should be generated.
        """
        if pre is None:
            pre = []

        # self.shortest_vals will be set by the GramFuzzer and will
        # contain a list of value options that have a minimal reference
        # chain
        if shortest and self.shortest_vals is not None:
            return utils.val(rand.choice(self.shortest_vals), pre, shortest=shortest)
        else:
            return utils.val(rand.choice(self.values), pre, shortest=shortest)

def build ( self , pre = None , shortest = False ) : if pre is None : pre = [ ] if shortest and self . shortest_vals is not None : return utils . val ( rand . choice ( self . shortest_vals ) , pre , shortest = shortest ) else : return utils . val ( rand . choice ( self . values ) , pre , shortest = shortest )


def build(self, pre=None, shortest=False):
        """Build the current ``Opt`` instance

        :param list pre: The prerequisites list
        :param bool shortest: Whether or not the shortest reference-chain (most minimal) version of the field should be generated.
        """
        if pre is None:
            pre = []

        if shortest or rand.maybe(self.prob):
            raise errors.OptGram

        return super(Opt, self).build(pre, shortest=shortest)

def build ( self , pre = None , shortest = False ) : if pre is None : pre = [ ] if shortest or rand . maybe ( self . prob ) : raise errors . OptGram return super ( Opt , self ) . build ( pre , shortest = shortest )


def build(self, pre=None, shortest=False):
        """Build the ``Ref`` instance by fetching the rule from
        the GramFuzzer instance and building it

        :param list pre: The prerequisites list
        :param bool shortest: Whether or not the shortest reference-chain (most minimal) version of the field should be generated.
        """
        global REF_LEVEL
        REF_LEVEL += 1

        try:
            if pre is None:
                pre = []

            #print("{:04d} - {} - {}:{}".format(REF_LEVEL, shortest, self.cat, self.refname))

            definition = self.fuzzer.get_ref(self.cat, self.refname)
            res = utils.val(
                definition,
                pre,
                shortest=(shortest or REF_LEVEL >= self.max_recursion)
            )

            return res

        # this needs to happen no matter what
        finally:
            REF_LEVEL -= 1

def build ( self , pre = None , shortest = False ) : global REF_LEVEL REF_LEVEL += 1 try : if pre is None : pre = [ ] definition = self . fuzzer . get_ref ( self . cat , self . refname ) res = utils . val ( definition , pre , shortest = ( shortest or REF_LEVEL >= self . max_recursion ) ) return res finally : REF_LEVEL -= 1

Build the Ref instance by fetching the rule from the GramFuzzer instance and building it


def build(self, pre=None, shortest=False):
        """Build the STAR field.

        :param list pre: The prerequisites list
        :param bool shortest: Whether or not the shortest reference-chain (most minimal) version of the field should be generated.
        """
        if pre is None:
            pre = []

        if shortest:
            raise errors.OptGram
        elif rand.maybe():
            return super(STAR, self).build(pre, shortest=shortest)
        else:
            raise errors.OptGram

def build ( self , pre = None , shortest = False ) : if pre is None : pre = [ ] if shortest : raise errors . OptGram elif rand . maybe ( ) : return super ( STAR , self ) . build ( pre , shortest = shortest ) else : raise errors . OptGram


def shutdown(self, *args):
        """
        Shutdown the running process and the monitor
        """
        try:
            self._shutdown()
            if self.process:
                self.process.wait()
                self.process.stdout.close()
                self.process.stdin.close()
                self.process.stderr.close()
            self.finished = True
            self.send_testcase('', '127.0.0.1', self.config.ports["servers"]["TCASE_PORT"])
            self.logger.debug("[{0}] - PJFProcessMonitor successfully completed".format(time.strftime("%H:%M:%S")))
        except Exception as e:
            raise PJFBaseException(e.message if hasattr(e, "message") else str(e))


def run_and_monitor(self):
        """
        Run command once and check exit code
        """
        signal.signal(signal.SIGINT, self.shutdown)
        self.spawn(self.config.process_to_monitor, timeout=0)
        return self._is_sigsegv(self.return_code)

def run_and_monitor ( self ) : signal . signal ( signal . SIGINT , self . shutdown ) self . spawn ( self . config . process_to_monitor , timeout = 0 ) return self . _is_sigsegv ( self . return_code )


def start_monitor(self, standalone=True):
        """
        Run command in a loop and check exit status plus restart process when needed
        """
        try:
            self.start()
            cmdline = shlex.split(self.config.process_to_monitor)
            if standalone:
                signal.signal(signal.SIGINT, self.shutdown)
            self.process = subprocess.Popen(cmdline, stdin=PIPE, stdout=PIPE, stderr=PIPE)
            while self.process and not self.finished:
                self.process.wait()
                if self._is_sigsegv(self.process.returncode):
                    if self.config.debug:
                        print("[\033[92mINFO\033[0m] Process crashed with \033[91mSIGSEGV\033[0m, waiting for testcase...")
                    while not self.got_testcase():
                        time.sleep(1)
                    self.save_testcase(self.testcase[-10:])  # just take last 10 testcases
                if self.process:
                    self.process = subprocess.Popen(cmdline, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        except OSError:
            self.shutdown()
            self.process = False
            self.got_testcase = lambda: True
            raise PJFProcessExecutionError("Binary <%s> does not exist" % cmdline[0])
        except Exception as e:
            raise PJFBaseException("Unknown error please send log to author")


def randfloat(a, b=None):
    """Return a random float

    :param float a: Either the minimum value (inclusive) if ``b`` is set, or
    the maximum value if ``b`` is not set (non-inclusive, in which case the minimum
    is implicitly 0.0)
    :param float b: The maximum value to generate (non-inclusive)
    :returns: float
    """
    if b is None:
        max_ = a
        min_ = 0.0
    else:
        min_ = a
        max_ = b

    diff = max_ - min_
    res = _random()
    res *= diff
    res += min_
    return res

def randfloat ( a , b = None ) : if b is None : max_ = a min_ = 0.0 else : min_ = a max_ = b diff = max_ - min_ res = _random ( ) res *= diff res += min_ return res


def add_definition(self, cat, def_name, def_val, no_prune=False, gram_file="default"):
        """Add a new rule definition named ``def_name`` having value ``def_value`` to
        the category ``cat``.

        :param str cat: The category to add the rule to
        :param str def_name: The name of the rule definition
        :param def_val: The value of the rule definition
        :param bool no_prune: If the rule should explicitly *NOT*
            be pruned even if it has been determined to be unreachable (default=``False``)
        :param str gram_file: The file the rule was defined in (default=``"default"``).
        """
        self._rules_processed = False

        self.add_to_cat_group(cat, gram_file, def_name)

        if no_prune:
            self.no_prunes.setdefault(cat, {}).setdefault(def_name, True)

        if self._staged_defs is not None:
            # if we're tracking changes during rule generation, add any new rules
            # to _staged_defs so they can be reverted if something goes wrong
            self._staged_defs.append((cat, def_name, def_val))
        else:
            self.defs.setdefault(cat, {}).setdefault(def_name, deque()).append(def_val)

def add_definition ( self , cat , def_name , def_val , no_prune = False , gram_file = "default" ) : self . _rules_processed = False self . add_to_cat_group ( cat , gram_file , def_name ) if no_prune : self . no_prunes . setdefault ( cat , { } ) . setdefault ( def_name , True ) if self . _staged_defs is not None : self . _staged_defs . append ( ( cat , def_name , def_val ) ) else : self . defs . setdefault ( cat , { } ) . setdefault ( def_name , deque ( ) ) . append ( def_val )

Add a new rule definition named def_name having value def_value to the category cat .