def get_parameters(self):
        """Returns a list of parameters"""
        if self.plugin_class is None:
            sig = inspect.signature(self.func)
            for index, parameter in enumerate(sig.parameters.values()):
                if not parameter.kind in [parameter.POSITIONAL_ONLY, parameter.KEYWORD_ONLY, parameter.POSITIONAL_OR_KEYWORD]:
                    raise RuntimeError("Task {} contains an unsupported {} parameter".format(parameter, parameter.kind))

                yield parameter

        else:
            var_keyword_seen = set()

            for cls in inspect.getmro(self.plugin_class):
                if issubclass(cls, BasePlugin) and hasattr(cls, self.func.__name__):
                    func = getattr(cls, self.func.__name__)
                    logger.debug("Found method %s from class %s", func, cls)
                    var_keyword_found = False
                    sig = inspect.signature(func)
                    for index, parameter in enumerate(sig.parameters.values()):
                        if index == 0:
                            # skip "self" parameter
                            continue

                        if parameter.kind == inspect.Parameter.VAR_KEYWORD:
                            # found "**kwargs" parameter.  we will continue to the next class in the mro
                            # to add any keyword parameters we have not yet used (i.e. whose name
                            # we have not yet seen)
                            var_keyword_found = True
                            continue

                        if parameter.kind in [parameter.POSITIONAL_ONLY, parameter.VAR_POSITIONAL]:
                            raise RuntimeError("Task {} contains an unsupported parameter \"{}\"".format(func, parameter))

                        if not parameter.name in var_keyword_seen:
                            var_keyword_seen.add(parameter.name)

                            logger.debug("Found parameter %s (%s)", parameter, parameter.kind)
                            yield parameter

                    # we only need to look at the next class in the mro
                    # when "**kwargs" is found
                    if not var_keyword_found:
                        break


def get_configuration(self, key, default=None):
        """Returns the configuration for KEY"""
        if key in self.config:
            return self.config.get(key)
        else:
            return default

def get_configuration ( self , key , default = None ) : if key in self . config : return self . config . get ( key ) else : return default


def get_plugin_instance(plugin_class, *args, **kwargs):
    """Returns an instance of a fully initialized plugin class

    Every plugin class is kept in a plugin cache, effectively making
    every plugin into a singleton object.

    When a plugin has a yaz.dependency decorator, it will be called
    as well, before the instance is returned.
    """
    assert issubclass(plugin_class, BasePlugin), type(plugin_class)

    global _yaz_plugin_instance_cache

    qualname = plugin_class.__qualname__
    if not qualname in _yaz_plugin_instance_cache:
        plugin_class = get_plugin_list()[qualname]
        _yaz_plugin_instance_cache[qualname] = plugin = plugin_class(*args, **kwargs)

        # find any yaz.dependency decorators, and call them when necessary
        funcs = [func
                 for _, func
                 in inspect.getmembers(plugin)
                 if inspect.ismethod(func) and hasattr(func, "yaz_dependency_config")]

        for func in funcs:
            signature = inspect.signature(func)
            assert all(parameter.kind is parameter.POSITIONAL_OR_KEYWORD and issubclass(parameter.annotation, BasePlugin) for parameter in signature.parameters.values()), "All parameters for {} must type hint to a BasePlugin".format(func)
            func(*[get_plugin_instance(parameter.annotation)
                   for parameter
                   in signature.parameters.values()])

    return _yaz_plugin_instance_cache[qualname]


def xml_to_json(root):
    """Convert an Open511 XML document or document fragment to JSON.

    Takes an lxml Element object. Returns a dict ready to be JSON-serialized."""
    j = {}

    if len(root) == 0:  # Tag with no children, return str/int
        return _maybe_intify(root.text)

    if len(root) == 1 and root[0].tag.startswith('{' + NS_GML):  # GML
        return gml_to_geojson(root[0])

    if root.tag == 'open511':
        j['meta'] = {'version': root.get('version')}

    for elem in root:
        name = elem.tag
        if name == 'link' and elem.get('rel'):
            name = elem.get('rel') + '_url'
            if name == 'self_url':
                name = 'url'
            if root.tag == 'open511':
                j['meta'][name] = elem.get('href')
                continue
        elif name.startswith('{' + NS_PROTECTED):
            name = '!' + name[name.index('}') + 1:]
        elif name[0] == '{':
            # Namespace!
            name = '+' + name[name.index('}') + 1:]

        if name in j:
            continue  # duplicate
        elif elem.tag == 'link' and not elem.text:
            j[name] = elem.get('href')
        elif len(elem):
            if name == 'grouped_events':
                # An array of URLs
                j[name] = [xml_link_to_json(child, to_dict=False) for child in elem]
            elif name in ('attachments', 'media_files'):
                # An array of JSON objects
                j[name] = [xml_link_to_json(child, to_dict=True) for child in elem]
            elif all((name == pluralize(child.tag) for child in elem)):
                # <something><somethings> serializes to a JSON array
                j[name] = [xml_to_json(child) for child in elem]
            else:
                j[name] = xml_to_json(elem)
        else:
            if root.tag == 'open511' and name.endswith('s') and not elem.text:
                # Special case: an empty e.g. <events /> container at the root level
                # should be serialized to [], not null
                j[name] = []
            else:
                j[name] = _maybe_intify(elem.text)

    return j

def xml_to_json ( root ) : j = { } if len ( root ) == 0 : return _maybe_intify ( root . text ) if len ( root ) == 1 and root [ 0 ] . tag . startswith ( '{' + NS_GML ) : return gml_to_geojson ( root [ 0 ] ) if root . tag == 'open511' : j [ 'meta' ] = { 'version' : root . get ( 'version' ) } for elem in root : name = elem . tag if name == 'link' and elem . get ( 'rel' ) : name = elem . get ( 'rel' ) + '_url' if name == 'self_url' : name = 'url' if root . tag == 'open511' : j [ 'meta' ] [ name ] = elem . get ( 'href' ) continue elif name . startswith ( '{' + NS_PROTECTED ) : name = '!' + name [ name . index ( '}' ) + 1 : ] elif name [ 0 ] == '{' : name = '+' + name [ name . index ( '}' ) + 1 : ] if name in j : continue elif elem . tag == 'link' and not elem . text : j [ name ] = elem . get ( 'href' ) elif len ( elem ) : if name == 'grouped_events' : j [ name ] = [ xml_link_to_json ( child , to_dict = False ) for child in elem ] elif name in ( 'attachments' , 'media_files' ) : j [ name ] = [ xml_link_to_json ( child , to_dict = True ) for child in elem ] elif all ( ( name == pluralize ( child . tag ) for child in elem ) ) : j [ name ] = [ xml_to_json ( child ) for child in elem ] else : j [ name ] = xml_to_json ( elem ) else : if root . tag == 'open511' and name . endswith ( 's' ) and not elem . text : j [ name ] = [ ] else : j [ name ] = _maybe_intify ( elem . text ) return j

Convert an Open511 XML document or document fragment to JSON .


def gml_to_geojson(el):
    """Given an lxml Element of a GML geometry, returns a dict in GeoJSON format."""
    if el.get('srsName') not in ('urn:ogc:def:crs:EPSG::4326', None):
        if el.get('srsName') == 'EPSG:4326':
            return _gmlv2_to_geojson(el)
        else:
            raise NotImplementedError("Unrecognized srsName %s" % el.get('srsName'))
    tag = el.tag.replace('{%s}' % NS_GML, '')
    if tag == 'Point':
        coordinates = _reverse_gml_coords(el.findtext('{%s}pos' % NS_GML))[0]
    elif tag == 'LineString':
        coordinates = _reverse_gml_coords(el.findtext('{%s}posList' % NS_GML))
    elif tag == 'Polygon':
        coordinates = []
        for ring in el.xpath('gml:exterior/gml:LinearRing/gml:posList', namespaces=NSMAP) \
                + el.xpath('gml:interior/gml:LinearRing/gml:posList', namespaces=NSMAP):
            coordinates.append(_reverse_gml_coords(ring.text))
    elif tag in ('MultiPoint', 'MultiLineString', 'MultiPolygon'):
        single_type = tag[5:]
        member_tag = single_type[0].lower() + single_type[1:] + 'Member'
        coordinates = [
            gml_to_geojson(member)['coordinates']
            for member in el.xpath('gml:%s/gml:%s' % (member_tag, single_type), namespaces=NSMAP)
        ]
    else:
        raise NotImplementedError

    return {
        'type': tag,
        'coordinates': coordinates
    }

Given an lxml Element of a GML geometry returns a dict in GeoJSON format .


def _gmlv2_to_geojson(el):
    """Translates a deprecated GML 2.0 geometry to GeoJSON"""
    tag = el.tag.replace('{%s}' % NS_GML, '')
    if tag == 'Point':
        coordinates = [float(c) for c in el.findtext('{%s}coordinates' % NS_GML).split(',')]
    elif tag == 'LineString':
        coordinates = [
            [float(x) for x in pair.split(',')]
            for pair in el.findtext('{%s}coordinates' % NS_GML).split(' ')
        ]
    elif tag == 'Polygon':
        coordinates = []
        for ring in el.xpath('gml:outerBoundaryIs/gml:LinearRing/gml:coordinates', namespaces=NSMAP) \
                + el.xpath('gml:innerBoundaryIs/gml:LinearRing/gml:coordinates', namespaces=NSMAP):
            coordinates.append([
                [float(x) for x in pair.split(',')]
                for pair in ring.text.split(' ')
            ])
    elif tag in ('MultiPoint', 'MultiLineString', 'MultiPolygon', 'MultiCurve'):
        if tag == 'MultiCurve':
            single_type = 'LineString'
            member_tag = 'curveMember'
        else:
            single_type = tag[5:]
            member_tag = single_type[0].lower() + single_type[1:] + 'Member'
        coordinates = [
            gml_to_geojson(member)['coordinates']
            for member in el.xpath('gml:%s/gml:%s' % (member_tag, single_type), namespaces=NSMAP)
        ]
    else:
        raise NotImplementedError

    return {
        'type': tag,
        'coordinates': coordinates
    }

Translates a deprecated GML 2 . 0 geometry to GeoJSON


def deparagraph(element, doc):
    """Panflute filter function that converts content wrapped in a Para to
    Plain.

    Use this filter with pandoc as::

        pandoc [..] --filter=lsstprojectmeta-deparagraph

    Only lone paragraphs are affected. Para elements with siblings (like a
    second Para) are left unaffected.

    This filter is useful for processing strings like titles or author names so
    that the output isn't wrapped in paragraph tags. For example, without
    this filter, pandoc converts a string ``"The title"`` to
    ``
The title
`` in HTML. These ``
`` tags aren't useful if you
    intend to put the title text in ``
`` tags using your own templating
    system.
    """
    if isinstance(element, Para):
        # Check if siblings exist; don't process the paragraph in that case.
        if element.next is not None:
            return element
        elif element.prev is not None:
            return element

        # Remove the Para wrapper from the lone paragraph.
        # `Plain` is a container that isn't rendered as a paragraph.
        return Plain(*element.content)

def deparagraph ( element , doc ) : if isinstance ( element , Para ) : if element . next is not None : return element elif element . prev is not None : return element return Plain ( * element . content )

Panflute filter function that converts content wrapped in a Para to Plain .


def all_subclasses(cls):
    """ Recursively generate of all the subclasses of class cls. """
    for subclass in cls.__subclasses__():
        yield subclass
        for subc in all_subclasses(subclass):
            yield subc

def all_subclasses ( cls ) : for subclass in cls . __subclasses__ ( ) : yield subclass for subc in all_subclasses ( subclass ) : yield subc

Recursively generate of all the subclasses of class cls .


def unique_justseen(iterable, key=None):
    "List unique elements, preserving order. Remember only the element just seen."
    # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B
    # unique_justseen('ABBCcAD', str.lower) --> A B C A D
    try:
        # PY2 support
        from itertools import imap as map
    except ImportError:
        from builtins import map

    return map(next, map(operator.itemgetter(1), itertools.groupby(iterable, key)))

List unique elements preserving order . Remember only the element just seen .


def generic_masked(arr, attrs=None, minv=None, maxv=None, mask_nan=True):
    """
    Returns a masked array with anything outside of values masked.
    The minv and maxv parameters take precendence over any dict values.
    The valid_range attribute takes precendence over the valid_min and
    valid_max attributes.
    """
    attrs = attrs or {}

    if 'valid_min' in attrs:
        minv = safe_attribute_typing(arr.dtype, attrs['valid_min'])
    if 'valid_max' in attrs:
        maxv = safe_attribute_typing(arr.dtype, attrs['valid_max'])
    if 'valid_range' in attrs:
        vr = attrs['valid_range']
        minv = safe_attribute_typing(arr.dtype, vr[0])
        maxv = safe_attribute_typing(arr.dtype, vr[1])

    # Get the min/max of values that the hardware supports
    try:
        info = np.iinfo(arr.dtype)
    except ValueError:
        info = np.finfo(arr.dtype)

    minv = minv if minv is not None else info.min
    maxv = maxv if maxv is not None else info.max

    if mask_nan is True:
        arr = np.ma.fix_invalid(arr)

    return np.ma.masked_outside(
        arr,
        minv,
        maxv
    )

def generic_masked ( arr , attrs = None , minv = None , maxv = None , mask_nan = True ) : attrs = attrs or { } if 'valid_min' in attrs : minv = safe_attribute_typing ( arr . dtype , attrs [ 'valid_min' ] ) if 'valid_max' in attrs : maxv = safe_attribute_typing ( arr . dtype , attrs [ 'valid_max' ] ) if 'valid_range' in attrs : vr = attrs [ 'valid_range' ] minv = safe_attribute_typing ( arr . dtype , vr [ 0 ] ) maxv = safe_attribute_typing ( arr . dtype , vr [ 1 ] ) try : info = np . iinfo ( arr . dtype ) except ValueError : info = np . finfo ( arr . dtype ) minv = minv if minv is not None else info . min maxv = maxv if maxv is not None else info . max if mask_nan is True : arr = np . ma . fix_invalid ( arr ) return np . ma . masked_outside ( arr , minv , maxv )

Returns a masked array with anything outside of values masked . The minv and maxv parameters take precendence over any dict values . The valid_range attribute takes precendence over the valid_min and valid_max attributes .


def default(self, obj):
        """If input object is an ndarray it will be converted into a list
        """
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        elif isinstance(obj, np.generic):
            return np.asscalar(obj)
        # Let the base class default method raise the TypeError
        return json.JSONEncoder(self, obj)

def default ( self , obj ) : if isinstance ( obj , np . ndarray ) : return obj . tolist ( ) elif isinstance ( obj , np . generic ) : return np . asscalar ( obj ) return json . JSONEncoder ( self , obj )


def default(self, obj):
        """If input object is an ndarray it will be converted into a dict
        holding dtype, shape and the data, base64 encoded.
        """
        if isinstance(obj, np.ndarray):
            if obj.flags['C_CONTIGUOUS']:
                obj_data = obj.data
            else:
                cont_obj = np.ascontiguousarray(obj)
                assert(cont_obj.flags['C_CONTIGUOUS'])
                obj_data = cont_obj.data
            data_b64 = base64.b64encode(obj_data)
            return dict(__ndarray__=data_b64,
                        dtype=str(obj.dtype),
                        shape=obj.shape)
        elif isinstance(obj, np.generic):
            return np.asscalar(obj)
        # Let the base class default method raise the TypeError
        return json.JSONEncoder(self, obj)

def default ( self , obj ) : if isinstance ( obj , np . ndarray ) : if obj . flags [ 'C_CONTIGUOUS' ] : obj_data = obj . data else : cont_obj = np . ascontiguousarray ( obj ) assert ( cont_obj . flags [ 'C_CONTIGUOUS' ] ) obj_data = cont_obj . data data_b64 = base64 . b64encode ( obj_data ) return dict ( __ndarray__ = data_b64 , dtype = str ( obj . dtype ) , shape = obj . shape ) elif isinstance ( obj , np . generic ) : return np . asscalar ( obj ) return json . JSONEncoder ( self , obj )

If input object is an ndarray it will be converted into a dict holding dtype shape and the data base64 encoded .


def update_desc_lsib_path(desc):
    '''
        leftSibling
        previousSibling
        leftSib
        prevSib
        lsib
        psib

        have the same parent,and on the left
    '''
    if(desc['sib_seq']>0):
        lsib_path = copy.deepcopy(desc['path'])
        lsib_path[-1] = desc['sib_seq']-1
        desc['lsib_path'] = lsib_path
    else:
        pass
    return(desc)

def update_desc_lsib_path ( desc ) : if ( desc [ 'sib_seq' ] > 0 ) : lsib_path = copy . deepcopy ( desc [ 'path' ] ) lsib_path [ - 1 ] = desc [ 'sib_seq' ] - 1 desc [ 'lsib_path' ] = lsib_path else : pass return ( desc )

leftSibling previousSibling leftSib prevSib lsib psib have the same parent and on the left


def update_desc_rsib_path(desc,sibs_len):
    '''
        rightSibling
        nextSibling
        rightSib
        nextSib
        rsib
        nsib

        have the same parent,and on the right
    '''
    if(desc['sib_seq']<(sibs_len-1)):
        rsib_path = copy.deepcopy(desc['path'])
        rsib_path[-1] = desc['sib_seq']+1
        desc['rsib_path'] = rsib_path
    else:
        pass
    return(desc)

def update_desc_rsib_path ( desc , sibs_len ) : if ( desc [ 'sib_seq' ] < ( sibs_len - 1 ) ) : rsib_path = copy . deepcopy ( desc [ 'path' ] ) rsib_path [ - 1 ] = desc [ 'sib_seq' ] + 1 desc [ 'rsib_path' ] = rsib_path else : pass return ( desc )

rightSibling nextSibling rightSib nextSib rsib nsib have the same parent and on the right


def update_desc_lcin_path(desc,pdesc_level):
    '''
        leftCousin
        previousCousin
        leftCin
        prevCin
        lcin
        pcin

        parents are neighbors,and on the left
    '''
    parent_breadth = desc['parent_breadth_path'][-1]
    if(desc['sib_seq']==0):
        if(parent_breadth==0):
            pass
        else:
            parent_lsib_breadth = parent_breadth - 1
            plsib_desc = pdesc_level[parent_lsib_breadth]
            if(plsib_desc['leaf']):
                pass
            else:
                lcin_path = copy.deepcopy(plsib_desc['path'])
                lcin_path.append(plsib_desc['sons_count'] - 1)
                desc['lcin_path'] = lcin_path
    else:
        pass
    return(desc)

def update_desc_lcin_path ( desc , pdesc_level ) : parent_breadth = desc [ 'parent_breadth_path' ] [ - 1 ] if ( desc [ 'sib_seq' ] == 0 ) : if ( parent_breadth == 0 ) : pass else : parent_lsib_breadth = parent_breadth - 1 plsib_desc = pdesc_level [ parent_lsib_breadth ] if ( plsib_desc [ 'leaf' ] ) : pass else : lcin_path = copy . deepcopy ( plsib_desc [ 'path' ] ) lcin_path . append ( plsib_desc [ 'sons_count' ] - 1 ) desc [ 'lcin_path' ] = lcin_path else : pass return ( desc )

leftCousin previousCousin leftCin prevCin lcin pcin parents are neighbors and on the left


def update_desc_rcin_path(desc,sibs_len,pdesc_level):
    '''
        rightCousin
        nextCousin
        rightCin
        nextCin
        rcin
        ncin

        parents are neighbors,and on the right
    '''
    psibs_len = pdesc_level.__len__()
    parent_breadth = desc['parent_breadth_path'][-1]
    if(desc['sib_seq']==(sibs_len - 1)):
        if(parent_breadth==(psibs_len -1)):
            pass
        else:
            parent_rsib_breadth = parent_breadth + 1
            prsib_desc = pdesc_level[parent_rsib_breadth]
            #because from left to right to handle each level
            #sons_count will only be updated in the next-round
            if(prsib_desc['leaf']):
                pass
            else:
                rcin_path = copy.deepcopy(prsib_desc['path'])
                rcin_path.append(0)
                desc['rcin_path'] = rcin_path
    else:
        pass
    return(desc)

def update_desc_rcin_path ( desc , sibs_len , pdesc_level ) : psibs_len = pdesc_level . __len__ ( ) parent_breadth = desc [ 'parent_breadth_path' ] [ - 1 ] if ( desc [ 'sib_seq' ] == ( sibs_len - 1 ) ) : if ( parent_breadth == ( psibs_len - 1 ) ) : pass else : parent_rsib_breadth = parent_breadth + 1 prsib_desc = pdesc_level [ parent_rsib_breadth ] if ( prsib_desc [ 'leaf' ] ) : pass else : rcin_path = copy . deepcopy ( prsib_desc [ 'path' ] ) rcin_path . append ( 0 ) desc [ 'rcin_path' ] = rcin_path else : pass return ( desc )

rightCousin nextCousin rightCin nextCin rcin ncin parents are neighbors and on the right


def child_begin_handler(self,scache,*args):
        '''
            _creat_child_desc
            update depth,parent_breadth_path,parent_path,sib_seq,path,lsib_path,rsib_path,lcin_path,rcin_path
        '''
        pdesc = self.pdesc
        depth = scache.depth
        sib_seq = self.sib_seq
        sibs_len = self.sibs_len
        pdesc_level = scache.pdesc_level
        desc = copy.deepcopy(pdesc)
        desc = reset_parent_desc_template(desc)
        desc['depth'] = depth
        desc['parent_breadth_path'] = copy.deepcopy(desc['breadth_path'])
        desc['sib_seq'] = sib_seq
        desc['parent_path'] = copy.deepcopy(desc['path'])
        desc['path'].append(sib_seq)
        update_desc_lsib_path(desc)
        update_desc_rsib_path(desc,sibs_len)
        if(depth == 1):
            pass
        else:
            update_desc_lcin_path(desc,pdesc_level)
            update_desc_rcin_path(desc,sibs_len,pdesc_level)
        return(desc)

def child_begin_handler ( self , scache , * args ) : pdesc = self . pdesc depth = scache . depth sib_seq = self . sib_seq sibs_len = self . sibs_len pdesc_level = scache . pdesc_level desc = copy . deepcopy ( pdesc ) desc = reset_parent_desc_template ( desc ) desc [ 'depth' ] = depth desc [ 'parent_breadth_path' ] = copy . deepcopy ( desc [ 'breadth_path' ] ) desc [ 'sib_seq' ] = sib_seq desc [ 'parent_path' ] = copy . deepcopy ( desc [ 'path' ] ) desc [ 'path' ] . append ( sib_seq ) update_desc_lsib_path ( desc ) update_desc_rsib_path ( desc , sibs_len ) if ( depth == 1 ) : pass else : update_desc_lcin_path ( desc , pdesc_level ) update_desc_rcin_path ( desc , sibs_len , pdesc_level ) return ( desc )


def child_end_handler(self,scache):
        '''
            _upgrade_breadth_info
            update breadth, breadth_path, and add desc to desc_level
        '''
        desc = self.desc
        desc_level = scache.desc_level
        breadth = desc_level.__len__()
        desc['breadth'] = breadth
        desc['breadth_path'].append(breadth)
        desc_level.append(desc)

def child_end_handler ( self , scache ) : desc = self . desc desc_level = scache . desc_level breadth = desc_level . __len__ ( ) desc [ 'breadth' ] = breadth desc [ 'breadth_path' ] . append ( breadth ) desc_level . append ( desc )

_upgrade_breadth_info update breadth breadth_path and add desc to desc_level


def parse(self, source):
        """Parse command content from the LaTeX source.

        Parameters
        ----------
        source : `str`
            The full source of the tex document.

        Yields
        ------
        parsed_command : `ParsedCommand`
            Yields parsed commands instances for each occurence of the command
            in the source.
        """
        command_regex = self._make_command_regex(self.name)
        for match in re.finditer(command_regex, source):
            self._logger.debug(match)
            start_index = match.start(0)
            yield self._parse_command(source, start_index)

def parse ( self , source ) : command_regex = self . _make_command_regex ( self . name ) for match in re . finditer ( command_regex , source ) : self . _logger . debug ( match ) start_index = match . start ( 0 ) yield self . _parse_command ( source , start_index )

Parse command content from the LaTeX source .


def _parse_command(self, source, start_index):
        """Parse a single command.

        Parameters
        ----------
        source : `str`
            The full source of the tex document.
        start_index : `int`
            Character index in ``source`` where the command begins.

        Returns
        -------
        parsed_command : `ParsedCommand`
            The parsed command from the source at the given index.
        """
        parsed_elements = []

        # Index of the parser in the source
        running_index = start_index

        for element in self.elements:
            opening_bracket = element['bracket']
            closing_bracket = self._brackets[opening_bracket]

            # Find the opening bracket.
            element_start = None
            element_end = None
            for i, c in enumerate(source[running_index:], start=running_index):
                if c == element['bracket']:
                    element_start = i
                    break
                elif c == '\n':
                    # No starting bracket on the line.
                    if element['required'] is True:
                        # Try to parse a single single-word token after the
                        # command, like '\input file'
                        content = self._parse_whitespace_argument(
                            source[running_index:],
                            self.name)
                        return ParsedCommand(
                            self.name,
                            [{'index': element['index'],
                              'name': element['name'],
                              'content': content.strip()}],
                            start_index,
                            source[start_index:i])
                    else:
                        # Give up on finding an optional element
                        break

            # Handle cases when the opening bracket is never found.
            if element_start is None and element['required'] is False:
                # Optional element not found. Continue to next element,
                # not advancing the running_index of the parser.
                continue
            elif element_start is None and element['required'] is True:
                message = ('Parsing command {0} at index {1:d}, '
                           'did not detect element {2:d}'.format(
                               self.name,
                               start_index,
                               element['index']))
                raise CommandParserError(message)

            # Find the closing bracket, keeping track of the number of times
            # the same type of bracket was opened and closed.
            balance = 1
            for i, c in enumerate(source[element_start + 1:],
                                  start=element_start + 1):
                if c == opening_bracket:
                    balance += 1
                elif c == closing_bracket:
                    balance -= 1

                if balance == 0:
                    element_end = i
                    break

            if balance > 0:
                message = ('Parsing command {0} at index {1:d}, '
                           'did not find closing bracket for required '
                           'command element {2:d}'.format(
                               self.name,
                               start_index,
                               element['index']))
                raise CommandParserError(message)

            # Package the parsed element's content.
            element_content = source[element_start + 1:element_end]
            parsed_element = {
                'index': element['index'],
                'name': element['name'],
                'content': element_content.strip()
            }
            parsed_elements.append(parsed_element)

            running_index = element_end + 1

        command_source = source[start_index:running_index]
        parsed_command = ParsedCommand(self.name, parsed_elements,
                                       start_index, command_source)
        return parsed_command


def _parse_whitespace_argument(source, name):
        r"""Attempt to parse a single token on the first line of this source.

        This method is used for parsing whitespace-delimited arguments, like
        ``\input file``. The source should ideally contain `` file`` along
        with a newline character.

        >>> source = 'Line 1\n' r'\input test.tex' '\nLine 2'
        >>> LatexCommand._parse_whitespace_argument(source, 'input')
        'test.tex'

        Bracket delimited arguments (``\input{test.tex}``) are handled in
        the normal logic of `_parse_command`.
        """
        # First match the command name itself so that we find the argument
        # *after* the command
        command_pattern = r'\\(' + name + r')(?:[\s{[%])'
        command_match = re.search(command_pattern, source)
        if command_match is not None:
            # Trim `source` so we only look after the command
            source = source[command_match.end(1):]

        # Find the whitespace-delimited argument itself.
        pattern = r'(?P<content>\S+)(?:[ %\t\n]+)'
        match = re.search(pattern, source)
        if match is None:
            message = (
                'When parsing {}, did not find whitespace-delimited command '
                'argument'
            )
            raise CommandParserError(message.format(name))
        content = match.group('content')
        content.strip()
        return content

r Attempt to parse a single token on the first line of this source .


def list_from_document(cls, doc):
        """Returns a list of TMDDEventConverter elements.

        doc is an XML Element containing one or more <FEU> events
        """
        objs = []
        for feu in doc.xpath('//FEU'):
            detail_els = feu.xpath('event-element-details/event-element-detail')
            for idx, detail in enumerate(detail_els):
                objs.append(cls(feu, detail, id_suffix=idx, number_in_group=len(detail_els)))
        return objs

Returns a list of TMDDEventConverter elements .


def clone(src, dst_path, skip_globals, skip_dimensions, skip_variables):
    """
        Mostly ripped from nc3tonc4 in netCDF4-python.
        Added ability to skip dimension and variables.
        Removed all of the unpacking logic for shorts.
    """

    if os.path.exists(dst_path):
        os.unlink(dst_path)
    dst = netCDF4.Dataset(dst_path, 'w')

    # Global attributes
    for attname in src.ncattrs():
        if attname not in skip_globals:
            setattr(dst, attname, getattr(src, attname))

    # Dimensions
    unlimdim     = None
    unlimdimname = False
    for dimname, dim in src.dimensions.items():

        # Skip what we need to
        if dimname in skip_dimensions:
            continue

        if dim.isunlimited():
            unlimdim     = dim
            unlimdimname = dimname
            dst.createDimension(dimname, None)
        else:
            dst.createDimension(dimname, len(dim))

    # Variables
    for varname, ncvar in src.variables.items():

        # Skip what we need to
        if varname in skip_variables:
            continue

        hasunlimdim = False
        if unlimdimname and unlimdimname in ncvar.dimensions:
            hasunlimdim = True

        filler = None
        if hasattr(ncvar, '_FillValue'):
            filler = ncvar._FillValue

        if ncvar.chunking == "contiguous":
            var = dst.createVariable(varname, ncvar.dtype, ncvar.dimensions, fill_value=filler)
        else:
            var = dst.createVariable(varname, ncvar.dtype, ncvar.dimensions, fill_value=filler, chunksizes=ncvar.chunking())

        # Attributes
        for attname in ncvar.ncattrs():
            if attname == '_FillValue':
                continue
            else:
                setattr(var, attname, getattr(ncvar, attname))

        # Data
        nchunk = 1000
        if hasunlimdim:
            if nchunk:
                start = 0
                stop = len(unlimdim)
                step = nchunk
                if step < 1:
                    step = 1
                for n in range(start, stop, step):
                    nmax = n + nchunk
                    if nmax > len(unlimdim):
                        nmax = len(unlimdim)
                    idata = ncvar[n:nmax]
                    var[n:nmax] = idata
            else:
                idata = ncvar[:]
                var[0:len(unlimdim)] = idata
        else:
            idata = ncvar[:]
            var[:] = idata

        dst.sync()

    src.close()
    dst.close()

def clone ( src , dst_path , skip_globals , skip_dimensions , skip_variables ) : if os . path . exists ( dst_path ) : os . unlink ( dst_path ) dst = netCDF4 . Dataset ( dst_path , 'w' ) for attname in src . ncattrs ( ) : if attname not in skip_globals : setattr ( dst , attname , getattr ( src , attname ) ) unlimdim = None unlimdimname = False for dimname , dim in src . dimensions . items ( ) : if dimname in skip_dimensions : continue if dim . isunlimited ( ) : unlimdim = dim unlimdimname = dimname dst . createDimension ( dimname , None ) else : dst . createDimension ( dimname , len ( dim ) ) for varname , ncvar in src . variables . items ( ) : if varname in skip_variables : continue hasunlimdim = False if unlimdimname and unlimdimname in ncvar . dimensions : hasunlimdim = True filler = None if hasattr ( ncvar , '_FillValue' ) : filler = ncvar . _FillValue if ncvar . chunking == "contiguous" : var = dst . createVariable ( varname , ncvar . dtype , ncvar . dimensions , fill_value = filler ) else : var = dst . createVariable ( varname , ncvar . dtype , ncvar . dimensions , fill_value = filler , chunksizes = ncvar . chunking ( ) ) for attname in ncvar . ncattrs ( ) : if attname == '_FillValue' : continue else : setattr ( var , attname , getattr ( ncvar , attname ) ) nchunk = 1000 if hasunlimdim : if nchunk : start = 0 stop = len ( unlimdim ) step = nchunk if step < 1 : step = 1 for n in range ( start , stop , step ) : nmax = n + nchunk if nmax > len ( unlimdim ) : nmax = len ( unlimdim ) idata = ncvar [ n : nmax ] var [ n : nmax ] = idata else : idata = ncvar [ : ] var [ 0 : len ( unlimdim ) ] = idata else : idata = ncvar [ : ] var [ : ] = idata dst . sync ( ) src . close ( ) dst . close ( )

Mostly ripped from nc3tonc4 in netCDF4 - python . Added ability to skip dimension and variables . Removed all of the unpacking logic for shorts .


def get_dataframe_from_variable(nc, data_var):
    """ Returns a Pandas DataFrame of the data.
        This always returns positive down depths
    """
    time_var = nc.get_variables_by_attributes(standard_name='time')[0]

    depth_vars = nc.get_variables_by_attributes(axis=lambda v: v is not None and v.lower() == 'z')
    depth_vars += nc.get_variables_by_attributes(standard_name=lambda v: v in ['height', 'depth' 'surface_altitude'], positive=lambda x: x is not None)

    # Find the correct depth variable
    depth_var = None
    for d in depth_vars:
        try:
            if d._name in data_var.coordinates.split(" ") or d._name in data_var.dimensions:
                depth_var = d
                break
        except AttributeError:
            continue

    times  = netCDF4.num2date(time_var[:], units=time_var.units, calendar=getattr(time_var, 'calendar', 'standard'))
    original_times_size = times.size

    if depth_var is None and hasattr(data_var, 'sensor_depth'):
        depth_type = get_type(data_var.sensor_depth)
        depths = np.asarray([data_var.sensor_depth] * len(times)).flatten()
        values = data_var[:].flatten()
    elif depth_var is None:
        depths = np.asarray([np.nan] * len(times)).flatten()
        depth_type = get_type(depths)
        values = data_var[:].flatten()
    else:
        depths = depth_var[:]
        depth_type = get_type(depths)
        if len(data_var.shape) > 1:
            times = np.repeat(times, depths.size)
            depths = np.tile(depths, original_times_size)
            values = data_var[:, :].flatten()
        else:
            values = data_var[:].flatten()

        if getattr(depth_var, 'positive', 'down').lower() == 'up':
            logger.warning("Converting depths to positive down before returning the DataFrame")
            depths = depths * -1

    # https://github.com/numpy/numpy/issues/4595
    # We can't call astype on a MaskedConstant
    if (
        isinstance(depths, np.ma.core.MaskedConstant) or
        (hasattr(depths, 'mask') and depths.mask.all())
    ):
        depths = np.asarray([np.nan] * len(times)).flatten()

    df = pd.DataFrame({ 'time':   times,
                        'value':  values.astype(data_var.dtype),
                        'unit':   data_var.units if hasattr(data_var, 'units') else np.nan,
                        'depth':  depths.astype(depth_type) })

    df.set_index([pd.DatetimeIndex(df['time']), pd.Float64Index(df['depth'])], inplace=True)
    return df

Returns a Pandas DataFrame of the data . This always returns positive down depths


def load(cls, query_name):
        """Load a pre-made query.

        These queries are distributed with lsstprojectmeta. See
        :file:`lsstrojectmeta/data/githubv4/README.rst` inside the
        package repository for details on available queries.

        Parameters
        ----------
        query_name : `str`
            Name of the query, such as ``'technote_repo'``.

        Returns
        -------
        github_query : `GitHubQuery
            A GitHub query or mutation object that you can pass to
            `github_request` to execute the request itself.
        """
        template_path = os.path.join(
            os.path.dirname(__file__),
            '../data/githubv4',
            query_name + '.graphql')

        with open(template_path) as f:
            query_data = f.read()

        return cls(query_data, name=query_name)

def load ( cls , query_name ) : template_path = os . path . join ( os . path . dirname ( __file__ ) , '../data/githubv4' , query_name + '.graphql' ) with open ( template_path ) as f : query_data = f . read ( ) return cls ( query_data , name = query_name )


def read_git_commit_timestamp_for_file(filepath, repo_path=None, repo=None):
    """Obtain the timestamp for the most recent commit to a given file in a
    Git repository.

    Parameters
    ----------
    filepath : `str`
        Absolute or repository-relative path for a file.
    repo_path : `str`, optional
        Path to the Git repository. Leave as `None` to use the current working
        directory or if a ``repo`` argument is provided.
    repo : `git.Repo`, optional
        A `git.Repo` instance.

    Returns
    -------
    commit_timestamp : `datetime.datetime`
        The datetime of the most recent commit to the given file.

    Raises
    ------
    IOError
        Raised if the ``filepath`` does not exist in the Git repository.
    """
    logger = logging.getLogger(__name__)

    if repo is None:
        repo = git.repo.base.Repo(path=repo_path,
                                  search_parent_directories=True)
    repo_path = repo.working_tree_dir

    head_commit = repo.head.commit

    # filepath relative to the repo path
    logger.debug('Using Git repo at %r', repo_path)
    filepath = os.path.relpath(
        os.path.abspath(filepath),
        start=repo_path)
    logger.debug('Repo-relative filepath is %r', filepath)

    # Most recent commit datetime of the given file.
    # Don't use head_commit.iter_parents because then it skips the
    # commit of a file that's added but never modified.
    for commit in head_commit.iter_items(repo,
                                         head_commit,
                                         [filepath],
                                         skip=0):
        return commit.committed_datetime

    # Only get here if git could not find the file path in the history
    raise IOError('File {} not found'.format(filepath))

def
read_git_commit_timestamp_for_file
(
filepath
,
repo_path
=
,
repo
=
)
:
logger
=
logging
.
getLogger
(
__name__
)
if
repo
is
:
repo
=
git
.
repo
.
base
.
Repo
(
path
=
repo_path
,
search_parent_directories
=
True
)
repo_path
=
repo
.
working_tree_dir
head_commit
=
repo
.
head
.
commit
logger
.
debug
(
'Using Git repo at %r'
,
repo_path
)
filepath
=
os
.
path
.
relpath
(
os
.
path
.
abspath
(
filepath
)
,
start
=
repo_path
)
logger
.
debug
(
'Repo-relative filepath is %r'
,
filepath
)
for
commit
in
head_commit
.
iter_items
(
repo
,
head_commit
,
[
filepath
]
,
skip
=
0
)
:
return
commit
.
committed_datetime
raise
IOError
(
'File {} not found'
.
format
(
filepath
)
)

Obtain the timestamp for the most recent commit to a given file in a Git repository .


def get_content_commit_date(extensions, acceptance_callback=None,
                            root_dir='.'):
    """Get the datetime for the most recent commit to a project that
    affected certain types of content.

    Parameters
    ----------
    extensions : sequence of 'str'
        Extensions of files to consider in getting the most recent commit
        date. For example, ``('rst', 'svg', 'png')`` are content extensions
        for a Sphinx project. **Extension comparision is case sensitive.** add
        uppercase variants to match uppercase extensions.
    acceptance_callback : callable
        Callable function whose sole argument is a file path, and returns
        `True` or `False` depending on whether the file's commit date should
        be considered or not. This callback is only run on files that are
        included by ``extensions``. Thus this callback is a way to exclude
        specific files that would otherwise be included by their extension.
    root_dir : 'str`, optional
        Only content contained within this root directory is considered.
        This directory must be, or be contained by, a Git repository. This is
        the current working directory by default.

    Returns
    -------
    commit_date : `datetime.datetime`
        Datetime of the most recent content commit.

    Raises
    ------
    RuntimeError
        Raised if no content files are found.
    """
    logger = logging.getLogger(__name__)

    def _null_callback(_):
        return True

    if acceptance_callback is None:
        acceptance_callback = _null_callback

    # Cache the repo object for each query
    root_dir = os.path.abspath(root_dir)
    repo = git.repo.base.Repo(path=root_dir, search_parent_directories=True)

    # Iterate over all files with all file extensions, looking for the
    # newest commit datetime.
    newest_datetime = None
    iters = [_iter_filepaths_with_extension(ext, root_dir=root_dir)
             for ext in extensions]
    for content_path in itertools.chain(*iters):
        content_path = os.path.abspath(os.path.join(root_dir, content_path))

        if acceptance_callback(content_path):
            logger.debug('Found content path %r', content_path)
            try:
                commit_datetime = read_git_commit_timestamp_for_file(
                    content_path, repo=repo)
                logger.debug('Commit timestamp of %r is %s',
                             content_path, commit_datetime)
            except IOError:
                logger.warning(
                    'Count not get commit for %r, skipping',
                    content_path)
                continue

            if not newest_datetime or commit_datetime > newest_datetime:
                # Seed initial newest_datetime
                # or set a newer newest_datetime
                newest_datetime = commit_datetime
                logger.debug('Newest commit timestamp is %s', newest_datetime)

        logger.debug('Final commit timestamp is %s', newest_datetime)

    if newest_datetime is None:
        raise RuntimeError('No content files found in {}'.format(root_dir))

    return newest_datetime

def
get_content_commit_date
(
extensions
,
acceptance_callback
=
,
root_dir
=
'.'
)
:
logger
=
logging
.
getLogger
(
__name__
)
def
_null_callback
(
_
)
:
return
True
if
acceptance_callback
is
:
acceptance_callback
=
_null_callback
root_dir
=
os
.
path
.
abspath
(
root_dir
)
repo
=
git
.
repo
.
base
.
Repo
(
path
=
root_dir
,
search_parent_directories
=
True
)
newest_datetime
=
iters
=
[
_iter_filepaths_with_extension
(
ext
,
root_dir
=
root_dir
)
for
ext
in
extensions
]
for
content_path
in
itertools
.
chain
(
*
iters
)
:
content_path
=
os
.
path
.
abspath
(
os
.
path
.
join
(
root_dir
,
content_path
)
)
if
acceptance_callback
(
content_path
)
:
logger
.
debug
(
'Found content path %r'
,
content_path
)
try
:
commit_datetime
=
read_git_commit_timestamp_for_file
(
content_path
,
repo
=
repo
)
logger
.
debug
(
'Commit timestamp of %r is %s'
,
content_path
,
commit_datetime
)
except
IOError
:
logger
.
warning
(
'Count not get commit for %r, skipping'
,
content_path
)
continue
if
not
newest_datetime
or
commit_datetime
>
newest_datetime
:
newest_datetime
=
commit_datetime
logger
.
debug
(
'Newest commit timestamp is %s'
,
newest_datetime
)
logger
.
debug
(
'Final commit timestamp is %s'
,
newest_datetime
)
if
newest_datetime
is
:
raise
RuntimeError
(
'No content files found in {}'
.
format
(
root_dir
)
)
return
newest_datetime

Get the datetime for the most recent commit to a project that affected certain types of content .


def _iter_filepaths_with_extension(extname, root_dir='.'):
    """Iterative over relative filepaths of files in a directory, and
    sub-directories, with the given extension.

    Parameters
    ----------
    extname : `str`
        Extension name (such as 'txt' or 'rst'). Extension comparison is
        case sensitive.
    root_dir : 'str`, optional
        Root directory. Current working directory by default.

    Yields
    ------
    filepath : `str`
        File path, relative to ``root_dir``, with the given extension.
    """
    # needed for comparison with os.path.splitext
    if not extname.startswith('.'):
        extname = '.' + extname

    root_dir = os.path.abspath(root_dir)

    for dirname, sub_dirnames, filenames in os.walk(root_dir):
        for filename in filenames:
            if os.path.splitext(filename)[-1] == extname:
                full_filename = os.path.join(dirname, filename)
                rel_filepath = os.path.relpath(full_filename, start=root_dir)
                yield rel_filepath

def _iter_filepaths_with_extension ( extname , root_dir = '.' ) : if not extname . startswith ( '.' ) : extname = '.' + extname root_dir = os . path . abspath ( root_dir ) for dirname , sub_dirnames , filenames in os . walk ( root_dir ) : for filename in filenames : if os . path . splitext ( filename ) [ - 1 ] == extname : full_filename = os . path . join ( dirname , filename ) rel_filepath = os . path . relpath ( full_filename , start = root_dir ) yield rel_filepath

Iterative over relative filepaths of files in a directory and sub - directories with the given extension .


def get_variables_by_attributes(self, **kwargs):
        """ Returns variables that match specific conditions.

        * Can pass in key=value parameters and variables are returned that
        contain all of the matches.  For example,

        >>> # Get variables with x-axis attribute.
        >>> vs = nc.get_variables_by_attributes(axis='X')
        >>> # Get variables with matching "standard_name" attribute.
        >>> nc.get_variables_by_attributes(standard_name='northward_sea_water_velocity')

        * Can pass in key=callable parameter and variables are returned if the
        callable returns True.  The callable should accept a single parameter,
        the attribute value.  None is given as the attribute value when the
        attribute does not exist on the variable. For example,

        >>> # Get Axis variables.
        >>> vs = nc.get_variables_by_attributes(axis=lambda v: v in ['X', 'Y', 'Z', 'T'])
        >>> # Get variables that don't have an "axis" attribute.
        >>> vs = nc.get_variables_by_attributes(axis=lambda v: v is None)
        >>> # Get variables that have a "grid_mapping" attribute.
        >>> vs = nc.get_variables_by_attributes(grid_mapping=lambda v: v is not None)

        """
        vs = []

        has_value_flag  = False
        for vname in self.variables:
            var = self.variables[vname]
            for k, v in kwargs.items():
                if callable(v):
                    has_value_flag = v(getattr(var, k, None))
                    if has_value_flag is False:
                        break
                elif hasattr(var, k) and getattr(var, k) == v:
                    has_value_flag = True
                else:
                    has_value_flag = False
                    break

            if has_value_flag is True:
                vs.append(self.variables[vname])

        return vs

def get_variables_by_attributes ( self , ** kwargs ) : vs = [ ] has_value_flag = False for vname in self . variables : var = self . variables [ vname ] for k , v in kwargs . items ( ) : if callable ( v ) : has_value_flag = v ( getattr ( var , k , None ) ) if has_value_flag is False : break elif hasattr ( var , k ) and getattr ( var , k ) == v : has_value_flag = True else : has_value_flag = False break if has_value_flag is True : vs . append ( self . variables [ vname ] ) return vs

Returns variables that match specific conditions .


def json_attributes(self, vfuncs=None):
        """
        vfuncs can be any callable that accepts a single argument, the
        Variable object, and returns a dictionary of new attributes to
        set. These will overwrite existing attributes
        """

        vfuncs = vfuncs or []

        js = {'global': {}}

        for k in self.ncattrs():
            js['global'][k] = self.getncattr(k)

        for varname, var in self.variables.items():
            js[varname] = {}
            for k in var.ncattrs():
                z = var.getncattr(k)
                try:
                    assert not np.isnan(z).all()
                    js[varname][k] = z
                except AssertionError:
                    js[varname][k] = None
                except TypeError:
                    js[varname][k] = z

            for vf in vfuncs:
                try:
                    js[varname].update(vfuncs(var))
                except BaseException:
                    logger.exception("Could not apply custom variable attribue function")

        return json.loads(json.dumps(js, cls=BasicNumpyEncoder))

vfuncs can be any callable that accepts a single argument the Variable object and returns a dictionary of new attributes to set . These will overwrite existing attributes


def ensure_pandoc(func):
    """Decorate a function that uses pypandoc to ensure that pandoc is
    installed if necessary.
    """
    logger = logging.getLogger(__name__)

    @functools.wraps(func)
    def _install_and_run(*args, **kwargs):
        try:
            # First try to run pypandoc function
            result = func(*args, **kwargs)
        except OSError:
            # Install pandoc and retry
            message = "pandoc needed but not found. Now installing it for you."
            logger.warning(message)
            # This version of pandoc is known to be compatible with both
            # pypandoc.download_pandoc and the functionality that
            # lsstprojectmeta needs. Travis CI tests are useful for ensuring
            # download_pandoc works.
            pypandoc.download_pandoc(version='1.19.1')
            logger.debug("pandoc download complete")

            result = func(*args, **kwargs)

        return result

    return _install_and_run

Decorate a function that uses pypandoc to ensure that pandoc is installed if necessary .


def convert_text(content, from_fmt, to_fmt, deparagraph=False, mathjax=False,
                 smart=True, extra_args=None):
    """Convert text from one markup format to another using pandoc.

    This function is a thin wrapper around `pypandoc.convert_text`.

    Parameters
    ----------
    content : `str`
        Original content.

    from_fmt : `str`
        Format of the original ``content``. Format identifier must be one of
        those known by Pandoc. See https://pandoc.org/MANUAL.html for details.

    to_fmt : `str`
        Output format for the content.

    deparagraph : `bool`, optional
        If `True`, then the
        `lsstprojectmeta.pandoc.filters.deparagraph.deparagraph` filter is
        used to remove paragraph (``
``, for example) tags around a single
        paragraph of content. That filter does not affect content that
        consists of multiple blocks (several paragraphs, or lists, for
        example). Default is `False`.

        For example, **without** this filter Pandoc will convert
        the string ``"Title text"`` to ``"
Title text
"`` in HTML. The
        paragraph tags aren't useful if you intend to wrap the converted
        content in different tags, like ``
``, using your own templating
        system.

        **With** this filter, Pandoc will convert the string ``"Title text"``
        to ``"Title text"`` in HTML.

    mathjax : `bool`, optional
        If `True` then Pandoc will markup output content to work with MathJax.
        Default is False.

    smart : `bool`, optional
        If `True` (default) then ascii characters will be converted to unicode
        characters like smart quotes and em dashes.

    extra_args : `list`, optional
        Sequence of Pandoc arguments command line arguments (such as
        ``'--normalize'``). The ``deparagraph``, ``mathjax``, and ``smart``
        arguments are convenience arguments that are equivalent to items
        in ``extra_args``.

    Returns
    -------
    output : `str`
        Content in the output (``to_fmt``) format.

    Notes
    -----
    This function will automatically install Pandoc if it is not available.
    See `ensure_pandoc`.
    """
    logger = logging.getLogger(__name__)

    if extra_args is not None:
        extra_args = list(extra_args)
    else:
        extra_args = []

    if mathjax:
        extra_args.append('--mathjax')

    if smart:
        extra_args.append('--smart')

    if deparagraph:
        extra_args.append('--filter=lsstprojectmeta-deparagraph')

    extra_args.append('--wrap=none')

    # de-dupe extra args
    extra_args = set(extra_args)

    logger.debug('Running pandoc from %s to %s with extra_args %s',
                 from_fmt, to_fmt, extra_args)

    output = pypandoc.convert_text(content, to_fmt, format=from_fmt,
                                   extra_args=extra_args)
    return output

Convert text from one markup format to another using pandoc .


def convert_lsstdoc_tex(
        content, to_fmt, deparagraph=False, mathjax=False,
        smart=True, extra_args=None):
    """Convert lsstdoc-class LaTeX to another markup format.

    This function is a thin wrapper around `convert_text` that automatically
    includes common lsstdoc LaTeX macros.

    Parameters
    ----------
    content : `str`
        Original content.

    to_fmt : `str`
        Output format for the content (see https://pandoc.org/MANUAL.html).
        For example, 'html5'.

    deparagraph : `bool`, optional
        If `True`, then the
        `lsstprojectmeta.pandoc.filters.deparagraph.deparagraph` filter is
        used to remove paragraph (``
``, for example) tags around a single
        paragraph of content. That filter does not affect content that
        consists of multiple blocks (several paragraphs, or lists, for
        example). Default is `False`.

        For example, **without** this filter Pandoc will convert
        the string ``"Title text"`` to ``"
Title text
"`` in HTML. The
        paragraph tags aren't useful if you intend to wrap the converted
        content in different tags, like ``
``, using your own templating
        system.

        **With** this filter, Pandoc will convert the string ``"Title text"``
        to ``"Title text"`` in HTML.

    mathjax : `bool`, optional
        If `True` then Pandoc will markup output content to work with MathJax.
        Default is False.

    smart : `bool`, optional
        If `True` (default) then ascii characters will be converted to unicode
        characters like smart quotes and em dashes.

    extra_args : `list`, optional
        Sequence of Pandoc arguments command line arguments (such as
        ``'--normalize'``). The ``deparagraph``, ``mathjax``, and ``smart``
        arguments are convenience arguments that are equivalent to items
        in ``extra_args``.

    Returns
    -------
    output : `str`
        Content in the output (``to_fmt``) format.

    Notes
    -----
    This function will automatically install Pandoc if it is not available.
    See `ensure_pandoc`.
    """
    augmented_content = '\n'.join((LSSTDOC_MACROS, content))
    return convert_text(
        augmented_content, 'latex', to_fmt,
        deparagraph=deparagraph, mathjax=mathjax,
        smart=smart, extra_args=extra_args)

def convert_lsstdoc_tex ( content , to_fmt , deparagraph = False , mathjax = False , smart = True , extra_args = None ) : augmented_content = '\n' . join ( ( LSSTDOC_MACROS , content ) ) return convert_text ( augmented_content , 'latex' , to_fmt , deparagraph = deparagraph , mathjax = mathjax , smart = smart , extra_args = extra_args )

Convert lsstdoc - class LaTeX to another markup format .


def decode_jsonld(jsonld_text):
    """Decode a JSON-LD dataset, including decoding datetime
    strings into `datetime.datetime` objects.

    Parameters
    ----------
    encoded_dataset : `str`
        The JSON-LD dataset encoded as a string.

    Returns
    -------
    jsonld_dataset : `dict`
        A JSON-LD dataset.

    Examples
    --------

    >>> doc = '{"dt": "2018-01-01T12:00:00Z"}'
    >>> decode_jsonld(doc)
    {'dt': datetime.datetime(2018, 1, 1, 12, 0, tzinfo=datetime.timezone.utc)}
    """
    decoder = json.JSONDecoder(object_pairs_hook=_decode_object_pairs)
    return decoder.decode(jsonld_text)

def decode_jsonld ( jsonld_text ) : decoder = json . JSONDecoder ( object_pairs_hook = _decode_object_pairs ) return decoder . decode ( jsonld_text )

Decode a JSON - LD dataset including decoding datetime strings into datetime . datetime objects .


def default(self, obj):
        """Encode values as JSON strings.

        This method overrides the default implementation from
        `json.JSONEncoder`.
        """
        if isinstance(obj, datetime.datetime):
            return self._encode_datetime(obj)

        # Fallback to the default encoding
        return json.JSONEncoder.default(self, obj)

def default ( self , obj ) : if isinstance ( obj , datetime . datetime ) : return self . _encode_datetime ( obj ) return json . JSONEncoder . default ( self , obj )


def find_repos(self, depth=10):
        '''Get all git repositories within this environment'''

        repos = []

        for root, subdirs, files in walk_dn(self.root, depth=depth):
            if 'modules' in root:
                continue
            if '.git' in subdirs:
                repos.append(root)

        return repos

def find_repos ( self , depth = 10 ) : repos = [ ] for root , subdirs , files in walk_dn ( self . root , depth = depth ) : if 'modules' in root : continue if '.git' in subdirs : repos . append ( root ) return repos


def install(self, package):
        '''Install a python package using pip'''

        logger.debug('Installing ' + package)
        shell.run(self.pip_path, 'install',  package)

def
install
(
self
,
package
)
:
logger
.
debug
(
'Installing '
+
package
)
shell
.
run
(
self
.
pip_path
,
'install'
,
package
)


def upgrade(self, package):
        '''Update a python package using pip'''

        logger.debug('Upgrading ' + package)
        shell.run(self.pip_path, 'install', '--upgrade', '--no-deps', package)
        shell.run(self.pip_path, 'install', package)

def
upgrade
(
self
,
package
)
:
logger
.
debug
(
'Upgrading '
+
package
)
shell
.
run
(
self
.
pip_path
,
'install'
,
'--upgrade'
,
'--no-deps'
,
package
)
shell
.
run
(
self
.
pip_path
,
'install'
,
package
)


def df_quantile(df, nb=100):
    """Returns the nb quantiles for datas in a dataframe
    """
    quantiles = np.linspace(0, 1., nb)
    res = pd.DataFrame()
    for q in quantiles:
        res = res.append(df.quantile(q), ignore_index=True)
    return res

def df_quantile ( df , nb = 100 ) : quantiles = np . linspace ( 0 , 1. , nb ) res = pd . DataFrame ( ) for q in quantiles : res = res . append ( df . quantile ( q ) , ignore_index = True ) return res


def rmse(a, b):
    """Returns the root mean square error betwwen a and b
    """
    return np.sqrt(np.square(a - b).mean())

def rmse ( a , b ) : return np . sqrt ( np . square ( a - b ) . mean ( ) )


def nmse(a, b):
    """Returns the normalized mean square error of a and b
    """
    return np.square(a - b).mean() / (a.mean() * b.mean())

def nmse ( a , b ) : return np . square ( a - b ) . mean ( ) / ( a . mean ( ) * b . mean ( ) )


def mfbe(a, b):
    """Returns the mean fractionalized bias error
    """
    return 2 * bias(a, b) / (a.mean() + b.mean())

def mfbe ( a , b ) : return 2 * bias ( a , b ) / ( a . mean ( ) + b . mean ( ) )


def foex(a, b):
    """Returns the factor of exceedance
    """
    return (np.sum(a > b, dtype=float) / len(a) - 0.5) * 100

def foex ( a , b ) : return ( np . sum ( a > b , dtype = float ) / len ( a ) - 0.5 ) * 100


def correlation(a, b):
    """Computes the correlation between a and b, says the Pearson's correlation
    coefficient R
    """
    diff1 = a - a.mean()
    diff2 = b - b.mean()
    return (diff1 * diff2).mean() / (np.sqrt(np.square(diff1).mean() * np.square(diff2).mean()))

def correlation ( a , b ) : diff1 = a - a . mean ( ) diff2 = b - b . mean ( ) return ( diff1 * diff2 ) . mean ( ) / ( np . sqrt ( np . square ( diff1 ) . mean ( ) * np . square ( diff2 ) . mean ( ) ) )

Computes the correlation between a and b says the Pearson s correlation coefficient R


def gmb(a, b):
    """Geometric mean bias
    """
    return np.exp(np.log(a).mean() - np.log(b).mean())

def gmb ( a , b ) : return np . exp ( np . log ( a ) . mean ( ) - np . log ( b ) . mean ( ) )


def gmv(a, b):
    """Geometric mean variance
    """
    return np.exp(np.square(np.log(a) - np.log(b)).mean())

def gmv ( a , b ) : return np . exp ( np . square ( np . log ( a ) - np . log ( b ) ) . mean ( ) )


def fmt(a, b):
    """Figure of merit in time
    """
    return 100 * np.min([a, b], axis=0).sum() / np.max([a, b], axis=0).sum()

def fmt ( a , b ) : return 100 * np . min ( [ a , b ] , axis = 0 ) . sum ( ) / np . max ( [ a , b ] , axis = 0 ) . sum ( )


def fullStats(a, b):
    """Performs several stats on a against b, typically a is the predictions
    array, and b the observations array

    Returns:
        A dataFrame of stat name, stat description, result
    """

    stats = [
        ['bias', 'Bias', bias(a, b)],
        ['stderr', 'Standard Deviation Error', stderr(a, b)],
        ['mae', 'Mean Absolute Error', mae(a, b)],
        ['rmse', 'Root Mean Square Error', rmse(a, b)],
        ['nmse', 'Normalized Mean Square Error', nmse(a, b)],
        ['mfbe', 'Mean Fractionalized bias Error', mfbe(a, b)],
        ['fa2', 'Factor of Two', fa(a, b, 2)],
        ['foex', 'Factor of Exceedance', foex(a, b)],
        ['correlation', 'Correlation R', correlation(a, b)],
        ['determination', 'Coefficient of Determination r2', determination(a, b)],
        ['gmb', 'Geometric Mean Bias', gmb(a, b)],
        ['gmv', 'Geometric Mean Variance', gmv(a, b)],
        ['fmt', 'Figure of Merit in Time', fmt(a, b)]
    ]
    rec = np.rec.fromrecords(stats, names=('stat', 'description', 'result'))
    df = pd.DataFrame.from_records(rec, index='stat')
    return df

def
fullStats
(
a
,
b
)
:
stats
=
[
[
'bias'
,
'Bias'
,
bias
(
a
,
b
)
]
,
[
'stderr'
,
'Standard Deviation Error'
,
stderr
(
a
,
b
)
]
,
[
'mae'
,
'Mean Absolute Error'
,
mae
(
a
,
b
)
]
,
[
'rmse'
,
'Root Mean Square Error'
,
rmse
(
a
,
b
)
]
,
[
'nmse'
,
'Normalized Mean Square Error'
,
nmse
(
a
,
b
)
]
,
[
'mfbe'
,
'Mean Fractionalized bias Error'
,
mfbe
(
a
,
b
)
]
,
[
'fa2'
,
'Factor of Two'
,
fa
(
a
,
b
,
2
)
]
,
[
'foex'
,
'Factor of Exceedance'
,
foex
(
a
,
b
)
]
,
[
'correlation'
,
'Correlation R'
,
correlation
(
a
,
b
)
]
,
[
'determination'
,
'Coefficient of Determination r2'
,
determination
(
a
,
b
)
]
,
[
'gmb'
,
'Geometric Mean Bias'
,
gmb
(
a
,
b
)
]
,
[
'gmv'
,
'Geometric Mean Variance'
,
gmv
(
a
,
b
)
]
,
[
'fmt'
,
'Figure of Merit in Time'
,
fmt
(
a
,
b
)
]
]
rec
=
np
.
rec
.
fromrecords
(
stats
,
names
=
(
'stat'
,
'description'
,
'result'
)
)
df
=
pd
.
DataFrame
.
from_records
(
rec
,
index
=
'stat'
)
return
df

Performs several stats on a against b typically a is the predictions array and b the observations array


def site_path(self):
        '''Path to environments site-packages'''

        if platform == 'win':
            return unipath(self.path, 'Lib', 'site-packages')

        py_ver = 'python{0}'.format(sys.version[:3])
        return unipath(self.path, 'lib', py_ver, 'site-packages')

def site_path ( self ) : if platform == 'win' : return unipath ( self . path , 'Lib' , 'site-packages' ) py_ver = 'python{0}' . format ( sys . version [ : 3 ] ) return unipath ( self . path , 'lib' , py_ver , 'site-packages' )


def _pre_activate(self):
        '''
        Prior to activating, store everything necessary to deactivate this
        environment.
        '''

        if 'CPENV_CLEAN_ENV' not in os.environ:
            if platform == 'win':
                os.environ['PROMPT'] = '$P$G'
            else:
                os.environ['PS1'] = '\\u@\\h:\\w\\$'
            clean_env_path = utils.get_store_env_tmp()
            os.environ['CPENV_CLEAN_ENV'] = clean_env_path
            utils.store_env(path=clean_env_path)
        else:
            utils.restore_env_from_file(os.environ['CPENV_CLEAN_ENV'])

def _pre_activate ( self ) : if 'CPENV_CLEAN_ENV' not in os . environ : if platform == 'win' : os . environ [ 'PROMPT' ] = '$P$G' else : os . environ [ 'PS1' ] = '\\u@\\h:\\w\\$' clean_env_path = utils . get_store_env_tmp ( ) os . environ [ 'CPENV_CLEAN_ENV' ] = clean_env_path utils . store_env ( path = clean_env_path ) else : utils . restore_env_from_file ( os . environ [ 'CPENV_CLEAN_ENV' ] )

Prior to activating store everything necessary to deactivate this environment .


def _activate(self):
        '''
        Do some serious mangling to the current python environment...
        This is necessary to activate an environment via python.
        '''

        old_syspath = set(sys.path)
        site.addsitedir(self.site_path)
        site.addsitedir(self.bin_path)
        new_syspaths = set(sys.path) - old_syspath
        for path in new_syspaths:
            sys.path.remove(path)
            sys.path.insert(1, path)

        if not hasattr(sys, 'real_prefix'):
            sys.real_prefix = sys.prefix

        sys.prefix = self.path

def _activate ( self ) : old_syspath = set ( sys . path ) site . addsitedir ( self . site_path ) site . addsitedir ( self . bin_path ) new_syspaths = set ( sys . path ) - old_syspath for path in new_syspaths : sys . path . remove ( path ) sys . path . insert ( 1 , path ) if not hasattr ( sys , 'real_prefix' ) : sys . real_prefix = sys . prefix sys . prefix = self . path

Do some serious mangling to the current python environment ... This is necessary to activate an environment via python .


def remove(self):
        '''
        Remove this environment
        '''
        self.run_hook('preremove')
        utils.rmtree(self.path)
        self.run_hook('postremove')

def remove ( self ) : self . run_hook ( 'preremove' ) utils . rmtree ( self . path ) self . run_hook ( 'postremove' )


def command(self):
        '''Command used to launch this application module'''

        cmd = self.config.get('command', None)
        if cmd is None:
            return

        cmd = cmd[platform]
        return cmd['path'], cmd['args']

def command ( self ) : cmd = self . config . get ( 'command' , None ) if cmd is None : return cmd = cmd [ platform ] return cmd [ 'path' ] , cmd [ 'args' ]


def create(name_or_path=None, config=None):
    '''Create a virtual environment. You can pass either the name of a new
    environment to create in your CPENV_HOME directory OR specify a full path
    to create an environment outisde your CPENV_HOME.

    Create an environment in CPENV_HOME::

        >>> cpenv.create('myenv')

    Create an environment elsewhere::

        >>> cpenv.create('~/custom_location/myenv')

    :param name_or_path: Name or full path of environment
    :param config: Environment configuration including dependencies etc...
    '''

    # Get the real path of the environment
    if utils.is_system_path(name_or_path):
        path = unipath(name_or_path)
    else:
        path = unipath(get_home_path(), name_or_path)

    if os.path.exists(path):
        raise OSError('{} already exists'.format(path))

    env = VirtualEnvironment(path)
    utils.ensure_path_exists(env.path)

    if config:
        if utils.is_git_repo(config):
            Git('').clone(config, env.path)
        else:
            shutil.copy2(config, env.config_path)
    else:
        with open(env.config_path, 'w') as f:
            f.write(defaults.environment_config)

    utils.ensure_path_exists(env.hook_path)
    utils.ensure_path_exists(env.modules_path)

    env.run_hook('precreate')

    virtualenv.create_environment(env.path)
    if not utils.is_home_environment(env.path):
        EnvironmentCache.add(env)
        EnvironmentCache.save()

    try:
        env.update()
    except:
        utils.rmtree(path)
        logger.debug('Failed to update, rolling back...')
        raise
    else:
        env.run_hook('postcreate')

    return env

def
create
(
name_or_path
=
,
config
=
)
:
if
utils
.
is_system_path
(
name_or_path
)
:
path
=
unipath
(
name_or_path
)
else
:
path
=
unipath
(
get_home_path
(
)
,
name_or_path
)
if
os
.
path
.
exists
(
path
)
:
raise
OSError
(
'{} already exists'
.
format
(
path
)
)
env
=
VirtualEnvironment
(
path
)
utils
.
ensure_path_exists
(
env
.
path
)
if
config
:
if
utils
.
is_git_repo
(
config
)
:
Git
(
''
)
.
clone
(
config
,
env
.
path
)
else
:
shutil
.
copy2
(
config
,
env
.
config_path
)
else
:
with
open
(
env
.
config_path
,
'w'
)
as
f
:
f
.
write
(
defaults
.
environment_config
)
utils
.
ensure_path_exists
(
env
.
hook_path
)
utils
.
ensure_path_exists
(
env
.
modules_path
)
env
.
run_hook
(
'precreate'
)
virtualenv
.
create_environment
(
env
.
path
)
if
not
utils
.
is_home_environment
(
env
.
path
)
:
EnvironmentCache
.
add
(
env
)
EnvironmentCache
.
save
(
)
try
:
env
.
update
(
)
except
:
utils
.
rmtree
(
path
)
logger
.
debug
(
'Failed to update, rolling back...'
)
raise
else
:
env
.
run_hook
(
'postcreate'
)
return
env

Create a virtual environment . You can pass either the name of a new environment to create in your CPENV_HOME directory OR specify a full path to create an environment outisde your CPENV_HOME .


def remove(name_or_path):
    '''Remove an environment or module

    :param name_or_path: name or path to environment or module
    '''

    r = resolve(name_or_path)
    r.resolved[0].remove()

    EnvironmentCache.discard(r.resolved[0])
    EnvironmentCache.save()

def remove ( name_or_path ) : r = resolve ( name_or_path ) r . resolved [ 0 ] . remove ( ) EnvironmentCache . discard ( r . resolved [ 0 ] ) EnvironmentCache . save ( )


def launch(module_name, *args, **kwargs):
    '''Activates and launches a module

    :param module_name: name of module to launch
    '''

    r = resolve(module_name)
    r.activate()
    mod = r.resolved[0]
    mod.launch(*args, **kwargs)

def launch ( module_name , * args , ** kwargs ) : r = resolve ( module_name ) r . activate ( ) mod = r . resolved [ 0 ] mod . launch ( * args , ** kwargs )


def deactivate():
    '''Deactivates an environment by restoring all env vars to a clean state
    stored prior to activating environments
    '''

    if 'CPENV_ACTIVE' not in os.environ or 'CPENV_CLEAN_ENV' not in os.environ:
        raise EnvironmentError('Can not deactivate environment...')

    utils.restore_env_from_file(os.environ['CPENV_CLEAN_ENV'])

def
deactivate
(
)
:
if
'CPENV_ACTIVE'
not
in
os
.
environ
or
'CPENV_CLEAN_ENV'
not
in
os
.
environ
:
raise
EnvironmentError
(
'Can not deactivate environment...'
)
utils
.
restore_env_from_file
(
os
.
environ
[
'CPENV_CLEAN_ENV'
]
)


def get_modules():
    '''Returns a list of available modules.'''

    modules = set()

    cwd = os.getcwd()
    for d in os.listdir(cwd):

        if d == 'module.yml':
            modules.add(Module(cwd))

        path = unipath(cwd, d)
        if utils.is_module(path):
            modules.add(Module(cwd))

    module_paths = get_module_paths()
    for module_path in module_paths:
        for d in os.listdir(module_path):

            path = unipath(module_path, d)
            if utils.is_module(path):
                modules.add(Module(path))

    return sorted(list(modules), key=lambda x: x.name)

def get_modules ( ) : modules = set ( ) cwd = os . getcwd ( ) for d in os . listdir ( cwd ) : if d == 'module.yml' : modules . add ( Module ( cwd ) ) path = unipath ( cwd , d ) if utils . is_module ( path ) : modules . add ( Module ( cwd ) ) module_paths = get_module_paths ( ) for module_path in module_paths : for d in os . listdir ( module_path ) : path = unipath ( module_path , d ) if utils . is_module ( path ) : modules . add ( Module ( path ) ) return sorted ( list ( modules ) , key = lambda x : x . name )


def add_active_module(module):
    '''Add a module to CPENV_ACTIVE_MODULES environment variable'''

    modules = set(get_active_modules())
    modules.add(module)
    new_modules_path = os.pathsep.join([m.path for m in modules])
    os.environ['CPENV_ACTIVE_MODULES'] = str(new_modules_path)

def add_active_module ( module ) : modules = set ( get_active_modules ( ) ) modules . add ( module ) new_modules_path = os . pathsep . join ( [ m . path for m in modules ] ) os . environ [ 'CPENV_ACTIVE_MODULES' ] = str ( new_modules_path )


def rem_active_module(module):
    '''Remove a module from CPENV_ACTIVE_MODULES environment variable'''

    modules = set(get_active_modules())
    modules.discard(module)
    new_modules_path = os.pathsep.join([m.path for m in modules])
    os.environ['CPENV_ACTIVE_MODULES'] = str(new_modules_path)

def rem_active_module ( module ) : modules = set ( get_active_modules ( ) ) modules . discard ( module ) new_modules_path = os . pathsep . join ( [ m . path for m in modules ] ) os . environ [ 'CPENV_ACTIVE_MODULES' ] = str ( new_modules_path )


def format_objects(objects, children=False, columns=None, header=True):
    '''Format a list of environments and modules for terminal output'''

    columns = columns or ('NAME', 'TYPE', 'PATH')
    objects = sorted(objects, key=_type_and_name)
    data = []
    for obj in objects:
        if isinstance(obj, cpenv.VirtualEnvironment):
            data.append(get_info(obj))
            modules = obj.get_modules()
            if children and modules:
                for mod in modules:
                    data.append(get_info(mod, indent=2, root=obj.path))
        else:
            data.append(get_info(obj))

    maxes = [len(max(col, key=len)) for col in zip(*data)]
    tmpl = '{:%d}  {:%d}  {:%d}' % tuple(maxes)
    lines = []
    if header:
        lines.append('\n' + bold_blue(tmpl.format(*columns)))

    for obj_data in data:
        lines.append(tmpl.format(*obj_data))

    return '\n'.join(lines)

def
format_objects
(
objects
,
children
=
False
,
columns
=
,
header
=
True
)
:
columns
=
columns
or
(
'NAME'
,
'TYPE'
,
'PATH'
)
objects
=
sorted
(
objects
,
key
=
_type_and_name
)
data
=
[
]
for
obj
in
objects
:
if
isinstance
(
obj
,
cpenv
.
VirtualEnvironment
)
:
data
.
append
(
get_info
(
obj
)
)
modules
=
obj
.
get_modules
(
)
if
children
and
modules
:
for
mod
in
modules
:
data
.
append
(
get_info
(
mod
,
indent
=
2
,
root
=
obj
.
path
)
)
else
:
data
.
append
(
get_info
(
obj
)
)
maxes
=
[
len
(
max
(
col
,
key
=
len
)
)
for
col
in
zip
(
*
data
)
]
tmpl
=
'{:%d}  {:%d}  {:%d}'
%
tuple
(
maxes
)
lines
=
[
]
if
header
:
lines
.
append
(
'\n'
+
bold_blue
(
tmpl
.
format
(
*
columns
)
)
)
for
obj_data
in
data
:
lines
.
append
(
tmpl
.
format
(
*
obj_data
)
)
return
'\n'
.
join
(
lines
)


def info():
    '''Show context info'''

    env = cpenv.get_active_env()
    modules = []
    if env:
        modules = env.get_modules()
    active_modules = cpenv.get_active_modules()

    if not env and not modules and not active_modules:
        click.echo('\nNo active modules...')
        return

    click.echo(bold('\nActive modules'))
    if env:
        click.echo(format_objects([env] + active_modules))

        available_modules = set(modules) - set(active_modules)
        if available_modules:

            click.echo(
                bold('\nInactive modules in {}\n').format(cyan(env.name))
            )
            click.echo(format_objects(available_modules, header=False))

    else:
        click.echo(format_objects(active_modules))

    available_shared_modules = set(cpenv.get_modules()) - set(active_modules)
    if not available_shared_modules:
        return

    click.echo(bold('\nInactive shared modules \n'))
    click.echo(format_objects(available_shared_modules, header=False))

def
info
(
)
:
env
=
cpenv
.
get_active_env
(
)
modules
=
[
]
if
env
:
modules
=
env
.
get_modules
(
)
active_modules
=
cpenv
.
get_active_modules
(
)
if
not
env
and
not
modules
and
not
active_modules
:
click
.
echo
(
'\nNo active modules...'
)
return
click
.
echo
(
bold
(
'\nActive modules'
)
)
if
env
:
click
.
echo
(
format_objects
(
[
env
]
+
active_modules
)
)
available_modules
=
set
(
modules
)
set
(
active_modules
)
if
available_modules
:
click
.
echo
(
bold
(
'\nInactive modules in {}\n'
)
.
format
(
cyan
(
env
.
name
)
)
)
click
.
echo
(
format_objects
(
available_modules
,
header
=
False
)
)
else
:
click
.
echo
(
format_objects
(
active_modules
)
)
available_shared_modules
=
set
(
cpenv
.
get_modules
(
)
)
set
(
active_modules
)
if
not
available_shared_modules
:
return
click
.
echo
(
bold
(
'\nInactive shared modules \n'
)
)
click
.
echo
(
format_objects
(
available_shared_modules
,
header
=
False
)
)


def activate(paths, skip_local, skip_shared):
    '''Activate an environment'''

    if not paths:
        ctx = click.get_current_context()
        if cpenv.get_active_env():
            ctx.invoke(info)
            return

        click.echo(ctx.get_help())
        examples = (
            '\nExamples: \n'
            '    cpenv activate my_env\n'
            '    cpenv activate ./relative/path/to/my_env\n'
            '    cpenv activate my_env my_module\n'
        )
        click.echo(examples)
        return

    if skip_local:
        cpenv.module_resolvers.remove(cpenv.resolver.module_resolver)
        cpenv.module_resolvers.remove(cpenv.resolver.active_env_module_resolver)

    if skip_shared:
        cpenv.module_resolvers.remove(cpenv.resolver.modules_path_resolver)

    try:
        r = cpenv.resolve(*paths)
    except cpenv.ResolveError as e:
        click.echo('\n' + str(e))
        return

    resolved = set(r.resolved)
    active_modules = set()
    env = cpenv.get_active_env()
    if env:
        active_modules.add(env)
    active_modules.update(cpenv.get_active_modules())

    new_modules = resolved - active_modules
    old_modules = active_modules & resolved

    if old_modules and not new_modules:
        click.echo(
            '\nModules already active: '
            + bold(' '.join([obj.name for obj in old_modules]))
        )
        return

    if env and contains_env(new_modules):
        click.echo('\nUse bold(exit) to leave your active environment first.')
        return

    click.echo('\nResolved the following modules...')
    click.echo(format_objects(r.resolved))
    r.activate()
    click.echo(blue('\nLaunching subshell...'))

    modules = sorted(resolved | active_modules, key=_type_and_name)
    prompt = ':'.join([obj.name for obj in modules])
    shell.launch(prompt)


def create(name_or_path, config):
    '''Create a new environment.'''

    if not name_or_path:
        ctx = click.get_current_context()
        click.echo(ctx.get_help())
        examples = (
            '\nExamples:\n'
            '    cpenv create my_env\n'
            '    cpenv create ./relative/path/to/my_env\n'
            '    cpenv create my_env --config ./relative/path/to/config\n'
            '    cpenv create my_env --config git@github.com:user/config.git\n'
        )
        click.echo(examples)
        return

    click.echo(
        blue('Creating a new virtual environment ' + name_or_path)
    )
    try:
        env = cpenv.create(name_or_path, config)
    except Exception as e:
        click.echo(bold_red('FAILED TO CREATE ENVIRONMENT!'))
        click.echo(e)
    else:
        click.echo(bold_green('Successfully created environment!'))
    click.echo(blue('Launching subshell'))

    cpenv.activate(env)
    shell.launch(env.name)


def remove(name_or_path):
    '''Remove an environment'''

    click.echo()
    try:
        r = cpenv.resolve(name_or_path)
    except cpenv.ResolveError as e:
        click.echo(e)
        return

    obj = r.resolved[0]
    if not isinstance(obj, cpenv.VirtualEnvironment):
        click.echo('{} is a module. Use `cpenv module remove` instead.')
        return

    click.echo(format_objects([obj]))
    click.echo()

    user_confirmed = click.confirm(
        red('Are you sure you want to remove this environment?')
    )
    if user_confirmed:
        click.echo('Attempting to remove...', nl=False)

        try:
            obj.remove()
        except Exception as e:
            click.echo(bold_red('FAIL'))
            click.echo(e)
        else:
            click.echo(bold_green('OK!'))


def add(path):
    '''Add an environment to the cache. Allows you to activate the environment
    by name instead of by full path'''

    click.echo('\nAdding {} to cache......'.format(path), nl=False)
    try:
        r = cpenv.resolve(path)
    except Exception as e:
        click.echo(bold_red('FAILED'))
        click.echo(e)
        return

    if isinstance(r.resolved[0], cpenv.VirtualEnvironment):
        EnvironmentCache.add(r.resolved[0])
        EnvironmentCache.save()
        click.echo(bold_green('OK!'))

def
add
(
path
)
:
click
.
echo
(
'\nAdding {} to cache......'
.
format
(
path
)
,
nl
=
False
)
try
:
r
=
cpenv
.
resolve
(
path
)
except
Exception
as
e
:
click
.
echo
(
bold_red
(
'FAILED'
)
)
click
.
echo
(
e
)
return
if
isinstance
(
r
.
resolved
[
0
]
,
cpenv
.
VirtualEnvironment
)
:
EnvironmentCache
.
add
(
r
.
resolved
[
0
]
)
EnvironmentCache
.
save
(
)
click
.
echo
(
bold_green
(
'OK!'
)
)

Add an environment to the cache . Allows you to activate the environment by name instead of by full path


def remove(path):
    '''Remove a cached environment. Removed paths will no longer be able to
    be activated by name'''

    r = cpenv.resolve(path)
    if isinstance(r.resolved[0], cpenv.VirtualEnvironment):
        EnvironmentCache.discard(r.resolved[0])
        EnvironmentCache.save()

def remove ( path ) : r = cpenv . resolve ( path ) if isinstance ( r . resolved [ 0 ] , cpenv . VirtualEnvironment ) : EnvironmentCache . discard ( r . resolved [ 0 ] ) EnvironmentCache . save ( )

Remove a cached environment . Removed paths will no longer be able to be activated by name


def create(name_or_path, config):
    '''Create a new template module.

    You can also specify a filesystem path like "./modules/new_module"
    '''

    click.echo('Creating module {}...'.format(name_or_path), nl=False)
    try:
        module = cpenv.create_module(name_or_path, config)
    except Exception as e:
        click.echo(bold_red('FAILED'))
        raise
    else:
        click.echo(bold_green('OK!'))
        click.echo('Browse to your new module and make some changes.')
        click.echo("When you're ready add the module to an environment:")
        click.echo('    cpenv module add my_module ./path/to/my_module')
        click.echo('Or track your module on git and add it directly from the repo:')
        click.echo('    cpenv module add my_module git@github.com:user/my_module.git')


def add(name, path, branch, type):
    '''Add a module to an environment. PATH can be a git repository path or
    a filesystem path. '''

    if not name and not path:
        ctx = click.get_current_context()
        click.echo(ctx.get_help())
        examples = (
            '\nExamples:\n'
            '    cpenv module add my_module ./path/to/my_module\n'
            '    cpenv module add my_module git@github.com:user/my_module.git'
            '    cpenv module add my_module git@github.com:user/my_module.git --branch=master --type=shared'
        )
        click.echo(examples)
        return

    if not name:
        click.echo('Missing required argument: name')
        return

    if not path:
        click.echo('Missing required argument: path')

    env = cpenv.get_active_env()
    if type=='local':
        if not env:
            click.echo('\nActivate an environment to add a local module.\n')
            return

        if click.confirm('\nAdd {} to active env {}?'.format(name, env.name)):
            click.echo('Adding module...', nl=False)
            try:
                env.add_module(name, path, branch)
            except:
                click.echo(bold_red('FAILED'))
                raise
            else:
                click.echo(bold_green('OK!'))

        return

    module_paths = cpenv.get_module_paths()
    click.echo('\nAvailable module paths:\n')
    for i, mod_path in enumerate(module_paths):
        click.echo('    {}. {}'.format(i, mod_path))
    choice = click.prompt(
        'Where do you want to add your module?',
        type=int,
        default=0
    )
    module_root = module_paths[choice]
    module_path = utils.unipath(module_root, name)
    click.echo('Creating module {}...'.format(module_path), nl=False)
    try:
        cpenv.create_module(module_path, path, branch)
    except:
        click.echo(bold_red('FAILED'))
        raise
    else:
        click.echo(bold_green('OK!'))

Add a module to an environment . PATH can be a git repository path or a filesystem path .


def localize(name):
    '''Copy a global module to the active environment.'''

    env = cpenv.get_active_env()
    if not env:
        click.echo('You need to activate an environment first.')
        return

    try:
        r = cpenv.resolve(name)
    except cpenv.ResolveError as e:
        click.echo('\n' + str(e))

    module = r.resolved[0]
    if isinstance(module, cpenv.VirtualEnvironment):
        click.echo('\nCan only localize a module not an environment')
        return

    active_modules = cpenv.get_active_modules()
    if module in active_modules:
        click.echo('\nCan not localize an active module.')
        return

    if module in env.get_modules():
        click.echo('\n{} is already local to {}'.format(module.name, env.name))
        return

    if click.confirm('\nAdd {} to env {}?'.format(module.name, env.name)):
        click.echo('Adding module...', nl=False)
        try:
            module = env.add_module(module.name, module.path)
        except:
            click.echo(bold_red('FAILED'))
            raise
        else:
            click.echo(bold_green('OK!'))

    click.echo('\nActivate the localize module:')
    click.echo('    cpenv activate {} {}'.format(env.name, module.name))

Copy a global module to the active environment .


def path_resolver(resolver, path):
    '''Resolves VirtualEnvironments with a relative or absolute path'''

    path = unipath(path)

    if is_environment(path):
        return VirtualEnvironment(path)

    raise ResolveError

def path_resolver ( resolver , path ) : path = unipath ( path ) if is_environment ( path ) : return VirtualEnvironment ( path ) raise ResolveError


def home_resolver(resolver, path):
    '''Resolves VirtualEnvironments in CPENV_HOME'''

    from .api import get_home_path

    path = unipath(get_home_path(), path)

    if is_environment(path):
        return VirtualEnvironment(path)

    raise ResolveError

def home_resolver ( resolver , path ) : from . api import get_home_path path = unipath ( get_home_path ( ) , path ) if is_environment ( path ) : return VirtualEnvironment ( path ) raise ResolveError


def cache_resolver(resolver, path):
    '''Resolves VirtualEnvironments in EnvironmentCache'''

    env = resolver.cache.find(path)
    if env:
        return env

    raise ResolveError

def cache_resolver ( resolver , path ) : env = resolver . cache . find ( path ) if env : return env raise ResolveError


def module_resolver(resolver, path):
    '''Resolves module in previously resolved environment.'''

    if resolver.resolved:

        if isinstance(resolver.resolved[0], VirtualEnvironment):
            env = resolver.resolved[0]
            mod = env.get_module(path)

            if mod:
                return mod

    raise ResolveError

def module_resolver ( resolver , path ) : if resolver . resolved : if isinstance ( resolver . resolved [ 0 ] , VirtualEnvironment ) : env = resolver . resolved [ 0 ] mod = env . get_module ( path ) if mod : return mod raise ResolveError

Resolves module in previously resolved environment .


def active_env_module_resolver(resolver, path):
    '''Resolves modules in currently active environment.'''

    from .api import get_active_env

    env = get_active_env()
    if not env:
        raise ResolveError

    mod = env.get_module(path)
    if not mod:
        raise ResolveError

    return mod

def active_env_module_resolver ( resolver , path ) : from . api import get_active_env env = get_active_env ( ) if not env : raise ResolveError mod = env . get_module ( path ) if not mod : raise ResolveError return mod

Resolves modules in currently active environment .


def redirect_resolver(resolver, path):
    '''Resolves environment from .cpenv file...recursively walks up the tree
    in attempt to find a .cpenv file'''

    if not os.path.exists(path):
        raise ResolveError

    if os.path.isfile(path):
        path = os.path.dirname(path)

    for root, _, _ in walk_up(path):
        if is_redirecting(root):
            env_paths = redirect_to_env_paths(unipath(root, '.cpenv'))
            r = Resolver(*env_paths)
            return r.resolve()

    raise ResolveError

def redirect_resolver ( resolver , path ) : if not os . path . exists ( path ) : raise ResolveError if os . path . isfile ( path ) : path = os . path . dirname ( path ) for root , _ , _ in walk_up ( path ) : if is_redirecting ( root ) : env_paths = redirect_to_env_paths ( unipath ( root , '.cpenv' ) ) r = Resolver ( * env_paths ) return r . resolve ( ) raise ResolveError

Resolves environment from . cpenv file ... recursively walks up the tree in attempt to find a . cpenv file


def transpose(a, axes=None):
    """Returns a view of the array with axes transposed.

    For a 1-D array, this has no effect.
    For a 2-D array, this is the usual matrix transpose.
    For an n-D array, if axes are given, their order indicates how the
    axes are permuted

    Args:
      a (array_like): Input array.
      axes (list of int, optional): By default, reverse the dimensions,
        otherwise permute the axes according to the values given.
    """
    if isinstance(a, np.ndarray):
        return np.transpose(a, axes)
    elif isinstance(a, RemoteArray):
        return a.transpose(*axes)
    elif isinstance(a, Remote):
        return _remote_to_array(a).transpose(*axes)
    elif isinstance(a, DistArray):
        if axes is None:
            axes = range(a.ndim - 1, -1, -1)
        axes = list(axes)
        if len(set(axes)) < len(axes):
            raise ValueError("repeated axis in transpose")
        if sorted(axes) != list(range(a.ndim)):
            raise ValueError("axes don't match array")
        distaxis = a._distaxis
        new_distaxis = axes.index(distaxis)
        new_subarrays = [ra.transpose(*axes) for ra in a._subarrays]
        return DistArray(new_subarrays, new_distaxis)
    else:
        return np.transpose(a, axes)

def
transpose
(
a
,
axes
=
)
:
if
isinstance
(
a
,
np
.
ndarray
)
:
return
np
.
transpose
(
a
,
axes
)
elif
isinstance
(
a
,
RemoteArray
)
:
return
a
.
transpose
(
*
axes
)
elif
isinstance
(
a
,
Remote
)
:
return
_remote_to_array
(
a
)
.
transpose
(
*
axes
)
elif
isinstance
(
a
,
DistArray
)
:
if
axes
is
:
axes
=
range
(
a
.
ndim
1
,
1
,
1
)
axes
=
list
(
axes
)
if
len
(
set
(
axes
)
)
<
len
(
axes
)
:
raise
ValueError
(
"repeated axis in transpose"
)
if
sorted
(
axes
)
!=
list
(
range
(
a
.
ndim
)
)
:
raise
ValueError
(
"axes don't match array"
)
distaxis
=
a
.
_distaxis
new_distaxis
=
axes
.
index
(
distaxis
)
new_subarrays
=
[
ra
.
transpose
(
*
axes
)
for
ra
in
a
.
_subarrays
]
return
DistArray
(
new_subarrays
,
new_distaxis
)
else
:
return
np
.
transpose
(
a
,
axes
)

Returns a view of the array with axes transposed .


def rollaxis(a, axis, start=0):
    """Roll the specified axis backwards, until it lies in a given position.

    Args:
      a (array_like): Input array.
      axis (int): The axis to roll backwards.  The positions of the other axes
        do not change relative to one another.
      start (int, optional): The axis is rolled until it lies before this
        position.  The default, 0, results in a "complete" roll.

    Returns:
      res (ndarray)
    """
    if isinstance(a, np.ndarray):
        return np.rollaxis(a, axis, start)
    if axis not in range(a.ndim):
        raise ValueError(
                'rollaxis: axis (%d) must be >=0 and < %d' % (axis, a.ndim))
    if start not in range(a.ndim + 1):
        raise ValueError(
                'rollaxis: start (%d) must be >=0 and < %d' % (axis, a.ndim+1))
    axes = list(range(a.ndim))
    axes.remove(axis)
    axes.insert(start, axis)
    return transpose(a, axes)

Roll the specified axis backwards until it lies in a given position .


def expand_dims(a, axis):
    """Insert a new axis, corresponding to a given position in the array shape

    Args:
      a (array_like): Input array.
      axis (int): Position (amongst axes) where new axis is to be inserted.
    """
    if hasattr(a, 'expand_dims') and hasattr(type(a), '__array_interface__'):
        return a.expand_dims(axis)
    else:
        return np.expand_dims(a, axis)

def expand_dims ( a , axis ) : if hasattr ( a , 'expand_dims' ) and hasattr ( type ( a ) , '__array_interface__' ) : return a . expand_dims ( axis ) else : return np . expand_dims ( a , axis )

Insert a new axis corresponding to a given position in the array shape


def concatenate(tup, axis=0):
    """Join a sequence of arrays together.
    Will aim to join `ndarray`, `RemoteArray`, and `DistArray` without moving
    their data, if they happen to be on different engines.

    Args:
      tup (sequence of array_like): Arrays to be concatenated. They must have
        the same shape, except in the dimension corresponding to `axis`.
      axis (int, optional): The axis along which the arrays will be joined.

    Returns:
      res: `ndarray`, if inputs were all local
           `RemoteArray`, if inputs were all on the same remote engine
           `DistArray`, if inputs were already scattered on different engines
    """
    from distob import engine
    if len(tup) is 0:
        raise ValueError('need at least one array to concatenate')
    first = tup[0]
    others = tup[1:]
    # allow subclasses to provide their own implementations of concatenate:
    if (hasattr(first, 'concatenate') and
            hasattr(type(first), '__array_interface__')):
        return first.concatenate(others, axis)
    # convert all arguments to arrays/RemoteArrays if they are not already:
    arrays = []
    for ar in tup:
        if isinstance(ar, DistArray):
            if axis == ar._distaxis:
                arrays.extend(ar._subarrays)
            else:
                # Since not yet implemented arrays distributed on more than
                # one axis, will fetch and re-scatter on the new axis:
                arrays.append(gather(ar))
        elif isinstance(ar, RemoteArray):
            arrays.append(ar)
        elif isinstance(ar, Remote):
            arrays.append(_remote_to_array(ar))
        elif hasattr(type(ar), '__array_interface__'):
            # then treat as a local ndarray
            arrays.append(ar)
        else:
            arrays.append(np.array(ar))
    if all(isinstance(ar, np.ndarray) for ar in arrays):
        return np.concatenate(arrays, axis)
    total_length = 0
    # validate dimensions are same, except for axis of concatenation:
    commonshape = list(arrays[0].shape)
    commonshape[axis] = None # ignore this axis for shape comparison
    for ar in arrays:
        total_length += ar.shape[axis]
        shp = list(ar.shape)
        shp[axis] = None
        if shp != commonshape:
            raise ValueError('incompatible shapes for concatenation')
    # set sensible target block size if splitting subarrays further:
    blocksize = ((total_length - 1) // engine.nengines) + 1
    rarrays = []
    for ar in arrays:
        if isinstance(ar, DistArray):
            rarrays.extend(ar._subarrays)
        elif isinstance(ar, RemoteArray):
            rarrays.append(ar)
        else:
            da = _scatter_ndarray(ar, axis, blocksize)
            for ra in da._subarrays:
                rarrays.append(ra)
            del da
    del arrays
    # At this point rarrays is a list of RemoteArray to be concatenated
    eid = rarrays[0]._id.engine
    if all(ra._id.engine == eid for ra in rarrays):
        # Arrays to be joined are all on the same engine
        if eid == engine.eid:
            # Arrays are all local
            return concatenate([gather(r) for r in rarrays], axis)
        else:
            return call(concatenate, rarrays, axis)
    else:
        # Arrays to be joined are on different engines.
        # TODO: consolidate any consecutive arrays already on same engine
        return DistArray(rarrays, axis)

Join a sequence of arrays together . Will aim to join ndarray RemoteArray and DistArray without moving their data if they happen to be on different engines .