def add_to_cat_group(self, cat, cat_group, def_name):
        """Associate the provided rule definition name ``def_name`` with the
        category group ``cat_group`` in the category ``cat``.

        :param str cat: The category the rule definition was declared in
        :param str cat_group: The group within the category the rule belongs to
        :param str def_name: The name of the rule definition
        """
        self.cat_groups.setdefault(cat, {}).setdefault(cat_group, deque()).append(def_name)

def add_to_cat_group ( self , cat , cat_group , def_name ) : self . cat_groups . setdefault ( cat , { } ) . setdefault ( cat_group , deque ( ) ) . append ( def_name )

Associate the provided rule definition name def_name with the category group cat_group in the category cat .


def gen(self, num, cat=None, cat_group=None, preferred=None, preferred_ratio=0.5, max_recursion=None, auto_process=True):
        """Generate ``num`` rules from category ``cat``, optionally specifying
        preferred category groups ``preferred`` that should be preferred at
        probability ``preferred_ratio`` over other randomly-chosen rule definitions.

        :param int num: The number of rules to generate
        :param str cat: The name of the category to generate ``num`` rules from
        :param str cat_group: The category group (ie python file) to generate rules from. This
            was added specifically to make it easier to generate data based on the name
            of the file the grammar was defined in, and is intended to work with the
            ``TOP_CAT`` values that may be defined in a loaded grammar file.
        :param list preferred: A list of preferred category groups to generate rules from
        :param float preferred_ratio: The percent probability that the preferred
            groups will be chosen over randomly choosen rule definitions from category ``cat``.
        :param int max_recursion: The maximum amount to allow references to recurse
        :param bool auto_process: Whether rules should be automatically pruned and
            shortest reference paths determined. See :any:`gramfuzz.GramFuzzer.preprocess_rules`
            for what would automatically be done.
        """
        import gramfuzz.fields
        gramfuzz.fields.REF_LEVEL = 1

        if cat is None and cat_group is None:
            raise gramfuzz.errors.GramFuzzError("cat and cat_group are None, one must be set")

        if cat is None and cat_group is not None:
            if cat_group not in self.cat_group_defaults:
                raise gramfuzz.errors.GramFuzzError(
                    "cat_group {!r} did not define a TOP_CAT variable"
                )
            cat = self.cat_group_defaults[cat_group]
            if not isinstance(cat, basestring):
                raise gramfuzz.errors.GramFuzzError(
                    "cat_group {!r}'s TOP_CAT variable was not a string"
                )

        if auto_process and self._rules_processed == False:
            self.preprocess_rules()

        if max_recursion is not None:
            self.set_max_recursion(max_recursion)

        if preferred is None:
            preferred = []

        res = deque()
        cat_defs = self.defs[cat]

        # optimizations
        _res_append = res.append
        _res_extend = res.extend
        _choice = rand.choice
        _maybe = rand.maybe
        _val = utils.val

        keys = self.defs[cat].keys()

        self._last_pref_keys = self._get_pref_keys(cat, preferred)
        # be sure to set this *after* fetching the pref keys (above^)
        self._last_prefs = preferred

        total_errors = deque()
        total_gend = 0
        while total_gend < num:
            # use a rule definition from one of the preferred category
            # groups
            if len(self._last_pref_keys) > 0 and _maybe(preferred_ratio):
                rand_key = _choice(self._last_pref_keys)
                if rand_key not in cat_defs:
                    # TODO this means it was removed / pruned b/c it was unreachable??
                    # TODO look into this more
                    rand_key = _choice(list(keys))

            # else just choose a key at random from the category
            else:
                rand_key = _choice(list(keys))

            # pruning failed, this rule is not defined/reachable
            if rand_key not in cat_defs:
                continue

            v = _choice(cat_defs[rand_key])

            # not used directly by GramFuzzer, but could be useful
            # to subclasses of GramFuzzer
            info = {}

            pre = deque()
            self.pre_revert(info)
            val_res = None

            try:
                val_res = _val(v, pre)
            except errors.GramFuzzError as e:
                raise
                #total_errors.append(e)
                #self.revert(info)
                #continue
            except RuntimeError as e:
                print("RUNTIME ERROR")
                self.revert(info)
                continue

            if val_res is not None:
                _res_extend(pre)
                _res_append(val_res)

                total_gend += 1
                self.post_revert(cat, res, total_gend, num, info)

        return res

Generate num rules from category cat optionally specifying preferred category groups preferred that should be preferred at probability preferred_ratio over other randomly - chosen rule definitions .


def fuzz_elements(self, element):
        """
        Fuzz all elements inside the object
        """
        try:
            if type(element) == dict:
                tmp_element = {}
                for key in element:
                    if len(self.config.parameters) > 0:
                        if self.config.exclude_parameters:
                            fuzz = key not in self.config.parameters
                        else:
                            fuzz = key in self.config.parameters
                    else:
                        fuzz = True
                    if fuzz:
                        if type(element[key]) == dict:
                            tmp_element.update({key: self.fuzz_elements(element[key])})
                        elif type(element[key]) == list:
                            tmp_element.update({key: self.fuzz_elements(element[key])})
                        else:
                            tmp_element.update({key: self.mutator.fuzz(element[key])})
                    else:
                        tmp_element.update({key: self.fuzz_elements(element[key])})
                element = tmp_element
                del tmp_element
            elif type(element) == list:
                arr = []
                for key in element:
                    if type(key) == dict:
                        arr.append(self.fuzz_elements(key))
                    elif type(key) == list:
                        arr.append(self.fuzz_elements(key))
                    else:
                        if len(self.config.parameters) <= 0:
                            arr.append(self.mutator.fuzz(key))
                        else:
                            arr.append(key)
                element = arr
                del arr
        except Exception as e:
            raise PJFBaseException(e.message if hasattr(e, "message") else str(e))
        return element

def fuzz_elements ( self , element ) : try : if type ( element ) == dict : tmp_element = { } for key in element : if len ( self . config . parameters ) > 0 : if self . config . exclude_parameters : fuzz = key not in self . config . parameters else : fuzz = key in self . config . parameters else : fuzz = True if fuzz : if type ( element [ key ] ) == dict : tmp_element . update ( { key : self . fuzz_elements ( element [ key ] ) } ) elif type ( element [ key ] ) == list : tmp_element . update ( { key : self . fuzz_elements ( element [ key ] ) } ) else : tmp_element . update ( { key : self . mutator . fuzz ( element [ key ] ) } ) else : tmp_element . update ( { key : self . fuzz_elements ( element [ key ] ) } ) element = tmp_element del tmp_element elif type ( element ) == list : arr = [ ] for key in element : if type ( key ) == dict : arr . append ( self . fuzz_elements ( key ) ) elif type ( key ) == list : arr . append ( self . fuzz_elements ( key ) ) else : if len ( self . config . parameters ) <= 0 : arr . append ( self . mutator . fuzz ( key ) ) else : arr . append ( key ) element = arr del arr except Exception as e : raise PJFBaseException ( e . message if hasattr ( e , "message" ) else str ( e ) ) return element


def fuzzed(self):
        """
        Get a printable fuzzed object
        """
        try:
            if self.config.strong_fuzz:
                fuzzer = PJFMutators(self.config)
                if self.config.url_encode:
                    if sys.version_info >= (3, 0):
                        return urllib.parse.quote(fuzzer.fuzz(json.dumps(self.config.json)))
                    else:
                        return urllib.quote(fuzzer.fuzz(json.dumps(self.config.json)))
                else:
                    if type(self.config.json) in [list, dict]:
                        return fuzzer.fuzz(json.dumps(self.config.json))
                    else:
                        return fuzzer.fuzz(self.config.json)
            else:
                if self.config.url_encode:
                    if sys.version_info >= (3, 0):
                        return urllib.parse.quote(self.get_fuzzed(self.config.indent, self.config.utf8))
                    else:
                        return urllib.quote(self.get_fuzzed(self.config.indent, self.config.utf8))
                else:
                    return self.get_fuzzed(self.config.indent, self.config.utf8)
        except Exception as e:
            raise PJFBaseException(e.message if hasattr(e, "message") else str(e))

def fuzzed ( self ) : try : if self . config . strong_fuzz : fuzzer = PJFMutators ( self . config ) if self . config . url_encode : if sys . version_info >= ( 3 , 0 ) : return urllib . parse . quote ( fuzzer . fuzz ( json . dumps ( self . config . json ) ) ) else : return urllib . quote ( fuzzer . fuzz ( json . dumps ( self . config . json ) ) ) else : if type ( self . config . json ) in [ list , dict ] : return fuzzer . fuzz ( json . dumps ( self . config . json ) ) else : return fuzzer . fuzz ( self . config . json ) else : if self . config . url_encode : if sys . version_info >= ( 3 , 0 ) : return urllib . parse . quote ( self . get_fuzzed ( self . config . indent , self . config . utf8 ) ) else : return urllib . quote ( self . get_fuzzed ( self . config . indent , self . config . utf8 ) ) else : return self . get_fuzzed ( self . config . indent , self . config . utf8 ) except Exception as e : raise PJFBaseException ( e . message if hasattr ( e , "message" ) else str ( e ) )


def get_fuzzed(self, indent=False, utf8=False):
        """
        Return the fuzzed object
        """
        try:
            if "array" in self.json:
                return self.fuzz_elements(dict(self.json))["array"]
            else:
                return self.fuzz_elements(dict(self.json))
        except Exception as e:
            raise PJFBaseException(e.message if hasattr(e, "message") else str(e))

def get_fuzzed ( self , indent = False , utf8 = False ) : try : if "array" in self . json : return self . fuzz_elements ( dict ( self . json ) ) [ "array" ] else : return self . fuzz_elements ( dict ( self . json ) ) except Exception as e : raise PJFBaseException ( e . message if hasattr ( e , "message" ) else str ( e ) )


def mutate_object_decorate(self, func):
        """
        Mutate a generic object based on type
        """
        def mutate():
            obj = func()
            return self.Mutators.get_mutator(obj, type(obj))
        return mutate

def mutate_object_decorate ( self , func ) : def mutate ( ) : obj = func ( ) return self . Mutators . get_mutator ( obj , type ( obj ) ) return mutate


def sigterm_handler(self, signum, frame):
        """ When we get term signal
        if we are waiting and got a sigterm, we just exit.
        if we have a child running, we pass the signal first to the child
        then we exit.

        :param signum:
        :param frame:
        :return:
        """
        assert(self.state in ('WAITING', 'RUNNING', 'PAUSED'))
        logger.debug("our state %s", self.state)
        if self.state == 'WAITING':
            return self.ioloop.stop()

        if self.state == 'RUNNING':
            logger.debug('already running sending signal to child - %s',
                         self.sprocess.pid)
            os.kill(self.sprocess.pid, signum)
        self.ioloop.stop()

When we get term signal if we are waiting and got a sigterm we just exit . if we have a child running we pass the signal first to the child then we exit .


def cli_command_quit(self, msg):
        """\
        kills the child and exits
        """
        if self.state == State.RUNNING and self.sprocess and self.sprocess.proc:
            self.sprocess.proc.kill()
        else:
            sys.exit(0)

def cli_command_quit ( self , msg ) : if self . state == State . RUNNING and self . sprocess and self . sprocess . proc : self . sprocess . proc . kill ( ) else : sys . exit ( 0 )


def cli_command_pause(self, msg):
        """\
        if we have a running child we kill it and set our state to paused
        if we don't have a running child, we set our state to paused
        this will pause all the nodes in single-beat cluster

        its useful when you deploy some code and don't want your child to spawn
        randomly

        :param msg:
        :return:
        """
        info = ''
        if self.state == State.RUNNING and self.sprocess and self.sprocess.proc:
            self.sprocess.set_exit_callback(self.proc_exit_cb_noop)
            self.sprocess.proc.kill()
            info = 'killed'
            # TODO: check if process is really dead etc.
        self.state = State.PAUSED
        return info

def cli_command_pause ( self , msg ) : info = '' if self . state == State . RUNNING and self . sprocess and self . sprocess . proc : self . sprocess . set_exit_callback ( self . proc_exit_cb_noop ) self . sprocess . proc . kill ( ) info = 'killed' self . state = State . PAUSED return info

\ if we have a running child we kill it and set our state to paused if we don t have a running child we set our state to paused this will pause all the nodes in single - beat cluster


def cli_command_resume(self, msg):
        """\
        sets state to waiting - so we resume spawning children
        """
        if self.state == State.PAUSED:
            self.state = State.WAITING

def cli_command_resume ( self , msg ) : if self . state == State . PAUSED : self . state = State . WAITING


def cli_command_stop(self, msg):
        """\
        stops the running child process - if its running
        it will re-spawn in any single-beat node after sometime

        :param msg:
        :return:
        """
        info = ''
        if self.state == State.RUNNING and self.sprocess and self.sprocess.proc:
            self.state = State.PAUSED
            self.sprocess.set_exit_callback(self.proc_exit_cb_state_set)
            self.sprocess.proc.kill()
            info = 'killed'
            # TODO: check if process is really dead etc.
        return info

def cli_command_stop ( self , msg ) : info = '' if self . state == State . RUNNING and self . sprocess and self . sprocess . proc : self . state = State . PAUSED self . sprocess . set_exit_callback ( self . proc_exit_cb_state_set ) self . sprocess . proc . kill ( ) info = 'killed' return info

\ stops the running child process - if its running it will re - spawn in any single - beat node after sometime


def cli_command_restart(self, msg):
        """\
        restart the subprocess
        i. we set our state to RESTARTING - on restarting we still send heartbeat
        ii. we kill the subprocess
        iii. we start again
        iv. if its started we set our state to RUNNING, else we set it to WAITING

        :param msg:
        :return:
        """
        info = ''
        if self.state == State.RUNNING and self.sprocess and self.sprocess.proc:
            self.state = State.RESTARTING
            self.sprocess.set_exit_callback(self.proc_exit_cb_restart)
            self.sprocess.proc.kill()
            info = 'killed'
            # TODO: check if process is really dead etc.
        return info

def cli_command_restart ( self , msg ) : info = '' if self . state == State . RUNNING and self . sprocess and self . sprocess . proc : self . state = State . RESTARTING self . sprocess . set_exit_callback ( self . proc_exit_cb_restart ) self . sprocess . proc . kill ( ) info = 'killed' return info

\ restart the subprocess i . we set our state to RESTARTING - on restarting we still send heartbeat ii . we kill the subprocess iii . we start again iv . if its started we set our state to RUNNING else we set it to WAITING


def getEvents(self):
        """
        Retrieve a list of events since the last poll.  Multiple calls may be needed to retrieve all events.

        If no events occur, the API will block for up to 30 seconds, after which an empty list is returned.  As soon as
        an event is received in this time, it is returned immediately.

        Returns:
            :class:`.SkypeEvent` list: a list of events, possibly empty
        """
        events = []
        for json in self.conn.endpoints["self"].getEvents():
            events.append(SkypeEvent.fromRaw(self, json))
        return events

def getEvents ( self ) : events = [ ] for json in self . conn . endpoints [ "self" ] . getEvents ( ) : events . append ( SkypeEvent . fromRaw ( self , json ) ) return events

Retrieve a list of events since the last poll . Multiple calls may be needed to retrieve all events .


def setMood(self, mood):
        """
        Update the activity message for the current user.

        Args:
            mood (str): new mood message
        """
        self.conn("POST", "{0}/users/{1}/profile/partial".format(SkypeConnection.API_USER, self.userId),
                  auth=SkypeConnection.Auth.SkypeToken, json={"payload": {"mood": mood or ""}})
        self.user.mood = SkypeUser.Mood(plain=mood) if mood else None

def setMood ( self , mood ) : self . conn ( "POST" , "{0}/users/{1}/profile/partial" . format ( SkypeConnection . API_USER , self . userId ) , auth = SkypeConnection . Auth . SkypeToken , json = { "payload" : { "mood" : mood or "" } } ) self . user . mood = SkypeUser . Mood ( plain = mood ) if mood else None

Update the activity message for the current user .


def setAvatar(self, image):
        """
        Update the profile picture for the current user.

        Args:
            image (file): a file-like object to read the image from
        """
        self.conn("PUT", "{0}/users/{1}/profile/avatar".format(SkypeConnection.API_USER, self.userId),
                  auth=SkypeConnection.Auth.SkypeToken, data=image.read())

def setAvatar ( self , image ) : self . conn ( "PUT" , "{0}/users/{1}/profile/avatar" . format ( SkypeConnection . API_USER , self . userId ) , auth = SkypeConnection . Auth . SkypeToken , data = image . read ( ) )

Update the profile picture for the current user .


def getUrlMeta(self, url):
        """
        Retrieve various metadata associated with a URL, as seen by Skype.

        Args:
            url (str): address to ping for info

        Returns:
            dict: metadata for the website queried
        """
        return self.conn("GET", SkypeConnection.API_URL, params={"url": url},
                         auth=SkypeConnection.Auth.Authorize).json()

def getUrlMeta ( self , url ) : return self . conn ( "GET" , SkypeConnection . API_URL , params = { "url" : url } , auth = SkypeConnection . Auth . Authorize ) . json ( )

Retrieve various metadata associated with a URL as seen by Skype .


def contact(self, id):
        """
        Retrieve all details for a specific contact, including fields such as birthday and mood.

        Args:
            id (str): user identifier to lookup

        Returns:
            SkypeContact: resulting contact object
        """
        try:
            json = self.skype.conn("POST", "{0}/users/batch/profiles".format(SkypeConnection.API_USER),
                                   json={"usernames": [id]}, auth=SkypeConnection.Auth.SkypeToken).json()
            contact = SkypeContact.fromRaw(self.skype, json[0])
            if contact.id not in self.contactIds:
                self.contactIds.append(contact.id)
            return self.merge(contact)
        except SkypeApiException as e:
            if len(e.args) >= 2 and getattr(e.args[1], "status_code", None) == 403:
                # Not a contact, so no permission to retrieve information.
                return None
            raise

def contact ( self , id ) : try : json = self . skype . conn ( "POST" , "{0}/users/batch/profiles" . format ( SkypeConnection . API_USER ) , json = { "usernames" : [ id ] } , auth = SkypeConnection . Auth . SkypeToken ) . json ( ) contact = SkypeContact . fromRaw ( self . skype , json [ 0 ] ) if contact . id not in self . contactIds : self . contactIds . append ( contact . id ) return self . merge ( contact ) except SkypeApiException as e : if len ( e . args ) >= 2 and getattr ( e . args [ 1 ] , "status_code" , None ) == 403 : return None raise

Retrieve all details for a specific contact including fields such as birthday and mood .


def user(self, id):
        """
        Retrieve public information about a user.

        Args:
            id (str): user identifier to lookup

        Returns:
            SkypeUser: resulting user object
        """
        json = self.skype.conn("POST", "{0}/batch/profiles".format(SkypeConnection.API_PROFILE),
                               auth=SkypeConnection.Auth.SkypeToken, json={"usernames": [id]}).json()
        if json and "status" not in json[0]:
            return self.merge(SkypeUser.fromRaw(self.skype, json[0]))
        else:
            return None

def user ( self , id ) : json = self . skype . conn ( "POST" , "{0}/batch/profiles" . format ( SkypeConnection . API_PROFILE ) , auth = SkypeConnection . Auth . SkypeToken , json = { "usernames" : [ id ] } ) . json ( ) if json and "status" not in json [ 0 ] : return self . merge ( SkypeUser . fromRaw ( self . skype , json [ 0 ] ) ) else : return None

Retrieve public information about a user .


def bots(self):
        """
        Retrieve a list of all known bots.

        Returns:
            SkypeBotUser list: resulting bot user objects
        """
        json = self.skype.conn("GET", "{0}/agents".format(SkypeConnection.API_BOT),
                               auth=SkypeConnection.Auth.SkypeToken).json().get("agentDescriptions", [])
        return [self.merge(SkypeBotUser.fromRaw(self.skype, raw)) for raw in json]

def bots ( self ) : json = self . skype . conn ( "GET" , "{0}/agents" . format ( SkypeConnection . API_BOT ) , auth = SkypeConnection . Auth . SkypeToken ) . json ( ) . get ( "agentDescriptions" , [ ] ) return [ self . merge ( SkypeBotUser . fromRaw ( self . skype , raw ) ) for raw in json ]


def bot(self, id):
        """
        Retrieve a single bot.

        Args:
            id (str): UUID or username of the bot

        Returns:
            SkypeBotUser: resulting bot user object
        """
        json = self.skype.conn("GET", "{0}/agents".format(SkypeConnection.API_BOT), params={"agentId": id},
                               auth=SkypeConnection.Auth.SkypeToken).json().get("agentDescriptions", [])
        return self.merge(SkypeBotUser.fromRaw(self.skype, json[0])) if json else None

def bot ( self , id ) : json = self . skype . conn ( "GET" , "{0}/agents" . format ( SkypeConnection . API_BOT ) , params = { "agentId" : id } , auth = SkypeConnection . Auth . SkypeToken ) . json ( ) . get ( "agentDescriptions" , [ ] ) return self . merge ( SkypeBotUser . fromRaw ( self . skype , json [ 0 ] ) ) if json else None


def search(self, query):
        """
        Search the Skype Directory for a user.

        Args:
            query (str): name to search for

        Returns:
            SkypeUser list: collection of possible results
        """
        results = self.skype.conn("GET", SkypeConnection.API_DIRECTORY,
                                  auth=SkypeConnection.Auth.SkypeToken,
                                  params={"searchstring": query, "requestId": "0"}).json().get("results", [])
        return [SkypeUser.fromRaw(self.skype, json.get("nodeProfileData", {})) for json in results]

def search ( self , query ) : results = self . skype . conn ( "GET" , SkypeConnection . API_DIRECTORY , auth = SkypeConnection . Auth . SkypeToken , params = { "searchstring" : query , "requestId" : "0" } ) . json ( ) . get ( "results" , [ ] ) return [ SkypeUser . fromRaw ( self . skype , json . get ( "nodeProfileData" , { } ) ) for json in results ]


def requests(self):
        """
        Retrieve any pending contact requests.

        Returns:
            :class:`SkypeRequest` list: collection of requests
        """
        requests = []
        for json in self.skype.conn("GET", "{0}/users/{1}/invites"
                                           .format(SkypeConnection.API_CONTACTS, self.skype.userId),
                                    auth=SkypeConnection.Auth.SkypeToken).json().get("invite_list", []):
            for invite in json.get("invites", []):
                # Copy user identifier to each invite message.
                invite["userId"] = SkypeUtils.noPrefix(json.get("mri"))
                requests.append(SkypeRequest.fromRaw(self.skype, invite))
        return requests

def requests ( self ) : requests = [ ] for json in self . skype . conn ( "GET" , "{0}/users/{1}/invites" . format ( SkypeConnection . API_CONTACTS , self . skype . userId ) , auth = SkypeConnection . Auth . SkypeToken ) . json ( ) . get ( "invite_list" , [ ] ) : for invite in json . get ( "invites" , [ ] ) : invite [ "userId" ] = SkypeUtils . noPrefix ( json . get ( "mri" ) ) requests . append ( SkypeRequest . fromRaw ( self . skype , invite ) ) return requests


def fromRaw(cls, skype=None, raw={}):
        """
        Create a new instance based on the raw properties of an API response.

        This can be overridden to automatically create subclass instances based on the raw content.

        Args:
            skype (Skype): parent Skype instance
            raw (dict): raw object, as provided by the API

        Returns:
            SkypeObj: the new class instance
        """
        return cls(skype, raw, **cls.rawToFields(raw))

def fromRaw ( cls , skype = None , raw = { } ) : return cls ( skype , raw , ** cls . rawToFields ( raw ) )

Create a new instance based on the raw properties of an API response .


def merge(self, other):
        """
        Copy properties from other into self, skipping ``None`` values.  Also merges the raw data.

        Args:
            other (SkypeObj): second object to copy fields from
        """
        for attr in self.attrs:
            if not getattr(other, attr, None) is None:
                setattr(self, attr, getattr(other, attr))
        if other.raw:
            if not self.raw:
                self.raw = {}
            self.raw.update(other.raw)

def merge ( self , other ) : for attr in self . attrs : if not getattr ( other , attr , None ) is None : setattr ( self , attr , getattr ( other , attr ) ) if other . raw : if not self . raw : self . raw = { } self . raw . update ( other . raw )

Copy properties from other into self skipping None values . Also merges the raw data .


def merge(self, obj):
        """
        Add a given object to the cache, or update an existing entry to include more fields.

        Args:
            obj (SkypeObj): object to add to the cache
        """
        if obj.id in self.cache:
            self.cache[obj.id].merge(obj)
        else:
            self.cache[obj.id] = obj
        return self.cache[obj.id]

def merge ( self , obj ) : if obj . id in self . cache : self . cache [ obj . id ] . merge ( obj ) else : self . cache [ obj . id ] = obj return self . cache [ obj . id ]

Add a given object to the cache or update an existing entry to include more fields .


def syncStateCall(self, method, url, params={}, **kwargs):
        """
        Follow and track sync state URLs provided by an API endpoint, in order to implicitly handle pagination.

        In the first call, ``url`` and ``params`` are used as-is.  If a ``syncState`` endpoint is provided in the
        response, subsequent calls go to the latest URL instead.

        Args:
            method (str): HTTP request method
            url (str): full URL to connect to
            params (dict): query parameters to include in the URL
            kwargs (dict): any extra parameters to pass to :meth:`__call__`
        """
        try:
            states = self.syncStates[(method, url)]
        except KeyError:
            states = self.syncStates[(method, url)] = []
        if states:
            # We have a state link, use it to replace the URL and query string.
            url = states[-1]
            params = {}
        resp = self(method, url, params=params, **kwargs)
        try:
            json = resp.json()
        except ValueError:
            # Don't do anything if not a JSON response.
            pass
        else:
            # If a state link exists in the response, store it for later.
            state = json.get("_metadata", {}).get("syncState")
            if state:
                states.append(state)
        return resp

def syncStateCall ( self , method , url , params = { } , ** kwargs ) : try : states = self . syncStates [ ( method , url ) ] except KeyError : states = self . syncStates [ ( method , url ) ] = [ ] if states : url = states [ - 1 ] params = { } resp = self ( method , url , params = params , ** kwargs ) try : json = resp . json ( ) except ValueError : pass else : state = json . get ( "_metadata" , { } ) . get ( "syncState" ) if state : states . append ( state ) return resp

Follow and track sync state URLs provided by an API endpoint in order to implicitly handle pagination .


def readToken(self):
        """
        Attempt to re-establish a connection using previously acquired tokens.

        If the Skype token is valid but the registration token is invalid, a new endpoint will be registered.

        Raises:
            .SkypeAuthException: if the token file cannot be used to authenticate
        """
        if not self.tokenFile:
            raise SkypeAuthException("No token file specified")
        try:
            with open(self.tokenFile, "r") as f:
                lines = f.read().splitlines()
        except OSError:
            raise SkypeAuthException("Token file doesn't exist or not readable")
        try:
            user, skypeToken, skypeExpiry, regToken, regExpiry, msgsHost = lines
            skypeExpiry = datetime.fromtimestamp(int(skypeExpiry))
            regExpiry = datetime.fromtimestamp(int(regExpiry))
        except ValueError:
            raise SkypeAuthException("Token file is malformed")
        if datetime.now() >= skypeExpiry:
            raise SkypeAuthException("Token file has expired")
        self.userId = user
        self.tokens["skype"] = skypeToken
        self.tokenExpiry["skype"] = skypeExpiry
        if datetime.now() < regExpiry:
            self.tokens["reg"] = regToken
            self.tokenExpiry["reg"] = regExpiry
            self.msgsHost = msgsHost
        else:
            self.getRegToken()

Attempt to re - establish a connection using previously acquired tokens .


def writeToken(self):
        """
        Store details of the current connection in the named file.

        This can be used by :meth:`readToken` to re-authenticate at a later time.
        """
        # Write token file privately.
        with os.fdopen(os.open(self.tokenFile, os.O_WRONLY | os.O_CREAT, 0o600), "w") as f:
            # When opening files via os, truncation must be done manually.
            f.truncate()
            f.write(self.userId + "\n")
            f.write(self.tokens["skype"] + "\n")
            f.write(str(int(time.mktime(self.tokenExpiry["skype"].timetuple()))) + "\n")
            f.write(self.tokens["reg"] + "\n")
            f.write(str(int(time.mktime(self.tokenExpiry["reg"].timetuple()))) + "\n")
            f.write(self.msgsHost + "\n")

def writeToken ( self ) : with os . fdopen ( os . open ( self . tokenFile , os . O_WRONLY | os . O_CREAT , 0o600 ) , "w" ) as f : f . truncate ( ) f . write ( self . userId + "\n" ) f . write ( self . tokens [ "skype" ] + "\n" ) f . write ( str ( int ( time . mktime ( self . tokenExpiry [ "skype" ] . timetuple ( ) ) ) ) + "\n" ) f . write ( self . tokens [ "reg" ] + "\n" ) f . write ( str ( int ( time . mktime ( self . tokenExpiry [ "reg" ] . timetuple ( ) ) ) ) + "\n" ) f . write ( self . msgsHost + "\n" )

Store details of the current connection in the named file .


def verifyToken(self, auth):
        """
        Ensure the authentication token for the given auth method is still valid.

        Args:
            auth (Auth): authentication type to check

        Raises:
            .SkypeAuthException: if Skype auth is required, and the current token has expired and can't be renewed
        """
        if auth in (self.Auth.SkypeToken, self.Auth.Authorize):
            if "skype" not in self.tokenExpiry or datetime.now() >= self.tokenExpiry["skype"]:
                if not hasattr(self, "getSkypeToken"):
                    raise SkypeAuthException("Skype token expired, and no password specified")
                self.getSkypeToken()
        elif auth == self.Auth.RegToken:
            if "reg" not in self.tokenExpiry or datetime.now() >= self.tokenExpiry["reg"]:
                self.getRegToken()

Ensure the authentication token for the given auth method is still valid .


def refreshSkypeToken(self):
        """
        Take the existing Skype token and refresh it, to extend the expiry time without other credentials.

        Raises:
            .SkypeAuthException: if the login request is rejected
            .SkypeApiException: if the login form can't be processed
        """
        self.tokens["skype"], self.tokenExpiry["skype"] = SkypeRefreshAuthProvider(self).auth(self.tokens["skype"])
        self.getRegToken()

def refreshSkypeToken ( self ) : self . tokens [ "skype" ] , self . tokenExpiry [ "skype" ] = SkypeRefreshAuthProvider ( self ) . auth ( self . tokens [ "skype" ] ) self . getRegToken ( )

Take the existing Skype token and refresh it to extend the expiry time without other credentials .


def getUserId(self):
        """
        Ask Skype for the authenticated user's identifier, and store it on the connection object.
        """
        self.userId = self("GET", "{0}/users/self/profile".format(self.API_USER),
                           auth=self.Auth.SkypeToken).json().get("username")

def getUserId ( self ) : self . userId = self ( "GET" , "{0}/users/self/profile" . format ( self . API_USER ) , auth = self . Auth . SkypeToken ) . json ( ) . get ( "username" )

Ask Skype for the authenticated user s identifier and store it on the connection object .


def getRegToken(self):
        """
        Acquire a new registration token.

        Once successful, all tokens and expiry times are written to the token file (if specified on initialisation).
        """
        self.verifyToken(self.Auth.SkypeToken)
        token, expiry, msgsHost, endpoint = SkypeRegistrationTokenProvider(self).auth(self.tokens["skype"])
        self.tokens["reg"] = token
        self.tokenExpiry["reg"] = expiry
        self.msgsHost = msgsHost
        if endpoint:
            endpoint.config()
            self.endpoints["main"] = endpoint
        self.syncEndpoints()
        if self.tokenFile:
            self.writeToken()

def getRegToken ( self ) : self . verifyToken ( self . Auth . SkypeToken ) token , expiry , msgsHost , endpoint = SkypeRegistrationTokenProvider ( self ) . auth ( self . tokens [ "skype" ] ) self . tokens [ "reg" ] = token self . tokenExpiry [ "reg" ] = expiry self . msgsHost = msgsHost if endpoint : endpoint . config ( ) self . endpoints [ "main" ] = endpoint self . syncEndpoints ( ) if self . tokenFile : self . writeToken ( )


def syncEndpoints(self):
        """
        Retrieve all current endpoints for the connected user.
        """
        self.endpoints["all"] = []
        for json in self("GET", "{0}/users/ME/presenceDocs/messagingService".format(self.msgsHost),
                         params={"view": "expanded"}, auth=self.Auth.RegToken).json().get("endpointPresenceDocs", []):
            id = json.get("link", "").split("/")[7]
            self.endpoints["all"].append(SkypeEndpoint(self, id))

Retrieve all current endpoints for the connected user .


def checkUser(self, user):
        """
        Query a username or email address to see if a corresponding Microsoft account exists.

        Args:
            user (str): username or email address of an account

        Returns:
            bool: whether the account exists
        """
        return not self.conn("POST", "{0}/GetCredentialType.srf".format(SkypeConnection.API_MSACC),
                             json={"username": user}).json().get("IfExistsResult")

def checkUser ( self , user ) : return not self . conn ( "POST" , "{0}/GetCredentialType.srf" . format ( SkypeConnection . API_MSACC ) , json = { "username" : user } ) . json ( ) . get ( "IfExistsResult" )

Query a username or email address to see if a corresponding Microsoft account exists .


def auth(self, token):
        """
        Take an existing Skype token and refresh it, to extend the expiry time without other credentials.

        Args:
            token (str): existing Skype token

        Returns:
            (str, datetime.datetime) tuple: Skype token, and associated expiry if known

        Raises:
            .SkypeAuthException: if the login request is rejected
            .SkypeApiException: if the login form can't be processed
        """
        t = self.sendToken(token)
        return self.getToken(t)

def auth ( self , token ) : t = self . sendToken ( token ) return self . getToken ( t )

Take an existing Skype token and refresh it to extend the expiry time without other credentials .


def auth(self, skypeToken):
        """
        Request a new registration token using a current Skype token.

        Args:
            skypeToken (str): existing Skype token

        Returns:
            (str, datetime.datetime, str, SkypeEndpoint) tuple: registration token, associated expiry if known,
                                                                resulting endpoint hostname, endpoint if provided

        Raises:
            .SkypeAuthException: if the login request is rejected
            .SkypeApiException: if the login form can't be processed
        """
        token = expiry = endpoint = None
        msgsHost = SkypeConnection.API_MSGSHOST
        while not token:
            secs = int(time.time())
            hash = self.getMac256Hash(str(secs))
            headers = {"LockAndKey": "appId=msmsgs@msnmsgr.com; time={0}; lockAndKeyResponse={1}".format(secs, hash),
                       "Authentication": "skypetoken=" + skypeToken, "BehaviorOverride": "redirectAs404"}
            endpointResp = self.conn("POST", "{0}/users/ME/endpoints".format(msgsHost), codes=(200, 201, 404),
                                     headers=headers, json={"endpointFeatures": "Agent"})
            regTokenHead = endpointResp.headers.get("Set-RegistrationToken")
            locHead = endpointResp.headers.get("Location")
            if locHead:
                locParts = re.search(r"(https://[^/]+/v1)/users/ME/endpoints(/(%7B[a-z0-9\-]+%7D))?", locHead).groups()
                if locParts[2]:
                    endpoint = SkypeEndpoint(self.conn, locParts[2].replace("%7B", "{").replace("%7D", "}"))
                if not locParts[0] == msgsHost:
                    # Skype is requiring the use of a different hostname.
                    msgsHost = locHead.rsplit("/", 4 if locParts[2] else 3)[0]
                    # Don't accept the token if present, we need to re-register first.
                    continue
            if regTokenHead:
                token = re.search(r"(registrationToken=[a-z0-9\+/=]+)", regTokenHead, re.I).group(1)
                regExpiry = re.search(r"expires=(\d+)", regTokenHead).group(1)
                expiry = datetime.fromtimestamp(int(regExpiry))
                regEndMatch = re.search(r"endpointId=({[a-z0-9\-]+})", regTokenHead)
                if regEndMatch:
                    endpoint = SkypeEndpoint(self.conn, regEndMatch.group(1))
            if not endpoint and endpointResp.status_code == 200 and endpointResp.json():
                # Use the most recent endpoint listed in the JSON response.
                endpoint = SkypeEndpoint(self.conn, endpointResp.json()[0]["id"])
        return token, expiry, msgsHost, endpoint

Request a new registration token using a current Skype token .


def config(self, name="skype"):
        """
        Configure this endpoint to allow setting presence.

        Args:
            name (str): display name for this endpoint
        """
        self.conn("PUT", "{0}/users/ME/endpoints/{1}/presenceDocs/messagingService"
                         .format(self.conn.msgsHost, self.id),
                  auth=SkypeConnection.Auth.RegToken,
                  json={"id": "messagingService",
                        "type": "EndpointPresenceDoc",
                        "selfLink": "uri",
                        "privateInfo": {"epname": name},
                        "publicInfo": {"capabilities": "",
                                       "type": 1,
                                       "skypeNameVersion": "skype.com",
                                       "nodeInfo": "xx",
                                       "version": "908/1.30.0.128"}})

Configure this endpoint to allow setting presence .


def ping(self, timeout=12):
        """
        Send a keep-alive request for the endpoint.

        Args:
            timeout (int): maximum amount of time for the endpoint to stay active
        """
        self.conn("POST", "{0}/users/ME/endpoints/{1}/active".format(self.conn.msgsHost, self.id),
                  auth=SkypeConnection.Auth.RegToken, json={"timeout": timeout})

def ping ( self , timeout = 12 ) : self . conn ( "POST" , "{0}/users/ME/endpoints/{1}/active" . format ( self . conn . msgsHost , self . id ) , auth = SkypeConnection . Auth . RegToken , json = { "timeout" : timeout } )

Send a keep - alive request for the endpoint .


def recent(self):
        """
        Retrieve a selection of conversations with the most recent activity, and store them in the cache.

        Each conversation is only retrieved once, so subsequent calls will retrieve older conversations.

        Returns:
            :class:`SkypeChat` list: collection of recent conversations
        """
        url = "{0}/users/ME/conversations".format(self.skype.conn.msgsHost)
        params = {"startTime": 0,
                  "view": "msnp24Equivalent",
                  "targetType": "Passport|Skype|Lync|Thread"}
        resp = self.skype.conn.syncStateCall("GET", url, params, auth=SkypeConnection.Auth.RegToken).json()
        chats = {}
        for json in resp.get("conversations", []):
            cls = SkypeSingleChat
            if "threadProperties" in json:
                info = self.skype.conn("GET", "{0}/threads/{1}".format(self.skype.conn.msgsHost, json.get("id")),
                                       auth=SkypeConnection.Auth.RegToken,
                                       params={"view": "msnp24Equivalent"}).json()
                json.update(info)
                cls = SkypeGroupChat
            chats[json.get("id")] = self.merge(cls.fromRaw(self.skype, json))
        return chats

def recent ( self ) : url = "{0}/users/ME/conversations" . format ( self . skype . conn . msgsHost ) params = { "startTime" : 0 , "view" : "msnp24Equivalent" , "targetType" : "Passport|Skype|Lync|Thread" } resp = self . skype . conn . syncStateCall ( "GET" , url , params , auth = SkypeConnection . Auth . RegToken ) . json ( ) chats = { } for json in resp . get ( "conversations" , [ ] ) : cls = SkypeSingleChat if "threadProperties" in json : info = self . skype . conn ( "GET" , "{0}/threads/{1}" . format ( self . skype . conn . msgsHost , json . get ( "id" ) ) , auth = SkypeConnection . Auth . RegToken , params = { "view" : "msnp24Equivalent" } ) . json ( ) json . update ( info ) cls = SkypeGroupChat chats [ json . get ( "id" ) ] = self . merge ( cls . fromRaw ( self . skype , json ) ) return chats

Retrieve a selection of conversations with the most recent activity and store them in the cache .


def chat(self, id):
        """
        Get a single conversation by identifier.

        Args:
            id (str): single or group chat identifier
        """
        json = self.skype.conn("GET", "{0}/users/ME/conversations/{1}".format(self.skype.conn.msgsHost, id),
                               auth=SkypeConnection.Auth.RegToken, params={"view": "msnp24Equivalent"}).json()
        cls = SkypeSingleChat
        if "threadProperties" in json:
            info = self.skype.conn("GET", "{0}/threads/{1}".format(self.skype.conn.msgsHost, json.get("id")),
                                   auth=SkypeConnection.Auth.RegToken, params={"view": "msnp24Equivalent"}).json()
            json.update(info)
            cls = SkypeGroupChat
        return self.merge(cls.fromRaw(self.skype, json))

def chat ( self , id ) : json = self . skype . conn ( "GET" , "{0}/users/ME/conversations/{1}" . format ( self . skype . conn . msgsHost , id ) , auth = SkypeConnection . Auth . RegToken , params = { "view" : "msnp24Equivalent" } ) . json ( ) cls = SkypeSingleChat if "threadProperties" in json : info = self . skype . conn ( "GET" , "{0}/threads/{1}" . format ( self . skype . conn . msgsHost , json . get ( "id" ) ) , auth = SkypeConnection . Auth . RegToken , params = { "view" : "msnp24Equivalent" } ) . json ( ) json . update ( info ) cls = SkypeGroupChat return self . merge ( cls . fromRaw ( self . skype , json ) )

Get a single conversation by identifier .


def create(self, members=(), admins=()):
        """
        Create a new group chat with the given users.

        The current user is automatically added to the conversation as an admin.  Any other admin identifiers must also
        be present in the member list.

        Args:
            members (str list): user identifiers to initially join the conversation
            admins (str list): user identifiers to gain admin privileges
        """
        memberObjs = [{"id": "8:{0}".format(self.skype.userId), "role": "Admin"}]
        for id in members:
            if id == self.skype.userId:
                continue
            memberObjs.append({"id": "8:{0}".format(id), "role": "Admin" if id in admins else "User"})
        resp = self.skype.conn("POST", "{0}/threads".format(self.skype.conn.msgsHost),
                               auth=SkypeConnection.Auth.RegToken, json={"members": memberObjs})
        return self.chat(resp.headers["Location"].rsplit("/", 1)[1])

def create ( self , members = ( ) , admins = ( ) ) : memberObjs = [ { "id" : "8:{0}" . format ( self . skype . userId ) , "role" : "Admin" } ] for id in members : if id == self . skype . userId : continue memberObjs . append ( { "id" : "8:{0}" . format ( id ) , "role" : "Admin" if id in admins else "User" } ) resp = self . skype . conn ( "POST" , "{0}/threads" . format ( self . skype . conn . msgsHost ) , auth = SkypeConnection . Auth . RegToken , json = { "members" : memberObjs } ) return self . chat ( resp . headers [ "Location" ] . rsplit ( "/" , 1 ) [ 1 ] )

Create a new group chat with the given users .


def userToId(url):
        """
        Extract the username from a contact URL.

        Matches addresses containing ``users/<user>`` or ``users/ME/contacts/<user>``.

        Args:
            url (str): Skype API URL

        Returns:
            str: extracted identifier
        """
        match = re.search(r"users(/ME/contacts)?/[0-9]+:([^/]+)", url)
        return match.group(2) if match else None

def userToId ( url ) : match = re . search ( r"users(/ME/contacts)?/[0-9]+:([^/]+)" , url ) return match . group ( 2 ) if match else None

Extract the username from a contact URL .


def chatToId(url):
        """
        Extract the conversation ID from a conversation URL.

        Matches addresses containing ``conversations/<chat>``.

        Args:
            url (str): Skype API URL

        Returns:
            str: extracted identifier
        """
        match = re.search(r"conversations/([0-9]+:[^/]+)", url)
        return match.group(1) if match else None

def chatToId ( url ) : match = re . search ( r"conversations/([0-9]+:[^/]+)" , url ) return match . group ( 1 ) if match else None

Extract the conversation ID from a conversation URL .


def exhaust(fn, transform=None, *args, **kwargs):
        """
        Repeatedly call a function, starting with init, until false-y, yielding each item in turn.

        The ``transform`` parameter can be used to map a collection to another format, for example iterating over a
        :class:`dict` by value rather than key.

        Use with state-synced functions to retrieve all results.

        Args:
            fn (method): function to call
            transform (method): secondary function to convert result into an iterable
            args (list): positional arguments to pass to ``fn``
            kwargs (dict): keyword arguments to pass to ``fn``

        Returns:
            generator: generator of objects produced from the method
        """
        while True:
            iterRes = fn(*args, **kwargs)
            if iterRes:
                for item in transform(iterRes) if transform else iterRes:
                    yield item
            else:
                break

def exhaust ( fn , transform = None , * args , ** kwargs ) : while True : iterRes = fn ( * args , ** kwargs ) if iterRes : for item in transform ( iterRes ) if transform else iterRes : yield item else : break

Repeatedly call a function starting with init until false - y yielding each item in turn .


def u(text, encoding='utf-8'):
    "Return unicode text, no matter what"

    if isinstance(text, six.binary_type):
        text = text.decode(encoding)

    # it's already unicode
    text = text.replace('\r\n', '\n')
    return text

def
u
(
text
,
encoding
=
'utf-8'
)
:
"Return unicode text, no matter what"
if
isinstance
(
text
,
six
.
binary_type
)
:
text
=
text
.
decode
(
encoding
)
text
=
text
.
replace
(
'\r\n'
,
'\n'
)
return
text


def detect_format(text, handlers):
    """
    Figure out which handler to use, based on metadata.
    Returns a handler instance or None.

    ``text`` should be unicode text about to be parsed.

    ``handlers`` is a dictionary where keys are opening delimiters
    and values are handler instances.
    """
    for pattern, handler in handlers.items():
        if pattern.match(text):
            return handler

    # nothing matched, give nothing back
    return None

def detect_format ( text , handlers ) : for pattern , handler in handlers . items ( ) : if pattern . match ( text ) : return handler return None

Figure out which handler to use based on metadata . Returns a handler instance or None .


def parse(text, encoding='utf-8', handler=None, **defaults):
    """
    Parse text with frontmatter, return metadata and content.
    Pass in optional metadata defaults as keyword args.

    If frontmatter is not found, returns an empty metadata dictionary
    (or defaults) and original text content.

    ::

        >>> with open('tests/hello-world.markdown') as f:
        ...     metadata, content = frontmatter.parse(f.read())
        >>> print(metadata['title'])
        Hello, world!

    """
    # ensure unicode first
    text = u(text, encoding).strip()

    # metadata starts with defaults
    metadata = defaults.copy()

    # this will only run if a handler hasn't been set higher up
    handler = handler or detect_format(text, handlers)
    if handler is None:
        return metadata, text

    # split on the delimiters
    try:
        fm, content = handler.split(text)
    except ValueError:
        # if we can't split, bail
        return metadata, text

    # parse, now that we have frontmatter
    fm = handler.load(fm)
    if isinstance(fm, dict):
        metadata.update(fm)

    return metadata, content.strip()

def parse ( text , encoding = 'utf-8' , handler = None , ** defaults ) : text = u ( text , encoding ) . strip ( ) metadata = defaults . copy ( ) handler = handler or detect_format ( text , handlers ) if handler is None : return metadata , text try : fm , content = handler . split ( text ) except ValueError : return metadata , text fm = handler . load ( fm ) if isinstance ( fm , dict ) : metadata . update ( fm ) return metadata , content . strip ( )

Parse text with frontmatter return metadata and content . Pass in optional metadata defaults as keyword args .


def to_dict(self):
        "Post as a dict, for serializing"
        d = self.metadata.copy()
        d['content'] = self.content
        return d

def
to_dict
(
self
)
:
"Post as a dict, for serializing"
d
=
self
.
metadata
.
copy
(
)
d
[
'content'
]
=
self
.
content
return
d


def load(self, fm, **kwargs):
        """
        Parse YAML front matter. This uses yaml.SafeLoader by default.
        """
        kwargs.setdefault('Loader', SafeLoader)
        return yaml.load(fm, **kwargs)

def load ( self , fm , ** kwargs ) : kwargs . setdefault ( 'Loader' , SafeLoader ) return yaml . load ( fm , ** kwargs )

Parse YAML front matter . This uses yaml . SafeLoader by default .


def export(self, metadata, **kwargs):
        """
        Export metadata as YAML. This uses yaml.SafeDumper by default.
        """
        kwargs.setdefault('Dumper', SafeDumper)
        kwargs.setdefault('default_flow_style', False)
        kwargs.setdefault('allow_unicode', True)

        metadata = yaml.dump(metadata, **kwargs).strip()
        return u(metadata)

def export ( self , metadata , ** kwargs ) : kwargs . setdefault ( 'Dumper' , SafeDumper ) kwargs . setdefault ( 'default_flow_style' , False ) kwargs . setdefault ( 'allow_unicode' , True ) metadata = yaml . dump ( metadata , ** kwargs ) . strip ( ) return u ( metadata )

Export metadata as YAML . This uses yaml . SafeDumper by default .


def export(self, metadata, **kwargs):
        "Turn metadata into JSON"
        kwargs.setdefault('indent', 4)
        metadata = json.dumps(metadata, **kwargs)
        return u(metadata)

def
export
(
self
,
metadata
,
**
kwargs
)
:
"Turn metadata into JSON"
kwargs
.
setdefault
(
'indent'
,
4
)
metadata
=
json
.
dumps
(
metadata
,
**
kwargs
)
return
u
(
metadata
)


def _match(self):
        """Return the match object for the current list."""
        cache_match, cache_string = self._match_cache
        string = self.string
        if cache_string == string:
            return cache_match
        cache_match = fullmatch(
            LIST_PATTERN_FORMAT.replace(b'{pattern}', self.pattern.encode()),
            self._shadow,
            MULTILINE,
        )
        self._match_cache = cache_match, string
        return cache_match

def _match ( self ) : cache_match , cache_string = self . _match_cache string = self . string if cache_string == string : return cache_match cache_match = fullmatch ( LIST_PATTERN_FORMAT . replace ( b'{pattern}' , self . pattern . encode ( ) ) , self . _shadow , MULTILINE , ) self . _match_cache = cache_match , string return cache_match

Return the match object for the current list .


def items(self) -> List[str]:
        """Return items as a list of strings.

        Don't include sub-items and the start pattern.
        """
        items = []  # type: List[str]
        append = items.append
        string = self.string
        match = self._match
        ms = match.start()
        for s, e in match.spans('item'):
            append(string[s - ms:e - ms])
        return items

def items ( self ) -> List [ str ] : items = [ ] append = items . append string = self . string match = self . _match ms = match . start ( ) for s , e in match . spans ( 'item' ) : append ( string [ s - ms : e - ms ] ) return items


def sublists(
        self, i: int = None, pattern: str = None
    ) -> List['WikiList']:
        """Return the Lists inside the item with the given index.

        :param i: The index if the item which its sub-lists are desired.
            The performance is likely to be better if `i` is None.

        :param pattern: The starting symbol for the desired sub-lists.
            The `pattern` of the current list will be automatically added
            as prefix.
            Although this parameter is optional, but specifying it can improve
            the performance.
        """
        patterns = (r'\#', r'\*', '[:;]') if pattern is None \
            else (pattern,)  # type: Tuple[str, ...]
        self_pattern = self.pattern
        lists = self.lists
        sublists = []  # type: List['WikiList']
        sublists_append = sublists.append
        if i is None:
            # Any sublist is acceptable
            for pattern in patterns:
                for lst in lists(self_pattern + pattern):
                    sublists_append(lst)
            return sublists
        # Only return sub-lists that are within the given item
        match = self._match
        fullitem_spans = match.spans('fullitem')
        ss = self._span[0]
        ms = match.start()
        s, e = fullitem_spans[i]
        e -= ms - ss
        s -= ms - ss
        for pattern in patterns:
            for lst in lists(self_pattern + pattern):
                # noinspection PyProtectedMember
                ls, le = lst._span
                if s < ls and le <= e:
                    sublists_append(lst)
        return sublists

def sublists ( self , i : int = None , pattern : str = None ) -> List [ 'WikiList' ] : patterns = ( r'\#' , r'\*' , '[:;]' ) if pattern is None else ( pattern , ) self_pattern = self . pattern lists = self . lists sublists = [ ] sublists_append = sublists . append if i is None : for pattern in patterns : for lst in lists ( self_pattern + pattern ) : sublists_append ( lst ) return sublists match = self . _match fullitem_spans = match . spans ( 'fullitem' ) ss = self . _span [ 0 ] ms = match . start ( ) s , e = fullitem_spans [ i ] e -= ms - ss s -= ms - ss for pattern in patterns : for lst in lists ( self_pattern + pattern ) : ls , le = lst . _span if s < ls and le <= e : sublists_append ( lst ) return sublists

Return the Lists inside the item with the given index .


def convert(self, newstart: str) -> None:
        """Convert to another list type by replacing starting pattern."""
        match = self._match
        ms = match.start()
        for s, e in reversed(match.spans('pattern')):
            self[s - ms:e - ms] = newstart
        self.pattern = escape(newstart)

def convert ( self , newstart : str ) -> None : match = self . _match ms = match . start ( ) for s , e in reversed ( match . spans ( 'pattern' ) ) : self [ s - ms : e - ms ] = newstart self . pattern = escape ( newstart )

Convert to another list type by replacing starting pattern .


def arguments(self) -> List[Argument]:
        """Parse template content. Create self.name and self.arguments."""
        shadow = self._shadow
        split_spans = self._args_matcher(shadow).spans('arg')
        if not split_spans:
            return []
        arguments = []
        arguments_append = arguments.append
        type_to_spans = self._type_to_spans
        ss, se = span = self._span
        type_ = id(span)
        lststr = self._lststr
        string = lststr[0]
        arg_spans = type_to_spans.setdefault(type_, [])
        span_tuple_to_span_get = {(s[0], s[1]): s for s in arg_spans}.get
        for arg_self_start, arg_self_end in split_spans:
            s, e = arg_span = [ss + arg_self_start, ss + arg_self_end]
            old_span = span_tuple_to_span_get((s, e))
            if old_span is None:
                insort(arg_spans, arg_span)
            else:
                arg_span = old_span
            arg = Argument(lststr, type_to_spans, arg_span, type_)
            arg._shadow_cache = (
                string[s:e], shadow[arg_self_start:arg_self_end])
            arguments_append(arg)
        return arguments

def arguments ( self ) -> List [ Argument ] : shadow = self . _shadow split_spans = self . _args_matcher ( shadow ) . spans ( 'arg' ) if not split_spans : return [ ] arguments = [ ] arguments_append = arguments . append type_to_spans = self . _type_to_spans ss , se = span = self . _span type_ = id ( span ) lststr = self . _lststr string = lststr [ 0 ] arg_spans = type_to_spans . setdefault ( type_ , [ ] ) span_tuple_to_span_get = { ( s [ 0 ] , s [ 1 ] ) : s for s in arg_spans } . get for arg_self_start , arg_self_end in split_spans : s , e = arg_span = [ ss + arg_self_start , ss + arg_self_end ] old_span = span_tuple_to_span_get ( ( s , e ) ) if old_span is None : insort ( arg_spans , arg_span ) else : arg_span = old_span arg = Argument ( lststr , type_to_spans , arg_span , type_ ) arg . _shadow_cache = ( string [ s : e ] , shadow [ arg_self_start : arg_self_end ] ) arguments_append ( arg ) return arguments

Parse template content . Create self . name and self . arguments .


def lists(self, pattern: str = None) -> List[WikiList]:
        """Return the lists in all arguments.

        For performance reasons it is usually preferred to get a specific
        Argument and use the `lists` method of that argument instead.
        """
        return [
            lst for arg in self.arguments for lst in arg.lists(pattern) if lst]

def lists ( self , pattern : str = None ) -> List [ WikiList ] : return [ lst for arg in self . arguments for lst in arg . lists ( pattern ) if lst ]


def _plant_trie(strings: _List[str]) -> dict:
    """Create a Trie out of a list of words and return an atomic regex pattern.

    The corresponding Regex should match much faster than a simple Regex union.
    """
    # plant the trie
    trie = {}
    for string in strings:
        d = trie
        for char in string:
            d[char] = char in d and d[char] or {}
            d = d[char]
        d[''] = None  # EOS
    return trie

def _plant_trie ( strings : _List [ str ] ) -> dict : trie = { } for string in strings : d = trie for char in string : d [ char ] = char in d and d [ char ] or { } d = d [ char ] d [ '' ] = None return trie

Create a Trie out of a list of words and return an atomic regex pattern .


def _pattern(trie: dict) -> str:
    """Convert a trie to a regex pattern."""
    if '' in trie:
        if len(trie) == 1:
            return ''
        optional = True
        del trie['']
    else:
        optional = False

    subpattern_to_chars = _defaultdict(list)

    for char, sub_trie in trie.items():
        subpattern = _pattern(sub_trie)
        subpattern_to_chars[subpattern].append(char)

    alts = []
    for subpattern, chars in subpattern_to_chars.items():
        if len(chars) == 1:
            alts.append(chars[0] + subpattern)
        else:
            chars.sort(reverse=True)
            alts.append('[' + ''.join(chars) + ']' + subpattern)

    if len(alts) == 1:
        result = alts[0]
        if optional:
            if len(result) == 1:
                result += '?+'
            else:  # more than one character in alts[0]
                result = '(?:' + result + ')?+'
    else:
        alts.sort(reverse=True)
        result = '(?>' + '|'.join(alts) + ')'
        if optional:
            result += '?+'
    return result

def _pattern ( trie : dict ) -> str : if '' in trie : if len ( trie ) == 1 : return '' optional = True del trie [ '' ] else : optional = False subpattern_to_chars = _defaultdict ( list ) for char , sub_trie in trie . items ( ) : subpattern = _pattern ( sub_trie ) subpattern_to_chars [ subpattern ] . append ( char ) alts = [ ] for subpattern , chars in subpattern_to_chars . items ( ) : if len ( chars ) == 1 : alts . append ( chars [ 0 ] + subpattern ) else : chars . sort ( reverse = True ) alts . append ( '[' + '' . join ( chars ) + ']' + subpattern ) if len ( alts ) == 1 : result = alts [ 0 ] if optional : if len ( result ) == 1 : result += '?+' else : result = '(?:' + result + ')?+' else : alts . sort ( reverse = True ) result = '(?>' + '|' . join ( alts ) + ')' if optional : result += '?+' return result


def _check_index(self, key: Union[slice, int]) -> (int, int):
        """Return adjusted start and stop index as tuple.

        Used in  __setitem__ and __delitem__.
        """
        ss, se = self._span
        if isinstance(key, int):
            if key < 0:
                key += se - ss
                if key < 0:
                    raise IndexError('index out of range')
            elif key >= se - ss:
                raise IndexError('index out of range')
            start = ss + key
            return start, start + 1
        # isinstance(key, slice)
        if key.step is not None:
            raise NotImplementedError(
                'step is not implemented for string setter.')
        start, stop = key.start or 0, key.stop
        if start < 0:
            start += se - ss
            if start < 0:
                raise IndexError('start index out of range')
        if stop is None:
            stop = se - ss
        elif stop < 0:
            stop += se - ss
        if start > stop:
            raise IndexError(
                'stop index out of range or start is after the stop')
        return start + ss, stop + ss

Return adjusted start and stop index as tuple .


def insert(self, index: int, string: str) -> None:
        """Insert the given string before the specified index.

        This method has the same effect as ``self[index:index] = string``;
        it only avoids some condition checks as it rules out the possibility
        of the key being an slice, or the need to shrink any of the sub-spans.

        If parse is False, don't parse the inserted string.
        """
        ss, se = self._span
        lststr = self._lststr
        lststr0 = lststr[0]
        if index < 0:
            index += se - ss
            if index < 0:
                index = 0
        elif index > se - ss:  # Note that it is not >=. Index can be new.
            index = se - ss
        index += ss
        # Update lststr
        lststr[0] = lststr0[:index] + string + lststr0[index:]
        string_len = len(string)
        # Update spans
        self._insert_update(
            index=index,
            length=string_len)
        # Remember newly added spans by the string.
        type_to_spans = self._type_to_spans
        for type_, spans in parse_to_spans(
            bytearray(string, 'ascii', 'replace')
        ).items():
            for s, e in spans:
                insort(type_to_spans[type_], [index + s, index + e])

def insert ( self , index : int , string : str ) -> None : ss , se = self . _span lststr = self . _lststr lststr0 = lststr [ 0 ] if index < 0 : index += se - ss if index < 0 : index = 0 elif index > se - ss : index = se - ss index += ss lststr [ 0 ] = lststr0 [ : index ] + string + lststr0 [ index : ] string_len = len ( string ) self . _insert_update ( index = index , length = string_len ) type_to_spans = self . _type_to_spans for type_ , spans in parse_to_spans ( bytearray ( string , 'ascii' , 'replace' ) ) . items ( ) : for s , e in spans : insort ( type_to_spans [ type_ ] , [ index + s , index + e ] )

Insert the given string before the specified index .


def _atomic_partition(self, char: int) -> Tuple[str, str, str]:
        """Partition self.string where `char`'s not in atomic sub-spans."""
        s, e = self._span
        index = self._shadow.find(char)
        if index == -1:
            return self._lststr[0][s:e], '', ''
        lststr0 = self._lststr[0]
        return lststr0[s:s + index], chr(char), lststr0[s + index + 1:e]

def _atomic_partition ( self , char : int ) -> Tuple [ str , str , str ] : s , e = self . _span index = self . _shadow . find ( char ) if index == - 1 : return self . _lststr [ 0 ] [ s : e ] , '' , '' lststr0 = self . _lststr [ 0 ] return lststr0 [ s : s + index ] , chr ( char ) , lststr0 [ s + index + 1 : e ]

Partition self . string where char s not in atomic sub - spans .


def _subspans(self, type_: str) -> List[List[int]]:
        """Return all the sub-span including self._span."""
        return self._type_to_spans[type_]

def _subspans ( self , type_ : str ) -> List [ List [ int ] ] : return self . _type_to_spans [ type_ ]

Return all the sub - span including self . _span .


def _shrink_update(self, rmstart: int, rmstop: int) -> None:
        """Update self._type_to_spans according to the removed span.

        Warning: If an operation involves both _shrink_update and
        _insert_update, you might wanna consider doing the
        _insert_update before the _shrink_update as this function
        can cause data loss in self._type_to_spans.
        """
        # Note: The following algorithm won't work correctly if spans
        # are not sorted.
        # Note: No span should be removed from _type_to_spans.
        for spans in self._type_to_spans.values():
            i = len(spans) - 1
            while i >= 0:
                s, e = span = spans[i]
                if rmstop <= s:
                    # rmstart <= rmstop <= s <= e
                    rmlength = rmstop - rmstart
                    span[:] = s - rmlength, e - rmlength
                    i -= 1
                    continue
                break
            else:
                continue
            while True:
                if rmstart <= s:
                    if rmstop < e:
                        # rmstart < s <= rmstop < e
                        span[:] = rmstart, e + rmstart - rmstop
                        i -= 1
                        if i < 0:
                            break
                        s, e = span = spans[i]
                        continue
                    # rmstart <= s <= e < rmstop
                    spans.pop(i)[:] = -1, -1
                    i -= 1
                    if i < 0:
                        break
                    s, e = span = spans[i]
                    continue
                break
            while i >= 0:
                if e <= rmstart:
                    # s <= e <= rmstart <= rmstop
                    i -= 1
                    if i < 0:
                        break
                    s, e = span = spans[i]
                    continue
                # s <= rmstart <= rmstop <= e
                span[1] -= rmstop - rmstart
                i -= 1
                if i < 0:
                    break
                s, e = span = spans[i]
                continue

def _shrink_update ( self , rmstart : int , rmstop : int ) -> None : for spans in self . _type_to_spans . values ( ) : i = len ( spans ) - 1 while i >= 0 : s , e = span = spans [ i ] if rmstop <= s : rmlength = rmstop - rmstart span [ : ] = s - rmlength , e - rmlength i -= 1 continue break else : continue while True : if rmstart <= s : if rmstop < e : span [ : ] = rmstart , e + rmstart - rmstop i -= 1 if i < 0 : break s , e = span = spans [ i ] continue spans . pop ( i ) [ : ] = - 1 , - 1 i -= 1 if i < 0 : break s , e = span = spans [ i ] continue break while i >= 0 : if e <= rmstart : i -= 1 if i < 0 : break s , e = span = spans [ i ] continue span [ 1 ] -= rmstop - rmstart i -= 1 if i < 0 : break s , e = span = spans [ i ] continue

Update self . _type_to_spans according to the removed span .


def _insert_update(self, index: int, length: int) -> None:
        """Update self._type_to_spans according to the added length."""
        ss, se = self._span
        for spans in self._type_to_spans.values():
            for span in spans:
                if index < span[1] or span[1] == index == se:
                    span[1] += length
                    # index is before s, or at s but not on self_span
                    if index < span[0] or span[0] == index != ss:
                        span[0] += length

def _insert_update ( self , index : int , length : int ) -> None : ss , se = self . _span for spans in self . _type_to_spans . values ( ) : for span in spans : if index < span [ 1 ] or span [ 1 ] == index == se : span [ 1 ] += length if index < span [ 0 ] or span [ 0 ] == index != ss : span [ 0 ] += length

Update self . _type_to_spans according to the added length .


def nesting_level(self) -> int:
        """Return the nesting level of self.

        The minimum nesting_level is 0. Being part of any Template or
        ParserFunction increases the level by one.
        """
        ss, se = self._span
        level = 0
        type_to_spans = self._type_to_spans
        for type_ in ('Template', 'ParserFunction'):
            spans = type_to_spans[type_]
            for s, e in spans[:bisect(spans, [ss + 1])]:
                if se <= e:
                    level += 1
        return level

def nesting_level ( self ) -> int : ss , se = self . _span level = 0 type_to_spans = self . _type_to_spans for type_ in ( 'Template' , 'ParserFunction' ) : spans = type_to_spans [ type_ ] for s , e in spans [ : bisect ( spans , [ ss + 1 ] ) ] : if se <= e : level += 1 return level


def _shadow(self) -> bytearray:
        """Return a copy of self.string with specific sub-spans replaced.

        Comments blocks are replaced by spaces. Other sub-spans are replaced
        by underscores.

        The replaced sub-spans are: (
            'Template', 'WikiLink', 'ParserFunction', 'ExtensionTag',
            'Comment',
        )

        This function is called upon extracting tables or extracting the data
        inside them.
        """
        ss, se = self._span
        string = self._lststr[0][ss:se]
        cached_string, shadow = getattr(
            self, '_shadow_cache', (None, None))
        if cached_string == string:
            return shadow
        # In the old method the existing spans were used to create the shadow.
        # But it was slow because there can be thousands of spans and iterating
        # over them to find the relevant sub-spans could take a significant
        # amount of time. The new method tries to parse the self.string which
        # is usually much more faster because there are usually far less
        # sub-spans for individual objects.
        shadow = bytearray(string, 'ascii', 'replace')
        if self._type in SPAN_PARSER_TYPES:
            head = shadow[:2]
            tail = shadow[-2:]
            shadow[:2] = shadow[-2:] = b'__'
            parse_to_spans(shadow)
            shadow[:2] = head
            shadow[-2:] = tail
        else:
            parse_to_spans(shadow)
        self._shadow_cache = string, shadow
        return shadow

def _shadow ( self ) -> bytearray : ss , se = self . _span string = self . _lststr [ 0 ] [ ss : se ] cached_string , shadow = getattr ( self , '_shadow_cache' , ( None , None ) ) if cached_string == string : return shadow shadow = bytearray ( string , 'ascii' , 'replace' ) if self . _type in SPAN_PARSER_TYPES : head = shadow [ : 2 ] tail = shadow [ - 2 : ] shadow [ : 2 ] = shadow [ - 2 : ] = b'__' parse_to_spans ( shadow ) shadow [ : 2 ] = head shadow [ - 2 : ] = tail else : parse_to_spans ( shadow ) self . _shadow_cache = string , shadow return shadow

Return a copy of self . string with specific sub - spans replaced .


def _ext_link_shadow(self):
        """Replace the invalid chars of SPAN_PARSER_TYPES with b'_'.

        For comments, all characters are replaced, but for ('Template',
        'ParserFunction', 'Parameter') only invalid characters are replaced.
        """
        ss, se = self._span
        string = self._lststr[0][ss:se]
        byte_array = bytearray(string, 'ascii', 'replace')
        subspans = self._subspans
        for type_ in 'Template', 'ParserFunction', 'Parameter':
            for s, e in subspans(type_):
                byte_array[s:e] = b'  ' + INVALID_EXT_CHARS_SUB(
                    b' ', byte_array[s + 2:e - 2]) + b'  '
        for s, e in subspans('Comment'):
            byte_array[s:e] = (e - s) * b'_'
        return byte_array

def
_ext_link_shadow
(
self
)
:
ss
,
se
=
self
.
_span
string
=
self
.
_lststr
[
0
]
[
ss
:
se
]
byte_array
=
bytearray
(
string
,
'ascii'
,
'replace'
)
subspans
=
self
.
_subspans
for
type_
in
'Template'
,
'ParserFunction'
,
'Parameter'
:
for
s
,
e
in
subspans
(
type_
)
:
byte_array
[
s
:
e
]
=
b'  '
+
INVALID_EXT_CHARS_SUB
(
b' '
,
byte_array
[
s
+
2
:
e
2
]
)
+
b'  '
for
s
,
e
in
subspans
(
'Comment'
)
:
byte_array
[
s
:
e
]
=
(
e
s
)
*
b'_'
return
byte_array

Replace the invalid chars of SPAN_PARSER_TYPES with b _ .


def _pp_type_to_spans(self) -> Dict[str, List[List[int]]]:
        """Create the arguments for the parse function used in pformat method.

        Only return sub-spans and change the them to fit the new scope, i.e
        self.string.
        """
        ss, se = self._span
        if ss == 0 and se == len(self._lststr[0]):
            return deepcopy(self._type_to_spans)
        return {
            type_: [
                [s - ss, e - ss] for s, e in spans[bisect(spans, [ss]):]
                if e <= se
            ] for type_, spans in self._type_to_spans.items()}

def _pp_type_to_spans ( self ) -> Dict [ str , List [ List [ int ] ] ] : ss , se = self . _span if ss == 0 and se == len ( self . _lststr [ 0 ] ) : return deepcopy ( self . _type_to_spans ) return { type_ : [ [ s - ss , e - ss ] for s , e in spans [ bisect ( spans , [ ss ] ) : ] if e <= se ] for type_ , spans in self . _type_to_spans . items ( ) }

Create the arguments for the parse function used in pformat method .


def pprint(self, indent: str = '    ', remove_comments=False):
        """Deprecated, use self.pformat instead."""
        warn(
            'pprint method is deprecated, use pformat instead.',
            DeprecationWarning,
        )
        return self.pformat(indent, remove_comments)


def parameters(self) -> List['Parameter']:
        """Return a list of parameter objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            Parameter(_lststr, _type_to_spans, span, 'Parameter')
            for span in self._subspans('Parameter')]

def parameters ( self ) -> List [ 'Parameter' ] : _lststr = self . _lststr _type_to_spans = self . _type_to_spans return [ Parameter ( _lststr , _type_to_spans , span , 'Parameter' ) for span in self . _subspans ( 'Parameter' ) ]


def parser_functions(self) -> List['ParserFunction']:
        """Return a list of parser function objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            ParserFunction(_lststr, _type_to_spans, span, 'ParserFunction')
            for span in self._subspans('ParserFunction')]

def parser_functions ( self ) -> List [ 'ParserFunction' ] : _lststr = self . _lststr _type_to_spans = self . _type_to_spans return [ ParserFunction ( _lststr , _type_to_spans , span , 'ParserFunction' ) for span in self . _subspans ( 'ParserFunction' ) ]

Return a list of parser function objects .


def templates(self) -> List['Template']:
        """Return a list of templates as template objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            Template(_lststr, _type_to_spans, span, 'Template')
            for span in self._subspans('Template')]

def templates ( self ) -> List [ 'Template' ] : _lststr = self . _lststr _type_to_spans = self . _type_to_spans return [ Template ( _lststr , _type_to_spans , span , 'Template' ) for span in self . _subspans ( 'Template' ) ]

Return a list of templates as template objects .


def wikilinks(self) -> List['WikiLink']:
        """Return a list of wikilink objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            WikiLink(_lststr, _type_to_spans, span, 'WikiLink')
            for span in self._subspans('WikiLink')]

def wikilinks ( self ) -> List [ 'WikiLink' ] : _lststr = self . _lststr _type_to_spans = self . _type_to_spans return [ WikiLink ( _lststr , _type_to_spans , span , 'WikiLink' ) for span in self . _subspans ( 'WikiLink' ) ]


def comments(self) -> List['Comment']:
        """Return a list of comment objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            Comment(_lststr, _type_to_spans, span, 'Comment')
            for span in self._subspans('Comment')]

def comments ( self ) -> List [ 'Comment' ] : _lststr = self . _lststr _type_to_spans = self . _type_to_spans return [ Comment ( _lststr , _type_to_spans , span , 'Comment' ) for span in self . _subspans ( 'Comment' ) ]


def external_links(self) -> List['ExternalLink']:
        """Return a list of found external link objects.

        Note:
            Templates adjacent to external links are considered part of the
            link. In reality, this depends on the contents of the template:

            >>> WikiText(
            ...    'http://example.com{{dead link}}'
            ...).external_links[0].url
            'http://example.com{{dead link}}'

            >>> WikiText(
            ...    '[http://example.com{{space template}} text]'
            ...).external_links[0].url
            'http://example.com{{space template}}'
        """
        external_links = []  # type: List['ExternalLink']
        external_links_append = external_links.append
        type_to_spans = self._type_to_spans
        lststr = self._lststr
        ss, se = self._span
        spans = type_to_spans.setdefault('ExternalLink', [])
        if not spans:
            # All the added spans will be new.
            spans_append = spans.append
            for m in EXTERNAL_LINK_FINDITER(self._ext_link_shadow):
                s, e = m.span()
                span = [ss + s, ss + e]
                spans_append(span)
                external_links_append(
                    ExternalLink(lststr, type_to_spans, span, 'ExternalLink'))
            return external_links
        # There are already some ExternalLink spans. Use the already existing
        # ones when the detected span is one of those.
        span_tuple_to_span_get = {(s[0], s[1]): s for s in spans}.get
        for m in EXTERNAL_LINK_FINDITER(self._ext_link_shadow):
            s, e = m.span()
            span = s, e = [s + ss, e + ss]
            old_span = span_tuple_to_span_get((s, e))
            if old_span is None:
                insort(spans, span)
            else:
                span = old_span
            external_links_append(
                ExternalLink(lststr, type_to_spans, span, 'ExternalLink'))
        return external_links

def external_links ( self ) -> List [ 'ExternalLink' ] : external_links = [ ] external_links_append = external_links . append type_to_spans = self . _type_to_spans lststr = self . _lststr ss , se = self . _span spans = type_to_spans . setdefault ( 'ExternalLink' , [ ] ) if not spans : spans_append = spans . append for m in EXTERNAL_LINK_FINDITER ( self . _ext_link_shadow ) : s , e = m . span ( ) span = [ ss + s , ss + e ] spans_append ( span ) external_links_append ( ExternalLink ( lststr , type_to_spans , span , 'ExternalLink' ) ) return external_links span_tuple_to_span_get = { ( s [ 0 ] , s [ 1 ] ) : s for s in spans } . get for m in EXTERNAL_LINK_FINDITER ( self . _ext_link_shadow ) : s , e = m . span ( ) span = s , e = [ s + ss , e + ss ] old_span = span_tuple_to_span_get ( ( s , e ) ) if old_span is None : insort ( spans , span ) else : span = old_span external_links_append ( ExternalLink ( lststr , type_to_spans , span , 'ExternalLink' ) ) return external_links

Return a list of found external link objects .


def sections(self) -> List['Section']:
        """Return a list of section in current wikitext.

        The first section will always be the lead section, even if it is an
        empty string.
        """
        sections = []  # type: List['Section']
        sections_append = sections.append
        type_to_spans = self._type_to_spans
        lststr = self._lststr
        ss, se = _span = self._span
        type_spans = type_to_spans.setdefault('Section', [])
        full_match = SECTIONS_FULLMATCH(self._shadow)
        section_spans = full_match.spans('section')
        levels = [len(eq) for eq in full_match.captures('equals')]
        if not type_spans:
            # All spans are new
            spans_append = type_spans.append
            for current_index, (current_level, (s, e)) in enumerate(
                zip(levels, section_spans), 1
            ):
                # Add text of the current_section to any parent section.
                # Note that section 0 is not a parent for any subsection.
                for section_index, section_level in enumerate(
                    levels[current_index:], current_index
                ):
                    if current_level and section_level > current_level:
                        e = section_spans[section_index][1]
                    else:
                        break
                span = [ss + s, ss + e]
                spans_append(span)
                sections_append(
                    Section(lststr, type_to_spans, span, 'Section'))
            return sections
        # There are already some spans. Instead of appending new spans
        # use them when the detected span already exists.
        span_tuple_to_span = {(s[0], s[1]): s for s in type_spans}.get
        for current_index, (current_level, (s, e)) in enumerate(
            zip(levels, section_spans), 1
        ):
            # Add text of the current_section to any parent section.
            # Note that section 0 is not a parent for any subsection.
            for section_index, section_level in enumerate(
                levels[current_index:], current_index
            ):
                if current_level and section_level > current_level:
                    e = section_spans[section_index][1]
                else:
                    break
            s, e = ss + s, ss + e
            old_span = span_tuple_to_span((s, e))
            if old_span is None:
                span = [s, e]
                insort(type_spans, span)
            else:
                span = old_span
            sections_append(Section(lststr, type_to_spans, span, 'Section'))
        return sections

def sections ( self ) -> List [ 'Section' ] : sections = [ ] sections_append = sections . append type_to_spans = self . _type_to_spans lststr = self . _lststr ss , se = _span = self . _span type_spans = type_to_spans . setdefault ( 'Section' , [ ] ) full_match = SECTIONS_FULLMATCH ( self . _shadow ) section_spans = full_match . spans ( 'section' ) levels = [ len ( eq ) for eq in full_match . captures ( 'equals' ) ] if not type_spans : spans_append = type_spans . append for current_index , ( current_level , ( s , e ) ) in enumerate ( zip ( levels , section_spans ) , 1 ) : for section_index , section_level in enumerate ( levels [ current_index : ] , current_index ) : if current_level and section_level > current_level : e = section_spans [ section_index ] [ 1 ] else : break span = [ ss + s , ss + e ] spans_append ( span ) sections_append ( Section ( lststr , type_to_spans , span , 'Section' ) ) return sections span_tuple_to_span = { ( s [ 0 ] , s [ 1 ] ) : s for s in type_spans } . get for current_index , ( current_level , ( s , e ) ) in enumerate ( zip ( levels , section_spans ) , 1 ) : for section_index , section_level in enumerate ( levels [ current_index : ] , current_index ) : if current_level and section_level > current_level : e = section_spans [ section_index ] [ 1 ] else : break s , e = ss + s , ss + e old_span = span_tuple_to_span ( ( s , e ) ) if old_span is None : span = [ s , e ] insort ( type_spans , span ) else : span = old_span sections_append ( Section ( lststr , type_to_spans , span , 'Section' ) ) return sections

Return a list of section in current wikitext .


def tables(self) -> List['Table']:
        """Return a list of found table objects."""
        tables = []  # type: List['Table']
        tables_append = tables.append
        type_to_spans = self._type_to_spans
        lststr = self._lststr
        shadow = self._shadow[:]
        ss, se = self._span
        spans = type_to_spans.setdefault('Table', [])
        if not spans:
            # All the added spans will be new.
            m = True  # type: Any
            while m:
                m = False
                for m in TABLE_FINDITER(shadow):
                    ms, me = m.span()
                    # Ignore leading whitespace using len(m[1]).
                    span = [ss + ms + len(m[1]), ss + me]
                    spans.append(span)
                    tables_append(Table(lststr, type_to_spans, span, 'Table'))
                    shadow[ms:me] = b'_' * (me - ms)
            return tables
        # There are already exists some spans. Try to use the already existing
        # before appending new spans.
        span_tuple_to_span_get = {(s[0], s[1]): s for s in spans}.get
        m = True
        while m:
            m = False
            for m in TABLE_FINDITER(shadow):
                ms, me = m.span()
                # Ignore leading whitespace using len(m[1]).
                s, e = ss + ms + len(m[1]), ss + me
                old_span = span_tuple_to_span_get((s, e))
                if old_span is None:
                    span = [s, e]
                    insort(spans, span)
                else:
                    span = old_span
                tables_append(Table(lststr, type_to_spans, span, 'Table'))
                shadow[ms:me] = b'_' * (me - ms)
        return tables

def tables ( self ) -> List [ 'Table' ] : tables = [ ] tables_append = tables . append type_to_spans = self . _type_to_spans lststr = self . _lststr shadow = self . _shadow [ : ] ss , se = self . _span spans = type_to_spans . setdefault ( 'Table' , [ ] ) if not spans : m = True while m : m = False for m in TABLE_FINDITER ( shadow ) : ms , me = m . span ( ) span = [ ss + ms + len ( m [ 1 ] ) , ss + me ] spans . append ( span ) tables_append ( Table ( lststr , type_to_spans , span , 'Table' ) ) shadow [ ms : me ] = b'_' * ( me - ms ) return tables span_tuple_to_span_get = { ( s [ 0 ] , s [ 1 ] ) : s for s in spans } . get m = True while m : m = False for m in TABLE_FINDITER ( shadow ) : ms , me = m . span ( ) s , e = ss + ms + len ( m [ 1 ] ) , ss + me old_span = span_tuple_to_span_get ( ( s , e ) ) if old_span is None : span = [ s , e ] insort ( spans , span ) else : span = old_span tables_append ( Table ( lststr , type_to_spans , span , 'Table' ) ) shadow [ ms : me ] = b'_' * ( me - ms ) return tables


def lists(self, pattern: str = None) -> List['WikiList']:
        r"""Return a list of WikiList objects.

        :param pattern: The starting pattern for list items.
            Return all types of lists (ol, ul, and dl) if pattern is None.
            If pattern is not None, it will be passed to the regex engine,
            remember to escape the `*` character. Examples:

                - `\#` means top-level ordered lists
                - `\#\*` means unordred lists inside an ordered one
                - Currently definition lists are not well supported, but you
                    can use `[:;]` as their pattern.

            Tips and tricks:

                Be careful when using the following patterns as they will
                probably cause malfunction in the `sublists` method of the
                resultant List. (However don't worry about them if you are
                not going to use the `sublists` method.)

                - Use `\*+` as a pattern and nested unordered lists will be
                    treated as flat.
                - Use `\*\s*` as pattern to rtstrip `items` of the list.

                Although the pattern parameter is optional, but specifying it
                can improve the performance.
        """
        lists = []
        lists_append = lists.append
        lststr = self._lststr
        type_to_spans = self._type_to_spans
        spans = type_to_spans.setdefault('WikiList', [])
        span_tuple_to_span_get = {(s[0], s[1]): s for s in spans}.get
        shadow, ss = self._lists_shadow_ss
        for pattern in \
                (r'\#', r'\*', '[:;]') if pattern is None else (pattern,):
            for m in finditer(
                LIST_PATTERN_FORMAT.replace(b'{pattern}', pattern.encode()),
                shadow, MULTILINE
            ):
                ms, me = m.span()
                s, e = ss + ms, ss + me
                old_span = span_tuple_to_span_get((s, e))
                if old_span is None:
                    span = [s, e]
                    insort(spans, span)
                else:
                    span = old_span
                lists_append(WikiList(
                    lststr, pattern, m, type_to_spans, span, 'WikiList'))
        return lists

def lists ( self , pattern : str = None ) -> List [ 'WikiList' ] : r lists = [ ] lists_append = lists . append lststr = self . _lststr type_to_spans = self . _type_to_spans spans = type_to_spans . setdefault ( 'WikiList' , [ ] ) span_tuple_to_span_get = { ( s [ 0 ] , s [ 1 ] ) : s for s in spans } . get shadow , ss = self . _lists_shadow_ss for pattern in ( r'\#' , r'\*' , '[:;]' ) if pattern is None else ( pattern , ) : for m in finditer ( LIST_PATTERN_FORMAT . replace ( b'{pattern}' , pattern . encode ( ) ) , shadow , MULTILINE ) : ms , me = m . span ( ) s , e = ss + ms , ss + me old_span = span_tuple_to_span_get ( ( s , e ) ) if old_span is None : span = [ s , e ] insort ( spans , span ) else : span = old_span lists_append ( WikiList ( lststr , pattern , m , type_to_spans , span , 'WikiList' ) ) return lists


def tags(self, name=None) -> List['Tag']:
        """Return all tags with the given name."""
        lststr = self._lststr
        type_to_spans = self._type_to_spans
        if name:
            if name in _tag_extensions:
                string = lststr[0]
                return [
                    Tag(lststr, type_to_spans, span, 'ExtensionTag')
                    for span in type_to_spans['ExtensionTag']
                    if string.startswith('<' + name, span[0])]
            tags = []  # type: List['Tag']
        else:
            # There is no name, add all extension tags. Before using shadow.
            tags = [
                Tag(lststr, type_to_spans, span, 'ExtensionTag')
                for span in type_to_spans['ExtensionTag']]
        tags_append = tags.append
        # Get the left-most start tag, match it to right-most end tag
        # and so on.
        ss = self._span[0]
        shadow = self._shadow
        if name:
            # There is a name but it is not in TAG_EXTENSIONS.
            reversed_start_matches = reversed([m for m in regex_compile(
                START_TAG_PATTERN.replace(
                    rb'{name}', rb'(?P<name>' + name.encode() + rb')')
            ).finditer(shadow)])
            end_search = regex_compile(END_TAG_PATTERN .replace(
                b'{name}', name.encode())).search
        else:
            reversed_start_matches = reversed(
                [m for m in START_TAG_FINDITER(shadow)])
        shadow_copy = shadow[:]
        spans = type_to_spans.setdefault('Tag', [])
        span_tuple_to_span_get = {(s[0], s[1]): s for s in spans}.get
        spans_append = spans.append
        for start_match in reversed_start_matches:
            if start_match['self_closing']:
                # Don't look for the end tag
                s, e = start_match.span()
                span = [ss + s, ss + e]
            else:
                # look for the end-tag
                if name:
                    # the end_search is already available
                    # noinspection PyUnboundLocalVariable
                    end_match = end_search(shadow_copy, start_match.end())
                else:
                    # build end_search according to start tag name
                    end_match = search(
                        END_TAG_PATTERN.replace(
                            b'{name}', start_match['name']),
                        shadow_copy)
                if end_match:
                    s, e = end_match.span()
                    shadow_copy[s:e] = b'_' * (e - s)
                    span = [ss + start_match.start(), ss + e]
                else:
                    # Assume start-only tag.
                    s, e = start_match.span()
                    span = [ss + s, ss + e]
            old_span = span_tuple_to_span_get((span[0], span[1]))
            if old_span is None:
                spans_append(span)
            else:
                span = old_span
            tags_append(Tag(lststr, type_to_spans, span, 'Tag'))
        return sorted(tags, key=attrgetter('_span'))

def tags ( self , name = None ) -> List [ 'Tag' ] : lststr = self . _lststr type_to_spans = self . _type_to_spans if name : if name in _tag_extensions : string = lststr [ 0 ] return [ Tag ( lststr , type_to_spans , span , 'ExtensionTag' ) for span in type_to_spans [ 'ExtensionTag' ] if string . startswith ( '<' + name , span [ 0 ] ) ] tags = [ ] else : tags = [ Tag ( lststr , type_to_spans , span , 'ExtensionTag' ) for span in type_to_spans [ 'ExtensionTag' ] ] tags_append = tags . append ss = self . _span [ 0 ] shadow = self . _shadow if name : reversed_start_matches = reversed ( [ m for m in regex_compile ( START_TAG_PATTERN . replace ( rb'{name}' , rb'(?P<name>' + name . encode ( ) + rb')' ) ) . finditer ( shadow ) ] ) end_search = regex_compile ( END_TAG_PATTERN . replace ( b'{name}' , name . encode ( ) ) ) . search else : reversed_start_matches = reversed ( [ m for m in START_TAG_FINDITER ( shadow ) ] ) shadow_copy = shadow [ : ] spans = type_to_spans . setdefault ( 'Tag' , [ ] ) span_tuple_to_span_get = { ( s [ 0 ] , s [ 1 ] ) : s for s in spans } . get spans_append = spans . append for start_match in reversed_start_matches : if start_match [ 'self_closing' ] : s , e = start_match . span ( ) span = [ ss + s , ss + e ] else : if name : end_match = end_search ( shadow_copy , start_match . end ( ) ) else : end_match = search ( END_TAG_PATTERN . replace ( b'{name}' , start_match [ 'name' ] ) , shadow_copy ) if end_match : s , e = end_match . span ( ) shadow_copy [ s : e ] = b'_' * ( e - s ) span = [ ss + start_match . start ( ) , ss + e ] else : s , e = start_match . span ( ) span = [ ss + s , ss + e ] old_span = span_tuple_to_span_get ( ( span [ 0 ] , span [ 1 ] ) ) if old_span is None : spans_append ( span ) else : span = old_span tags_append ( Tag ( lststr , type_to_spans , span , 'Tag' ) ) return sorted ( tags , key = attrgetter ( '_span' ) )