def train(self, text):
        r"""Generate a probability dict from the provided text.

        Text to 0-order probability statistics as a dict

        Parameters
        ----------
        text : str
            The text data over which to calculate probability statistics. This
            must not contain the NUL (0x00) character because that is used to
            indicate the end of data.

        Example
        -------
        >>> ac = Arithmetic()
        >>> ac.train('the quick brown fox jumped over the lazy dog')
        >>> ac.get_probs()
        {' ': (Fraction(0, 1), Fraction(8, 45)),
         'o': (Fraction(8, 45), Fraction(4, 15)),
         'e': (Fraction(4, 15), Fraction(16, 45)),
         'u': (Fraction(16, 45), Fraction(2, 5)),
         't': (Fraction(2, 5), Fraction(4, 9)),
         'r': (Fraction(4, 9), Fraction(22, 45)),
         'h': (Fraction(22, 45), Fraction(8, 15)),
         'd': (Fraction(8, 15), Fraction(26, 45)),
         'z': (Fraction(26, 45), Fraction(3, 5)),
         'y': (Fraction(3, 5), Fraction(28, 45)),
         'x': (Fraction(28, 45), Fraction(29, 45)),
         'w': (Fraction(29, 45), Fraction(2, 3)),
         'v': (Fraction(2, 3), Fraction(31, 45)),
         'q': (Fraction(31, 45), Fraction(32, 45)),
         'p': (Fraction(32, 45), Fraction(11, 15)),
         'n': (Fraction(11, 15), Fraction(34, 45)),
         'm': (Fraction(34, 45), Fraction(7, 9)),
         'l': (Fraction(7, 9), Fraction(4, 5)),
         'k': (Fraction(4, 5), Fraction(37, 45)),
         'j': (Fraction(37, 45), Fraction(38, 45)),
         'i': (Fraction(38, 45), Fraction(13, 15)),
         'g': (Fraction(13, 15), Fraction(8, 9)),
         'f': (Fraction(8, 9), Fraction(41, 45)),
         'c': (Fraction(41, 45), Fraction(14, 15)),
         'b': (Fraction(14, 15), Fraction(43, 45)),
         'a': (Fraction(43, 45), Fraction(44, 45)),
         '\x00': (Fraction(44, 45), Fraction(1, 1))}

        """
        text = text_type(text)
        if '\x00' in text:
            text = text.replace('\x00', ' ')
        counts = Counter(text)
        counts['\x00'] = 1
        tot_letters = sum(counts.values())

        tot = 0
        self._probs = {}
        prev = Fraction(0)
        for char, count in sorted(
            counts.items(), key=lambda x: (x[1], x[0]), reverse=True
        ):
            follow = Fraction(tot + count, tot_letters)
            self._probs[char] = (prev, follow)
            prev = follow
            tot = tot + count

def
train
(
self
,
text
)
:
r
text
=
text_type
(
text
)
if
'\x00'
in
text
:
text
=
text
.
replace
(
'\x00'
,
' '
)
counts
=
Counter
(
text
)
counts
[
'\x00'
]
=
1
tot_letters
=
sum
(
counts
.
values
(
)
)
tot
=
0
self
.
_probs
=
{
}
prev
=
Fraction
(
0
)
for
char
,
count
in
sorted
(
counts
.
items
(
)
,
key
=
lambda
x
:
(
x
[
1
]
,
x
[
0
]
)
,
reverse
=
True
)
:
follow
=
Fraction
(
tot
+
count
,
tot_letters
)
self
.
_probs
[
char
]
=
(
prev
,
follow
)
prev
=
follow
tot
=
tot
+
count

r Generate a probability dict from the provided text .


def encode(self, text):
        """Encode a text using arithmetic coding.

        Text and the 0-order probability statistics -> longval, nbits

        The encoded number is Fraction(longval, 2**nbits)

        Parameters
        ----------
        text : str
            A string to encode

        Returns
        -------
        tuple
            The arithmetically coded text

        Example
        -------
        >>> ac = Arithmetic('the quick brown fox jumped over the lazy dog')
        >>> ac.encode('align')
        (16720586181, 34)

        """
        text = text_type(text)
        if '\x00' in text:
            text = text.replace('\x00', ' ')
        minval = Fraction(0)
        maxval = Fraction(1)

        for char in text + '\x00':
            prob_range = self._probs[char]
            delta = maxval - minval
            maxval = minval + prob_range[1] * delta
            minval = minval + prob_range[0] * delta

        # I tried without the /2 just to check.  Doesn't work.
        # Keep scaling up until the error range is >= 1.  That
        # gives me the minimum number of bits needed to resolve
        # down to the end-of-data character.
        delta = (maxval - minval) / 2
        nbits = long(0)
        while delta < 1:
            nbits += 1
            delta *= 2
        # The below condition shouldn't ever be false
        if nbits == 0:  # pragma: no cover
            return 0, 0
        # using -1 instead of /2
        avg = (maxval + minval) * 2 ** (nbits - 1)
        # Could return a rational instead ...
        # the division truncation is deliberate
        return avg.numerator // avg.denominator, nbits

def
encode
(
self
,
text
)
:
text
=
text_type
(
text
)
if
'\x00'
in
text
:
text
=
text
.
replace
(
'\x00'
,
' '
)
minval
=
Fraction
(
0
)
maxval
=
Fraction
(
1
)
for
char
in
text
+
'\x00'
:
prob_range
=
self
.
_probs
[
char
]
delta
=
maxval
minval
maxval
=
minval
+
prob_range
[
1
]
*
delta
minval
=
minval
+
prob_range
[
0
]
*
delta
delta
=
(
maxval
minval
)
/
2
nbits
=
long
(
0
)
while
delta
<
1
:
nbits
+=
1
delta
*=
2
if
nbits
==
0
:
return
0
,
0
avg
=
(
maxval
+
minval
)
*
2
**
(
nbits
1
)
return
avg
.
numerator
//
avg
.
denominator
,
nbits


def corpus_importer(self, corpus, n_val=1, bos='_START_', eos='_END_'):
        r"""Fill in self.ngcorpus from a Corpus argument.

        Parameters
        ----------
        corpus :Corpus
            The Corpus from which to initialize the n-gram corpus
        n_val : int
            Maximum n value for n-grams
        bos : str
            String to insert as an indicator of beginning of sentence
        eos : str
            String to insert as an indicator of end of sentence

        Raises
        ------
        TypeError
            Corpus argument of the Corpus class required.

        Example
        -------
        >>> tqbf = 'The quick brown fox jumped over the lazy dog.\n'
        >>> tqbf += 'And then it slept.\n And the dog ran off.'
        >>> ngcorp = NGramCorpus()
        >>> ngcorp.corpus_importer(Corpus(tqbf))

        """
        if not corpus or not isinstance(corpus, Corpus):
            raise TypeError('Corpus argument of the Corpus class required.')

        sentences = corpus.sents()

        for sent in sentences:
            ngs = Counter(sent)
            for key in ngs.keys():
                self._add_to_ngcorpus(self.ngcorpus, [key], ngs[key])

            if n_val > 1:
                if bos and bos != '':
                    sent = [bos] + sent
                if eos and eos != '':
                    sent += [eos]
                for i in range(2, n_val + 1):
                    for j in range(len(sent) - i + 1):
                        self._add_to_ngcorpus(
                            self.ngcorpus, sent[j : j + i], 1
                        )

r Fill in self . ngcorpus from a Corpus argument .


def get_count(self, ngram, corpus=None):
        r"""Get the count of an n-gram in the corpus.

        Parameters
        ----------
        ngram : str
            The n-gram to retrieve the count of from the n-gram corpus
        corpus : Corpus
            The corpus

        Returns
        -------
        int
            The n-gram count

        Examples
        --------
        >>> tqbf = 'The quick brown fox jumped over the lazy dog.\n'
        >>> tqbf += 'And then it slept.\n And the dog ran off.'
        >>> ngcorp = NGramCorpus(Corpus(tqbf))
        >>> NGramCorpus(Corpus(tqbf)).get_count('the')
        2
        >>> NGramCorpus(Corpus(tqbf)).get_count('fox')
        1

        """
        if not corpus:
            corpus = self.ngcorpus

        # if ngram is empty, we're at our leaf node and should return the
        # value in None
        if not ngram:
            return corpus[None]

        # support strings or lists/tuples by splitting strings
        if isinstance(ngram, (text_type, str)):
            ngram = text_type(ngram).split()

        # if ngram is not empty, check whether the next element is in the
        # corpus; if so, recurse--if not, return 0
        if ngram[0] in corpus:
            return self.get_count(ngram[1:], corpus[ngram[0]])
        return 0

def get_count ( self , ngram , corpus = None ) : r if not corpus : corpus = self . ngcorpus if not ngram : return corpus [ None ] if isinstance ( ngram , ( text_type , str ) ) : ngram = text_type ( ngram ) . split ( ) if ngram [ 0 ] in corpus : return self . get_count ( ngram [ 1 : ] , corpus [ ngram [ 0 ] ] ) return 0

r Get the count of an n - gram in the corpus .


def _add_to_ngcorpus(self, corpus, words, count):
        """Build up a corpus entry recursively.

        Parameters
        ----------
        corpus : Corpus
            The corpus
        words : [str]
            Words to add to the corpus
        count : int
            Count of words

        """
        if words[0] not in corpus:
            corpus[words[0]] = Counter()

        if len(words) == 1:
            corpus[words[0]][None] += count
        else:
            self._add_to_ngcorpus(corpus[words[0]], words[1:], count)

def _add_to_ngcorpus ( self , corpus , words , count ) : if words [ 0 ] not in corpus : corpus [ words [ 0 ] ] = Counter ( ) if len ( words ) == 1 : corpus [ words [ 0 ] ] [ None ] += count else : self . _add_to_ngcorpus ( corpus [ words [ 0 ] ] , words [ 1 : ] , count )


def gng_importer(self, corpus_file):
        """Fill in self.ngcorpus from a Google NGram corpus file.

        Parameters
        ----------
        corpus_file : file
            The Google NGram file from which to initialize the n-gram corpus

        """
        with c_open(corpus_file, 'r', encoding='utf-8') as gng:
            for line in gng:
                line = line.rstrip().split('\t')
                words = line[0].split()

                self._add_to_ngcorpus(self.ngcorpus, words, int(line[2]))

def gng_importer ( self , corpus_file ) : with c_open ( corpus_file , 'r' , encoding = 'utf-8' ) as gng : for line in gng : line = line . rstrip ( ) . split ( '\t' ) words = line [ 0 ] . split ( ) self . _add_to_ngcorpus ( self . ngcorpus , words , int ( line [ 2 ] ) )

Fill in self . ngcorpus from a Google NGram corpus file .


def tf(self, term):
        r"""Return term frequency.

        Parameters
        ----------
        term : str
            The term for which to calculate tf

        Returns
        -------
        float
            The term frequency (tf)

        Raises
        ------
        ValueError
            tf can only calculate the frequency of individual words

        Examples
        --------
        >>> tqbf = 'The quick brown fox jumped over the lazy dog.\n'
        >>> tqbf += 'And then it slept.\n And the dog ran off.'
        >>> ngcorp = NGramCorpus(Corpus(tqbf))
        >>> NGramCorpus(Corpus(tqbf)).tf('the')
        1.3010299956639813
        >>> NGramCorpus(Corpus(tqbf)).tf('fox')
        1.0

        """
        if ' ' in term:
            raise ValueError(
                'tf can only calculate the term frequency of individual words'
            )
        tcount = self.get_count(term)
        if tcount == 0:
            return 0.0
        return 1 + log10(tcount)


def encode(self, word, terminator='\0'):
        r"""Return the Burrows-Wheeler transformed form of a word.

        Parameters
        ----------
        word : str
            The word to transform using BWT
        terminator : str
            A character added to signal the end of the string

        Returns
        -------
        str
            Word encoded by BWT

        Raises
        ------
        ValueError
            Specified terminator absent from code.

        Examples
        --------
        >>> bwt = BWT()
        >>> bwt.encode('align')
        'n\x00ilag'
        >>> bwt.encode('banana')
        'annb\x00aa'
        >>> bwt.encode('banana', '@')
        'annb@aa'

        """
        if word:
            if terminator in word:
                raise ValueError(
                    'Specified terminator, {}, already in word.'.format(
                        terminator if terminator != '\0' else '\\0'
                    )
                )
            else:
                word += terminator
                wordlist = sorted(
                    word[i:] + word[:i] for i in range(len(word))
                )
                return ''.join([w[-1] for w in wordlist])
        else:
            return terminator

r Return the Burrows - Wheeler transformed form of a word .


def decode(self, code, terminator='\0'):
        r"""Return a word decoded from BWT form.

        Parameters
        ----------
        code : str
            The word to transform from BWT form
        terminator : str
            A character added to signal the end of the string

        Returns
        -------
        str
            Word decoded by BWT

        Raises
        ------
        ValueError
            Specified terminator absent from code.

        Examples
        --------
        >>> bwt = BWT()
        >>> bwt.decode('n\x00ilag')
        'align'
        >>> bwt.decode('annb\x00aa')
        'banana'
        >>> bwt.decode('annb@aa', '@')
        'banana'

        """
        if code:
            if terminator not in code:
                raise ValueError(
                    'Specified terminator, {}, absent from code.'.format(
                        terminator if terminator != '\0' else '\\0'
                    )
                )
            else:
                wordlist = [''] * len(code)
                for i in range(len(code)):
                    wordlist = sorted(
                        code[i] + wordlist[i] for i in range(len(code))
                    )
                rows = [w for w in wordlist if w[-1] == terminator][0]
                return rows.rstrip(terminator)
        else:
            return ''


def dist_abs(self, src, tar):
        """Return the indel distance between two strings.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        int
            Indel distance

        Examples
        --------
        >>> cmp = Indel()
        >>> cmp.dist_abs('cat', 'hat')
        2
        >>> cmp.dist_abs('Niall', 'Neil')
        3
        >>> cmp.dist_abs('Colin', 'Cuilen')
        5
        >>> cmp.dist_abs('ATCG', 'TAGC')
        4

        """
        return self._lev.dist_abs(
            src, tar, mode='lev', cost=(1, 1, 9999, 9999)
        )

def dist_abs ( self , src , tar ) : return self . _lev . dist_abs ( src , tar , mode = 'lev' , cost = ( 1 , 1 , 9999 , 9999 ) )

Return the indel distance between two strings .


def dist(self, src, tar):
        """Return the normalized indel distance between two strings.

        This is equivalent to normalized Levenshtein distance, when only
        inserts and deletes are possible.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        float
            Normalized indel distance

        Examples
        --------
        >>> cmp = Indel()
        >>> round(cmp.dist('cat', 'hat'), 12)
        0.333333333333
        >>> round(cmp.dist('Niall', 'Neil'), 12)
        0.333333333333
        >>> round(cmp.dist('Colin', 'Cuilen'), 12)
        0.454545454545
        >>> cmp.dist('ATCG', 'TAGC')
        0.5

        """
        if src == tar:
            return 0.0
        return self.dist_abs(src, tar) / (len(src) + len(tar))

def dist ( self , src , tar ) : if src == tar : return 0.0 return self . dist_abs ( src , tar ) / ( len ( src ) + len ( tar ) )

Return the normalized indel distance between two strings .


def sim(self, src, tar, *args, **kwargs):
        """Return similarity.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison
        *args
            Variable length argument list.
        **kwargs
            Arbitrary keyword arguments.

        Returns
        -------
        float
            Similarity

        """
        return 1.0 - self.dist(src, tar, *args, **kwargs)

def sim ( self , src , tar , * args , ** kwargs ) : return 1.0 - self . dist ( src , tar , * args , ** kwargs )


def dist_abs(self, src, tar, *args, **kwargs):
        """Return absolute distance.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison
        *args
            Variable length argument list.
        **kwargs
            Arbitrary keyword arguments.

        Returns
        -------
        int
            Absolute distance

        """
        return self.dist(src, tar, *args, **kwargs)

def dist_abs ( self , src , tar , * args , ** kwargs ) : return self . dist ( src , tar , * args , ** kwargs )


def dist_baystat(src, tar, min_ss_len=None, left_ext=None, right_ext=None):
    """Return the Baystat distance.

    This is a wrapper for :py:meth:`Baystat.dist`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    min_ss_len : int
        Minimum substring length to be considered
    left_ext : int
        Left-side extension length
    right_ext : int
        Right-side extension length

    Returns
    -------
    float
        The Baystat distance

    Examples
    --------
    >>> round(dist_baystat('cat', 'hat'), 12)
    0.333333333333
    >>> dist_baystat('Niall', 'Neil')
    0.6
    >>> round(dist_baystat('Colin', 'Cuilen'), 12)
    0.833333333333
    >>> dist_baystat('ATCG', 'TAGC')
    1.0

    """
    return Baystat().dist(src, tar, min_ss_len, left_ext, right_ext)

def dist_baystat ( src , tar , min_ss_len = None , left_ext = None , right_ext = None ) : return Baystat ( ) . dist ( src , tar , min_ss_len , left_ext , right_ext )


def dist_tversky(src, tar, qval=2, alpha=1, beta=1, bias=None):
    """Return the Tversky distance between two strings.

    This is a wrapper for :py:meth:`Tversky.dist`.

    Parameters
    ----------
    src : str
        Source string (or QGrams/Counter objects) for comparison
    tar : str
        Target string (or QGrams/Counter objects) for comparison
    qval : int
        The length of each q-gram; 0 for non-q-gram version
    alpha : float
        Tversky index parameter as described above
    beta : float
        Tversky index parameter as described above
    bias : float
        The symmetric Tversky index bias parameter

    Returns
    -------
    float
        Tversky distance

    Examples
    --------
    >>> dist_tversky('cat', 'hat')
    0.6666666666666667
    >>> dist_tversky('Niall', 'Neil')
    0.7777777777777778
    >>> dist_tversky('aluminum', 'Catalan')
    0.9375
    >>> dist_tversky('ATCG', 'TAGC')
    1.0

    """
    return Tversky().dist(src, tar, qval, alpha, beta, bias)

def dist_tversky ( src , tar , qval = 2 , alpha = 1 , beta = 1 , bias = None ) : return Tversky ( ) . dist ( src , tar , qval , alpha , beta , bias )

Return the Tversky distance between two strings .


def lcsseq(self, src, tar):
        """Return the longest common subsequence of two strings.

        Based on the dynamic programming algorithm from
        http://rosettacode.org/wiki/Longest_common_subsequence
        :cite:`rosettacode:2018b`. This is licensed GFDL 1.2.

        Modifications include:
            conversion to a numpy array in place of a list of lists

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        str
            The longest common subsequence

        Examples
        --------
        >>> sseq = LCSseq()
        >>> sseq.lcsseq('cat', 'hat')
        'at'
        >>> sseq.lcsseq('Niall', 'Neil')
        'Nil'
        >>> sseq.lcsseq('aluminum', 'Catalan')
        'aln'
        >>> sseq.lcsseq('ATCG', 'TAGC')
        'AC'

        """
        lengths = np_zeros((len(src) + 1, len(tar) + 1), dtype=np_int)

        # row 0 and column 0 are initialized to 0 already
        for i, src_char in enumerate(src):
            for j, tar_char in enumerate(tar):
                if src_char == tar_char:
                    lengths[i + 1, j + 1] = lengths[i, j] + 1
                else:
                    lengths[i + 1, j + 1] = max(
                        lengths[i + 1, j], lengths[i, j + 1]
                    )

        # read the substring out from the matrix
        result = ''
        i, j = len(src), len(tar)
        while i != 0 and j != 0:
            if lengths[i, j] == lengths[i - 1, j]:
                i -= 1
            elif lengths[i, j] == lengths[i, j - 1]:
                j -= 1
            else:
                result = src[i - 1] + result
                i -= 1
                j -= 1
        return result

def lcsseq ( self , src , tar ) : lengths = np_zeros ( ( len ( src ) + 1 , len ( tar ) + 1 ) , dtype = np_int ) for i , src_char in enumerate ( src ) : for j , tar_char in enumerate ( tar ) : if src_char == tar_char : lengths [ i + 1 , j + 1 ] = lengths [ i , j ] + 1 else : lengths [ i + 1 , j + 1 ] = max ( lengths [ i + 1 , j ] , lengths [ i , j + 1 ] ) result = '' i , j = len ( src ) , len ( tar ) while i != 0 and j != 0 : if lengths [ i , j ] == lengths [ i - 1 , j ] : i -= 1 elif lengths [ i , j ] == lengths [ i , j - 1 ] : j -= 1 else : result = src [ i - 1 ] + result i -= 1 j -= 1 return result

Return the longest common subsequence of two strings .


def sim(self, src, tar):
        r"""Return the longest common subsequence similarity of two strings.

        Longest common subsequence similarity (:math:`sim_{LCSseq}`).

        This employs the LCSseq function to derive a similarity metric:
        :math:`sim_{LCSseq}(s,t) = \frac{|LCSseq(s,t)|}{max(|s|, |t|)}`

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        float
            LCSseq similarity

        Examples
        --------
        >>> sseq = LCSseq()
        >>> sseq.sim('cat', 'hat')
        0.6666666666666666
        >>> sseq.sim('Niall', 'Neil')
        0.6
        >>> sseq.sim('aluminum', 'Catalan')
        0.375
        >>> sseq.sim('ATCG', 'TAGC')
        0.5

        """
        if src == tar:
            return 1.0
        elif not src or not tar:
            return 0.0
        return len(self.lcsseq(src, tar)) / max(len(src), len(tar))

def sim ( self , src , tar ) : r if src == tar : return 1.0 elif not src or not tar : return 0.0 return len ( self . lcsseq ( src , tar ) ) / max ( len ( src ) , len ( tar ) )

r Return the longest common subsequence similarity of two strings .


def sim(self, src, tar):
        """Return the prefix similarity of two strings.

        Prefix similarity is the ratio of the length of the shorter term that
        exactly matches the longer term to the length of the shorter term,
        beginning at the start of both terms.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        float
            Prefix similarity

        Examples
        --------
        >>> cmp = Prefix()
        >>> cmp.sim('cat', 'hat')
        0.0
        >>> cmp.sim('Niall', 'Neil')
        0.25
        >>> cmp.sim('aluminum', 'Catalan')
        0.0
        >>> cmp.sim('ATCG', 'TAGC')
        0.0

        """
        if src == tar:
            return 1.0
        if not src or not tar:
            return 0.0
        min_word, max_word = (src, tar) if len(src) < len(tar) else (tar, src)
        min_len = len(min_word)
        for i in range(min_len, 0, -1):
            if min_word[:i] == max_word[:i]:
                return i / min_len
        return 0.0

def sim ( self , src , tar ) : if src == tar : return 1.0 if not src or not tar : return 0.0 min_word , max_word = ( src , tar ) if len ( src ) < len ( tar ) else ( tar , src ) min_len = len ( min_word ) for i in range ( min_len , 0 , - 1 ) : if min_word [ : i ] == max_word [ : i ] : return i / min_len return 0.0

Return the prefix similarity of two strings .


def docs_of_words(self):
        r"""Return the docs in the corpus, with sentences flattened.

        Each list within the corpus represents all the words of that document.
        Thus the sentence level of lists has been flattened.

        Returns
        -------
        [[str]]
            The docs in the corpus as a list of list of strs

        Example
        -------
        >>> tqbf = 'The quick brown fox jumped over the lazy dog.\n'
        >>> tqbf += 'And then it slept.\n And the dog ran off.'
        >>> corp = Corpus(tqbf)
        >>> corp.docs_of_words()
        [['The', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy',
        'dog.', 'And', 'then', 'it', 'slept.', 'And', 'the', 'dog', 'ran',
        'off.']]
        >>> len(corp.docs_of_words())
        1

        """
        return [
            [words for sents in doc for words in sents] for doc in self.corpus
        ]

def docs_of_words ( self ) : r return [ [ words for sents in doc for words in sents ] for doc in self . corpus ]

r Return the docs in the corpus with sentences flattened .


def raw(self):
        r"""Return the raw corpus.

        This is reconstructed by joining sub-components with the corpus' split
        characters

        Returns
        -------
        str
            The raw corpus

        Example
        -------
        >>> tqbf = 'The quick brown fox jumped over the lazy dog.\n'
        >>> tqbf += 'And then it slept.\n And the dog ran off.'
        >>> corp = Corpus(tqbf)
        >>> print(corp.raw())
        The quick brown fox jumped over the lazy dog.
        And then it slept.
        And the dog ran off.
        >>> len(corp.raw())
        85

        """
        doc_list = []
        for doc in self.corpus:
            sent_list = []
            for sent in doc:
                sent_list.append(' '.join(sent))
            doc_list.append(self.sent_split.join(sent_list))
            del sent_list
        return self.doc_split.join(doc_list)

def
raw
(
self
)
:
r
doc_list
=
[
]
for
doc
in
self
.
corpus
:
sent_list
=
[
]
for
sent
in
doc
:
sent_list
.
append
(
' '
.
join
(
sent
)
)
doc_list
.
append
(
self
.
sent_split
.
join
(
sent_list
)
)
del
sent_list
return
self
.
doc_split
.
join
(
doc_list
)


def idf(self, term, transform=None):
        r"""Calculate the Inverse Document Frequency of a term in the corpus.

        Parameters
        ----------
        term : str
            The term to calculate the IDF of
        transform : function
            A function to apply to each document term before checking for the
            presence of term

        Returns
        -------
        float
            The IDF

        Examples
        --------
        >>> tqbf = 'The quick brown fox jumped over the lazy dog.\n\n'
        >>> tqbf += 'And then it slept.\n\n And the dog ran off.'
        >>> corp = Corpus(tqbf)
        >>> print(corp.docs())
        [[['The', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy',
        'dog.']],
        [['And', 'then', 'it', 'slept.']],
        [['And', 'the', 'dog', 'ran', 'off.']]]
        >>> round(corp.idf('dog'), 10)
        0.4771212547
        >>> round(corp.idf('the'), 10)
        0.1760912591

        """
        docs_with_term = 0
        docs = self.docs_of_words()
        for doc in docs:
            doc_set = set(doc)
            if transform:
                transformed_doc = []
                for word in doc_set:
                    transformed_doc.append(transform(word))
                doc_set = set(transformed_doc)

            if term in doc_set:
                docs_with_term += 1

        if docs_with_term == 0:
            return float('inf')

        return log10(len(docs) / docs_with_term)

def idf ( self , term , transform = None ) : r docs_with_term = 0 docs = self . docs_of_words ( ) for doc in docs : doc_set = set ( doc ) if transform : transformed_doc = [ ] for word in doc_set : transformed_doc . append ( transform ( word ) ) doc_set = set ( transformed_doc ) if term in doc_set : docs_with_term += 1 if docs_with_term == 0 : return float ( 'inf' ) return log10 ( len ( docs ) / docs_with_term )

r Calculate the Inverse Document Frequency of a term in the corpus .


def stem(self, word):
        """Return Paice-Husk stem.

        Parameters
        ----------
        word : str
            The word to stem

        Returns
        -------
        str
            Word stem

        Examples
        --------
        >>> stmr = PaiceHusk()
        >>> stmr.stem('assumption')
        'assum'
        >>> stmr.stem('verifiable')
        'ver'
        >>> stmr.stem('fancies')
        'fant'
        >>> stmr.stem('fanciful')
        'fancy'
        >>> stmr.stem('torment')
        'tor'

        """
        terminate = False
        intact = True
        while not terminate:
            for n in range(6, 0, -1):
                if word[-n:] in self._rule_table[n]:
                    accept = False
                    if len(self._rule_table[n][word[-n:]]) < 4:
                        for rule in self._rule_table[n][word[-n:]]:
                            (
                                word,
                                accept,
                                intact,
                                terminate,
                            ) = self._apply_rule(word, rule, intact, terminate)
                            if accept:
                                break
                    else:
                        rule = self._rule_table[n][word[-n:]]
                        (word, accept, intact, terminate) = self._apply_rule(
                            word, rule, intact, terminate
                        )

                    if accept:
                        break
            else:
                break

        return word

def stem ( self , word ) : terminate = False intact = True while not terminate : for n in range ( 6 , 0 , - 1 ) : if word [ - n : ] in self . _rule_table [ n ] : accept = False if len ( self . _rule_table [ n ] [ word [ - n : ] ] ) < 4 : for rule in self . _rule_table [ n ] [ word [ - n : ] ] : ( word , accept , intact , terminate , ) = self . _apply_rule ( word , rule , intact , terminate ) if accept : break else : rule = self . _rule_table [ n ] [ word [ - n : ] ] ( word , accept , intact , terminate ) = self . _apply_rule ( word , rule , intact , terminate ) if accept : break else : break return word


def _language(self, name, name_mode):
        """Return the best guess language ID for the word and language choices.

        Parameters
        ----------
        name : str
            The term to guess the language of
        name_mode : str
            The name mode of the algorithm: ``gen`` (default),
            ``ash`` (Ashkenazi), or ``sep`` (Sephardic)

        Returns
        -------
        int
            Language ID

        """
        name = name.strip().lower()
        rules = BMDATA[name_mode]['language_rules']
        all_langs = (
            sum(_LANG_DICT[_] for _ in BMDATA[name_mode]['languages']) - 1
        )
        choices_remaining = all_langs
        for rule in rules:
            letters, languages, accept = rule
            if search(letters, name) is not None:
                if accept:
                    choices_remaining &= languages
                else:
                    choices_remaining &= (~languages) % (all_langs + 1)
        if choices_remaining == L_NONE:
            choices_remaining = L_ANY
        return choices_remaining

def _language ( self , name , name_mode ) : name = name . strip ( ) . lower ( ) rules = BMDATA [ name_mode ] [ 'language_rules' ] all_langs = ( sum ( _LANG_DICT [ _ ] for _ in BMDATA [ name_mode ] [ 'languages' ] ) - 1 ) choices_remaining = all_langs for rule in rules : letters , languages , accept = rule if search ( letters , name ) is not None : if accept : choices_remaining &= languages else : choices_remaining &= ( ~ languages ) % ( all_langs + 1 ) if choices_remaining == L_NONE : choices_remaining = L_ANY return choices_remaining

Return the best guess language ID for the word and language choices .


def _redo_language(
        self, term, name_mode, rules, final_rules1, final_rules2, concat
    ):
        """Reassess the language of the terms and call the phonetic encoder.

        Uses a split multi-word term.

        Parameters
        ----------
        term : str
            The term to encode via Beider-Morse
        name_mode : str
            The name mode of the algorithm: ``gen`` (default),
            ``ash`` (Ashkenazi), or ``sep`` (Sephardic)
        rules : tuple
            The set of initial phonetic transform regexps
        final_rules1 : tuple
            The common set of final phonetic transform regexps
        final_rules2 : tuple
            The specific set of final phonetic transform regexps
        concat : bool
            A flag to indicate concatenation

        Returns
        -------
        str
            A Beider-Morse phonetic code

        """
        language_arg = self._language(term, name_mode)
        return self._phonetic(
            term,
            name_mode,
            rules,
            final_rules1,
            final_rules2,
            language_arg,
            concat,
        )

def _redo_language ( self , term , name_mode , rules , final_rules1 , final_rules2 , concat ) : language_arg = self . _language ( term , name_mode ) return self . _phonetic ( term , name_mode , rules , final_rules1 , final_rules2 , language_arg , concat , )

Reassess the language of the terms and call the phonetic encoder .


def _apply_final_rules(self, phonetic, final_rules, language_arg, strip):
        """Apply a set of final rules to the phonetic encoding.

        Parameters
        ----------
        phonetic : str
            The term to which to apply the final rules
        final_rules : tuple
            The set of final phonetic transform regexps
        language_arg : int
            An integer representing the target language of the phonetic
            encoding
        strip : bool
            Flag to indicate whether to normalize the language attributes

        Returns
        -------
        str
            A Beider-Morse phonetic code

        """
        # optimization to save time
        if not final_rules:
            return phonetic

        # expand the result
        phonetic = self._expand_alternates(phonetic)
        phonetic_array = phonetic.split('|')

        for k in range(len(phonetic_array)):
            phonetic = phonetic_array[k]
            phonetic2 = ''
            phoneticx = self._normalize_lang_attrs(phonetic, True)

            i = 0
            while i < len(phonetic):
                found = False

                if phonetic[i] == '[':  # skip over language attribute
                    attrib_start = i
                    i += 1
                    while True:
                        if phonetic[i] == ']':
                            i += 1
                            phonetic2 += phonetic[attrib_start:i]
                            break
                        i += 1
                    continue

                for rule in final_rules:
                    pattern = rule[_PATTERN_POS]
                    pattern_length = len(pattern)
                    lcontext = rule[_LCONTEXT_POS]
                    rcontext = rule[_RCONTEXT_POS]

                    right = '^' + rcontext
                    left = lcontext + '$'

                    # check to see if next sequence in phonetic matches the
                    # string in the rule
                    if (pattern_length > len(phoneticx) - i) or phoneticx[
                        i : i + pattern_length
                    ] != pattern:
                        continue

                    # check that right context is satisfied
                    if rcontext != '':
                        if not search(right, phoneticx[i + pattern_length :]):
                            continue

                    # check that left context is satisfied
                    if lcontext != '':
                        if not search(left, phoneticx[:i]):
                            continue

                    # check for incompatible attributes
                    candidate = self._apply_rule_if_compat(
                        phonetic2, rule[_PHONETIC_POS], language_arg
                    )
                    # The below condition shouldn't ever be false
                    if candidate is not None:  # pragma: no branch
                        phonetic2 = candidate
                        found = True
                        break

                if not found:
                    # character in name for which there is no substitution in
                    # the table
                    phonetic2 += phonetic[i]
                    pattern_length = 1

                i += pattern_length

            phonetic_array[k] = self._expand_alternates(phonetic2)

        phonetic = '|'.join(phonetic_array)
        if strip:
            phonetic = self._normalize_lang_attrs(phonetic, True)

        if '|' in phonetic:
            phonetic = '(' + self._remove_dupes(phonetic) + ')'

        return phonetic

def _apply_final_rules ( self , phonetic , final_rules , language_arg , strip ) : if not final_rules : return phonetic phonetic = self . _expand_alternates ( phonetic ) phonetic_array = phonetic . split ( '|' ) for k in range ( len ( phonetic_array ) ) : phonetic = phonetic_array [ k ] phonetic2 = '' phoneticx = self . _normalize_lang_attrs ( phonetic , True ) i = 0 while i < len ( phonetic ) : found = False if phonetic [ i ] == '[' : attrib_start = i i += 1 while True : if phonetic [ i ] == ']' : i += 1 phonetic2 += phonetic [ attrib_start : i ] break i += 1 continue for rule in final_rules : pattern = rule [ _PATTERN_POS ] pattern_length = len ( pattern ) lcontext = rule [ _LCONTEXT_POS ] rcontext = rule [ _RCONTEXT_POS ] right = '^' + rcontext left = lcontext + '$' if ( pattern_length > len ( phoneticx ) - i ) or phoneticx [ i : i + pattern_length ] != pattern : continue if rcontext != '' : if not search ( right , phoneticx [ i + pattern_length : ] ) : continue if lcontext != '' : if not search ( left , phoneticx [ : i ] ) : continue candidate = self . _apply_rule_if_compat ( phonetic2 , rule [ _PHONETIC_POS ] , language_arg ) if candidate is not None : phonetic2 = candidate found = True break if not found : phonetic2 += phonetic [ i ] pattern_length = 1 i += pattern_length phonetic_array [ k ] = self . _expand_alternates ( phonetic2 ) phonetic = '|' . join ( phonetic_array ) if strip : phonetic = self . _normalize_lang_attrs ( phonetic , True ) if '|' in phonetic : phonetic = '(' + self . _remove_dupes ( phonetic ) + ')' return phonetic

Apply a set of final rules to the phonetic encoding .


def _expand_alternates(self, phonetic):
        """Expand phonetic alternates separated by |s.

        Parameters
        ----------
        phonetic : str
            A Beider-Morse phonetic encoding

        Returns
        -------
        str
            A Beider-Morse phonetic code

        """
        alt_start = phonetic.find('(')
        if alt_start == -1:
            return self._normalize_lang_attrs(phonetic, False)

        prefix = phonetic[:alt_start]
        alt_start += 1  # get past the (
        alt_end = phonetic.find(')', alt_start)
        alt_string = phonetic[alt_start:alt_end]
        alt_end += 1  # get past the )
        suffix = phonetic[alt_end:]
        alt_array = alt_string.split('|')
        result = ''

        for i in range(len(alt_array)):
            alt = alt_array[i]
            alternate = self._expand_alternates(prefix + alt + suffix)
            if alternate != '' and alternate != '[0]':
                if result != '':
                    result += '|'
                result += alternate

        return result

def _expand_alternates ( self , phonetic ) : alt_start = phonetic . find ( '(' ) if alt_start == - 1 : return self . _normalize_lang_attrs ( phonetic , False ) prefix = phonetic [ : alt_start ] alt_start += 1 alt_end = phonetic . find ( ')' , alt_start ) alt_string = phonetic [ alt_start : alt_end ] alt_end += 1 suffix = phonetic [ alt_end : ] alt_array = alt_string . split ( '|' ) result = '' for i in range ( len ( alt_array ) ) : alt = alt_array [ i ] alternate = self . _expand_alternates ( prefix + alt + suffix ) if alternate != '' and alternate != '[0]' : if result != '' : result += '|' result += alternate return result

Expand phonetic alternates separated by |s .


def _pnums_with_leading_space(self, phonetic):
        """Join prefixes & suffixes in cases of alternate phonetic values.

        Parameters
        ----------
        phonetic : str
            A Beider-Morse phonetic encoding

        Returns
        -------
        str
            A Beider-Morse phonetic code

        """
        alt_start = phonetic.find('(')
        if alt_start == -1:
            return ' ' + self._phonetic_number(phonetic)

        prefix = phonetic[:alt_start]
        alt_start += 1  # get past the (
        alt_end = phonetic.find(')', alt_start)
        alt_string = phonetic[alt_start:alt_end]
        alt_end += 1  # get past the )
        suffix = phonetic[alt_end:]
        alt_array = alt_string.split('|')
        result = ''
        for alt in alt_array:
            result += self._pnums_with_leading_space(prefix + alt + suffix)

        return result

def
_pnums_with_leading_space
(
self
,
phonetic
)
:
alt_start
=
phonetic
.
find
(
'('
)
if
alt_start
==
1
:
return
' '
+
self
.
_phonetic_number
(
phonetic
)
prefix
=
phonetic
[
:
alt_start
]
alt_start
+=
1
alt_end
=
phonetic
.
find
(
')'
,
alt_start
)
alt_string
=
phonetic
[
alt_start
:
alt_end
]
alt_end
+=
1
suffix
=
phonetic
[
alt_end
:
]
alt_array
=
alt_string
.
split
(
'|'
)
result
=
''
for
alt
in
alt_array
:
result
+=
self
.
_pnums_with_leading_space
(
prefix
+
alt
+
suffix
)
return
result

Join prefixes & suffixes in cases of alternate phonetic values .


def _phonetic_numbers(self, phonetic):
        """Prepare & join phonetic numbers.

        Split phonetic value on '-', run through _pnums_with_leading_space,
        and join with ' '

        Parameters
        ----------
        phonetic : str
            A Beider-Morse phonetic encoding

        Returns
        -------
        str
            A Beider-Morse phonetic code

        """
        phonetic_array = phonetic.split('-')  # for names with spaces in them
        result = ' '.join(
            [self._pnums_with_leading_space(i)[1:] for i in phonetic_array]
        )
        return result

def
_phonetic_numbers
(
self
,
phonetic
)
:
phonetic_array
=
phonetic
.
split
(
'-'
)
result
=
' '
.
join
(
[
self
.
_pnums_with_leading_space
(
i
)
[
1
:
]
for
i
in
phonetic_array
]
)
return
result


def _remove_dupes(self, phonetic):
        """Remove duplicates from a phonetic encoding list.

        Parameters
        ----------
        phonetic : str
            A Beider-Morse phonetic encoding

        Returns
        -------
        str
            A Beider-Morse phonetic code

        """
        alt_string = phonetic
        alt_array = alt_string.split('|')

        result = '|'
        for i in range(len(alt_array)):
            alt = alt_array[i]
            if alt and '|' + alt + '|' not in result:
                result += alt + '|'

        return result[1:-1]

def _remove_dupes ( self , phonetic ) : alt_string = phonetic alt_array = alt_string . split ( '|' ) result = '|' for i in range ( len ( alt_array ) ) : alt = alt_array [ i ] if alt and '|' + alt + '|' not in result : result += alt + '|' return result [ 1 : - 1 ]

Remove duplicates from a phonetic encoding list .


def _normalize_lang_attrs(self, text, strip):
        """Remove embedded bracketed attributes.

        This (potentially) bitwise-ands bracketed attributes together and adds
        to the end.
        This is applied to a single alternative at a time -- not to a
        parenthesized list.
        It removes all embedded bracketed attributes, logically-ands them
        together, and places them at the end.
        However if strip is true, this can indeed remove embedded bracketed
        attributes from a parenthesized list.

        Parameters
        ----------
        text : str
            A Beider-Morse phonetic encoding (in progress)
        strip : bool
            Remove the bracketed attributes (and throw away)

        Returns
        -------
        str
            A Beider-Morse phonetic code

        Raises
        ------
        ValueError
            No closing square bracket

        """
        uninitialized = -1  # all 1's
        attrib = uninitialized
        while '[' in text:
            bracket_start = text.find('[')
            bracket_end = text.find(']', bracket_start)
            if bracket_end == -1:
                raise ValueError(
                    'No closing square bracket: text=('
                    + text
                    + ') strip=('
                    + text_type(strip)
                    + ')'
                )
            attrib &= int(text[bracket_start + 1 : bracket_end])
            text = text[:bracket_start] + text[bracket_end + 1 :]

        if attrib == uninitialized or strip:
            return text
        elif attrib == 0:
            # means that the attributes were incompatible and there is no
            # alternative here
            return '[0]'
        return text + '[' + str(attrib) + ']'

def
_normalize_lang_attrs
(
self
,
text
,
strip
)
:
uninitialized
=
1
attrib
=
uninitialized
while
'['
in
text
:
bracket_start
=
text
.
find
(
'['
)
bracket_end
=
text
.
find
(
']'
,
bracket_start
)
if
bracket_end
==
1
:
raise
ValueError
(
'No closing square bracket: text=('
+
text
+
') strip=('
+
text_type
(
strip
)
+
')'
)
attrib
&=
int
(
text
[
bracket_start
+
1
:
bracket_end
]
)
text
=
text
[
:
bracket_start
]
+
text
[
bracket_end
+
1
:
]
if
attrib
==
uninitialized
or
strip
:
return
text
elif
attrib
==
0
:
return
'[0]'
return
text
+
'['
+
str
(
attrib
)
+
']'


def _apply_rule_if_compat(self, phonetic, target, language_arg):
        """Apply a phonetic regex if compatible.

        tests for compatible language rules

        to do so, apply the rule, expand the results, and detect alternatives
            with incompatible attributes

        then drop each alternative that has incompatible attributes and keep
            those that are compatible

        if there are no compatible alternatives left, return false

        otherwise return the compatible alternatives

        apply the rule

        Parameters
        ----------
        phonetic : str
            The Beider-Morse phonetic encoding (so far)
        target : str
            A proposed addition to the phonetic encoding
        language_arg : int
            An integer representing the target language of the phonetic
            encoding

        Returns
        -------
        str
            A candidate encoding

        """
        candidate = phonetic + target
        if '[' not in candidate:  # no attributes so we need test no further
            return candidate

        # expand the result, converting incompatible attributes to [0]
        candidate = self._expand_alternates(candidate)
        candidate_array = candidate.split('|')

        # drop each alternative that has incompatible attributes
        candidate = ''
        found = False

        for i in range(len(candidate_array)):
            this_candidate = candidate_array[i]
            if language_arg != 1:
                this_candidate = self._normalize_lang_attrs(
                    this_candidate + '[' + str(language_arg) + ']', False
                )
            if this_candidate != '[0]':
                found = True
                if candidate:
                    candidate += '|'
                candidate += this_candidate

        # return false if no compatible alternatives remain
        if not found:
            return None

        # return the result of applying the rule
        if '|' in candidate:
            candidate = '(' + candidate + ')'
        return candidate

def _apply_rule_if_compat ( self , phonetic , target , language_arg ) : candidate = phonetic + target if '[' not in candidate : return candidate candidate = self . _expand_alternates ( candidate ) candidate_array = candidate . split ( '|' ) candidate = '' found = False for i in range ( len ( candidate_array ) ) : this_candidate = candidate_array [ i ] if language_arg != 1 : this_candidate = self . _normalize_lang_attrs ( this_candidate + '[' + str ( language_arg ) + ']' , False ) if this_candidate != '[0]' : found = True if candidate : candidate += '|' candidate += this_candidate if not found : return None if '|' in candidate : candidate = '(' + candidate + ')' return candidate


def _language_index_from_code(self, code, name_mode):
        """Return the index value for a language code.

        This returns l_any if more than one code is specified or the code is
        out of bounds.

        Parameters
        ----------
        code : int
            The language code to interpret
        name_mode : str
            The name mode of the algorithm: ``gen`` (default),
            ``ash`` (Ashkenazi), or ``sep`` (Sephardic)

        Returns
        -------
        int
            Language code index

        """
        if code < 1 or code > sum(
            _LANG_DICT[_] for _ in BMDATA[name_mode]['languages']
        ):  # code out of range
            return L_ANY
        if (
            code & (code - 1)
        ) != 0:  # choice was more than one language; use any
            return L_ANY
        return code

def _language_index_from_code ( self , code , name_mode ) : if code < 1 or code > sum ( _LANG_DICT [ _ ] for _ in BMDATA [ name_mode ] [ 'languages' ] ) : return L_ANY if ( code & ( code - 1 ) ) != 0 : return L_ANY return code

Return the index value for a language code .


def dist_strcmp95(src, tar, long_strings=False):
    """Return the strcmp95 distance between two strings.

    This is a wrapper for :py:meth:`Strcmp95.dist`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    long_strings : bool
        Set to True to increase the probability of a match when the number of
        matched characters is large. This option allows for a little more
        tolerance when the strings are large. It is not an appropriate test
        when comparing fixed length fields such as phone and social security
        numbers.

    Returns
    -------
    float
        Strcmp95 distance

    Examples
    --------
    >>> round(dist_strcmp95('cat', 'hat'), 12)
    0.222222222222
    >>> round(dist_strcmp95('Niall', 'Neil'), 12)
    0.1545
    >>> round(dist_strcmp95('aluminum', 'Catalan'), 12)
    0.345238095238
    >>> round(dist_strcmp95('ATCG', 'TAGC'), 12)
    0.166666666667

    """
    return Strcmp95().dist(src, tar, long_strings)

def dist_strcmp95 ( src , tar , long_strings = False ) : return Strcmp95 ( ) . dist ( src , tar , long_strings )

Return the strcmp95 distance between two strings .


def encode(self, word):
        """Return the Naval Research Laboratory phonetic encoding of a word.

        Parameters
        ----------
        word : str
            The word to transform

        Returns
        -------
        str
            The NRL phonetic encoding

        Examples
        --------
        >>> pe = NRL()
        >>> pe.encode('the')
        'DHAX'
        >>> pe.encode('round')
        'rAWnd'
        >>> pe.encode('quick')
        'kwIHk'
        >>> pe.encode('eaten')
        'IYtEHn'
        >>> pe.encode('Smith')
        'smIHTH'
        >>> pe.encode('Larsen')
        'lAArsEHn'

        """

        def _to_regex(pattern, left_match=True):
            new_pattern = ''
            replacements = {
                '#': '[AEIOU]+',
                ':': '[BCDFGHJKLMNPQRSTVWXYZ]*',
                '^': '[BCDFGHJKLMNPQRSTVWXYZ]',
                '.': '[BDVGJLMNTWZ]',
                '%': '(ER|E|ES|ED|ING|ELY)',
                '+': '[EIY]',
                ' ': '^',
            }
            for char in pattern:
                new_pattern += (
                    replacements[char] if char in replacements else char
                )

            if left_match:
                new_pattern += '$'
                if '^' not in pattern:
                    new_pattern = '^.*' + new_pattern
            else:
                new_pattern = '^' + new_pattern.replace('^', '$')
                if '$' not in new_pattern:
                    new_pattern += '.*$'

            return new_pattern

        word = word.upper()

        pron = ''
        pos = 0
        while pos < len(word):
            left_orig = word[:pos]
            right_orig = word[pos:]
            first = word[pos] if word[pos] in self._rules else ' '
            for rule in self._rules[first]:
                left, match, right, out = rule
                if right_orig.startswith(match):
                    if left:
                        l_pattern = _to_regex(left, left_match=True)
                    if right:
                        r_pattern = _to_regex(right, left_match=False)
                    if (not left or re_match(l_pattern, left_orig)) and (
                        not right
                        or re_match(r_pattern, right_orig[len(match) :])
                    ):
                        pron += out
                        pos += len(match)
                        break
            else:
                pron += word[pos]
                pos += 1

        return pron

def
encode
(
self
,
word
)
:
def
_to_regex
(
pattern
,
left_match
=
True
)
:
new_pattern
=
''
replacements
=
{
'#'
:
'[AEIOU]+'
,
':'
:
'[BCDFGHJKLMNPQRSTVWXYZ]*'
,
'^'
:
'[BCDFGHJKLMNPQRSTVWXYZ]'
,
'.'
:
'[BDVGJLMNTWZ]'
,
'%'
:
'(ER|E|ES|ED|ING|ELY)'
,
'+'
:
'[EIY]'
,
' '
:
'^'
,
}
for
char
in
pattern
:
new_pattern
+=
(
replacements
[
char
]
if
char
in
replacements
else
char
)
if
left_match
:
new_pattern
+=
'$'
if
'^'
not
in
pattern
:
new_pattern
=
'^.*'
+
new_pattern
else
:
new_pattern
=
'^'
+
new_pattern
.
replace
(
'^'
,
'$'
)
if
'$'
not
in
new_pattern
:
new_pattern
+=
'.*$'
return
new_pattern
word
=
word
.
upper
(
)
pron
=
''
pos
=
0
while
pos
<
len
(
word
)
:
left_orig
=
word
[
:
pos
]
right_orig
=
word
[
pos
:
]
first
=
word
[
pos
]
if
word
[
pos
]
in
self
.
_rules
else
' '
for
rule
in
self
.
_rules
[
first
]
:
left
,
match
,
right
,
out
=
rule
if
right_orig
.
startswith
(
match
)
:
if
left
:
l_pattern
=
_to_regex
(
left
,
left_match
=
True
)
if
right
:
r_pattern
=
_to_regex
(
right
,
left_match
=
False
)
if
(
not
left
or
re_match
(
l_pattern
,
left_orig
)
)
and
(
not
right
or
re_match
(
r_pattern
,
right_orig
[
len
(
match
)
:
]
)
)
:
pron
+=
out
pos
+=
len
(
match
)
break
else
:
pron
+=
word
[
pos
]
pos
+=
1
return
pron

Return the Naval Research Laboratory phonetic encoding of a word .


def lcsstr(self, src, tar):
        """Return the longest common substring of two strings.

        Longest common substring (LCSstr).

        Based on the code from
        https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Longest_common_substring
        :cite:`Wikibooks:2018`.
        This is licensed Creative Commons: Attribution-ShareAlike 3.0.

        Modifications include:

            - conversion to a numpy array in place of a list of lists
            - conversion to Python 2/3-safe range from xrange via six

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        str
            The longest common substring

        Examples
        --------
        >>> sstr = LCSstr()
        >>> sstr.lcsstr('cat', 'hat')
        'at'
        >>> sstr.lcsstr('Niall', 'Neil')
        'N'
        >>> sstr.lcsstr('aluminum', 'Catalan')
        'al'
        >>> sstr.lcsstr('ATCG', 'TAGC')
        'A'

        """
        lengths = np_zeros((len(src) + 1, len(tar) + 1), dtype=np_int)
        longest, i_longest = 0, 0
        for i in range(1, len(src) + 1):
            for j in range(1, len(tar) + 1):
                if src[i - 1] == tar[j - 1]:
                    lengths[i, j] = lengths[i - 1, j - 1] + 1
                    if lengths[i, j] > longest:
                        longest = lengths[i, j]
                        i_longest = i
                else:
                    lengths[i, j] = 0
        return src[i_longest - longest : i_longest]

def lcsstr ( self , src , tar ) : lengths = np_zeros ( ( len ( src ) + 1 , len ( tar ) + 1 ) , dtype = np_int ) longest , i_longest = 0 , 0 for i in range ( 1 , len ( src ) + 1 ) : for j in range ( 1 , len ( tar ) + 1 ) : if src [ i - 1 ] == tar [ j - 1 ] : lengths [ i , j ] = lengths [ i - 1 , j - 1 ] + 1 if lengths [ i , j ] > longest : longest = lengths [ i , j ] i_longest = i else : lengths [ i , j ] = 0 return src [ i_longest - longest : i_longest ]

Return the longest common substring of two strings .


def sim(self, src, tar):
        r"""Return the longest common substring similarity of two strings.

        Longest common substring similarity (:math:`sim_{LCSstr}`).

        This employs the LCS function to derive a similarity metric:
        :math:`sim_{LCSstr}(s,t) = \frac{|LCSstr(s,t)|}{max(|s|, |t|)}`

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        float
            LCSstr similarity

        Examples
        --------
        >>> sim_lcsstr('cat', 'hat')
        0.6666666666666666
        >>> sim_lcsstr('Niall', 'Neil')
        0.2
        >>> sim_lcsstr('aluminum', 'Catalan')
        0.25
        >>> sim_lcsstr('ATCG', 'TAGC')
        0.25

        """
        if src == tar:
            return 1.0
        elif not src or not tar:
            return 0.0
        return len(self.lcsstr(src, tar)) / max(len(src), len(tar))

def sim ( self , src , tar ) : r if src == tar : return 1.0 elif not src or not tar : return 0.0 return len ( self . lcsstr ( src , tar ) ) / max ( len ( src ) , len ( tar ) )

r Return the longest common substring similarity of two strings .


def needleman_wunsch(src, tar, gap_cost=1, sim_func=sim_ident):
    """Return the Needleman-Wunsch score of two strings.

    This is a wrapper for :py:meth:`NeedlemanWunsch.dist_abs`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    gap_cost : float
        The cost of an alignment gap (1 by default)
    sim_func : function
        A function that returns the similarity of two characters (identity
        similarity by default)

    Returns
    -------
    float
        Needleman-Wunsch score

    Examples
    --------
    >>> needleman_wunsch('cat', 'hat')
    2.0
    >>> needleman_wunsch('Niall', 'Neil')
    1.0
    >>> needleman_wunsch('aluminum', 'Catalan')
    -1.0
    >>> needleman_wunsch('ATCG', 'TAGC')
    0.0

    """
    return NeedlemanWunsch().dist_abs(src, tar, gap_cost, sim_func)

def needleman_wunsch ( src , tar , gap_cost = 1 , sim_func = sim_ident ) : return NeedlemanWunsch ( ) . dist_abs ( src , tar , gap_cost , sim_func )

Return the Needleman - Wunsch score of two strings .


def sim_matrix(
        src,
        tar,
        mat=None,
        mismatch_cost=0,
        match_cost=1,
        symmetric=True,
        alphabet=None,
    ):
        """Return the matrix similarity of two strings.

        With the default parameters, this is identical to sim_ident.
        It is possible for sim_matrix to return values outside of the range
        :math:`[0, 1]`, if values outside that range are present in mat,
        mismatch_cost, or match_cost.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison
        mat : dict
            A dict mapping tuples to costs; the tuples are (src, tar) pairs of
            symbols from the alphabet parameter
        mismatch_cost : float
            The value returned if (src, tar) is absent from mat when src does
            not equal tar
        match_cost : float
            The value returned if (src, tar) is absent from mat when src equals
            tar
        symmetric : bool
            True if the cost of src not matching tar is identical to the cost
            of tar not matching src; in this case, the values in mat need only
            contain (src, tar) or (tar, src), not both
        alphabet : str
            A collection of tokens from which src and tar are drawn; if this is
            defined a ValueError is raised if either tar or src is not found in
            alphabet

        Returns
        -------
        float
            Matrix similarity

        Raises
        ------
        ValueError
            src value not in alphabet
        ValueError
            tar value not in alphabet

        Examples
        --------
        >>> NeedlemanWunsch.sim_matrix('cat', 'hat')
        0
        >>> NeedlemanWunsch.sim_matrix('hat', 'hat')
        1

        """
        if alphabet:
            alphabet = tuple(alphabet)
            for i in src:
                if i not in alphabet:
                    raise ValueError('src value not in alphabet')
            for i in tar:
                if i not in alphabet:
                    raise ValueError('tar value not in alphabet')

        if src == tar:
            if mat and (src, src) in mat:
                return mat[(src, src)]
            return match_cost
        if mat and (src, tar) in mat:
            return mat[(src, tar)]
        elif symmetric and mat and (tar, src) in mat:
            return mat[(tar, src)]
        return mismatch_cost

def
sim_matrix
(
src
,
tar
,
mat
=
,
mismatch_cost
=
0
,
match_cost
=
1
,
symmetric
=
True
,
alphabet
=
,
)
:
if
alphabet
:
alphabet
=
tuple
(
alphabet
)
for
i
in
src
:
if
i
not
in
alphabet
:
raise
ValueError
(
'src value not in alphabet'
)
for
i
in
tar
:
if
i
not
in
alphabet
:
raise
ValueError
(
'tar value not in alphabet'
)
if
src
==
tar
:
if
mat
and
(
src
,
src
)
in
mat
:
return
mat
[
(
src
,
src
)
]
return
match_cost
if
mat
and
(
src
,
tar
)
in
mat
:
return
mat
[
(
src
,
tar
)
]
elif
symmetric
and
mat
and
(
tar
,
src
)
in
mat
:
return
mat
[
(
tar
,
src
)
]
return
mismatch_cost

Return the matrix similarity of two strings .


def encode(self, word, max_length=-1):
        """Return the PhoneticSpanish coding of word.

        Parameters
        ----------
        word : str
            The word to transform
        max_length : int
            The length of the code returned (defaults to unlimited)

        Returns
        -------
        str
            The PhoneticSpanish code

        Examples
        --------
        >>> pe = PhoneticSpanish()
        >>> pe.encode('Perez')
        '094'
        >>> pe.encode('Martinez')
        '69364'
        >>> pe.encode('Gutierrez')
        '83994'
        >>> pe.encode('Santiago')
        '4638'
        >>> pe.encode('Nicolás')
        '6454'

        """
        # uppercase, normalize, and decompose, filter to A-Z minus vowels & W
        word = unicode_normalize('NFKD', text_type(word.upper()))
        word = ''.join(c for c in word if c in self._uc_set)

        # merge repeated Ls & Rs
        word = word.replace('LL', 'L')
        word = word.replace('R', 'R')

        # apply the Soundex algorithm
        sdx = word.translate(self._trans)

        if max_length > 0:
            sdx = (sdx + ('0' * max_length))[:max_length]

        return sdx

def encode ( self , word , max_length = - 1 ) : word = unicode_normalize ( 'NFKD' , text_type ( word . upper ( ) ) ) word = '' . join ( c for c in word if c in self . _uc_set ) word = word . replace ( 'LL' , 'L' ) word = word . replace ( 'R' , 'R' ) sdx = word . translate ( self . _trans ) if max_length > 0 : sdx = ( sdx + ( '0' * max_length ) ) [ : max_length ] return sdx

Return the PhoneticSpanish coding of word .


def dist(self, src, tar):
        """Return the NCD between two strings using BWT plus RLE.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        float
            Compression distance

        Examples
        --------
        >>> cmp = NCDbwtrle()
        >>> cmp.dist('cat', 'hat')
        0.75
        >>> cmp.dist('Niall', 'Neil')
        0.8333333333333334
        >>> cmp.dist('aluminum', 'Catalan')
        1.0
        >>> cmp.dist('ATCG', 'TAGC')
        0.8

        """
        if src == tar:
            return 0.0

        src_comp = self._rle.encode(self._bwt.encode(src))
        tar_comp = self._rle.encode(self._bwt.encode(tar))
        concat_comp = self._rle.encode(self._bwt.encode(src + tar))
        concat_comp2 = self._rle.encode(self._bwt.encode(tar + src))

        return (
            min(len(concat_comp), len(concat_comp2))
            - min(len(src_comp), len(tar_comp))
        ) / max(len(src_comp), len(tar_comp))

def dist ( self , src , tar ) : if src == tar : return 0.0 src_comp = self . _rle . encode ( self . _bwt . encode ( src ) ) tar_comp = self . _rle . encode ( self . _bwt . encode ( tar ) ) concat_comp = self . _rle . encode ( self . _bwt . encode ( src + tar ) ) concat_comp2 = self . _rle . encode ( self . _bwt . encode ( tar + src ) ) return ( min ( len ( concat_comp ) , len ( concat_comp2 ) ) - min ( len ( src_comp ) , len ( tar_comp ) ) ) / max ( len ( src_comp ) , len ( tar_comp ) )

Return the NCD between two strings using BWT plus RLE .


def to_tuple(self):
        """Cast to tuple.

        Returns
        -------
        tuple
            The confusion table as a 4-tuple (tp, tn, fp, fn)

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.to_tuple()
        (120, 60, 20, 30)

        """
        return self._tp, self._tn, self._fp, self._fn

def to_tuple ( self ) : return self . _tp , self . _tn , self . _fp , self . _fn


def to_dict(self):
        """Cast to dict.

        Returns
        -------
        dict
            The confusion table as a dict

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> import pprint
        >>> pprint.pprint(ct.to_dict())
        {'fn': 30, 'fp': 20, 'tn': 60, 'tp': 120}

        """
        return {'tp': self._tp, 'tn': self._tn, 'fp': self._fp, 'fn': self._fn}

def to_dict ( self ) : return { 'tp' : self . _tp , 'tn' : self . _tn , 'fp' : self . _fp , 'fn' : self . _fn }


def population(self):
        """Return population, N.

        Returns
        -------
        int
            The population (N) of the confusion table

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.population()
        230

        """
        return self._tp + self._tn + self._fp + self._fn

def population ( self ) : return self . _tp + self . _tn + self . _fp + self . _fn


def precision(self):
        r"""Return precision.

        Precision is defined as :math:`\frac{tp}{tp + fp}`

        AKA positive predictive value (PPV)

        Cf. https://en.wikipedia.org/wiki/Precision_and_recall

        Cf. https://en.wikipedia.org/wiki/Information_retrieval#Precision

        Returns
        -------
        float
            The precision of the confusion table

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.precision()
        0.8571428571428571

        """
        if self._tp + self._fp == 0:
            return float('NaN')
        return self._tp / (self._tp + self._fp)

def precision ( self ) : r if self . _tp + self . _fp == 0 : return float ( 'NaN' ) return self . _tp / ( self . _tp + self . _fp )


def precision_gain(self):
        r"""Return gain in precision.

        The gain in precision is defined as:
        :math:`G(precision) = \frac{precision}{random~ precision}`

        Cf. https://en.wikipedia.org/wiki/Gain_(information_retrieval)

        Returns
        -------
        float
            The gain in precision of the confusion table

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.precision_gain()
        1.3142857142857143

        """
        if self.population() == 0:
            return float('NaN')
        random_precision = self.cond_pos_pop() / self.population()
        return self.precision() / random_precision

def precision_gain ( self ) : r if self . population ( ) == 0 : return float ( 'NaN' ) random_precision = self . cond_pos_pop ( ) / self . population ( ) return self . precision ( ) / random_precision


def recall(self):
        r"""Return recall.

        Recall is defined as :math:`\frac{tp}{tp + fn}`

        AKA sensitivity

        AKA true positive rate (TPR)

        Cf. https://en.wikipedia.org/wiki/Precision_and_recall

        Cf. https://en.wikipedia.org/wiki/Sensitivity_(test)

        Cf. https://en.wikipedia.org/wiki/Information_retrieval#Recall

        Returns
        -------
        float
            The recall of the confusion table

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.recall()
        0.8

        """
        if self._tp + self._fn == 0:
            return float('NaN')
        return self._tp / (self._tp + self._fn)

def recall ( self ) : r if self . _tp + self . _fn == 0 : return float ( 'NaN' ) return self . _tp / ( self . _tp + self . _fn )


def specificity(self):
        r"""Return specificity.

        Specificity is defined as :math:`\frac{tn}{tn + fp}`

        AKA true negative rate (TNR)

        Cf. https://en.wikipedia.org/wiki/Specificity_(tests)

        Returns
        -------
        float
            The specificity of the confusion table

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.specificity()
        0.75

        """
        if self._tn + self._fp == 0:
            return float('NaN')
        return self._tn / (self._tn + self._fp)

def specificity ( self ) : r if self . _tn + self . _fp == 0 : return float ( 'NaN' ) return self . _tn / ( self . _tn + self . _fp )


def fallout(self):
        r"""Return fall-out.

        Fall-out is defined as :math:`\frac{fp}{fp + tn}`

        AKA false positive rate (FPR)

        Cf. https://en.wikipedia.org/wiki/Information_retrieval#Fall-out

        Returns
        -------
        float
            The fall-out of the confusion table

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.fallout()
        0.25

        """
        if self._fp + self._tn == 0:
            return float('NaN')
        return self._fp / (self._fp + self._tn)

def fallout ( self ) : r if self . _fp + self . _tn == 0 : return float ( 'NaN' ) return self . _fp / ( self . _fp + self . _tn )


def accuracy(self):
        r"""Return accuracy.

        Accuracy is defined as :math:`\frac{tp + tn}{population}`

        Cf. https://en.wikipedia.org/wiki/Accuracy

        Returns
        -------
        float
            The accuracy of the confusion table

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.accuracy()
        0.782608695652174

        """
        if self.population() == 0:
            return float('NaN')
        return (self._tp + self._tn) / self.population()

def accuracy ( self ) : r if self . population ( ) == 0 : return float ( 'NaN' ) return ( self . _tp + self . _tn ) / self . population ( )


def accuracy_gain(self):
        r"""Return gain in accuracy.

        The gain in accuracy is defined as:
        :math:`G(accuracy) = \frac{accuracy}{random~ accuracy}`

        Cf. https://en.wikipedia.org/wiki/Gain_(information_retrieval)

        Returns
        -------
        float
            The gain in accuracy of the confusion table

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.accuracy_gain()
        1.4325259515570934

        """
        if self.population() == 0:
            return float('NaN')
        random_accuracy = (self.cond_pos_pop() / self.population()) ** 2 + (
            self.cond_neg_pop() / self.population()
        ) ** 2
        return self.accuracy() / random_accuracy

def accuracy_gain ( self ) : r if self . population ( ) == 0 : return float ( 'NaN' ) random_accuracy = ( self . cond_pos_pop ( ) / self . population ( ) ) ** 2 + ( self . cond_neg_pop ( ) / self . population ( ) ) ** 2 return self . accuracy ( ) / random_accuracy


def pr_lmean(self):
        r"""Return logarithmic mean of precision & recall.

        The logarithmic mean is:
        0 if either precision or recall is 0,
        the precision if they are equal,
        otherwise :math:`\frac{precision - recall}
        {ln(precision) - ln(recall)}`

        Cf. https://en.wikipedia.org/wiki/Logarithmic_mean

        Returns
        -------
        float
            The logarithmic mean of the confusion table's precision & recall

        Example
        -------
        >>> ct = ConfusionTable(120, 60, 20, 30)
        >>> ct.pr_lmean()
        0.8282429171492667

        """
        precision = self.precision()
        recall = self.recall()
        if not precision or not recall:
            return 0.0
        elif precision == recall:
            return precision
        return (precision - recall) / (math.log(precision) - math.log(recall))

def pr_lmean ( self ) : r precision = self . precision ( ) recall = self . recall ( ) if not precision or not recall : return 0.0 elif precision == recall : return precision return ( precision - recall ) / ( math . log ( precision ) - math . log ( recall ) )

r Return logarithmic mean of precision & recall .


def stem(self, word):
        """Return CLEF German stem.

        Parameters
        ----------
        word : str
            The word to stem

        Returns
        -------
        str
            Word stem

        Examples
        --------
        >>> stmr = CLEFGerman()
        >>> stmr.stem('lesen')
        'lese'
        >>> stmr.stem('graues')
        'grau'
        >>> stmr.stem('buchstabieren')
        'buchstabier'

        """
        # lowercase, normalize, and compose
        word = normalize('NFC', text_type(word.lower()))

        # remove umlauts
        word = word.translate(self._umlauts)

        # remove plurals
        wlen = len(word) - 1

        if wlen > 3:
            if wlen > 5:
                if word[-3:] == 'nen':
                    return word[:-3]
            if wlen > 4:
                if word[-2:] in {'en', 'se', 'es', 'er'}:
                    return word[:-2]
            if word[-1] in {'e', 'n', 'r', 's'}:
                return word[:-1]
        return word

def stem ( self , word ) : word = normalize ( 'NFC' , text_type ( word . lower ( ) ) ) word = word . translate ( self . _umlauts ) wlen = len ( word ) - 1 if wlen > 3 : if wlen > 5 : if word [ - 3 : ] == 'nen' : return word [ : - 3 ] if wlen > 4 : if word [ - 2 : ] in { 'en' , 'se' , 'es' , 'er' } : return word [ : - 2 ] if word [ - 1 ] in { 'e' , 'n' , 'r' , 's' } : return word [ : - 1 ] return word


def dist_abs(self, src, tar, max_offset=5):
        """Return the "simplest" Sift4 distance between two terms.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison
        max_offset : int
            The number of characters to search for matching letters

        Returns
        -------
        int
            The Sift4 distance according to the simplest formula

        Examples
        --------
        >>> cmp = Sift4Simplest()
        >>> cmp.dist_abs('cat', 'hat')
        1
        >>> cmp.dist_abs('Niall', 'Neil')
        2
        >>> cmp.dist_abs('Colin', 'Cuilen')
        3
        >>> cmp.dist_abs('ATCG', 'TAGC')
        2

        """
        if not src:
            return len(tar)

        if not tar:
            return len(src)

        src_len = len(src)
        tar_len = len(tar)

        src_cur = 0
        tar_cur = 0
        lcss = 0
        local_cs = 0

        while (src_cur < src_len) and (tar_cur < tar_len):
            if src[src_cur] == tar[tar_cur]:
                local_cs += 1
            else:
                lcss += local_cs
                local_cs = 0
                if src_cur != tar_cur:
                    src_cur = tar_cur = max(src_cur, tar_cur)
                for i in range(max_offset):
                    if not (
                        (src_cur + i < src_len) or (tar_cur + i < tar_len)
                    ):
                        break
                    if (src_cur + i < src_len) and (
                        src[src_cur + i] == tar[tar_cur]
                    ):
                        src_cur += i
                        local_cs += 1
                        break
                    if (tar_cur + i < tar_len) and (
                        src[src_cur] == tar[tar_cur + i]
                    ):
                        tar_cur += i
                        local_cs += 1
                        break

            src_cur += 1
            tar_cur += 1

        lcss += local_cs
        return round(max(src_len, tar_len) - lcss)

def dist_abs ( self , src , tar , max_offset = 5 ) : if not src : return len ( tar ) if not tar : return len ( src ) src_len = len ( src ) tar_len = len ( tar ) src_cur = 0 tar_cur = 0 lcss = 0 local_cs = 0 while ( src_cur < src_len ) and ( tar_cur < tar_len ) : if src [ src_cur ] == tar [ tar_cur ] : local_cs += 1 else : lcss += local_cs local_cs = 0 if src_cur != tar_cur : src_cur = tar_cur = max ( src_cur , tar_cur ) for i in range ( max_offset ) : if not ( ( src_cur + i < src_len ) or ( tar_cur + i < tar_len ) ) : break if ( src_cur + i < src_len ) and ( src [ src_cur + i ] == tar [ tar_cur ] ) : src_cur += i local_cs += 1 break if ( tar_cur + i < tar_len ) and ( src [ src_cur ] == tar [ tar_cur + i ] ) : tar_cur += i local_cs += 1 break src_cur += 1 tar_cur += 1 lcss += local_cs return round ( max ( src_len , tar_len ) - lcss )

Return the simplest Sift4 distance between two terms .


def sim_typo(
    src, tar, metric='euclidean', cost=(1, 1, 0.5, 0.5), layout='QWERTY'
):
    """Return the normalized typo similarity between two strings.

    This is a wrapper for :py:meth:`Typo.sim`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    metric : str
        Supported values include: ``euclidean``, ``manhattan``,
        ``log-euclidean``, and ``log-manhattan``
    cost : tuple
        A 4-tuple representing the cost of the four possible edits: inserts,
        deletes, substitutions, and shift, respectively (by default:
        (1, 1, 0.5, 0.5)) The substitution & shift costs should be
        significantly less than the cost of an insertion & deletion unless a
        log metric is used.
    layout : str
        Name of the keyboard layout to use (Currently supported:
        ``QWERTY``, ``Dvorak``, ``AZERTY``, ``QWERTZ``)

    Returns
    -------
    float
        Normalized typo similarity

    Examples
    --------
    >>> round(sim_typo('cat', 'hat'), 12)
    0.472953716914
    >>> round(sim_typo('Niall', 'Neil'), 12)
    0.434971857071
    >>> round(sim_typo('Colin', 'Cuilen'), 12)
    0.430964390437
    >>> sim_typo('ATCG', 'TAGC')
    0.375

    """
    return Typo().sim(src, tar, metric, cost, layout)

def sim_typo ( src , tar , metric = 'euclidean' , cost = ( 1 , 1 , 0.5 , 0.5 ) , layout = 'QWERTY' ) : return Typo ( ) . sim ( src , tar , metric , cost , layout )

Return the normalized typo similarity between two strings .


def manhattan(src, tar, qval=2, normalized=False, alphabet=None):
    """Return the Manhattan distance between two strings.

    This is a wrapper for :py:meth:`Manhattan.dist_abs`.

    Parameters
    ----------
    src : str
        Source string (or QGrams/Counter objects) for comparison
    tar : str
        Target string (or QGrams/Counter objects) for comparison
    qval : int
        The length of each q-gram; 0 for non-q-gram version
    normalized : bool
        Normalizes to [0, 1] if True
    alphabet : collection or int
        The values or size of the alphabet

    Returns
    -------
    float
        The Manhattan distance

    Examples
    --------
    >>> manhattan('cat', 'hat')
    4.0
    >>> manhattan('Niall', 'Neil')
    7.0
    >>> manhattan('Colin', 'Cuilen')
    9.0
    >>> manhattan('ATCG', 'TAGC')
    10.0

    """
    return Manhattan().dist_abs(src, tar, qval, normalized, alphabet)

def manhattan ( src , tar , qval = 2 , normalized = False , alphabet = None ) : return Manhattan ( ) . dist_abs ( src , tar , qval , normalized , alphabet )

Return the Manhattan distance between two strings .


def dist_manhattan(src, tar, qval=2, alphabet=None):
    """Return the normalized Manhattan distance between two strings.

    This is a wrapper for :py:meth:`Manhattan.dist`.

    Parameters
    ----------
    src : str
        Source string (or QGrams/Counter objects) for comparison
    tar : str
        Target string (or QGrams/Counter objects) for comparison
    qval : int
        The length of each q-gram; 0 for non-q-gram version
    alphabet : collection or int
        The values or size of the alphabet

    Returns
    -------
    float
        The normalized Manhattan distance

    Examples
    --------
    >>> dist_manhattan('cat', 'hat')
    0.5
    >>> round(dist_manhattan('Niall', 'Neil'), 12)
    0.636363636364
    >>> round(dist_manhattan('Colin', 'Cuilen'), 12)
    0.692307692308
    >>> dist_manhattan('ATCG', 'TAGC')
    1.0

    """
    return Manhattan().dist(src, tar, qval, alphabet)

def dist_manhattan ( src , tar , qval = 2 , alphabet = None ) : return Manhattan ( ) . dist ( src , tar , qval , alphabet )

Return the normalized Manhattan distance between two strings .


def sim_manhattan(src, tar, qval=2, alphabet=None):
    """Return the normalized Manhattan similarity of two strings.

    This is a wrapper for :py:meth:`Manhattan.sim`.

    Parameters
    ----------
    src : str
        Source string (or QGrams/Counter objects) for comparison
    tar : str
        Target string (or QGrams/Counter objects) for comparison
    qval : int
        The length of each q-gram; 0 for non-q-gram version
    alphabet : collection or int
        The values or size of the alphabet

    Returns
    -------
    float
        The normalized Manhattan similarity

    Examples
    --------
    >>> sim_manhattan('cat', 'hat')
    0.5
    >>> round(sim_manhattan('Niall', 'Neil'), 12)
    0.363636363636
    >>> round(sim_manhattan('Colin', 'Cuilen'), 12)
    0.307692307692
    >>> sim_manhattan('ATCG', 'TAGC')
    0.0

    """
    return Manhattan().sim(src, tar, qval, alphabet)

def sim_manhattan ( src , tar , qval = 2 , alphabet = None ) : return Manhattan ( ) . sim ( src , tar , qval , alphabet )

Return the normalized Manhattan similarity of two strings .


def dist_jaro_winkler(
    src,
    tar,
    qval=1,
    mode='winkler',
    long_strings=False,
    boost_threshold=0.7,
    scaling_factor=0.1,
):
    """Return the Jaro or Jaro-Winkler distance between two strings.

    This is a wrapper for :py:meth:`JaroWinkler.dist`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    qval : int
        The length of each q-gram (defaults to 1: character-wise matching)
    mode : str
        Indicates which variant of this distance metric to compute:

            - ``winkler`` -- computes the Jaro-Winkler distance (default) which
              increases the score for matches near the start of the word
            - ``jaro`` -- computes the Jaro distance

    long_strings : bool
        Set to True to "Increase the probability of a match when the number of
        matched characters is large. This option allows for a little more
        tolerance when the strings are large. It is not an appropriate test
        when comparing fixedlength fields such as phone and social security
        numbers." (Used in 'winkler' mode only.)
    boost_threshold : float
        A value between 0 and 1, below which the Winkler boost is not applied
        (defaults to 0.7). (Used in 'winkler' mode only.)
    scaling_factor : float
        A value between 0 and 0.25, indicating by how much to boost scores for
        matching prefixes (defaults to 0.1). (Used in 'winkler' mode only.)

    Returns
    -------
    float
        Jaro or Jaro-Winkler distance

    Examples
    --------
    >>> round(dist_jaro_winkler('cat', 'hat'), 12)
    0.222222222222
    >>> round(dist_jaro_winkler('Niall', 'Neil'), 12)
    0.195
    >>> round(dist_jaro_winkler('aluminum', 'Catalan'), 12)
    0.39880952381
    >>> round(dist_jaro_winkler('ATCG', 'TAGC'), 12)
    0.166666666667

    >>> round(dist_jaro_winkler('cat', 'hat', mode='jaro'), 12)
    0.222222222222
    >>> round(dist_jaro_winkler('Niall', 'Neil', mode='jaro'), 12)
    0.216666666667
    >>> round(dist_jaro_winkler('aluminum', 'Catalan', mode='jaro'), 12)
    0.39880952381
    >>> round(dist_jaro_winkler('ATCG', 'TAGC', mode='jaro'), 12)
    0.166666666667

    """
    return JaroWinkler().dist(
        src, tar, qval, mode, long_strings, boost_threshold, scaling_factor
    )

def dist_jaro_winkler ( src , tar , qval = 1 , mode = 'winkler' , long_strings = False , boost_threshold = 0.7 , scaling_factor = 0.1 , ) : return JaroWinkler ( ) . dist ( src , tar , qval , mode , long_strings , boost_threshold , scaling_factor )

Return the Jaro or Jaro - Winkler distance between two strings .


def sim_hamming(src, tar, diff_lens=True):
    """Return the normalized Hamming similarity of two strings.

    This is a wrapper for :py:meth:`Hamming.sim`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    diff_lens : bool
        If True (default), this returns the Hamming distance for those
        characters that have a matching character in both strings plus the
        difference in the strings' lengths. This is equivalent to extending the
        shorter string with obligatorily non-matching characters. If False, an
        exception is raised in the case of strings of unequal lengths.

    Returns
    -------
    float
        The normalized Hamming similarity

    Examples
    --------
    >>> round(sim_hamming('cat', 'hat'), 12)
    0.666666666667
    >>> sim_hamming('Niall', 'Neil')
    0.4
    >>> sim_hamming('aluminum', 'Catalan')
    0.0
    >>> sim_hamming('ATCG', 'TAGC')
    0.0

    """
    return Hamming().sim(src, tar, diff_lens)

def sim_hamming ( src , tar , diff_lens = True ) : return Hamming ( ) . sim ( src , tar , diff_lens )

Return the normalized Hamming similarity of two strings .


def fingerprint(self, word):
        """Return the skeleton key.

        Parameters
        ----------
        word : str
            The word to transform into its skeleton key

        Returns
        -------
        str
            The skeleton key

        Examples
        --------
        >>> sk = SkeletonKey()
        >>> sk.fingerprint('The quick brown fox jumped over the lazy dog.')
        'THQCKBRWNFXJMPDVLZYGEUIOA'
        >>> sk.fingerprint('Christopher')
        'CHRSTPIOE'
        >>> sk.fingerprint('Niall')
        'NLIA'

        """
        word = unicode_normalize('NFKD', text_type(word.upper()))
        word = ''.join(c for c in word if c in self._letters)
        start = word[0:1]
        consonant_part = ''
        vowel_part = ''

        # add consonants & vowels to to separate strings
        # (omitting the first char & duplicates)
        for char in word[1:]:
            if char != start:
                if char in self._vowels:
                    if char not in vowel_part:
                        vowel_part += char
                elif char not in consonant_part:
                    consonant_part += char
        # return the first char followed by consonants followed by vowels
        return start + consonant_part + vowel_part

def fingerprint ( self , word ) : word = unicode_normalize ( 'NFKD' , text_type ( word . upper ( ) ) ) word = '' . join ( c for c in word if c in self . _letters ) start = word [ 0 : 1 ] consonant_part = '' vowel_part = '' for char in word [ 1 : ] : if char != start : if char in self . _vowels : if char not in vowel_part : vowel_part += char elif char not in consonant_part : consonant_part += char return start + consonant_part + vowel_part


def mean_pairwise_similarity(
    collection, metric=sim, mean_func=hmean, symmetric=False
):
    """Calculate the mean pairwise similarity of a collection of strings.

    Takes the mean of the pairwise similarity between each member of a
    collection, optionally in both directions (for asymmetric similarity
    metrics.

    Parameters
    ----------
    collection : list
        A collection of terms or a string that can be split
    metric : function
        A similarity metric function
    mean_func : function
        A mean function that takes a list of values and returns a float
    symmetric : bool
        Set to True if all pairwise similarities should be calculated in both
        directions

    Returns
    -------
    float
        The mean pairwise similarity of a collection of strings

    Raises
    ------
    ValueError
        mean_func must be a function
    ValueError
        metric must be a function
    ValueError
        collection is neither a string nor iterable type
    ValueError
        collection has fewer than two members

    Examples
    --------
    >>> round(mean_pairwise_similarity(['Christopher', 'Kristof',
    ... 'Christobal']), 12)
    0.519801980198
    >>> round(mean_pairwise_similarity(['Niall', 'Neal', 'Neil']), 12)
    0.545454545455

    """
    if not callable(mean_func):
        raise ValueError('mean_func must be a function')
    if not callable(metric):
        raise ValueError('metric must be a function')

    if hasattr(collection, 'split'):
        collection = collection.split()
    if not hasattr(collection, '__iter__'):
        raise ValueError('collection is neither a string nor iterable type')
    elif len(collection) < 2:
        raise ValueError('collection has fewer than two members')

    collection = list(collection)

    pairwise_values = []

    for i in range(len(collection)):
        for j in range(i + 1, len(collection)):
            pairwise_values.append(metric(collection[i], collection[j]))
            if symmetric:
                pairwise_values.append(metric(collection[j], collection[i]))

    return mean_func(pairwise_values)

Calculate the mean pairwise similarity of a collection of strings .


def pairwise_similarity_statistics(
    src_collection,
    tar_collection,
    metric=sim,
    mean_func=amean,
    symmetric=False,
):
    """Calculate the pairwise similarity statistics a collection of strings.

    Calculate pairwise similarities among members of two collections,
    returning the maximum, minimum, mean (according to a supplied function,
    arithmetic mean, by default), and (population) standard deviation
    of those similarities.

    Parameters
    ----------
    src_collection : list
        A collection of terms or a string that can be split
    tar_collection : list
        A collection of terms or a string that can be split
    metric : function
        A similarity metric function
    mean_func : function
        A mean function that takes a list of values and returns a float
    symmetric : bool
        Set to True if all pairwise similarities should be calculated in both
        directions

    Returns
    -------
    tuple
        The max, min, mean, and standard deviation of similarities

    Raises
    ------
    ValueError
        mean_func must be a function
    ValueError
        metric must be a function
    ValueError
        src_collection is neither a string nor iterable
    ValueError
        tar_collection is neither a string nor iterable

    Example
    -------
    >>> tuple(round(_, 12) for _ in pairwise_similarity_statistics(
    ... ['Christopher', 'Kristof', 'Christobal'], ['Niall', 'Neal', 'Neil']))
    (0.2, 0.0, 0.118614718615, 0.075070477184)

    """
    if not callable(mean_func):
        raise ValueError('mean_func must be a function')
    if not callable(metric):
        raise ValueError('metric must be a function')

    if hasattr(src_collection, 'split'):
        src_collection = src_collection.split()
    if not hasattr(src_collection, '__iter__'):
        raise ValueError('src_collection is neither a string nor iterable')

    if hasattr(tar_collection, 'split'):
        tar_collection = tar_collection.split()
    if not hasattr(tar_collection, '__iter__'):
        raise ValueError('tar_collection is neither a string nor iterable')

    src_collection = list(src_collection)
    tar_collection = list(tar_collection)

    pairwise_values = []

    for src in src_collection:
        for tar in tar_collection:
            pairwise_values.append(metric(src, tar))
            if symmetric:
                pairwise_values.append(metric(tar, src))

    return (
        max(pairwise_values),
        min(pairwise_values),
        mean_func(pairwise_values),
        std(pairwise_values, mean_func, 0),
    )

Calculate the pairwise similarity statistics a collection of strings .


def _sb_r1(self, term, r1_prefixes=None):
        """Return the R1 region, as defined in the Porter2 specification.

        Parameters
        ----------
        term : str
            The term to examine
        r1_prefixes : set
            Prefixes to consider

        Returns
        -------
        int
            Length of the R1 region

        """
        vowel_found = False
        if hasattr(r1_prefixes, '__iter__'):
            for prefix in r1_prefixes:
                if term[: len(prefix)] == prefix:
                    return len(prefix)

        for i in range(len(term)):
            if not vowel_found and term[i] in self._vowels:
                vowel_found = True
            elif vowel_found and term[i] not in self._vowels:
                return i + 1
        return len(term)

def _sb_r1 ( self , term , r1_prefixes = None ) : vowel_found = False if hasattr ( r1_prefixes , '__iter__' ) : for prefix in r1_prefixes : if term [ : len ( prefix ) ] == prefix : return len ( prefix ) for i in range ( len ( term ) ) : if not vowel_found and term [ i ] in self . _vowels : vowel_found = True elif vowel_found and term [ i ] not in self . _vowels : return i + 1 return len ( term )

Return the R1 region as defined in the Porter2 specification .


def _sb_r2(self, term, r1_prefixes=None):
        """Return the R2 region, as defined in the Porter2 specification.

        Parameters
        ----------
        term : str
            The term to examine
        r1_prefixes : set
            Prefixes to consider

        Returns
        -------
        int
            Length of the R1 region

        """
        r1_start = self._sb_r1(term, r1_prefixes)
        return r1_start + self._sb_r1(term[r1_start:])

def _sb_r2 ( self , term , r1_prefixes = None ) : r1_start = self . _sb_r1 ( term , r1_prefixes ) return r1_start + self . _sb_r1 ( term [ r1_start : ] )

Return the R2 region as defined in the Porter2 specification .


def _sb_ends_in_short_syllable(self, term):
        """Return True iff term ends in a short syllable.

        (...according to the Porter2 specification.)

        NB: This is akin to the CVC test from the Porter stemmer. The
        description is unfortunately poor/ambiguous.

        Parameters
        ----------
        term : str
            The term to examine

        Returns
        -------
        bool
            True iff term ends in a short syllable

        """
        if not term:
            return False
        if len(term) == 2:
            if term[-2] in self._vowels and term[-1] not in self._vowels:
                return True
        elif len(term) >= 3:
            if (
                term[-3] not in self._vowels
                and term[-2] in self._vowels
                and term[-1] in self._codanonvowels
            ):
                return True
        return False

def _sb_ends_in_short_syllable ( self , term ) : if not term : return False if len ( term ) == 2 : if term [ - 2 ] in self . _vowels and term [ - 1 ] not in self . _vowels : return True elif len ( term ) >= 3 : if ( term [ - 3 ] not in self . _vowels and term [ - 2 ] in self . _vowels and term [ - 1 ] in self . _codanonvowels ) : return True return False

Return True iff term ends in a short syllable .


def _sb_short_word(self, term, r1_prefixes=None):
        """Return True iff term is a short word.

        (...according to the Porter2 specification.)

        Parameters
        ----------
        term : str
            The term to examine
        r1_prefixes : set
            Prefixes to consider

        Returns
        -------
        bool
            True iff term is a short word

        """
        if self._sb_r1(term, r1_prefixes) == len(
            term
        ) and self._sb_ends_in_short_syllable(term):
            return True
        return False

def _sb_short_word ( self , term , r1_prefixes = None ) : if self . _sb_r1 ( term , r1_prefixes ) == len ( term ) and self . _sb_ends_in_short_syllable ( term ) : return True return False


def _sb_has_vowel(self, term):
        """Return Porter helper function _sb_has_vowel value.

        Parameters
        ----------
        term : str
            The term to examine

        Returns
        -------
        bool
            True iff a vowel exists in the term (as defined in the Porter
            stemmer definition)

        """
        for letter in term:
            if letter in self._vowels:
                return True
        return False

def _sb_has_vowel ( self , term ) : for letter in term : if letter in self . _vowels : return True return False

Return Porter helper function _sb_has_vowel value .