def run(self, model, tol=0.001, max_iters=999, verbose=True):
        """
        Runs EM iterations

        :param model: Normalization model (1: Gene->Allele->Isoform, 2: Gene->Isoform->Allele, 3: Gene->Isoform*Allele, 4: Gene*Isoform*Allele)
        :param tol: Tolerance for termination
        :param max_iters: Maximum number of iterations until termination
        :param verbose: Display information on how EM is running
        :return: Nothing (as it performs in-place operations)
        """
        orig_err_states = np.seterr(all='raise')
        np.seterr(under='ignore')
        if verbose:
            print
            print "Iter No  Time (hh:mm:ss)    Total change (TPM)  "
            print "-------  ---------------  ----------------------"
        num_iters = 0
        err_sum = 1000000.0
        time0 = time.time()
        target_err = 1000000.0 * tol
        while err_sum > target_err and num_iters < max_iters:
            prev_isoform_expression = self.get_allelic_expression().sum(axis=0)
            prev_isoform_expression *= (1000000.0 / prev_isoform_expression.sum())
            self.update_allelic_expression(model=model)
            curr_isoform_expression = self.get_allelic_expression().sum(axis=0)
            curr_isoform_expression *= (1000000.0 / curr_isoform_expression.sum())
            err = np.abs(curr_isoform_expression - prev_isoform_expression)
            err_sum = err.sum()
            num_iters += 1
            if verbose:
                time1 = time.time()
                delmin, s = divmod(int(time1 - time0), 60)
                h, m = divmod(delmin, 60)
                print " %5d      %4d:%02d:%02d     %9.1f / 1000000" % (num_iters, h, m, s, err_sum)


def report_read_counts(self, filename, grp_wise=False, reorder='as-is', notes=None):
        """
        Exports expected read counts

        :param filename: File name for output
        :param grp_wise: whether the report is at isoform level or gene level
        :param reorder: whether the report should be either 'decreasing' or 'increasing' order or just 'as-is'
        :return: Nothing but the method writes a file
        """
        expected_read_counts = self.probability.sum(axis=APM.Axis.READ)
        if grp_wise:
            lname = self.probability.gname
            expected_read_counts = expected_read_counts * self.grp_conv_mat
        else:
            lname = self.probability.lname
        total_read_counts = expected_read_counts.sum(axis=0)
        if reorder == 'decreasing':
            report_order = np.argsort(total_read_counts.flatten())
            report_order = report_order[::-1]
        elif reorder == 'increasing':
            report_order = np.argsort(total_read_counts.flatten())
        elif reorder == 'as-is':
            report_order = np.arange(len(lname))  # report in the original locus order
        cntdata = np.vstack((expected_read_counts, total_read_counts))
        fhout = open(filename, 'w')
        fhout.write("locus\t" + "\t".join(self.probability.hname) + "\ttotal")
        if notes is not None:
            fhout.write("\tnotes")
        fhout.write("\n")
        for locus_id in report_order:
            lname_cur = lname[locus_id]
            fhout.write("\t".join([lname_cur] + map(str, cntdata[:, locus_id].ravel())))
            if notes is not None:
                fhout.write("\t%s" % notes[lname_cur])
            fhout.write("\n")
        fhout.close()

def report_read_counts ( self , filename , grp_wise = False , reorder = 'as-is' , notes = None ) : expected_read_counts = self . probability . sum ( axis = APM . Axis . READ ) if grp_wise : lname = self . probability . gname expected_read_counts = expected_read_counts * self . grp_conv_mat else : lname = self . probability . lname total_read_counts = expected_read_counts . sum ( axis = 0 ) if reorder == 'decreasing' : report_order = np . argsort ( total_read_counts . flatten ( ) ) report_order = report_order [ : : - 1 ] elif reorder == 'increasing' : report_order = np . argsort ( total_read_counts . flatten ( ) ) elif reorder == 'as-is' : report_order = np . arange ( len ( lname ) ) cntdata = np . vstack ( ( expected_read_counts , total_read_counts ) ) fhout = open ( filename , 'w' ) fhout . write ( "locus\t" + "\t" . join ( self . probability . hname ) + "\ttotal" ) if notes is not None : fhout . write ( "\tnotes" ) fhout . write ( "\n" ) for locus_id in report_order : lname_cur = lname [ locus_id ] fhout . write ( "\t" . join ( [ lname_cur ] + map ( str , cntdata [ : , locus_id ] . ravel ( ) ) ) ) if notes is not None : fhout . write ( "\t%s" % notes [ lname_cur ] ) fhout . write ( "\n" ) fhout . close ( )


def report_depths(self, filename, tpm=True, grp_wise=False, reorder='as-is', notes=None):
        """
        Exports expected depths

        :param filename: File name for output
        :param grp_wise: whether the report is at isoform level or gene level
        :param reorder: whether the report should be either 'decreasing' or 'increasing' order or just 'as-is'
        :return: Nothing but the method writes a file
        """
        if grp_wise:
            lname = self.probability.gname
            depths = self.allelic_expression * self.grp_conv_mat
        else:
            lname = self.probability.lname
            depths = self.allelic_expression
        if tpm:
            depths *= (1000000.0 / depths.sum())
        total_depths = depths.sum(axis=0)
        if reorder == 'decreasing':
            report_order = np.argsort(total_depths.flatten())
            report_order = report_order[::-1]
        elif reorder == 'increasing':
            report_order = np.argsort(total_depths.flatten())
        elif reorder == 'as-is':
            report_order = np.arange(len(lname))  # report in the original locus order
        cntdata = np.vstack((depths, total_depths))
        fhout = open(filename, 'w')
        fhout.write("locus\t" + "\t".join(self.probability.hname) + "\ttotal")
        if notes is not None:
            fhout.write("\tnotes")
        fhout.write("\n")
        for locus_id in report_order:
            lname_cur = lname[locus_id]
            fhout.write("\t".join([lname_cur] + map(str, cntdata[:, locus_id].ravel())))
            if notes is not None:
                fhout.write("\t%s" % notes[lname_cur])
            fhout.write("\n")
        fhout.close()

def report_depths ( self , filename , tpm = True , grp_wise = False , reorder = 'as-is' , notes = None ) : if grp_wise : lname = self . probability . gname depths = self . allelic_expression * self . grp_conv_mat else : lname = self . probability . lname depths = self . allelic_expression if tpm : depths *= ( 1000000.0 / depths . sum ( ) ) total_depths = depths . sum ( axis = 0 ) if reorder == 'decreasing' : report_order = np . argsort ( total_depths . flatten ( ) ) report_order = report_order [ : : - 1 ] elif reorder == 'increasing' : report_order = np . argsort ( total_depths . flatten ( ) ) elif reorder == 'as-is' : report_order = np . arange ( len ( lname ) ) cntdata = np . vstack ( ( depths , total_depths ) ) fhout = open ( filename , 'w' ) fhout . write ( "locus\t" + "\t" . join ( self . probability . hname ) + "\ttotal" ) if notes is not None : fhout . write ( "\tnotes" ) fhout . write ( "\n" ) for locus_id in report_order : lname_cur = lname [ locus_id ] fhout . write ( "\t" . join ( [ lname_cur ] + map ( str , cntdata [ : , locus_id ] . ravel ( ) ) ) ) if notes is not None : fhout . write ( "\t%s" % notes [ lname_cur ] ) fhout . write ( "\n" ) fhout . close ( )


def export_posterior_probability(self, filename, title="Posterior Probability"):
        """
        Writes the posterior probability of read origin

        :param filename: File name for output
        :param title: The title of the posterior probability matrix
        :return: Nothing but the method writes a file in EMASE format (PyTables)
        """
        self.probability.save(h5file=filename, title=title)

def
export_posterior_probability
(
self
,
filename
,
title
=
"Posterior Probability"
)
:
self
.
probability
.
save
(
h5file
=
filename
,
title
=
title
)


def print_read(self, rid):
        """
        Prints nonzero rows of the read wanted

        """
        if self.rname is not None:
            print self.rname[rid]
            print '--'
        r = self.get_read_data(rid)
        aligned_loci = np.unique(r.nonzero()[1])
        for locus in aligned_loci:
            nzvec = r[:, locus].todense().transpose()[0].A.flatten()
            if self.lname is not None:
                print self.lname[locus],
            else:
                print locus,
            print nzvec

def print_read ( self , rid ) : if self . rname is not None : print self . rname [ rid ] print '--' r = self . get_read_data ( rid ) aligned_loci = np . unique ( r . nonzero ( ) [ 1 ] ) for locus in aligned_loci : nzvec = r [ : , locus ] . todense ( ) . transpose ( ) [ 0 ] . A . flatten ( ) if self . lname is not None : print self . lname [ locus ] , else : print locus , print nzvec


def _roman(data, scheme_map, **kw):
  """Transliterate `data` with the given `scheme_map`. This function is used
  when the source scheme is a Roman scheme.

  :param data: the data to transliterate
  :param scheme_map: a dict that maps between characters in the old scheme
                     and characters in the new scheme
  """
  vowels = scheme_map.vowels
  marks = scheme_map.marks
  virama = scheme_map.virama
  consonants = scheme_map.consonants
  non_marks_viraama = scheme_map.non_marks_viraama
  max_key_length_from_scheme = scheme_map.max_key_length_from_scheme
  to_roman = scheme_map.to_scheme.is_roman

  togglers = kw.pop('togglers', set())
  suspend_on = kw.pop('suspend_on', set())
  suspend_off = kw.pop('suspend_off', set())
  if kw:
    raise TypeError('Unexpected keyword argument %s' % list(kw.keys())[0])

  buf = []
  i = 0
  had_consonant = found = False
  len_data = len(data)
  append = buf.append

  # If true, don't transliterate. The toggle token is discarded.
  toggled = False
  # If true, don't transliterate. The suspend token is retained.
  # `suspended` overrides `toggled`.
  suspended = False

  while i <= len_data:
    # The longest token in the source scheme has length `max_key_length_from_scheme`. Iterate
    # over `data` while taking `max_key_length_from_scheme` characters at a time. If we don`t
    # find the character group in our scheme map, lop off a character and
    # try again.
    #
    # If we've finished reading through `data`, then `token` will be empty
    # and the loop below will be skipped.
    token = data[i:i + max_key_length_from_scheme]

    while token:
      if token in togglers:
        toggled = not toggled
        i += 2  # skip over the token
        found = True  # force the token to fill up again
        break

      if token in suspend_on:
        suspended = True
      elif token in suspend_off:
        suspended = False

      if toggled or suspended:
        token = token[:-1]
        continue

      # Catch the pattern CV, where C is a consonant and V is a vowel.
      # V should be rendered as a vowel mark, a.k.a. a "dependent"
      # vowel. But due to the nature of Brahmic scripts, 'a' is implicit
      # and has no vowel mark. If we see 'a', add nothing.
      if had_consonant and token in vowels:
        mark = marks.get(token, '')
        if mark:
          append(mark)
        elif to_roman:
          append(vowels[token])
        found = True

      # Catch any non_marks_viraama character, including consonants, punctuation,
      # and regular vowels. Due to the implicit 'a', we must explicitly
      # end any lingering consonants before we can handle the current
      # token.
      elif token in non_marks_viraama:
        if had_consonant:
          append(virama[''])
        append(non_marks_viraama[token])
        found = True

      if found:
        had_consonant = token in consonants
        i += len(token)
        break
      else:
        token = token[:-1]

    # We've exhausted the token; this must be some other character. Due to
    # the implicit 'a', we must explicitly end any lingering consonants
    # before we can handle the current token.
    if not found:
      if had_consonant:
        append(virama[''])
      if i < len_data:
        append(data[i])
        had_consonant = False
      i += 1

    found = False

  return ''.join(buf)

def
_roman
(
data
,
scheme_map
,
**
kw
)
:
vowels
=
scheme_map
.
vowels
marks
=
scheme_map
.
marks
virama
=
scheme_map
.
virama
consonants
=
scheme_map
.
consonants
non_marks_viraama
=
scheme_map
.
non_marks_viraama
max_key_length_from_scheme
=
scheme_map
.
max_key_length_from_scheme
to_roman
=
scheme_map
.
to_scheme
.
is_roman
togglers
=
kw
.
pop
(
'togglers'
,
set
(
)
)
suspend_on
=
kw
.
pop
(
'suspend_on'
,
set
(
)
)
suspend_off
=
kw
.
pop
(
'suspend_off'
,
set
(
)
)
if
kw
:
raise
TypeError
(
'Unexpected keyword argument %s'
%
list
(
kw
.
keys
(
)
)
[
0
]
)
buf
=
[
]
i
=
0
had_consonant
=
found
=
False
len_data
=
len
(
data
)
append
=
buf
.
append
toggled
=
False
suspended
=
False
while
i
<=
len_data
:
token
=
data
[
i
:
i
+
max_key_length_from_scheme
]
while
token
:
if
token
in
togglers
:
toggled
=
not
toggled
i
+=
2
found
=
True
break
if
token
in
suspend_on
:
suspended
=
True
elif
token
in
suspend_off
:
suspended
=
False
if
toggled
or
suspended
:
token
=
token
[
:
1
]
continue
if
had_consonant
and
token
in
vowels
:
mark
=
marks
.
get
(
token
,
''
)
if
mark
:
append
(
mark
)
elif
to_roman
:
append
(
vowels
[
token
]
)
found
=
True
elif
token
in
non_marks_viraama
:
if
had_consonant
:
append
(
virama
[
''
]
)
append
(
non_marks_viraama
[
token
]
)
found
=
True
if
found
:
had_consonant
=
token
in
consonants
i
+=
len
(
token
)
break
else
:
token
=
token
[
:
1
]
if
not
found
:
if
had_consonant
:
append
(
virama
[
''
]
)
if
i
<
len_data
:
append
(
data
[
i
]
)
had_consonant
=
False
i
+=
1
found
=
False
return
''
.
join
(
buf
)

Transliterate data with the given scheme_map . This function is used when the source scheme is a Roman scheme .


def _brahmic(data, scheme_map, **kw):
  """Transliterate `data` with the given `scheme_map`. This function is used
  when the source scheme is a Brahmic scheme.

  :param data: the data to transliterate
  :param scheme_map: a dict that maps between characters in the old scheme
                     and characters in the new scheme
  """
  if scheme_map.from_scheme.name == northern.GURMUKHI:
    data = northern.GurmukhiScheme.replace_tippi(text=data)
  marks = scheme_map.marks
  virama = scheme_map.virama
  consonants = scheme_map.consonants
  non_marks_viraama = scheme_map.non_marks_viraama
  to_roman = scheme_map.to_scheme.is_roman
  max_key_length_from_scheme = scheme_map.max_key_length_from_scheme

  buf = []
  i = 0
  to_roman_had_consonant = found = False
  append = buf.append
  # logging.debug(pprint.pformat(scheme_map.consonants))

  # We dont just translate each brAhmic character one after another in order to prefer concise transliterations when possible - for example ज्ञ -> jn in optitrans rather than j~n.
  while i <= len(data):
    # The longest token in the source scheme has length `max_key_length_from_scheme`. Iterate
    # over `data` while taking `max_key_length_from_scheme` characters at a time. If we don`t
    # find the character group in our scheme map, lop off a character and
    # try again.
    #
    # If we've finished reading through `data`, then `token` will be empty
    # and the loop below will be skipped.
    token = data[i:i + max_key_length_from_scheme]

    while token:
      if len(token) == 1:
        if token in marks:
          append(marks[token])
          found = True
        elif token in virama:
          append(virama[token])
          found = True
        else:
          if to_roman_had_consonant:
            append('a')
          append(non_marks_viraama.get(token, token))
          found = True
      else:
        if token in non_marks_viraama:
          if to_roman_had_consonant:
            append('a')
          append(non_marks_viraama.get(token))
          found = True

      if found:
        to_roman_had_consonant = to_roman and token in consonants
        i += len(token)
        break
      else:
        token = token[:-1]

    # Continuing the outer while loop.
    # We've exhausted the token; this must be some other character. Due to
    # the implicit 'a', we must explicitly end any lingering consonants
    # before we can handle the current token.
    if not found:
      if to_roman_had_consonant:
        append(next(iter(virama.values())))
      if i < len(data):
        append(data[i])
        to_roman_had_consonant = False
      i += 1

    found = False

  if to_roman_had_consonant:
    append('a')
  return ''.join(buf)

def _brahmic ( data , scheme_map , ** kw ) : if scheme_map . from_scheme . name == northern . GURMUKHI : data = northern . GurmukhiScheme . replace_tippi ( text = data ) marks = scheme_map . marks virama = scheme_map . virama consonants = scheme_map . consonants non_marks_viraama = scheme_map . non_marks_viraama to_roman = scheme_map . to_scheme . is_roman max_key_length_from_scheme = scheme_map . max_key_length_from_scheme buf = [ ] i = 0 to_roman_had_consonant = found = False append = buf . append while i <= len ( data ) : token = data [ i : i + max_key_length_from_scheme ] while token : if len ( token ) == 1 : if token in marks : append ( marks [ token ] ) found = True elif token in virama : append ( virama [ token ] ) found = True else : if to_roman_had_consonant : append ( 'a' ) append ( non_marks_viraama . get ( token , token ) ) found = True else : if token in non_marks_viraama : if to_roman_had_consonant : append ( 'a' ) append ( non_marks_viraama . get ( token ) ) found = True if found : to_roman_had_consonant = to_roman and token in consonants i += len ( token ) break else : token = token [ : - 1 ] if not found : if to_roman_had_consonant : append ( next ( iter ( virama . values ( ) ) ) ) if i < len ( data ) : append ( data [ i ] ) to_roman_had_consonant = False i += 1 found = False if to_roman_had_consonant : append ( 'a' ) return '' . join ( buf )

Transliterate data with the given scheme_map . This function is used when the source scheme is a Brahmic scheme .


def detect(text):
  """Detect the input's transliteration scheme.

    :param text: some text data, either a `unicode` or a `str` encoded
                 in UTF-8.
    """
  if sys.version_info < (3, 0):
    # Verify encoding
    try:
      text = text.decode('utf-8')
    except UnicodeError:
      pass

  # Brahmic schemes are all within a specific range of code points.
  for L in text:
    code = ord(L)
    if code >= BRAHMIC_FIRST_CODE_POINT:
      for name, start_code in BLOCKS:
        if start_code <= code <= BRAHMIC_LAST_CODE_POINT:
          return name

  # Romanizations
  if Regex.IAST_OR_KOLKATA_ONLY.search(text):
    if Regex.KOLKATA_ONLY.search(text):
      return Scheme.Kolkata
    else:
      return Scheme.IAST

  if Regex.ITRANS_ONLY.search(text):
    return Scheme.ITRANS

  if Regex.SLP1_ONLY.search(text):
    return Scheme.SLP1

  if Regex.VELTHUIS_ONLY.search(text):
    return Scheme.Velthuis

  if Regex.ITRANS_OR_VELTHUIS_ONLY.search(text):
    return Scheme.ITRANS

  return Scheme.HK

def detect ( text ) : if sys . version_info < ( 3 , 0 ) : try : text = text . decode ( 'utf-8' ) except UnicodeError : pass for L in text : code = ord ( L ) if code >= BRAHMIC_FIRST_CODE_POINT : for name , start_code in BLOCKS : if start_code <= code <= BRAHMIC_LAST_CODE_POINT : return name if Regex . IAST_OR_KOLKATA_ONLY . search ( text ) : if Regex . KOLKATA_ONLY . search ( text ) : return Scheme . Kolkata else : return Scheme . IAST if Regex . ITRANS_ONLY . search ( text ) : return Scheme . ITRANS if Regex . SLP1_ONLY . search ( text ) : return Scheme . SLP1 if Regex . VELTHUIS_ONLY . search ( text ) : return Scheme . Velthuis if Regex . ITRANS_OR_VELTHUIS_ONLY . search ( text ) : return Scheme . ITRANS return Scheme . HK

Detect the input s transliteration scheme .


def _setup():
    """Add a variety of default schemes."""
    s = str.split
    if sys.version_info < (3, 0):
        # noinspection PyUnresolvedReferences
        s = unicode.split

    def pop_all(some_dict, some_list):
        for scheme in some_list:
            some_dict.pop(scheme)
    global SCHEMES
    SCHEMES = copy.deepcopy(sanscript.SCHEMES)
    pop_all(SCHEMES, [sanscript.ORIYA, sanscript.BENGALI, sanscript.GUJARATI])
    SCHEMES[HK].update({
        'vowels': s("""a A i I u U R RR lR lRR E ai O au""") + s("""e o"""),
        'marks': s("""A i I u U R RR lR lRR E ai O au""") + s("""e o"""),
        'consonants': sanscript.SCHEMES[HK]['consonants'] + s("""n2 r2 zh""")
    })
    SCHEMES[ITRANS].update({
        'vowels': s("""a A i I u U R RR LLi LLI E ai O au""") + s("""e o"""),
        'marks': s("""A i I u U R RR LLi LLI E ai O au""") + s("""e o"""),
        'consonants': sanscript.SCHEMES[ITRANS]['consonants'] + s("""n2 r2 zh""")
    })
    pop_all(SCHEMES[ITRANS].synonym_map, s("""e o"""))
    SCHEMES[OPTITRANS].update({
        'vowels': s("""a A i I u U R RR LLi LLI E ai O au""") + s("""e o"""),
        'marks': s("""A i I u U R RR LLi LLI E ai O au""") + s("""e o"""),
        'consonants': sanscript.SCHEMES[OPTITRANS]['consonants'] + s("""n2 r2 zh""")
    })
    pop_all(SCHEMES[OPTITRANS].synonym_map, s("""e o"""))

def _setup ( ) : s = str . split if sys . version_info < ( 3 , 0 ) : s = unicode . split def pop_all ( some_dict , some_list ) : for scheme in some_list : some_dict . pop ( scheme ) global SCHEMES SCHEMES = copy . deepcopy ( sanscript . SCHEMES ) pop_all ( SCHEMES , [ sanscript . ORIYA , sanscript . BENGALI , sanscript . GUJARATI ] ) SCHEMES [ HK ] . update ( { 'vowels' : s ( ) + s ( ) , 'marks' : s ( ) + s ( ) , 'consonants' : sanscript . SCHEMES [ HK ] [ 'consonants' ] + s ( ) } ) SCHEMES [ ITRANS ] . update ( { 'vowels' : s ( ) + s ( ) , 'marks' : s ( ) + s ( ) , 'consonants' : sanscript . SCHEMES [ ITRANS ] [ 'consonants' ] + s ( ) } ) pop_all ( SCHEMES [ ITRANS ] . synonym_map , s ( ) ) SCHEMES [ OPTITRANS ] . update ( { 'vowels' : s ( ) + s ( ) , 'marks' : s ( ) + s ( ) , 'consonants' : sanscript . SCHEMES [ OPTITRANS ] [ 'consonants' ] + s ( ) } ) pop_all ( SCHEMES [ OPTITRANS ] . synonym_map , s ( ) )


def to_utf8(y):
    """
    converts an array of integers to utf8 string
    """

    out = []

    for x in y:

        if x < 0x080:
            out.append(x)
        elif x < 0x0800:
            out.append((x >> 6) | 0xC0)
            out.append((x & 0x3F) | 0x80)
        elif x < 0x10000:
            out.append((x >> 12) | 0xE0)
            out.append(((x >> 6) & 0x3F) | 0x80)
            out.append((x & 0x3F) | 0x80)
        else:
            out.append((x >> 18) | 0xF0)
            out.append((x >> 12) & 0x3F)
            out.append(((x >> 6) & 0x3F) | 0x80)
            out.append((x & 0x3F) | 0x80)

    return ''.join(map(chr, out))

def to_utf8 ( y ) : out = [ ] for x in y : if x < 0x080 : out . append ( x ) elif x < 0x0800 : out . append ( ( x >> 6 ) | 0xC0 ) out . append ( ( x & 0x3F ) | 0x80 ) elif x < 0x10000 : out . append ( ( x >> 12 ) | 0xE0 ) out . append ( ( ( x >> 6 ) & 0x3F ) | 0x80 ) out . append ( ( x & 0x3F ) | 0x80 ) else : out . append ( ( x >> 18 ) | 0xF0 ) out . append ( ( x >> 12 ) & 0x3F ) out . append ( ( ( x >> 6 ) & 0x3F ) | 0x80 ) out . append ( ( x & 0x3F ) | 0x80 ) return '' . join ( map ( chr , out ) )


def set_script(self, i):
        """
        set the value of delta to reflect the current codepage

        """

        if i in range(1, 10):
            n = i - 1
        else:
            raise IllegalInput("Invalid Value for ATR %s" % (hex(i)))

        if n > -1: # n = -1 is the default script ..
            self.curr_script = n
            self.delta = n * DELTA

        return

def
set_script
(
self
,
i
)
:
if
i
in
range
(
1
,
10
)
:
n
=
i
1
else
:
raise
IllegalInput
(
"Invalid Value for ATR %s"
%
(
hex
(
i
)
)
)
if
n
>
1
:
self
.
curr_script
=
n
self
.
delta
=
n
*
DELTA
return


def _unrecognised(chr):
    """ Handle unrecognised characters. """
    if options['handleUnrecognised'] == UNRECOGNISED_ECHO:
        return chr
    elif options['handleUnrecognised'] == UNRECOGNISED_SUBSTITUTE:
        return options['substituteChar']
    else:
        raise (KeyError, chr)

def _unrecognised ( chr ) : if options [ 'handleUnrecognised' ] == UNRECOGNISED_ECHO : return chr elif options [ 'handleUnrecognised' ] == UNRECOGNISED_SUBSTITUTE : return options [ 'substituteChar' ] else : raise ( KeyError , chr )


def _equivalent(self, char, prev, next, implicitA):
        """ Transliterate a Latin character equivalent to Devanagari.

        Add VIRAMA for ligatures.
        Convert standalone to dependent vowels.

        """
        result = []
        if char.isVowel == False:
            result.append(char.chr)
            if char.isConsonant \
            and ((next is not None and next.isConsonant) \
            or next is None):
                result.append(DevanagariCharacter._VIRAMA)
        else:
            if prev is None or prev.isConsonant == False:
                result.append(char.chr)
            else:
                if char._dependentVowel is not None:
                    result.append(char._dependentVowel)
        return result

def _equivalent ( self , char , prev , next , implicitA ) : result = [ ] if char . isVowel == False : result . append ( char . chr ) if char . isConsonant and ( ( next is not None and next . isConsonant ) or next is None ) : result . append ( DevanagariCharacter . _VIRAMA ) else : if prev is None or prev . isConsonant == False : result . append ( char . chr ) else : if char . _dependentVowel is not None : result . append ( char . _dependentVowel ) return result

Transliterate a Latin character equivalent to Devanagari . Add VIRAMA for ligatures . Convert standalone to dependent vowels .


def from_devanagari(self, data):
        """A convenience method"""
        from indic_transliteration import sanscript
        return sanscript.transliterate(data=data, _from=sanscript.DEVANAGARI, _to=self.name)

def from_devanagari ( self , data ) : from indic_transliteration import sanscript return sanscript . transliterate ( data = data , _from = sanscript . DEVANAGARI , _to = self . name )


def generate(grammar=None, num=1, output=sys.stdout, max_recursion=10, seed=None):
    """Load and generate ``num`` number of top-level rules from the specified grammar.

    :param list grammar: The grammar file to load and generate data from
    :param int num: The number of times to generate data
    :param output: The output destination (an open, writable stream-type object. default=``sys.stdout``)
    :param int max_recursion: The maximum reference-recursion when generating data (default=``10``)
    :param int seed: The seed to initialize the PRNG with. If None, will not initialize it.
    """
    if seed is not None:
        gramfuzz.rand.seed(seed)

    fuzzer = gramfuzz.GramFuzzer()
    fuzzer.load_grammar(grammar)

    cat_group = os.path.basename(grammar).replace(".py", "")

    results = fuzzer.gen(cat_group=cat_group, num=num, max_recursion=max_recursion)
    for res in results:
        output.write(res)

def generate ( grammar = None , num = 1 , output = sys . stdout , max_recursion = 10 , seed = None ) : if seed is not None : gramfuzz . rand . seed ( seed ) fuzzer = gramfuzz . GramFuzzer ( ) fuzzer . load_grammar ( grammar ) cat_group = os . path . basename ( grammar ) . replace ( ".py" , "" ) results = fuzzer . gen ( cat_group = cat_group , num = num , max_recursion = max_recursion ) for res in results : output . write ( res )

Load and generate num number of top - level rules from the specified grammar .


def build(self, pre=None, shortest=False):
        """Build the ``Quote`` instance

        :param list pre: The prerequisites list
        :param bool shortest: Whether or not the shortest reference-chain (most minimal) version of the field should be generated.
        """
        res = super(Q, self).build(pre, shortest=shortest)

        if self.escape:
            return repr(res)
        elif self.html_js_escape:
            return ("'" + res.encode("string_escape").replace("<", "\\x3c").replace(">", "\\x3e") + "'")
        else:
            return "".join([self.quote, res, self.quote])

def build ( self , pre = None , shortest = False ) : res = super ( Q , self ) . build ( pre , shortest = shortest ) if self . escape : return repr ( res ) elif self . html_js_escape : return ( "'" + res . encode ( "string_escape" ) . replace ( "<" , "\\x3c" ) . replace ( ">" , "\\x3e" ) + "'" ) else : return "" . join ( [ self . quote , res , self . quote ] )


def make_present_participles(verbs):
    """Make the list of verbs into present participles

    E.g.:

        empower -> empowering
        drive -> driving
    """
    res = []
    for verb in verbs:
        parts = verb.split()
        if parts[0].endswith("e"):
            parts[0] = parts[0][:-1] + "ing"
        else:
            parts[0] = parts[0] + "ing"
        res.append(" ".join(parts))
    return res

def
make_present_participles
(
verbs
)
:
res
=
[
]
for
verb
in
verbs
:
parts
=
verb
.
split
(
)
if
parts
[
0
]
.
endswith
(
"e"
)
:
parts
[
0
]
=
parts
[
0
]
[
:
1
]
+
"ing"
else
:
parts
[
0
]
=
parts
[
0
]
+
"ing"
res
.
append
(
" "
.
join
(
parts
)
)
return
res


def clear_sent_messages(self, offset=None):
        """ Deletes sent MailerMessage records """
        if offset is None:
            offset = getattr(settings, 'MAILQUEUE_CLEAR_OFFSET', defaults.MAILQUEUE_CLEAR_OFFSET)

        if type(offset) is int:
            offset = datetime.timedelta(hours=offset)

        delete_before = timezone.now() - offset
        self.filter(sent=True, last_attempt__lte=delete_before).delete()

def clear_sent_messages ( self , offset = None ) : if offset is None : offset = getattr ( settings , 'MAILQUEUE_CLEAR_OFFSET' , defaults . MAILQUEUE_CLEAR_OFFSET ) if type ( offset ) is int : offset = datetime . timedelta ( hours = offset ) delete_before = timezone . now ( ) - offset self . filter ( sent = True , last_attempt__lte = delete_before ) . delete ( )


def _loadNamelistIncludes(item, unique_glyphs, cache):
  """Load the includes of an encoding Namelist files.

  This is an implementation detail of readNamelist.
  """
  includes = item["includes"] = []
  charset = item["charset"] = set() | item["ownCharset"]

  noCharcode = item["noCharcode"] = set() | item["ownNoCharcode"]

  dirname =  os.path.dirname(item["fileName"])
  for include in item["header"]["includes"]:
    includeFile = os.path.join(dirname, include)
    try:
      includedItem = readNamelist(includeFile, unique_glyphs, cache)
    except NamelistRecursionError:
      continue
    if includedItem in includes:
      continue
    includes.append(includedItem)
    charset |= includedItem["charset"]
    noCharcode |= includedItem["ownNoCharcode"]
  return item

def _loadNamelistIncludes ( item , unique_glyphs , cache ) : includes = item [ "includes" ] = [ ] charset = item [ "charset" ] = set ( ) | item [ "ownCharset" ] noCharcode = item [ "noCharcode" ] = set ( ) | item [ "ownNoCharcode" ] dirname = os . path . dirname ( item [ "fileName" ] ) for include in item [ "header" ] [ "includes" ] : includeFile = os . path . join ( dirname , include ) try : includedItem = readNamelist ( includeFile , unique_glyphs , cache ) except NamelistRecursionError : continue if includedItem in includes : continue includes . append ( includedItem ) charset |= includedItem [ "charset" ] noCharcode |= includedItem [ "ownNoCharcode" ] return item

Load the includes of an encoding Namelist files .


def __readNamelist(cache, filename, unique_glyphs):
  """Return a dict with the data of an encoding Namelist file.

  This is an implementation detail of readNamelist.
  """
  if filename in cache:
    item = cache[filename]
  else:
    cps, header, noncodes = parseNamelist(filename)
    item = {
      "fileName": filename
    , "ownCharset": cps
    , "header": header
    , "ownNoCharcode": noncodes
    , "includes": None # placeholder
    , "charset": None # placeholder
    , "noCharcode": None
    }
    cache[filename] = item

  if unique_glyphs or item["charset"] is not None:
    return item

  # full-charset/includes are requested and not cached yet
  _loadNamelistIncludes(item, unique_glyphs, cache)
  return item

def __readNamelist ( cache , filename , unique_glyphs ) : if filename in cache : item = cache [ filename ] else : cps , header , noncodes = parseNamelist ( filename ) item = { "fileName" : filename , "ownCharset" : cps , "header" : header , "ownNoCharcode" : noncodes , "includes" : None , "charset" : None , "noCharcode" : None } cache [ filename ] = item if unique_glyphs or item [ "charset" ] is not None : return item _loadNamelistIncludes ( item , unique_glyphs , cache ) return item

Return a dict with the data of an encoding Namelist file .


def _readNamelist(currentlyIncluding, cache, namFilename, unique_glyphs):
  """ Detect infinite recursion and prevent it.

  This is an implementation detail of readNamelist.

  Raises NamelistRecursionError if namFilename is in the process of being included
  """
  # normalize
  filename = os.path.abspath(os.path.normcase(namFilename))
  if filename in currentlyIncluding:
    raise NamelistRecursionError(filename)
  currentlyIncluding.add(filename)
  try:
    result = __readNamelist(cache, filename, unique_glyphs)
  finally:
    currentlyIncluding.remove(filename)
  return result

def _readNamelist ( currentlyIncluding , cache , namFilename , unique_glyphs ) : filename = os . path . abspath ( os . path . normcase ( namFilename ) ) if filename in currentlyIncluding : raise NamelistRecursionError ( filename ) currentlyIncluding . add ( filename ) try : result = __readNamelist ( cache , filename , unique_glyphs ) finally : currentlyIncluding . remove ( filename ) return result

Detect infinite recursion and prevent it .


def codepointsInNamelist(namFilename, unique_glyphs=False, cache=None):
  """Returns the set of codepoints contained in a given Namelist file.

  This is a replacement CodepointsInSubset and implements the "#$ include"
  header format.

  Args:
    namFilename: The path to the  Namelist file.
    unique_glyphs: Optional, whether to only include glyphs unique to subset.
  Returns:
    A set containing the glyphs in the subset.
  """
  key = 'charset' if not unique_glyphs else 'ownCharset'

  internals_dir = os.path.dirname(os.path.abspath(__file__))
  target = os.path.join(internals_dir, namFilename)
  result = readNamelist(target, unique_glyphs, cache)
  return result[key]

def codepointsInNamelist ( namFilename , unique_glyphs = False , cache = None ) : key = 'charset' if not unique_glyphs else 'ownCharset' internals_dir = os . path . dirname ( os . path . abspath ( __file__ ) ) target = os . path . join ( internals_dir , namFilename ) result = readNamelist ( target , unique_glyphs , cache ) return result [ key ]

Returns the set of codepoints contained in a given Namelist file .


def get_orthographies(self, _library=library):
        ''' Returns list of CharsetInfo about supported orthographies '''
        results = []
        for charset in _library.charsets:
            if self._charsets:
                cn = getattr(charset, 'common_name', False)
                abbr = getattr(charset, 'abbreviation', False)
                nn = getattr(charset, 'short_name', False)
                naive = getattr(charset, 'native_name', False)

                if cn and cn.lower() in self._charsets:
                    results.append(charset)

                elif nn and nn.lower() in self._charsets:
                    results.append(charset)

                elif naive and naive.lower() in self._charsets:
                    results.append(charset)

                elif abbr and abbr.lower() in self._charsets:
                    results.append(charset)
            else:
                results.append(charset)

        for result in results:
            yield CharsetInfo(self, result)

def get_orthographies ( self , _library = library ) : results = [ ] for charset in _library . charsets : if self . _charsets : cn = getattr ( charset , 'common_name' , False ) abbr = getattr ( charset , 'abbreviation' , False ) nn = getattr ( charset , 'short_name' , False ) naive = getattr ( charset , 'native_name' , False ) if cn and cn . lower ( ) in self . _charsets : results . append ( charset ) elif nn and nn . lower ( ) in self . _charsets : results . append ( charset ) elif naive and naive . lower ( ) in self . _charsets : results . append ( charset ) elif abbr and abbr . lower ( ) in self . _charsets : results . append ( charset ) else : results . append ( charset ) for result in results : yield CharsetInfo ( self , result )


def generate_oauth2_headers(self):
        """Generates header for oauth2
        """
        encoded_credentials = base64.b64encode(('{0}:{1}'.format(self.consumer_key,self.consumer_secret)).encode('utf-8'))
        headers={
            'Authorization':'Basic {0}'.format(encoded_credentials.decode('utf-8')),
            'Content-Type': 'application/x-www-form-urlencoded'
        }

        return headers

def
generate_oauth2_headers
(
self
)
:
encoded_credentials
=
base64
.
b64encode
(
(
'{0}:{1}'
.
format
(
self
.
consumer_key
,
self
.
consumer_secret
)
)
.
encode
(
'utf-8'
)
)
headers
=
{
'Authorization'
:
'Basic {0}'
.
format
(
encoded_credentials
.
decode
(
'utf-8'
)
)
,
'Content-Type'
:
'application/x-www-form-urlencoded'
}
return
headers


def oauth2_access_parser(self, raw_access):
        """Parse oauth2 access
        """
        parsed_access = json.loads(raw_access.content.decode('utf-8'))
        self.access_token = parsed_access['access_token']
        self.token_type = parsed_access['token_type']
        self.refresh_token = parsed_access['refresh_token']
        self.guid = parsed_access['xoauth_yahoo_guid']

        credentials = {
            'access_token': self.access_token,
            'token_type': self.token_type,
            'refresh_token': self.refresh_token,
            'guid': self.guid
        }

        return credentials

def oauth2_access_parser ( self , raw_access ) : parsed_access = json . loads ( raw_access . content . decode ( 'utf-8' ) ) self . access_token = parsed_access [ 'access_token' ] self . token_type = parsed_access [ 'token_type' ] self . refresh_token = parsed_access [ 'refresh_token' ] self . guid = parsed_access [ 'xoauth_yahoo_guid' ] credentials = { 'access_token' : self . access_token , 'token_type' : self . token_type , 'refresh_token' : self . refresh_token , 'guid' : self . guid } return credentials


def refresh_access_token(self,):
        """Refresh access token
        """
        logger.debug("REFRESHING TOKEN")
        self.token_time = time.time()
        credentials = {
            'token_time': self.token_time
        }

        if self.oauth_version == 'oauth1':
            self.access_token, self.access_token_secret = self.oauth.get_access_token(self.access_token, self.access_token_secret, params={"oauth_session_handle": self.session_handle})
            credentials.update({
                'access_token': self.access_token,
                'access_token_secret': self.access_token_secret,
                'session_handle': self.session_handle,
                'token_time': self.token_time
            })
        else:
            headers = self.generate_oauth2_headers()

            raw_access = self.oauth.get_raw_access_token(data={"refresh_token": self.refresh_token, 'redirect_uri': self.callback_uri,'grant_type':'refresh_token'}, headers=headers)
            credentials.update(self.oauth2_access_parser(raw_access))

        return credentials

def
refresh_access_token
(
self
,
)
:
logger
.
debug
(
"REFRESHING TOKEN"
)
self
.
token_time
=
time
.
time
(
)
credentials
=
{
'token_time'
:
self
.
token_time
}
if
self
.
oauth_version
==
'oauth1'
:
self
.
access_token
,
self
.
access_token_secret
=
self
.
oauth
.
get_access_token
(
self
.
access_token
,
self
.
access_token_secret
,
params
=
{
"oauth_session_handle"
:
self
.
session_handle
}
)
credentials
.
update
(
{
'access_token'
:
self
.
access_token
,
'access_token_secret'
:
self
.
access_token_secret
,
'session_handle'
:
self
.
session_handle
,
'token_time'
:
self
.
token_time
}
)
else
:
headers
=
self
.
generate_oauth2_headers
(
)
raw_access
=
self
.
oauth
.
get_raw_access_token
(
data
=
{
"refresh_token"
:
self
.
refresh_token
,
'redirect_uri'
:
self
.
callback_uri
,
'grant_type'
:
'refresh_token'
}
,
headers
=
headers
)
credentials
.
update
(
self
.
oauth2_access_parser
(
raw_access
)
)
return
credentials


def get_data(filename):
    """Calls right function according to file extension
    """
    name, ext = get_file_extension(filename)
    func = json_get_data if ext == '.json' else yaml_get_data
    return func(filename)

def get_data ( filename ) : name , ext = get_file_extension ( filename ) func = json_get_data if ext == '.json' else yaml_get_data return func ( filename )


def write_data(data, filename):
    """Call right func to save data according to file extension
    """
    name, ext = get_file_extension(filename)
    func = json_write_data if ext == '.json' else yaml_write_data
    return func(data, filename)

def write_data ( data , filename ) : name , ext = get_file_extension ( filename ) func = json_write_data if ext == '.json' else yaml_write_data return func ( data , filename )


def json_write_data(json_data, filename):
    """Write json data into a file
    """
    with open(filename, 'w') as fp:
        json.dump(json_data, fp, indent=4, sort_keys=True, ensure_ascii=False)
        return True
    return False

def json_write_data ( json_data , filename ) : with open ( filename , 'w' ) as fp : json . dump ( json_data , fp , indent = 4 , sort_keys = True , ensure_ascii = False ) return True return False


def json_get_data(filename):
    """Get data from json file
    """
    with open(filename) as fp:
        json_data = json.load(fp)
        return json_data

    return False

def json_get_data ( filename ) : with open ( filename ) as fp : json_data = json . load ( fp ) return json_data return False


def yaml_get_data(filename):
    """Get data from .yml file
    """
    with open(filename, 'rb') as fd:
        yaml_data = yaml.load(fd)
        return yaml_data
    return False

def yaml_get_data ( filename ) : with open ( filename , 'rb' ) as fd : yaml_data = yaml . load ( fd ) return yaml_data return False


def yaml_write_data(yaml_data, filename):
    """Write data into a .yml file
    """
    with open(filename, 'w') as fd:
        yaml.dump(yaml_data, fd, default_flow_style=False)
        return True

    return False

def yaml_write_data ( yaml_data , filename ) : with open ( filename , 'w' ) as fd : yaml . dump ( yaml_data , fd , default_flow_style = False ) return True return False


def transform(self, X):
        '''
        Turns distances into RBF values.

        Parameters
        ----------
        X : array
            The raw pairwise distances.

        Returns
        -------
        X_rbf : array of same shape as X
            The distances in X passed through the RBF kernel.
        '''
        X = check_array(X)
        X_rbf = np.empty_like(X) if self.copy else X

        X_in = X
        if not self.squared:
            np.power(X_in, 2, out=X_rbf)
            X_in = X_rbf

        if self.scale_by_median:
            scale = self.median_ if self.squared else self.median_ ** 2
            gamma = self.gamma * scale
        else:
            gamma = self.gamma
        np.multiply(X_in, -gamma, out=X_rbf)

        np.exp(X_rbf, out=X_rbf)
        return X_rbf

def transform ( self , X ) : X = check_array ( X ) X_rbf = np . empty_like ( X ) if self . copy else X X_in = X if not self . squared : np . power ( X_in , 2 , out = X_rbf ) X_in = X_rbf if self . scale_by_median : scale = self . median_ if self . squared else self . median_ ** 2 gamma = self . gamma * scale else : gamma = self . gamma np . multiply ( X_in , - gamma , out = X_rbf ) np . exp ( X_rbf , out = X_rbf ) return X_rbf


def fit(self, X, y=None):
        '''
        Learn the linear transformation to clipped eigenvalues.

        Note that if min_eig isn't zero and any of the original eigenvalues
        were exactly zero, this will leave those eigenvalues as zero.

        Parameters
        ----------
        X : array, shape [n, n]
            The *symmetric* input similarities. If X is asymmetric, it will be
            treated as if it were symmetric based on its lower-triangular part.
        '''
        n = X.shape[0]
        if X.shape != (n, n):
            raise TypeError("Input must be a square matrix.")

        # TODO: only get negative eigs somehow?
        memory = get_memory(self.memory)
        vals, vecs = memory.cache(scipy.linalg.eigh, ignore=['overwrite_a'])(
            X, overwrite_a=not self.copy)
        vals = vals.reshape(-1, 1)

        if self.min_eig == 0:
            inner = vals > self.min_eig
        else:
            with np.errstate(divide='ignore'):
                inner = np.where(vals >= self.min_eig, 1,
                                 np.where(vals == 0, 0, self.min_eig / vals))

        self.clip_ = np.dot(vecs, inner * vecs.T)
        return self

def
fit
(
self
,
X
,
y
=
)
:
n
=
X
.
shape
[
0
]
if
X
.
shape
!=
(
n
,
n
)
:
raise
TypeError
(
"Input must be a square matrix."
)
memory
=
get_memory
(
self
.
memory
)
vals
,
vecs
=
memory
.
cache
(
scipy
.
linalg
.
eigh
,
ignore
=
[
'overwrite_a'
]
)
(
X
,
overwrite_a
=
not
self
.
copy
)
vals
=
vals
.
reshape
(
1
,
1
)
if
self
.
min_eig
==
0
:
inner
=
vals
>
self
.
min_eig
else
:
with
np
.
errstate
(
divide
=
'ignore'
)
:
inner
=
np
.
where
(
vals
>=
self
.
min_eig
,
1
,
np
.
where
(
vals
==
0
,
0
,
self
.
min_eig
/
vals
)
)
self
.
clip_
=
np
.
dot
(
vecs
,
inner
*
vecs
.
T
)
return
self

Learn the linear transformation to clipped eigenvalues .


def fit(self, X, y=None):
        '''
        Learn the linear transformation to flipped eigenvalues.

        Parameters
        ----------
        X : array, shape [n, n]
            The *symmetric* input similarities. If X is asymmetric, it will be
            treated as if it were symmetric based on its lower-triangular part.
        '''
        n = X.shape[0]
        if X.shape != (n, n):
            raise TypeError("Input must be a square matrix.")

        # TODO: only get negative eigs somehow?
        memory = get_memory(self.memory)
        vals, vecs = memory.cache(scipy.linalg.eigh, ignore=['overwrite_a'])(
            X, overwrite_a=not self.copy)
        vals = vals[:, None]

        self.flip_ = np.dot(vecs, np.sign(vals) * vecs.T)
        return self

def
fit
(
self
,
X
,
y
=
)
:
n
=
X
.
shape
[
0
]
if
X
.
shape
!=
(
n
,
n
)
:
raise
TypeError
(
"Input must be a square matrix."
)
memory
=
get_memory
(
self
.
memory
)
vals
,
vecs
=
memory
.
cache
(
scipy
.
linalg
.
eigh
,
ignore
=
[
'overwrite_a'
]
)
(
X
,
overwrite_a
=
not
self
.
copy
)
vals
=
vals
[
:
,
]
self
.
flip_
=
np
.
dot
(
vecs
,
np
.
sign
(
vals
)
*
vecs
.
T
)
return
self

Learn the linear transformation to flipped eigenvalues .


def transform(self, X):
        '''
        Transforms X according to the linear transformation corresponding to
        flipping the input eigenvalues.

        Parameters
        ----------
        X : array, shape [n_test, n]
            The test similarities to training points.

        Returns
        -------
        Xt : array, shape [n_test, n]
            The transformed test similarites to training points.
        '''
        n = self.flip_.shape[0]
        if X.ndim != 2 or X.shape[1] != n:
            msg = "X should have {} columns, the number of samples at fit time"
            raise TypeError(msg.format(self.flip_.shape[0]))
        return np.dot(X, self.flip_)

Transforms X according to the linear transformation corresponding to flipping the input eigenvalues .


def fit_transform(self, X, y=None):
        '''
        Flips the negative eigenvalues of X.

        Parameters
        ----------
        X : array, shape [n, n]
            The *symmetric* input similarities. If X is asymmetric, it will be
            treated as if it were symmetric based on its lower-triangular part.

        Returns
        -------
        Xt : array, shape [n, n]
            The transformed training similarities.
        '''
        n = X.shape[0]
        if X.shape != (n, n):
            raise TypeError("Input must be a square matrix.")

        memory = get_memory(self.memory)
        discard_X = not self.copy and self.negatives_likely
        vals, vecs = memory.cache(scipy.linalg.eigh, ignore=['overwrite_a'])(
            X, overwrite_a=discard_X)
        vals = vals[:, None]

        self.clip_ = np.dot(vecs, np.sign(vals) * vecs.T)

        if discard_X or vals[0, 0] < 0:
            del X
            np.abs(vals, out=vals)
            X = np.dot(vecs, vals * vecs.T)
            del vals, vecs

            # should be symmetric, but make sure because floats
            X = Symmetrize(copy=False).fit_transform(X)
        return X

def
fit_transform
(
self
,
X
,
y
=
)
:
n
=
X
.
shape
[
0
]
if
X
.
shape
!=
(
n
,
n
)
:
raise
TypeError
(
"Input must be a square matrix."
)
memory
=
get_memory
(
self
.
memory
)
discard_X
=
not
self
.
copy
and
self
.
negatives_likely
vals
,
vecs
=
memory
.
cache
(
scipy
.
linalg
.
eigh
,
ignore
=
[
'overwrite_a'
]
)
(
X
,
overwrite_a
=
discard_X
)
vals
=
vals
[
:
,
]
self
.
clip_
=
np
.
dot
(
vecs
,
np
.
sign
(
vals
)
*
vecs
.
T
)
if
discard_X
or
vals
[
0
,
0
]
<
0
:
del
X
np
.
abs
(
vals
,
out
=
vals
)
X
=
np
.
dot
(
vecs
,
vals
*
vecs
.
T
)
del
vals
,
vecs
X
=
Symmetrize
(
copy
=
False
)
.
fit_transform
(
X
)
return
X


def fit(self, X, y=None):
        '''
        Learn the transformation to shifted eigenvalues. Only depends
        on the input dimension.

        Parameters
        ----------
        X : array, shape [n, n]
            The *symmetric* input similarities.
        '''
        n = X.shape[0]
        if X.shape != (n, n):
            raise TypeError("Input must be a square matrix.")

        self.train_ = X

        memory = get_memory(self.memory)
        lo, = memory.cache(scipy.linalg.eigvalsh)(X, eigvals=(0, 0))
        self.shift_ = max(self.min_eig - lo, 0)

        return self

def
fit
(
self
,
X
,
y
=
)
:
n
=
X
.
shape
[
0
]
if
X
.
shape
!=
(
n
,
n
)
:
raise
TypeError
(
"Input must be a square matrix."
)
self
.
train_
=
X
memory
=
get_memory
(
self
.
memory
)
lo
,
=
memory
.
cache
(
scipy
.
linalg
.
eigvalsh
)
(
X
,
eigvals
=
(
0
,
0
)
)
self
.
shift_
=
max
(
self
.
min_eig
lo
,
0
)
return
self

Learn the transformation to shifted eigenvalues . Only depends on the input dimension .


def transform(self, X):
        '''
        Transforms X according to the linear transformation corresponding to
        shifting the input eigenvalues to all be at least ``self.min_eig``.

        Parameters
        ----------
        X : array, shape [n_test, n]
            The test similarities to training points.

        Returns
        -------
        Xt : array, shape [n_test, n]
            The transformed test similarites to training points. Only different
            from X if X is the training data.
        '''
        n = self.train_.shape[0]
        if X.ndim != 2 or X.shape[1] != n:
            msg = "X should have {} columns, the number of samples at fit time"
            raise TypeError(msg.format(n))

        if self.copy:
            X = X.copy()

        if self.shift_ != 0 and X is self.train_ or (
                X.shape == self.train_.shape and np.allclose(X, self.train_)):
            X[xrange(n), xrange(n)] += self.shift_
        return X

Transforms X according to the linear transformation corresponding to shifting the input eigenvalues to all be at least self . min_eig .


def fit(self, X, y=None):
        '''
        Picks the elements of the basis to use for the given data.

        Only depends on the dimension of X. If it's more convenient, you can
        pass a single integer for X, which is the dimension to use.

        Parameters
        ----------
        X : an integer, a :class:`Features` instance, or a list of bag features
            The input data, or just its dimension, since only the dimension is
            needed here.
        '''
        if is_integer(X):
            dim = X
        else:
            X = as_features(X)
            dim = X.dim
        M = self.smoothness

        # figure out the smooth-enough elements of our basis
        inds = np.mgrid[(slice(M + 1),) * dim].reshape(dim, (M + 1) ** dim).T
        self.inds_ = inds[(inds ** 2).sum(axis=1) <= M ** 2]
        return self

def fit ( self , X , y = None ) : if is_integer ( X ) : dim = X else : X = as_features ( X ) dim = X . dim M = self . smoothness inds = np . mgrid [ ( slice ( M + 1 ) , ) * dim ] . reshape ( dim , ( M + 1 ) ** dim ) . T self . inds_ = inds [ ( inds ** 2 ) . sum ( axis = 1 ) <= M ** 2 ] return self

Picks the elements of the basis to use for the given data .


def transform(self, X):
        '''
        Transform a list of bag features into its projection series
        representation.

        Parameters
        ----------
        X : :class:`skl_groups.features.Features` or list of bag feature arrays
            New data to transform. The data should all lie in [0, 1];
            use :class:`skl_groups.preprocessing.BagMinMaxScaler` if not.

        Returns
        -------
        X_new : integer array, shape ``[len(X), dim_]``
            X transformed into the new space.
        '''
        self._check_fitted()
        M = self.smoothness
        dim = self.dim_
        inds = self.inds_
        do_check = self.do_bounds_check

        X = as_features(X)
        if X.dim != dim:
            msg = "model fit for dimension {} but got dim {}"
            raise ValueError(msg.format(dim, X.dim))

        Xt = np.empty((len(X), self.inds_.shape[0]))
        Xt.fill(np.nan)

        if self.basis == 'cosine':  # TODO: put this in a C extension?
            coefs = (np.pi * np.arange(M + 1))[..., :]
            for i, bag in enumerate(X):
                if do_check:
                    if np.min(bag) < 0 or np.max(bag) > 1:
                        raise ValueError("Bag {} not in [0, 1]".format(i))

                # apply each phi func to each dataset point: n x dim x M
                phi = coefs * bag[..., np.newaxis]
                np.cos(phi, out=phi)
                phi[:, :, 1:] *= np.sqrt(2)

                # B is the evaluation of each tensor-prodded basis func
                # at each point: n x inds.shape[0]
                B = reduce(op.mul, (phi[:, i, inds[:, i]] for i in xrange(dim)))

                Xt[i, :] = np.mean(B, axis=0)
        else:
            raise ValueError("unknown basis '{}'".format(self.basis))

        return Xt

Transform a list of bag features into its projection series representation .


def get_version(self):
        """
        Get distribution version.

        This method is enhanced compared to original distutils implementation.
        If the version string is set to a special value then instead of using
        the actual value the real version is obtained by querying versiontools.

        If versiontools package is not installed then the version is obtained
        from the standard section of the ``PKG-INFO`` file. This file is
        automatically created by any source distribution. This method is less
        useful as it cannot take advantage of version control information that
        is automatically loaded by versiontools. It has the advantage of not
        requiring versiontools installation and that it does not depend on
        ``setup_requires`` feature of ``setuptools``.
        """
        if (self.name is not None and self.version is not None
            and self.version.startswith(":versiontools:")):
            return (self.__get_live_version() or self.__get_frozen_version()
                    or self.__fail_to_get_any_version())
        else:
            return self.__base.get_version(self)

def get_version ( self ) : if ( self . name is not None and self . version is not None and self . version . startswith ( ":versiontools:" ) ) : return ( self . __get_live_version ( ) or self . __get_frozen_version ( ) or self . __fail_to_get_any_version ( ) ) else : return self . __base . get_version ( self )


def __get_live_version(self):
        """
        Get a live version string using versiontools
        """
        try:
            import versiontools
        except ImportError:
            return None
        else:
            return str(versiontools.Version.from_expression(self.name))

def __get_live_version ( self ) : try : import versiontools except ImportError : return None else : return str ( versiontools . Version . from_expression ( self . name ) )


def fit(self, X, y=None, **params):
        '''
        Fit the transformer on the stacked points.

        Parameters
        ----------
        X : :class:`Features` or list of arrays of shape ``[n_samples[i], n_features]``
            Training set. If a Features object, it will be stacked.

        any other keyword argument :
            Passed on as keyword arguments to the transformer's ``fit()``.
        '''
        X = as_features(X, stack=True)
        self.transformer.fit(X.stacked_features, y, **params)
        return self

def fit ( self , X , y = None , ** params ) : X = as_features ( X , stack = True ) self . transformer . fit ( X . stacked_features , y , ** params ) return self

Fit the transformer on the stacked points .


def transform(self, X, **params):
        '''
        Transform the stacked points.

        Parameters
        ----------
        X : :class:`Features` or list of bag feature arrays
            New data to transform.

        any other keyword argument :
            Passed on as keyword arguments to the transformer's ``transform()``.

        Returns
        -------
        X_new : :class:`Features`
            Transformed features.
        '''
        X = as_features(X, stack=True)
        X_new = self.transformer.transform(X.stacked_features, **params)
        return self._gather_outputs(X, X_new)

def transform ( self , X , ** params ) : X = as_features ( X , stack = True ) X_new = self . transformer . transform ( X . stacked_features , ** params ) return self . _gather_outputs ( X , X_new )


def fit_transform(self, X, y=None, **params):
        '''
        Fit and transform the stacked points.

        Parameters
        ----------
        X : :class:`Features` or list of bag feature arrays
            Data to train on and transform.

        any other keyword argument :
            Passed on as keyword arguments to the transformer's ``transform()``.

        Returns
        -------
        X_new : :class:`Features`
            Transformed features.
        '''
        X = as_features(X, stack=True)
        X_new = self.transformer.fit_transform(X.stacked_features, y, **params)
        return self._gather_outputs(X, X_new)

def fit_transform ( self , X , y = None , ** params ) : X = as_features ( X , stack = True ) X_new = self . transformer . fit_transform ( X . stacked_features , y , ** params ) return self . _gather_outputs ( X , X_new )


def fit(self, X, y=None):
        """Compute the minimum and maximum to be used for later scaling.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            The data used to compute the per-feature minimum and maximum
            used for later scaling along the features axis.
        """
        X = check_array(X, copy=self.copy,
                        dtype=[np.float64, np.float32, np.float16, np.float128])

        feature_range = self.feature_range
        if feature_range[0] >= feature_range[1]:
            raise ValueError("Minimum of desired feature range must be smaller"
                             " than maximum. Got %s." % str(feature_range))
        if self.fit_feature_range is not None:
            fit_feature_range = self.fit_feature_range
            if fit_feature_range[0] >= fit_feature_range[1]:
                raise ValueError("Minimum of desired (fit) feature range must "
                                 "be smaller than maximum. Got %s."
                                 % str(feature_range))
            if (fit_feature_range[0] < feature_range[0] or
                    fit_feature_range[1] > feature_range[1]):
                raise ValueError("fit_feature_range must be a subset of "
                                 "feature_range. Got %s, fit %s."
                                 % (str(feature_range),
                                    str(fit_feature_range)))
            feature_range = fit_feature_range

        data_min = np.min(X, axis=0)
        data_range = np.max(X, axis=0) - data_min
        # Do not scale constant features
        data_range[data_range == 0.0] = 1.0
        self.scale_ = (feature_range[1] - feature_range[0]) / data_range
        self.min_ = feature_range[0] - data_min * self.scale_
        self.data_range = data_range
        self.data_min = data_min
        return self

Compute the minimum and maximum to be used for later scaling .


def transform(self, X):
        """Scaling features of X according to feature_range.

        Parameters
        ----------
        X : array-like with shape [n_samples, n_features]
            Input data that will be transformed.
        """
        X = check_array(X, copy=self.copy)
        X *= self.scale_
        X += self.min_
        if self.truncate:
            np.maximum(self.feature_range[0], X, out=X)
            np.minimum(self.feature_range[1], X, out=X)
        return X

def transform ( self , X ) : X = check_array ( X , copy = self . copy ) X *= self . scale_ X += self . min_ if self . truncate : np . maximum ( self . feature_range [ 0 ] , X , out = X ) np . minimum ( self . feature_range [ 1 ] , X , out = X ) return X

Scaling features of X according to feature_range .


def inverse_transform(self, X):
        """Undo the scaling of X according to feature_range.

        Note that if truncate is true, any truncated points will not
        be restored exactly.

        Parameters
        ----------
        X : array-like with shape [n_samples, n_features]
            Input data that will be transformed.
        """
        X = check_array(X, copy=self.copy)
        X -= self.min_
        X /= self.scale_
        return X

def inverse_transform ( self , X ) : X = check_array ( X , copy = self . copy ) X -= self . min_ X /= self . scale_ return X

Undo the scaling of X according to feature_range .


def fit(self, X, y=None):
        '''
        Choose the codewords based on a training set.

        Parameters
        ----------
        X : :class:`skl_groups.features.Features` or list of arrays of shape ``[n_samples[i], n_features]``
            Training set. If a Features object, it will be stacked.
        '''
        self.kmeans_fit_ = copy(self.kmeans)
        X = as_features(X, stack=True)
        self.kmeans_fit_.fit(X.stacked_features)
        return self

def fit ( self , X , y = None ) : self . kmeans_fit_ = copy ( self . kmeans ) X = as_features ( X , stack = True ) self . kmeans_fit_ . fit ( X . stacked_features ) return self

Choose the codewords based on a training set .


def transform(self, X):
        '''
        Transform a list of bag features into its bag-of-words representation.

        Parameters
        ----------
        X : :class:`skl_groups.features.Features` or list of bag feature arrays
            New data to transform.

        Returns
        -------
        X_new : integer array, shape [len(X), kmeans.n_clusters]
            X transformed into the new space.
        '''
        self._check_fitted()
        X = as_features(X, stack=True)
        assignments = self.kmeans_fit_.predict(X.stacked_features)
        return self._group_assignments(X, assignments)

def transform ( self , X ) : self . _check_fitted ( ) X = as_features ( X , stack = True ) assignments = self . kmeans_fit_ . predict ( X . stacked_features ) return self . _group_assignments ( X , assignments )

Transform a list of bag features into its bag - of - words representation .


def is_categorical_type(ary):
    "Checks whether the array is either integral or boolean."
    ary = np.asanyarray(ary)
    return is_integer_type(ary) or ary.dtype.kind == 'b'

Checks whether the array is either integral or boolean .


def as_integer_type(ary):
    '''
    Returns argument as an integer array, converting floats if convertable.
    Raises ValueError if it's a float array with nonintegral values.
    '''
    ary = np.asanyarray(ary)
    if is_integer_type(ary):
        return ary
    rounded = np.rint(ary)
    if np.any(rounded != ary):
        raise ValueError("argument array must contain only integers")
    return rounded.astype(int)

Returns argument as an integer array converting floats if convertable . Raises ValueError if it s a float array with nonintegral values .


def start(self, total):
        '''
        Signal the start of the process.

        Parameters
        ----------
        total : int
            The total number of steps in the process, or None if unknown.
        '''
        self.logger.info(json.dumps(['START', self.name, total]))

def start ( self , total ) : self . logger . info ( json . dumps ( [ 'START' , self . name , total ] ) )


def _build_indices(X, flann_args):
    "Builds FLANN indices for each bag."
    # TODO: should probably multithread this
    logger.info("Building indices...")
    indices = [None] * len(X)
    for i, bag in enumerate(plog(X, name="index building")):
        indices[i] = idx = FLANNIndex(**flann_args)
        idx.build_index(bag)
    return indices

def
_build_indices
(
X
,
flann_args
)
:
"Builds FLANN indices for each bag."
logger
.
info
(
"Building indices..."
)
indices
=
[
]
*
len
(
X
)
for
i
,
bag
in
enumerate
(
plog
(
X
,
name
=
"index building"
)
)
:
indices
[
i
]
=
idx
=
FLANNIndex
(
**
flann_args
)
idx
.
build_index
(
bag
)
return
indices


def _get_rhos(X, indices, Ks, max_K, save_all_Ks, min_dist):
    "Gets within-bag distances for each bag."
    logger.info("Getting within-bag distances...")

    if max_K >= X.n_pts.min():
        msg = "asked for K = {}, but there's a bag with only {} points"
        raise ValueError(msg.format(max_K, X.n_pts.min()))

    # need to throw away the closest neighbor, which will always be self
    # thus K=1 corresponds to column 1 in the result array
    which_Ks = slice(1, None) if save_all_Ks else Ks

    indices = plog(indices, name="within-bag distances")
    rhos = [None] * len(X)
    for i, (idx, bag) in enumerate(zip(indices, X)):
        r = np.sqrt(idx.nn_index(bag, max_K + 1)[1][:, which_Ks])
        np.maximum(min_dist, r, out=r)
        rhos[i] = r
    return rhos

Gets within - bag distances for each bag .


def linear(Ks, dim, num_q, rhos, nus):
    r'''
    Estimates the linear inner product \int p q between two distributions,
    based on kNN distances.
    '''
    return _get_linear(Ks, dim)(num_q, rhos, nus)

def linear ( Ks , dim , num_q , rhos , nus ) : r return _get_linear ( Ks , dim ) ( num_q , rhos , nus )

r Estimates the linear inner product \ int p q between two distributions based on kNN distances .


def quadratic(Ks, dim, rhos, required=None):
    r'''
    Estimates \int p^2 based on kNN distances.

    In here because it's used in the l2 distance, above.

    Returns array of shape (num_Ks,).
    '''
    # Estimated with alpha=1, beta=0:
    #   B_{k,d,1,0} is the same as B_{k,d,0,1} in linear()
    # and the full estimator is
    #   B / (n - 1) * mean(rho ^ -dim)
    N = rhos.shape[0]
    Ks = np.asarray(Ks)
    Bs = (Ks - 1) / np.pi ** (dim / 2) * gamma(dim / 2 + 1)  # shape (num_Ks,)
    est = Bs / (N - 1) * np.mean(rhos ** (-dim), axis=0)
    return est

def quadratic ( Ks , dim , rhos , required = None ) : r N = rhos . shape [ 0 ] Ks = np . asarray ( Ks ) Bs = ( Ks - 1 ) / np . pi ** ( dim / 2 ) * gamma ( dim / 2 + 1 ) est = Bs / ( N - 1 ) * np . mean ( rhos ** ( - dim ) , axis = 0 ) return est

r Estimates \ int p^2 based on kNN distances .


def topological_sort(deps):
    '''
    Topologically sort a DAG, represented by a dict of child => set of parents.
    The dependency dict is destroyed during operation.

    Uses the Kahn algorithm: http://en.wikipedia.org/wiki/Topological_sorting
    Not a particularly good implementation, but we're just running it on tiny
    graphs.
    '''
    order = []
    available = set()

    def _move_available():
        to_delete = []
        for n, parents in iteritems(deps):
            if not parents:
                available.add(n)
                to_delete.append(n)
        for n in to_delete:
            del deps[n]

    _move_available()
    while available:
        n = available.pop()
        order.append(n)
        for parents in itervalues(deps):
            parents.discard(n)
        _move_available()

    if available:
        raise ValueError("dependency cycle found")
    return order

def
topological_sort
(
deps
)
:
order
=
[
]
available
=
set
(
)
def
_move_available
(
)
:
to_delete
=
[
]
for
n
,
parents
in
iteritems
(
deps
)
:
if
not
parents
:
available
.
add
(
n
)
to_delete
.
append
(
n
)
for
n
in
to_delete
:
del
deps
[
n
]
_move_available
(
)
while
available
:
n
=
available
.
pop
(
)
order
.
append
(
n
)
for
parents
in
itervalues
(
deps
)
:
parents
.
discard
(
n
)
_move_available
(
)
if
available
:
raise
ValueError
(
"dependency cycle found"
)
return
order

Topologically sort a DAG represented by a dict of child = > set of parents . The dependency dict is destroyed during operation .


def _get_Ks(self):
        "Ks as an array and type-checked."
        Ks = as_integer_type(self.Ks)
        if Ks.ndim != 1:
            raise TypeError("Ks should be 1-dim, got shape {}".format(Ks.shape))
        if Ks.min() < 1:
            raise ValueError("Ks should be positive; got {}".format(Ks.min()))
        return Ks

def
_get_Ks
(
self
)
:
"Ks as an array and type-checked."
Ks
=
as_integer_type
(
self
.
Ks
)
if
Ks
.
ndim
!=
1
:
raise
TypeError
(
"Ks should be 1-dim, got shape {}"
.
format
(
Ks
.
shape
)
)
if
Ks
.
min
(
)
<
1
:
raise
ValueError
(
"Ks should be positive; got {}"
.
format
(
Ks
.
min
(
)
)
)
return
Ks


def _flann_args(self, X=None):
        "The dictionary of arguments to give to FLANN."
        args = {'cores': self._n_jobs}
        if self.flann_algorithm == 'auto':
            if X is None or X.dim > 5:
                args['algorithm'] = 'linear'
            else:
                args['algorithm'] = 'kdtree_single'
        else:
            args['algorithm'] = self.flann_algorithm
        if self.flann_args:
            args.update(self.flann_args)

        # check that arguments are correct
        try:
            FLANNParameters().update(args)
        except AttributeError as e:
            msg = "flann_args contains an invalid argument:\n  {}"
            raise TypeError(msg.format(e))

        return args

The dictionary of arguments to give to FLANN .


def fit(self, X, y=None, get_rhos=False):
        '''
        Sets up for divergence estimation "from" new data "to" X.
        Builds FLANN indices for each bag, and maybe gets within-bag distances.

        Parameters
        ----------
        X : list of arrays or :class:`skl_groups.features.Features`
            The bags to search "to".

        get_rhos : boolean, optional, default False
            Compute within-bag distances :attr:`rhos_`. These are only needed
            for some divergence functions or if do_sym is passed, and they'll
            be computed (and saved) during :meth:`transform` if they're not
            computed here.

            If you're using Jensen-Shannon divergence, a higher max_K may
            be needed once it sees the number of points in the transformed bags,
            so the computation here might be wasted.
        '''
        self.features_ = X = as_features(X, stack=True, bare=True)

        # if we're using a function that needs to pick its K vals itself,
        # then we need to set max_K here. when we transform(), might have to
        # re-do this :|
        Ks = self._get_Ks()
        _, _, _, max_K, save_all_Ks, _ = _choose_funcs(
            self.div_funcs, Ks, X.dim, X.n_pts, None, self.version)

        if max_K >= X.n_pts.min():
            msg = "asked for K = {}, but there's a bag with only {} points"
            raise ValueError(msg.format(max_K, X.n_pts.min()))

        memory = self.memory
        if isinstance(memory, string_types):
            memory = Memory(cachedir=memory, verbose=0)

        self.indices_ = id = memory.cache(_build_indices)(X, self._flann_args())
        if get_rhos:
            self.rhos_ = _get_rhos(X, id, Ks, max_K, save_all_Ks, self.min_dist)
        elif hasattr(self, 'rhos_'):
            del self.rhos_
        return self

Sets up for divergence estimation from new data to X . Builds FLANN indices for each bag and maybe gets within - bag distances .


def make_stacked(self):
        "If unstacked, convert to stacked. If stacked, do nothing."
        if self.stacked:
            return

        self._boundaries = bounds = np.r_[0, np.cumsum(self.n_pts)]
        self.stacked_features = stacked = np.vstack(self.features)
        self.features = np.array(
            [stacked[bounds[i-1]:bounds[i]] for i in xrange(1, len(bounds))],
            dtype=object)
        self.stacked = True

If unstacked convert to stacked . If stacked do nothing .


def copy(self, stack=False, copy_meta=False, memo=None):
        '''
        Copies the Feature object. Makes a copy of the features array.

        Parameters
        ----------
        stack : boolean, optional, default False
            Whether to stack the copy if this one is unstacked.

        copy_meta : boolean, optional, default False
            Also copy the metadata. If False, metadata in both points to the
            same object.
        '''
        if self.stacked:
            fs = deepcopy(self.stacked_features, memo)
            n_pts = self.n_pts.copy()
        elif stack:
            fs = np.vstack(self.features)
            n_pts = self.n_pts.copy()
        else:
            fs = deepcopy(self.features, memo)
            n_pts = None

        meta = deepcopy(self.meta, memo) if copy_meta else self.meta
        return Features(fs, n_pts, copy=False, **meta)

def copy ( self , stack = False , copy_meta = False , memo = None ) : if self . stacked : fs = deepcopy ( self . stacked_features , memo ) n_pts = self . n_pts . copy ( ) elif stack : fs = np . vstack ( self . features ) n_pts = self . n_pts . copy ( ) else : fs = deepcopy ( self . features , memo ) n_pts = None meta = deepcopy ( self . meta , memo ) if copy_meta else self . meta return Features ( fs , n_pts , copy = False , ** meta )

Copies the Feature object . Makes a copy of the features array .


def bare(self):
        "Make a Features object with no metadata; points to the same features."
        if not self.meta:
            return self
        elif self.stacked:
            return Features(self.stacked_features, self.n_pts, copy=False)
        else:
            return Features(self.features, copy=False)

Make a Features object with no metadata ; points to the same features .


def fit(self, X, y=None):
        '''
        Specify the data to which kernel values should be computed.

        Parameters
        ----------
        X : list of arrays or :class:`skl_groups.features.Features`
            The bags to compute "to".
        '''
        self.features_ = as_features(X, stack=True, bare=True)
        # TODO: could precompute things like squared norms if kernel == "rbf".
        # Probably should add support to sklearn instead of hacking it here.
        return self

def fit ( self , X , y = None ) : self . features_ = as_features ( X , stack = True , bare = True ) return self

Specify the data to which kernel values should be computed .


def transform(self, X):
        '''
        Transform a list of bag features into a matrix of its mean features.

        Parameters
        ----------
        X : :class:`skl_groups.features.Features` or list of bag feature arrays
            Data to transform.

        Returns
        -------
        X_new : array, shape ``[len(X), X.dim]``
            X transformed into its means.
        '''
        X = as_features(X)
        return np.vstack([np.mean(bag, axis=0) for bag in X])

def transform ( self , X ) : X = as_features ( X ) return np . vstack ( [ np . mean ( bag , axis = 0 ) for bag in X ] )

Transform a list of bag features into a matrix of its mean features .


def run(self):
        """Start listening to the server"""
        logger.info(u'Started listening')
        while not self._stop:
            xml = self._readxml()

            # Exit on invalid XML
            if xml is None:
                break

            # Raw xml only
            if not self.modelize:
                logger.info(u'Raw xml: %s' % xml)
                self.results.put(xml)
                continue

            # Model objects + raw xml as fallback
            if xml.tag == 'RECOGOUT':
                sentence = Sentence.from_shypo(xml.find('SHYPO'), self.encoding)
                logger.info(u'Modelized recognition: %r' % sentence)
                self.results.put(sentence)
            else:
                logger.info(u'Unmodelized xml: %s' % xml)
                self.results.put(xml)

        logger.info(u'Stopped listening')

def
run
(
self
)
:
logger
.
info
(
u'Started listening'
)
while
not
self
.
_stop
:
xml
=
self
.
_readxml
(
)
if
xml
is
:
break
if
not
self
.
modelize
:
logger
.
info
(
u'Raw xml: %s'
%
xml
)
self
.
results
.
put
(
xml
)
continue
if
xml
.
tag
==
'RECOGOUT'
:
sentence
=
Sentence
.
from_shypo
(
xml
.
find
(
'SHYPO'
)
,
self
.
encoding
)
logger
.
info
(
u'Modelized recognition: %r'
%
sentence
)
self
.
results
.
put
(
sentence
)
else
:
logger
.
info
(
u'Unmodelized xml: %s'
%
xml
)
self
.
results
.
put
(
xml
)
logger
.
info
(
u'Stopped listening'
)


def connect(self):
        """Connect to the server

        :raise ConnectionError: If socket cannot establish a connection

        """
        try:
            logger.info(u'Connecting %s:%d' % (self.host, self.port))
            self.sock.connect((self.host, self.port))
        except socket.error:
            raise ConnectionError()
        self.state = CONNECTED

def
connect
(
self
)
:
try
:
logger
.
info
(
u'Connecting %s:%d'
%
(
self
.
host
,
self
.
port
)
)
self
.
sock
.
connect
(
(
self
.
host
,
self
.
port
)
)
except
socket
.
error
:
raise
ConnectionError
(
)
self
.
state
=
CONNECTED


def disconnect(self):
        """Disconnect from the server"""
        logger.info(u'Disconnecting')
        self.sock.shutdown(socket.SHUT_RDWR)
        self.sock.close()
        self.state = DISCONNECTED

def disconnect ( self ) : logger . info ( u'Disconnecting' ) self . sock . shutdown ( socket . SHUT_RDWR ) self . sock . close ( ) self . state = DISCONNECTED


def send(self, command, timeout=5):
        """Send a command to the server

        :param string command: command to send

        """
        logger.info(u'Sending %s' % command)
        _, writable, __ = select.select([], [self.sock], [], timeout)
        if not writable:
            raise SendTimeoutError()
        writable[0].sendall(command + '\n')

def
send
(
self
,
command
,
timeout
=
5
)
:
logger
.
info
(
u'Sending %s'
%
command
)
_
,
writable
,
__
=
select
.
select
(
[
]
,
[
self
.
sock
]
,
[
]
,
timeout
)
if
not
writable
:
raise
SendTimeoutError
(
)
writable
[
0
]
.
sendall
(
command
+
'\n'
)


def _readline(self):
        """Read a line from the server. Data is read from the socket until a character ``\n`` is found

        :return: the read line
        :rtype: string

        """
        line = ''
        while 1:
            readable, _, __ = select.select([self.sock], [], [], 0.5)
            if self._stop:
                break
            if not readable:
                continue
            data = readable[0].recv(1)
            if data == '\n':
                break
            line += unicode(data, self.encoding)
        return line

def _readline ( self ) : line = '' while 1 : readable , _ , __ = select . select ( [ self . sock ] , [ ] , [ ] , 0.5 ) if self . _stop : break if not readable : continue data = readable [ 0 ] . recv ( 1 ) if data == '\n' : break line += unicode ( data , self . encoding ) return line

Read a line from the server . Data is read from the socket until a character \ n is found


def _readblock(self):
        """Read a block from the server. Lines are read until a character ``.`` is found

        :return: the read block
        :rtype: string

        """
        block = ''
        while not self._stop:
            line = self._readline()
            if line == '.':
                break
            block += line
        return block

def _readblock ( self ) : block = '' while not self . _stop : line = self . _readline ( ) if line == '.' : break block += line return block

Read a block from the server . Lines are read until a character . is found


def _readxml(self):
        """Read a block and return the result as XML

        :return: block as xml
        :rtype: xml.etree.ElementTree

        """
        block = re.sub(r'<(/?)s>', r'<\1s>', self._readblock())
        try:
            xml = XML(block)
        except ParseError:
            xml = None
        return xml

def _readxml ( self ) : block = re . sub ( r'<(/?)s>' , r'<\1s>' , self . _readblock ( ) ) try : xml = XML ( block ) except ParseError : xml = None return xml


def cli(id):
    """Analyse an OpenStreetMap changeset."""
    ch = Analyse(id)
    ch.full_analysis()
    click.echo(
        'Created: %s. Modified: %s. Deleted: %s' % (ch.create, ch.modify, ch.delete)
        )
    if ch.is_suspect:
        click.echo('The changeset {} is suspect! Reasons: {}'.format(
            id,
            ', '.join(ch.suspicion_reasons)
            ))
    else:
        click.echo('The changeset %s is not suspect!' % id)


def get_user_details(user_id):
    """Get information about number of changesets, blocks and mapping days of a
    user, using both the OSM API and the Mapbox comments APIself.
    """
    reasons = []
    try:
        url = OSM_USERS_API.format(user_id=requests.compat.quote(user_id))
        user_request = requests.get(url)
        if user_request.status_code == 200:
            user_data = user_request.content
            xml_data = ET.fromstring(user_data).getchildren()[0].getchildren()
            changesets = [i for i in xml_data if i.tag == 'changesets'][0]
            blocks = [i for i in xml_data if i.tag == 'blocks'][0]
            if int(changesets.get('count')) <= 5:
                reasons.append('New mapper')
            elif int(changesets.get('count')) <= 30:
                url = MAPBOX_USERS_API.format(
                    user_id=requests.compat.quote(user_id)
                    )
                user_request = requests.get(url)
                if user_request.status_code == 200:
                    mapping_days = int(
                        user_request.json().get('extra').get('mapping_days')
                        )
                    if mapping_days <= 5:
                        reasons.append('New mapper')
            if int(blocks.getchildren()[0].get('count')) > 1:
                reasons.append('User has multiple blocks')
    except Exception as e:
        message = 'Could not verify user of the changeset: {}, {}'
        print(message.format(user_id, str(e)))
    return reasons

Get information about number of changesets blocks and mapping days of a user using both the OSM API and the Mapbox comments APIself .


def changeset_info(changeset):
    """Return a dictionary with id, user, user_id, bounds, date of creation
    and all the tags of the changeset.

    Args:
        changeset: the XML string of the changeset.
    """
    keys = [tag.attrib.get('k') for tag in changeset.getchildren()]
    keys += ['id', 'user', 'uid', 'bbox', 'created_at']
    values = [tag.attrib.get('v') for tag in changeset.getchildren()]
    values += [
        changeset.get('id'), changeset.get('user'), changeset.get('uid'),
        get_bounds(changeset), changeset.get('created_at')
        ]

    return dict(zip(keys, values))

def changeset_info ( changeset ) : keys = [ tag . attrib . get ( 'k' ) for tag in changeset . getchildren ( ) ] keys += [ 'id' , 'user' , 'uid' , 'bbox' , 'created_at' ] values = [ tag . attrib . get ( 'v' ) for tag in changeset . getchildren ( ) ] values += [ changeset . get ( 'id' ) , changeset . get ( 'user' ) , changeset . get ( 'uid' ) , get_bounds ( changeset ) , changeset . get ( 'created_at' ) ] return dict ( zip ( keys , values ) )

Return a dictionary with id user user_id bounds date of creation and all the tags of the changeset .


def get_changeset(changeset):
    """Get the changeset using the OSM API and return the content as a XML
    ElementTree.

    Args:
        changeset: the id of the changeset.
    """
    url = 'https://www.openstreetmap.org/api/0.6/changeset/{}/download'.format(
        changeset
        )
    return ET.fromstring(requests.get(url).content)

Get the changeset using the OSM API and return the content as a XML ElementTree .


def get_metadata(changeset):
    """Get the metadata of a changeset using the OSM API and return it as a XML
    ElementTree.

    Args:
        changeset: the id of the changeset.
    """
    url = 'https://www.openstreetmap.org/api/0.6/changeset/{}'.format(changeset)
    return ET.fromstring(requests.get(url).content).getchildren()[0]

Get the metadata of a changeset using the OSM API and return it as a XML ElementTree .


def get_area(self, geojson):
        """Read the first feature from the geojson and return it as a Polygon
        object.
        """
        geojson = json.load(open(geojson, 'r'))
        self.area = Polygon(geojson['features'][0]['geometry']['coordinates'][0])

def get_area ( self , geojson ) : geojson = json . load ( open ( geojson , 'r' ) ) self . area = Polygon ( geojson [ 'features' ] [ 0 ] [ 'geometry' ] [ 'coordinates' ] [ 0 ] )

Read the first feature from the geojson and return it as a Polygon object .


def filter(self):
        """Filter the changesets that intersects with the geojson geometry."""
        self.content = [
            ch
            for ch in self.xml.getchildren()
            if get_bounds(ch).intersects(self.area)
            ]

def filter ( self ) : self . content = [ ch for ch in self . xml . getchildren ( ) if get_bounds ( ch ) . intersects ( self . area ) ]

Filter the changesets that intersects with the geojson geometry .


def set_fields(self, changeset):
        """Set the fields of this class with the metadata of the analysed
        changeset.
        """
        self.id = int(changeset.get('id'))
        self.user = changeset.get('user')
        self.uid = changeset.get('uid')
        self.editor = changeset.get('created_by', None)
        self.review_requested = changeset.get('review_requested', False)
        self.host = changeset.get('host', 'Not reported')
        self.bbox = changeset.get('bbox').wkt
        self.comment = changeset.get('comment', 'Not reported')
        self.source = changeset.get('source', 'Not reported')
        self.imagery_used = changeset.get('imagery_used', 'Not reported')
        self.date = datetime.strptime(
            changeset.get('created_at'),
            '%Y-%m-%dT%H:%M:%SZ'
            )
        self.suspicion_reasons = []
        self.is_suspect = False
        self.powerfull_editor = False

def
set_fields
(
self
,
changeset
)
:
self
.
id
=
int
(
changeset
.
get
(
'id'
)
)
self
.
user
=
changeset
.
get
(
'user'
)
self
.
uid
=
changeset
.
get
(
'uid'
)
self
.
editor
=
changeset
.
get
(
'created_by'
,
)
self
.
review_requested
=
changeset
.
get
(
'review_requested'
,
False
)
self
.
host
=
changeset
.
get
(
'host'
,
'Not reported'
)
self
.
bbox
=
changeset
.
get
(
'bbox'
)
.
wkt
self
.
comment
=
changeset
.
get
(
'comment'
,
'Not reported'
)
self
.
source
=
changeset
.
get
(
'source'
,
'Not reported'
)
self
.
imagery_used
=
changeset
.
get
(
'imagery_used'
,
'Not reported'
)
self
.
date
=
datetime
.
strptime
(
changeset
.
get
(
'created_at'
)
,
'%Y-%m-%dT%H:%M:%SZ'
)
self
.
suspicion_reasons
=
[
]
self
.
is_suspect
=
False
self
.
powerfull_editor
=
False

Set the fields of this class with the metadata of the analysed changeset .


def label_suspicious(self, reason):
        """Add suspicion reason and set the suspicious flag."""
        self.suspicion_reasons.append(reason)
        self.is_suspect = True

def label_suspicious ( self , reason ) : self . suspicion_reasons . append ( reason ) self . is_suspect = True

Add suspicion reason and set the suspicious flag .


def full_analysis(self):
        """Execute the count and verify_words methods."""
        self.count()
        self.verify_words()
        self.verify_user()

        if self.review_requested == 'yes':
            self.label_suspicious('Review requested')

def
full_analysis
(
self
)
:
self
.
count
(
)
self
.
verify_words
(
)
self
.
verify_user
(
)
if
self
.
review_requested
==
'yes'
:
self
.
label_suspicious
(
'Review requested'
)

Execute the count and verify_words methods .


def verify_words(self):
        """Verify the fields source, imagery_used and comment of the changeset
        for some suspect words.
        """
        if self.comment:
            if find_words(self.comment, self.suspect_words, self.excluded_words):
                self.label_suspicious('suspect_word')

        if self.source:
            for word in self.illegal_sources:
                if word in self.source.lower():
                    self.label_suspicious('suspect_word')
                    break

        if self.imagery_used:
            for word in self.illegal_sources:
                if word in self.imagery_used.lower():
                    self.label_suspicious('suspect_word')
                    break

        self.suspicion_reasons = list(set(self.suspicion_reasons))

def verify_words ( self ) : if self . comment : if find_words ( self . comment , self . suspect_words , self . excluded_words ) : self . label_suspicious ( 'suspect_word' ) if self . source : for word in self . illegal_sources : if word in self . source . lower ( ) : self . label_suspicious ( 'suspect_word' ) break if self . imagery_used : for word in self . illegal_sources : if word in self . imagery_used . lower ( ) : self . label_suspicious ( 'suspect_word' ) break self . suspicion_reasons = list ( set ( self . suspicion_reasons ) )

Verify the fields source imagery_used and comment of the changeset for some suspect words .


def verify_editor(self):
        """Verify if the software used in the changeset is a powerfull_editor.
        """
        powerful_editors = [
            'josm', 'level0', 'merkaartor', 'qgis', 'arcgis', 'upload.py',
            'osmapi', 'Services_OpenStreetMap'
            ]
        if self.editor is not None:
            for editor in powerful_editors:
                if editor in self.editor.lower():
                    self.powerfull_editor = True
                    break

            if 'iD' in self.editor:
                trusted_hosts = [
                    'www.openstreetmap.org/id',
                    'www.openstreetmap.org/edit',
                    'improveosm.org',
                    'strava.github.io/iD',
                    'preview.ideditor.com/release',
                    'preview.ideditor.com/master',
                    'hey.mapbox.com/iD-internal',
                    'projets.pavie.info/id-indoor',
                    'maps.mapcat.com/edit',
                    'id.softek.ir'
                    ]
                if self.host.split('://')[-1].strip('/') not in trusted_hosts:
                    self.label_suspicious('Unknown iD instance')
        else:
            self.powerfull_editor = True
            self.label_suspicious('Software editor was not declared')

def
verify_editor
(
self
)
:
powerful_editors
=
[
'josm'
,
'level0'
,
'merkaartor'
,
'qgis'
,
'arcgis'
,
'upload.py'
,
'osmapi'
,
'Services_OpenStreetMap'
]
if
self
.
editor
is
not
:
for
editor
in
powerful_editors
:
if
editor
in
self
.
editor
.
lower
(
)
:
self
.
powerfull_editor
=
True
break
if
'iD'
in
self
.
editor
:
trusted_hosts
=
[
'www.openstreetmap.org/id'
,
'www.openstreetmap.org/edit'
,
'improveosm.org'
,
'strava.github.io/iD'
,
'preview.ideditor.com/release'
,
'preview.ideditor.com/master'
,
'hey.mapbox.com/iD-internal'
,
'projets.pavie.info/id-indoor'
,
'maps.mapcat.com/edit'
,
'id.softek.ir'
]
if
self
.
host
.
split
(
'://'
)
[
1
]
.
strip
(
'/'
)
not
in
trusted_hosts
:
self
.
label_suspicious
(
'Unknown iD instance'
)
else
:
self
.
powerfull_editor
=
True
self
.
label_suspicious
(
'Software editor was not declared'
)

Verify if the software used in the changeset is a powerfull_editor .


def count(self):
        """Count the number of elements created, modified and deleted by the
        changeset and analyses if it is a possible import, mass modification or
        a mass deletion.
        """
        xml = get_changeset(self.id)
        actions = [action.tag for action in xml.getchildren()]
        self.create = actions.count('create')
        self.modify = actions.count('modify')
        self.delete = actions.count('delete')
        self.verify_editor()

        try:
            if (self.create / len(actions) > self.percentage and
                    self.create > self.create_threshold and
                    (self.powerfull_editor or self.create > self.top_threshold)):
                self.label_suspicious('possible import')
            elif (self.modify / len(actions) > self.percentage and
                    self.modify > self.modify_threshold):
                self.label_suspicious('mass modification')
            elif ((self.delete / len(actions) > self.percentage and
                    self.delete > self.delete_threshold) or
                    self.delete > self.top_threshold):
                self.label_suspicious('mass deletion')
        except ZeroDivisionError:
            print('It seems this changeset was redacted')

def
count
(
self
)
:
xml
=
get_changeset
(
self
.
id
)
actions
=
[
action
.
tag
for
action
in
xml
.
getchildren
(
)
]
self
.
create
=
actions
.
count
(
'create'
)
self
.
modify
=
actions
.
count
(
'modify'
)
self
.
delete
=
actions
.
count
(
'delete'
)
self
.
verify_editor
(
)
try
:
if
(
self
.
create
/
len
(
actions
)
>
self
.
percentage
and
self
.
create
>
self
.
create_threshold
and
(
self
.
powerfull_editor
or
self
.
create
>
self
.
top_threshold
)
)
:
self
.
label_suspicious
(
'possible import'
)
elif
(
self
.
modify
/
len
(
actions
)
>
self
.
percentage
and
self
.
modify
>
self
.
modify_threshold
)
:
self
.
label_suspicious
(
'mass modification'
)
elif
(
(
self
.
delete
/
len
(
actions
)
>
self
.
percentage
and
self
.
delete
>
self
.
delete_threshold
)
or
self
.
delete
>
self
.
top_threshold
)
:
self
.
label_suspicious
(
'mass deletion'
)
except
ZeroDivisionError
:
print
(
'It seems this changeset was redacted'
)

Count the number of elements created modified and deleted by the changeset and analyses if it is a possible import mass modification or a mass deletion .


def _unwrap_stream(uri, timeout, scanner, requests_session):
    """
    Get a stream URI from a playlist URI, ``uri``.
    Unwraps nested playlists until something that's not a playlist is found or
    the ``timeout`` is reached.
    """

    original_uri = uri
    seen_uris = set()
    deadline = time.time() + timeout

    while time.time() < deadline:
        if uri in seen_uris:
            logger.info(
                'Unwrapping stream from URI (%s) failed: '
                'playlist referenced itself', uri)
            return None
        else:
            seen_uris.add(uri)

        logger.debug('Unwrapping stream from URI: %s', uri)

        try:
            scan_timeout = deadline - time.time()
            if scan_timeout < 0:
                logger.info(
                    'Unwrapping stream from URI (%s) failed: '
                    'timed out in %sms', uri, timeout)
                return None
            scan_result = scanner.scan(uri, timeout=scan_timeout)
        except exceptions.ScannerError as exc:
            logger.debug('GStreamer failed scanning URI (%s): %s', uri, exc)
            scan_result = None

        if scan_result is not None and not (
                scan_result.mime.startswith('text/') or
                scan_result.mime.startswith('application/')):
            logger.debug(
                'Unwrapped potential %s stream: %s', scan_result.mime, uri)
            return uri

        download_timeout = deadline - time.time()
        if download_timeout < 0:
            logger.info(
                'Unwrapping stream from URI (%s) failed: timed out in %sms',
                uri, timeout)
            return None
        content = http.download(
            requests_session, uri, timeout=download_timeout)

        if content is None:
            logger.info(
                'Unwrapping stream from URI (%s) failed: '
                'error downloading URI %s', original_uri, uri)
            return None

        uris = playlists.parse(content)
        if not uris:
            logger.debug(
                'Failed parsing URI (%s) as playlist; found potential stream.',
                uri)
            return uri

        # TODO Test streams and return first that seems to be playable
        logger.debug(
            'Parsed playlist (%s) and found new URI: %s', uri, uris[0])
        uri = uris[0]

Get a stream URI from a playlist URI uri . Unwraps nested playlists until something that s not a playlist is found or the timeout is reached .