def _ends_in_doubled_cons(self, term):
        """Return Porter helper function _ends_in_doubled_cons value.

        Parameters
        ----------
        term : str
            The word to check for a final doubled consonant

        Returns
        -------
        bool
            True iff the stem ends in a doubled consonant (as defined in the
            Porter stemmer definition)

        """
        return (
            len(term) > 1
            and term[-1] not in self._vowels
            and term[-2] == term[-1]
        )

def _ends_in_doubled_cons ( self , term ) : return ( len ( term ) > 1 and term [ - 1 ] not in self . _vowels and term [ - 2 ] == term [ - 1 ] )

Return Porter helper function _ends_in_doubled_cons value .


def _ends_in_cvc(self, term):
        """Return Porter helper function _ends_in_cvc value.

        Parameters
        ----------
        term : str
            The word to scan for cvc

        Returns
        -------
        bool
            True iff the stem ends in cvc (as defined in the Porter stemmer
            definition)

        """
        return len(term) > 2 and (
            term[-1] not in self._vowels
            and term[-2] in self._vowels
            and term[-3] not in self._vowels
            and term[-1] not in tuple('wxY')
        )

def _ends_in_cvc ( self , term ) : return len ( term ) > 2 and ( term [ - 1 ] not in self . _vowels and term [ - 2 ] in self . _vowels and term [ - 3 ] not in self . _vowels and term [ - 1 ] not in tuple ( 'wxY' ) )

Return Porter helper function _ends_in_cvc value .


def filter_symlog(y, base=10.0):
    """Symmetrical logarithmic scale.

    Optional arguments:

    *base*:
        The base of the logarithm.
    """
    log_base = np.log(base)
    sign = np.sign(y)
    logs = np.log(np.abs(y) / log_base)
    return sign * logs

def filter_symlog ( y , base = 10.0 ) : log_base = np . log ( base ) sign = np . sign ( y ) logs = np . log ( np . abs ( y ) / log_base ) return sign * logs


def usage_function(parser):
    """Show usage and available curve functions."""
    parser.print_usage()
    print('')
    print('available functions:')
    for function in sorted(FUNCTION):
        doc = FUNCTION[function].__doc__.strip().splitlines()[0]
        print('    %-12s %s' % (function + ':', doc))

    return 0

def
usage_function
(
parser
)
:
parser
.
print_usage
(
)
print
(
''
)
print
(
'available functions:'
)
for
function
in
sorted
(
FUNCTION
)
:
doc
=
FUNCTION
[
function
]
.
__doc__
.
strip
(
)
.
splitlines
(
)
[
0
]
print
(
'    %-12s %s'
%
(
function
+
':'
,
doc
)
)
return
0

Show usage and available curve functions .


def usage_palette(parser):
    """Show usage and available palettes."""
    parser.print_usage()
    print('')
    print('available palettes:')
    for palette in sorted(PALETTE):
        print('    %-12s' % (palette,))

    return 0

def
usage_palette
(
parser
)
:
parser
.
print_usage
(
)
print
(
''
)
print
(
'available palettes:'
)
for
palette
in
sorted
(
PALETTE
)
:
print
(
'    %-12s'
%
(
palette
,
)
)
return
0


def size(self):
        """Get the current terminal size."""
        for fd in range(3):
            cr = self._ioctl_GWINSZ(fd)
            if cr:
                break
        if not cr:
            try:
                fd = os.open(os.ctermid(), os.O_RDONLY)
                cr = self._ioctl_GWINSZ(fd)
                os.close(fd)
            except Exception:
                pass

        if not cr:
            env = os.environ
            cr = (env.get('LINES', 25), env.get('COLUMNS', 80))

        return int(cr[1]), int(cr[0])

def size ( self ) : for fd in range ( 3 ) : cr = self . _ioctl_GWINSZ ( fd ) if cr : break if not cr : try : fd = os . open ( os . ctermid ( ) , os . O_RDONLY ) cr = self . _ioctl_GWINSZ ( fd ) os . close ( fd ) except Exception : pass if not cr : env = os . environ cr = ( env . get ( 'LINES' , 25 ) , env . get ( 'COLUMNS' , 80 ) ) return int ( cr [ 1 ] ) , int ( cr [ 0 ] )


def color(self, index):
        """Get the escape sequence for indexed color ``index``.

        The ``index`` is a color index in the 256 color space. The color space
        consists of:

        * 0x00-0x0f: default EGA colors
        * 0x10-0xe7: 6x6x6 RGB cubes
        * 0xe8-0xff: gray scale ramp
        """
        if self.colors == 16:
            if index >= 8:
                return self.csi('bold') + self.csi('setaf', index - 8)
            else:
                return self.csi('sgr0') + self.csi('setaf', index)
        else:
            return self.csi('setaf', index)

def color ( self , index ) : if self . colors == 16 : if index >= 8 : return self . csi ( 'bold' ) + self . csi ( 'setaf' , index - 8 ) else : return self . csi ( 'sgr0' ) + self . csi ( 'setaf' , index ) else : return self . csi ( 'setaf' , index )

Get the escape sequence for indexed color index .


def csi(self, capname, *args):
        """Return the escape sequence for the selected Control Sequence."""
        value = curses.tigetstr(capname)
        if value is None:
            return b''
        else:
            return curses.tparm(value, *args)

def csi ( self , capname , * args ) : value = curses . tigetstr ( capname ) if value is None : return b'' else : return curses . tparm ( value , * args )

Return the escape sequence for the selected Control Sequence .


def csi_wrap(self, value, capname, *args):
        """Return a value wrapped in the selected CSI and does a reset."""
        if isinstance(value, str):
            value = value.encode('utf-8')
        return b''.join([
            self.csi(capname, *args),
            value,
            self.csi('sgr0'),
        ])

def csi_wrap ( self , value , capname , * args ) : if isinstance ( value , str ) : value = value . encode ( 'utf-8' ) return b'' . join ( [ self . csi ( capname , * args ) , value , self . csi ( 'sgr0' ) , ] )

Return a value wrapped in the selected CSI and does a reset .


def consume(self, istream, ostream, batch=False):
        """Read points from istream and output to ostream."""
        datapoints = []  # List of 2-tuples

        if batch:
            sleep = max(0.01, self.option.sleep)
            fd = istream.fileno()
            while True:
                try:
                    if select.select([fd], [], [], sleep):
                        try:
                            line = istream.readline()
                            if line == '':
                                break
                            datapoints.append(self.consume_line(line))
                        except ValueError:
                            continue

                        if self.option.sort_by_column:
                            datapoints = sorted(datapoints, key=itemgetter(self.option.sort_by_column - 1))

                        if len(datapoints) > 1:
                            datapoints = datapoints[-self.maximum_points:]
                            self.update([dp[0] for dp in datapoints], [dp[1] for dp in datapoints])
                            self.render(ostream)

                        time.sleep(sleep)

                except KeyboardInterrupt:
                    break

        else:
            for line in istream:
                try:
                    datapoints.append(self.consume_line(line))
                except ValueError:
                    pass

            if self.option.sort_by_column:
                datapoints = sorted(datapoints, key=itemgetter(self.option.sort_by_column - 1))

            self.update([dp[0] for dp in datapoints], [dp[1] for dp in datapoints])
            self.render(ostream)

def consume ( self , istream , ostream , batch = False ) : datapoints = [ ] if batch : sleep = max ( 0.01 , self . option . sleep ) fd = istream . fileno ( ) while True : try : if select . select ( [ fd ] , [ ] , [ ] , sleep ) : try : line = istream . readline ( ) if line == '' : break datapoints . append ( self . consume_line ( line ) ) except ValueError : continue if self . option . sort_by_column : datapoints = sorted ( datapoints , key = itemgetter ( self . option . sort_by_column - 1 ) ) if len ( datapoints ) > 1 : datapoints = datapoints [ - self . maximum_points : ] self . update ( [ dp [ 0 ] for dp in datapoints ] , [ dp [ 1 ] for dp in datapoints ] ) self . render ( ostream ) time . sleep ( sleep ) except KeyboardInterrupt : break else : for line in istream : try : datapoints . append ( self . consume_line ( line ) ) except ValueError : pass if self . option . sort_by_column : datapoints = sorted ( datapoints , key = itemgetter ( self . option . sort_by_column - 1 ) ) self . update ( [ dp [ 0 ] for dp in datapoints ] , [ dp [ 1 ] for dp in datapoints ] ) self . render ( ostream )

Read points from istream and output to ostream .


def consume_line(self, line):
        """Consume data from a line."""
        data = RE_VALUE_KEY.split(line.strip(), 1)
        if len(data) == 1:
            return float(data[0]), None
        else:
            return float(data[0]), data[1].strip()

def consume_line ( self , line ) : data = RE_VALUE_KEY . split ( line . strip ( ) , 1 ) if len ( data ) == 1 : return float ( data [ 0 ] ) , None else : return float ( data [ 0 ] ) , data [ 1 ] . strip ( )


def update(self, points, values=None):
        """Add a set of data points."""
        self.values = values or [None] * len(points)

        if np is None:
            if self.option.function:
                warnings.warn('numpy not available, function ignored')
            self.points = points
            self.minimum = min(self.points)
            self.maximum = max(self.points)
            self.current = self.points[-1]

        else:
            self.points = self.apply_function(points)
            self.minimum = np.min(self.points)
            self.maximum = np.max(self.points)
            self.current = self.points[-1]

        if self.maximum == self.minimum:
            self.extents = 1
        else:
            self.extents = (self.maximum - self.minimum)
            self.extents = (self.maximum - self.minimum)

def
update
(
self
,
points
,
values
=
)
:
self
.
values
=
values
or
[
]
*
len
(
points
)
if
np
is
:
if
self
.
option
.
function
:
warnings
.
warn
(
'numpy not available, function ignored'
)
self
.
points
=
points
self
.
minimum
=
min
(
self
.
points
)
self
.
maximum
=
max
(
self
.
points
)
self
.
current
=
self
.
points
[
1
]
else
:
self
.
points
=
self
.
apply_function
(
points
)
self
.
minimum
=
np
.
min
(
self
.
points
)
self
.
maximum
=
np
.
max
(
self
.
points
)
self
.
current
=
self
.
points
[
1
]
if
self
.
maximum
==
self
.
minimum
:
self
.
extents
=
1
else
:
self
.
extents
=
(
self
.
maximum
self
.
minimum
)
self
.
extents
=
(
self
.
maximum
self
.
minimum
)


def color_ramp(self, size):
        """Generate a color ramp for the current screen height."""
        color = PALETTE.get(self.option.palette, {})
        color = color.get(self.term.colors, None)
        color_ramp = []
        if color is not None:
            ratio = len(color) / float(size)
            for i in range(int(size)):
                color_ramp.append(self.term.color(color[int(ratio * i)]))

        return color_ramp

def color_ramp ( self , size ) : color = PALETTE . get ( self . option . palette , { } ) color = color . get ( self . term . colors , None ) color_ramp = [ ] if color is not None : ratio = len ( color ) / float ( size ) for i in range ( int ( size ) ) : color_ramp . append ( self . term . color ( color [ int ( ratio * i ) ] ) ) return color_ramp

Generate a color ramp for the current screen height .


def human(self, size, base=1000, units=' kMGTZ'):
        """Convert the input ``size`` to human readable, short form."""
        sign = '+' if size >= 0 else '-'
        size = abs(size)
        if size < 1000:
            return '%s%d' % (sign, size)
        for i, suffix in enumerate(units):
            unit = 1000 ** (i + 1)
            if size < unit:
                return ('%s%.01f%s' % (
                    sign,
                    size / float(unit) * base,
                    suffix,
                )).strip()
        raise OverflowError

def
human
(
self
,
size
,
base
=
1000
,
units
=
' kMGTZ'
)
:
sign
=
'+'
if
size
>=
0
else
'-'
size
=
abs
(
size
)
if
size
<
1000
:
return
'%s%d'
%
(
sign
,
size
)
for
i
,
suffix
in
enumerate
(
units
)
:
unit
=
1000
**
(
i
+
1
)
if
size
<
unit
:
return
(
'%s%.01f%s'
%
(
sign
,
size
/
float
(
unit
)
*
base
,
suffix
,
)
)
.
strip
(
)
raise
OverflowError

Convert the input size to human readable short form .


def apply_function(self, points):
        """Run the filter function on the provided points."""
        if not self.option.function:
            return points

        if np is None:
            raise ImportError('numpy is not available')

        if ':' in self.option.function:
            function, arguments = self.option.function.split(':', 1)
            arguments = arguments.split(',')
        else:
            function = self.option.function
            arguments = []

        # Resolve arguments
        arguments = list(map(self._function_argument, arguments))

        # Resolve function
        filter_function = FUNCTION.get(function)

        if filter_function is None:
            raise TypeError('Invalid function "%s"' % (function,))

        else:
            # We wrap in ``list()`` to consume generators and iterators, as
            # ``np.array`` doesn't do this for us.
            return filter_function(np.array(list(points)), *arguments)

def
apply_function
(
self
,
points
)
:
if
not
self
.
option
.
function
:
return
points
if
np
is
:
raise
ImportError
(
'numpy is not available'
)
if
':'
in
self
.
option
.
function
:
function
,
arguments
=
self
.
option
.
function
.
split
(
':'
,
1
)
arguments
=
arguments
.
split
(
','
)
else
:
function
=
self
.
option
.
function
arguments
=
[
]
arguments
=
list
(
map
(
self
.
_function_argument
,
arguments
)
)
filter_function
=
FUNCTION
.
get
(
function
)
if
filter_function
is
:
raise
TypeError
(
'Invalid function "%s"'
%
(
function
,
)
)
else
:
return
filter_function
(
np
.
array
(
list
(
points
)
)
,
*
arguments
)

Run the filter function on the provided points .


def line(self, p1, p2, resolution=1):
        """Resolve the points to make a line between two points."""
        xdiff = max(p1.x, p2.x) - min(p1.x, p2.x)
        ydiff = max(p1.y, p2.y) - min(p1.y, p2.y)
        xdir = [-1, 1][int(p1.x <= p2.x)]
        ydir = [-1, 1][int(p1.y <= p2.y)]
        r = int(round(max(xdiff, ydiff)))
        if r == 0:
            return

        for i in range((r + 1) * resolution):
            x = p1.x
            y = p1.y

            if xdiff:
                x += (float(i) * xdiff) / r * xdir / resolution
            if ydiff:
                y += (float(i) * ydiff) / r * ydir / resolution

            yield Point((x, y))

def line ( self , p1 , p2 , resolution = 1 ) : xdiff = max ( p1 . x , p2 . x ) - min ( p1 . x , p2 . x ) ydiff = max ( p1 . y , p2 . y ) - min ( p1 . y , p2 . y ) xdir = [ - 1 , 1 ] [ int ( p1 . x <= p2 . x ) ] ydir = [ - 1 , 1 ] [ int ( p1 . y <= p2 . y ) ] r = int ( round ( max ( xdiff , ydiff ) ) ) if r == 0 : return for i in range ( ( r + 1 ) * resolution ) : x = p1 . x y = p1 . y if xdiff : x += ( float ( i ) * xdiff ) / r * xdir / resolution if ydiff : y += ( float ( i ) * ydiff ) / r * ydir / resolution yield Point ( ( x , y ) )

Resolve the points to make a line between two points .


def set_text(self, point, text):
        """Set a text value in the screen canvas."""
        if not self.option.legend:
            return

        if not isinstance(point, Point):
            point = Point(point)

        for offset, char in enumerate(str(text)):
            self.screen.canvas[point.y][point.x + offset] = char

def set_text ( self , point , text ) : if not self . option . legend : return if not isinstance ( point , Point ) : point = Point ( point ) for offset , char in enumerate ( str ( text ) ) : self . screen . canvas [ point . y ] [ point . x + offset ] = char


def render(self, stream):
        """Render graph to stream."""
        encoding = self.option.encoding or self.term.encoding or "utf8"

        if self.option.color:
            ramp = self.color_ramp(self.size.y)[::-1]
        else:
            ramp = None

        if self.cycle >= 1 and self.lines:
            stream.write(self.term.csi('cuu', self.lines))

        zero = int(self.null / 4)  # Zero crossing
        lines = 0
        for y in range(self.screen.size.y):
            if y == zero and self.size.y > 1:
                stream.write(self.term.csi('smul'))
            if ramp:
                stream.write(ramp[y])

            for x in range(self.screen.size.x):
                point = Point((x, y))
                if point in self.screen:
                    value = self.screen[point]
                    if isinstance(value, int):
                        stream.write(chr(self.base + value).encode(encoding))
                    else:
                        stream.write(self.term.csi('sgr0'))
                        stream.write(self.term.csi_wrap(
                            value.encode(encoding),
                            'bold'
                        ))
                        if y == zero and self.size.y > 1:
                            stream.write(self.term.csi('smul'))
                        if ramp:
                            stream.write(ramp[y])
                else:
                    stream.write(b' ')

            if y == zero and self.size.y > 1:
                stream.write(self.term.csi('rmul'))
            if ramp:
                stream.write(self.term.csi('sgr0'))

            stream.write(b'\n')
            lines += 1
        stream.flush()

        self.cycle = self.cycle + 1
        self.lines = lines

def
render
(
self
,
stream
)
:
encoding
=
self
.
option
.
encoding
or
self
.
term
.
encoding
or
"utf8"
if
self
.
option
.
color
:
ramp
=
self
.
color_ramp
(
self
.
size
.
y
)
[
:
:
1
]
else
:
ramp
=
if
self
.
cycle
>=
1
and
self
.
lines
:
stream
.
write
(
self
.
term
.
csi
(
'cuu'
,
self
.
lines
)
)
zero
=
int
(
self
.
/
4
)
lines
=
0
for
y
in
range
(
self
.
screen
.
size
.
y
)
:
if
y
==
zero
and
self
.
size
.
y
>
1
:
stream
.
write
(
self
.
term
.
csi
(
'smul'
)
)
if
ramp
:
stream
.
write
(
ramp
[
y
]
)
for
x
in
range
(
self
.
screen
.
size
.
x
)
:
point
=
Point
(
(
x
,
y
)
)
if
point
in
self
.
screen
:
value
=
self
.
screen
[
point
]
if
isinstance
(
value
,
int
)
:
stream
.
write
(
chr
(
self
.
base
+
value
)
.
encode
(
encoding
)
)
else
:
stream
.
write
(
self
.
term
.
csi
(
'sgr0'
)
)
stream
.
write
(
self
.
term
.
csi_wrap
(
value
.
encode
(
encoding
)
,
'bold'
)
)
if
y
==
zero
and
self
.
size
.
y
>
1
:
stream
.
write
(
self
.
term
.
csi
(
'smul'
)
)
if
ramp
:
stream
.
write
(
ramp
[
y
]
)
else
:
stream
.
write
(
b' '
)
if
y
==
zero
and
self
.
size
.
y
>
1
:
stream
.
write
(
self
.
term
.
csi
(
'rmul'
)
)
if
ramp
:
stream
.
write
(
self
.
term
.
csi
(
'sgr0'
)
)
stream
.
write
(
b'\n'
)
lines
+=
1
stream
.
flush
(
)
self
.
cycle
=
self
.
cycle
+
1
self
.
lines
=
lines


def _normalised_numpy(self):
        """Normalised data points using numpy."""
        dx = (self.screen.width / float(len(self.points)))
        oy = (self.screen.height)
        points = np.array(self.points) - self.minimum
        points = points * 4.0 / self.extents * self.size.y
        for x, y in enumerate(points):
            yield Point((
                dx * x,
                min(oy, oy - y),
            ))

def _normalised_numpy ( self ) : dx = ( self . screen . width / float ( len ( self . points ) ) ) oy = ( self . screen . height ) points = np . array ( self . points ) - self . minimum points = points * 4.0 / self . extents * self . size . y for x , y in enumerate ( points ) : yield Point ( ( dx * x , min ( oy , oy - y ) , ) )


def _normalised_python(self):
        """Normalised data points using pure Python."""
        dx = (self.screen.width / float(len(self.points)))
        oy = (self.screen.height)
        for x, point in enumerate(self.points):
            y = (point - self.minimum) * 4.0 / self.extents * self.size.y
            yield Point((
                dx * x,
                min(oy, oy - y),
            ))

def _normalised_python ( self ) : dx = ( self . screen . width / float ( len ( self . points ) ) ) oy = ( self . screen . height ) for x , point in enumerate ( self . points ) : y = ( point - self . minimum ) * 4.0 / self . extents * self . size . y yield Point ( ( dx * x , min ( oy , oy - y ) , ) )

Normalised data points using pure Python .


def null(self):
        """Zero crossing value."""
        if not self.option.axis:
            return -1
        else:
            return self.screen.height - (
                -self.minimum * 4.0 / self.extents * self.size.y
            )

def null ( self ) : if not self . option . axis : return - 1 else : return self . screen . height - ( - self . minimum * 4.0 / self . extents * self . size . y )


def mem_size(self):
        '''used when allocating memory ingame'''

        data_len = self._data_mem_size
        node_count = len(list(self.xml_doc.iter(tag=etree.Element)))

        if self.compressed:
            size = 52 * node_count + data_len + 630
        else:
            tags_len = 0
            for e in self.xml_doc.iter(tag=etree.Element):
                e_len = max(len(e.tag), 8)
                e_len = (e_len + 3) & ~3
                tags_len += e_len

            size = 56 * node_count + data_len + 630 + tags_len

        # debugging
        #print('nodes:{} ({}) data:{} ({})'.format(node_count,hex(node_count), data_len, hex(data_len)))

        return (size + 8) & ~7

def mem_size ( self ) : data_len = self . _data_mem_size node_count = len ( list ( self . xml_doc . iter ( tag = etree . Element ) ) ) if self . compressed : size = 52 * node_count + data_len + 630 else : tags_len = 0 for e in self . xml_doc . iter ( tag = etree . Element ) : e_len = max ( len ( e . tag ) , 8 ) e_len = ( e_len + 3 ) & ~ 3 tags_len += e_len size = 56 * node_count + data_len + 630 + tags_len return ( size + 8 ) & ~ 7


def _load_class(class_path, default):
    """ Loads the class from the class_path string """
    if class_path is None:
        return default

    component = class_path.rsplit('.', 1)
    result_processor = getattr(
        importlib.import_module(component[0]),
        component[1],
        default
    ) if len(component) > 1 else default

    return result_processor

def _load_class ( class_path , default ) : if class_path is None : return default component = class_path . rsplit ( '.' , 1 ) result_processor = getattr ( importlib . import_module ( component [ 0 ] ) , component [ 1 ] , default ) if len ( component ) > 1 else default return result_processor


def _process_pagination_values(request):
    """ process pagination requests from request parameter """
    size = 20
    page = 0
    from_ = 0
    if "page_size" in request.POST:
        size = int(request.POST["page_size"])
        max_page_size = getattr(settings, "SEARCH_MAX_PAGE_SIZE", 100)
        # The parens below are superfluous, but make it much clearer to the reader what is going on
        if not (0 < size <= max_page_size):  # pylint: disable=superfluous-parens
            raise ValueError(_('Invalid page size of {page_size}').format(page_size=size))

        if "page_index" in request.POST:
            page = int(request.POST["page_index"])
            from_ = page * size
    return size, from_, page

def
_process_pagination_values
(
request
)
:
size
=
20
page
=
0
from_
=
0
if
"page_size"
in
request
.
POST
:
size
=
int
(
request
.
POST
[
"page_size"
]
)
max_page_size
=
getattr
(
settings
,
"SEARCH_MAX_PAGE_SIZE"
,
100
)
if
not
(
0
<
size
<=
max_page_size
)
:
raise
ValueError
(
_
(
'Invalid page size of {page_size}'
)
.
format
(
page_size
=
size
)
)
if
"page_index"
in
request
.
POST
:
page
=
int
(
request
.
POST
[
"page_index"
]
)
from_
=
page
*
size
return
size
,
from_
,
page


def _process_field_values(request):
    """ Create separate dictionary of supported filter values provided """
    return {
        field_key: request.POST[field_key]
        for field_key in request.POST
        if field_key in course_discovery_filter_fields()
    }

def _process_field_values ( request ) : return { field_key : request . POST [ field_key ] for field_key in request . POST if field_key in course_discovery_filter_fields ( ) }


def do_search(request, course_id=None):
    """
    Search view for http requests

    Args:
        request (required) - django request object
        course_id (optional) - course_id within which to restrict search

    Returns:
        http json response with the following fields
            "took" - how many seconds the operation took
            "total" - how many results were found
            "max_score" - maximum score from these results
            "results" - json array of result documents

            or

            "error" - displayable information about an error that occured on the server

    POST Params:
        "search_string" (required) - text upon which to search
        "page_size" (optional)- how many results to return per page (defaults to 20, with maximum cutoff at 100)
        "page_index" (optional) - for which page (zero-indexed) to include results (defaults to 0)
    """

    # Setup search environment
    SearchInitializer.set_search_enviroment(request=request, course_id=course_id)

    results = {
        "error": _("Nothing to search")
    }
    status_code = 500

    search_term = request.POST.get("search_string", None)

    try:
        if not search_term:
            raise ValueError(_('No search term provided for search'))

        size, from_, page = _process_pagination_values(request)

        # Analytics - log search request
        track.emit(
            'edx.course.search.initiated',
            {
                "search_term": search_term,
                "page_size": size,
                "page_number": page,
            }
        )

        results = perform_search(
            search_term,
            user=request.user,
            size=size,
            from_=from_,
            course_id=course_id
        )

        status_code = 200

        # Analytics - log search results before sending to browser
        track.emit(
            'edx.course.search.results_displayed',
            {
                "search_term": search_term,
                "page_size": size,
                "page_number": page,
                "results_count": results["total"],
            }
        )

    except ValueError as invalid_err:
        results = {
            "error": six.text_type(invalid_err)
        }
        log.debug(six.text_type(invalid_err))

    except QueryParseError:
        results = {
            "error": _('Your query seems malformed. Check for unmatched quotes.')
        }

    # Allow for broad exceptions here - this is an entry point from external reference
    except Exception as err:  # pylint: disable=broad-except
        results = {
            "error": _('An error occurred when searching for "{search_string}"').format(search_string=search_term)
        }
        log.exception(
            'Search view exception when searching for %s for user %s: %r',
            search_term,
            request.user.id,
            err
        )

    return JsonResponse(results, status=status_code)


def course_discovery(request):
    """
    Search for courses

    Args:
        request (required) - django request object

    Returns:
        http json response with the following fields
            "took" - how many seconds the operation took
            "total" - how many results were found
            "max_score" - maximum score from these resutls
            "results" - json array of result documents

            or

            "error" - displayable information about an error that occured on the server

    POST Params:
        "search_string" (optional) - text with which to search for courses
        "page_size" (optional)- how many results to return per page (defaults to 20, with maximum cutoff at 100)
        "page_index" (optional) - for which page (zero-indexed) to include results (defaults to 0)
    """
    results = {
        "error": _("Nothing to search")
    }
    status_code = 500

    search_term = request.POST.get("search_string", None)

    try:
        size, from_, page = _process_pagination_values(request)
        field_dictionary = _process_field_values(request)

        # Analytics - log search request
        track.emit(
            'edx.course_discovery.search.initiated',
            {
                "search_term": search_term,
                "page_size": size,
                "page_number": page,
            }
        )

        results = course_discovery_search(
            search_term=search_term,
            size=size,
            from_=from_,
            field_dictionary=field_dictionary,
        )

        # Analytics - log search results before sending to browser
        track.emit(
            'edx.course_discovery.search.results_displayed',
            {
                "search_term": search_term,
                "page_size": size,
                "page_number": page,
                "results_count": results["total"],
            }
        )

        status_code = 200

    except ValueError as invalid_err:
        results = {
            "error": six.text_type(invalid_err)
        }
        log.debug(six.text_type(invalid_err))

    except QueryParseError:
        results = {
            "error": _('Your query seems malformed. Check for unmatched quotes.')
        }

    # Allow for broad exceptions here - this is an entry point from external reference
    except Exception as err:  # pylint: disable=broad-except
        results = {
            "error": _('An error occurred when searching for "{search_string}"').format(search_string=search_term)
        }
        log.exception(
            'Search view exception when searching for %s for user %s: %r',
            search_term,
            request.user.id,
            err
        )

    return JsonResponse(results, status=status_code)


def _translate_hits(es_response):
    """ Provide resultset in our desired format from elasticsearch results """

    def translate_result(result):
        """ Any conversion from ES result syntax into our search engine syntax """
        translated_result = copy.copy(result)
        data = translated_result.pop("_source")

        translated_result.update({
            "data": data,
            "score": translated_result["_score"]
        })

        return translated_result

    def translate_facet(result):
        """ Any conversion from ES facet syntax into our search engine sytax """
        terms = {term["term"]: term["count"] for term in result["terms"]}
        return {
            "terms": terms,
            "total": result["total"],
            "other": result["other"],
        }

    results = [translate_result(hit) for hit in es_response["hits"]["hits"]]
    response = {
        "took": es_response["took"],
        "total": es_response["hits"]["total"],
        "max_score": es_response["hits"]["max_score"],
        "results": results,
    }

    if "facets" in es_response:
        response["facets"] = {facet: translate_facet(es_response["facets"][facet]) for facet in es_response["facets"]}

    return response

def _translate_hits ( es_response ) : def translate_result ( result ) : translated_result = copy . copy ( result ) data = translated_result . pop ( "_source" ) translated_result . update ( { "data" : data , "score" : translated_result [ "_score" ] } ) return translated_result def translate_facet ( result ) : terms = { term [ "term" ] : term [ "count" ] for term in result [ "terms" ] } return { "terms" : terms , "total" : result [ "total" ] , "other" : result [ "other" ] , } results = [ translate_result ( hit ) for hit in es_response [ "hits" ] [ "hits" ] ] response = { "took" : es_response [ "took" ] , "total" : es_response [ "hits" ] [ "total" ] , "max_score" : es_response [ "hits" ] [ "max_score" ] , "results" : results , } if "facets" in es_response : response [ "facets" ] = { facet : translate_facet ( es_response [ "facets" ] [ facet ] ) for facet in es_response [ "facets" ] } return response


def _get_filter_field(field_name, field_value):
    """ Return field to apply into filter, if an array then use a range, otherwise look for a term match """
    filter_field = None
    if isinstance(field_value, ValueRange):
        range_values = {}
        if field_value.lower:
            range_values.update({"gte": field_value.lower_string})
        if field_value.upper:
            range_values.update({"lte": field_value.upper_string})
        filter_field = {
            "range": {
                field_name: range_values
            }
        }
    elif _is_iterable(field_value):
        filter_field = {
            "terms": {
                field_name: field_value
            }
        }
    else:
        filter_field = {
            "term": {
                field_name: field_value
            }
        }
    return filter_field

def _get_filter_field ( field_name , field_value ) : filter_field = None if isinstance ( field_value , ValueRange ) : range_values = { } if field_value . lower : range_values . update ( { "gte" : field_value . lower_string } ) if field_value . upper : range_values . update ( { "lte" : field_value . upper_string } ) filter_field = { "range" : { field_name : range_values } } elif _is_iterable ( field_value ) : filter_field = { "terms" : { field_name : field_value } } else : filter_field = { "term" : { field_name : field_value } } return filter_field

Return field to apply into filter if an array then use a range otherwise look for a term match


def _process_field_queries(field_dictionary):
    """
    We have a field_dictionary - we want to match the values for an elasticsearch "match" query
    This is only potentially useful when trying to tune certain search operations
    """
    def field_item(field):
        """ format field match as "match" item for elasticsearch query """
        return {
            "match": {
                field: field_dictionary[field]
            }
        }

    return [field_item(field) for field in field_dictionary]

def _process_field_queries ( field_dictionary ) : def field_item ( field ) : return { "match" : { field : field_dictionary [ field ] } } return [ field_item ( field ) for field in field_dictionary ]

We have a field_dictionary - we want to match the values for an elasticsearch match query This is only potentially useful when trying to tune certain search operations


def _process_filters(filter_dictionary):
    """
    We have a filter_dictionary - this means that if the field is included
    and matches, then we can include, OR if the field is undefined, then we
    assume it is safe to include
    """
    def filter_item(field):
        """ format elasticsearch filter to pass if value matches OR field is not included """
        if filter_dictionary[field] is not None:
            return {
                "or": [
                    _get_filter_field(field, filter_dictionary[field]),
                    {
                        "missing": {
                            "field": field
                        }
                    }
                ]
            }

        return {
            "missing": {
                "field": field
            }
        }

    return [filter_item(field) for field in filter_dictionary]

def _process_filters ( filter_dictionary ) : def filter_item ( field ) : if filter_dictionary [ field ] is not None : return { "or" : [ _get_filter_field ( field , filter_dictionary [ field ] ) , { "missing" : { "field" : field } } ] } return { "missing" : { "field" : field } } return [ filter_item ( field ) for field in filter_dictionary ]

We have a filter_dictionary - this means that if the field is included and matches then we can include OR if the field is undefined then we assume it is safe to include


def _process_exclude_dictionary(exclude_dictionary):
    """
    Based on values in the exclude_dictionary generate a list of term queries that
    will filter out unwanted results.
    """
    # not_properties will hold the generated term queries.
    not_properties = []
    for exclude_property in exclude_dictionary:
        exclude_values = exclude_dictionary[exclude_property]
        if not isinstance(exclude_values, list):
            exclude_values = [exclude_values]
        not_properties.extend([{"term": {exclude_property: exclude_value}} for exclude_value in exclude_values])

    # Returning a query segment with an empty list freaks out ElasticSearch,
    #   so just return an empty segment.
    if not not_properties:
        return {}

    return {
        "not": {
            "filter": {
                "or": not_properties
            }
        }
    }

def _process_exclude_dictionary ( exclude_dictionary ) : not_properties = [ ] for exclude_property in exclude_dictionary : exclude_values = exclude_dictionary [ exclude_property ] if not isinstance ( exclude_values , list ) : exclude_values = [ exclude_values ] not_properties . extend ( [ { "term" : { exclude_property : exclude_value } } for exclude_value in exclude_values ] ) if not not_properties : return { } return { "not" : { "filter" : { "or" : not_properties } } }

Based on values in the exclude_dictionary generate a list of term queries that will filter out unwanted results .


def _process_facet_terms(facet_terms):
    """ We have a list of terms with which we return facets """
    elastic_facets = {}
    for facet in facet_terms:
        facet_term = {"field": facet}
        if facet_terms[facet]:
            for facet_option in facet_terms[facet]:
                facet_term[facet_option] = facet_terms[facet][facet_option]

        elastic_facets[facet] = {
            "terms": facet_term
        }

    return elastic_facets

def _process_facet_terms ( facet_terms ) : elastic_facets = { } for facet in facet_terms : facet_term = { "field" : facet } if facet_terms [ facet ] : for facet_option in facet_terms [ facet ] : facet_term [ facet_option ] = facet_terms [ facet ] [ facet_option ] elastic_facets [ facet ] = { "terms" : facet_term } return elastic_facets


def get_mappings(cls, index_name, doc_type):
        """ fetch mapped-items structure from cache """
        return cache.get(cls.get_cache_item_name(index_name, doc_type), {})

def get_mappings ( cls , index_name , doc_type ) : return cache . get ( cls . get_cache_item_name ( index_name , doc_type ) , { } )

fetch mapped - items structure from cache


def set_mappings(cls, index_name, doc_type, mappings):
        """ set new mapped-items structure into cache """
        cache.set(cls.get_cache_item_name(index_name, doc_type), mappings)

def set_mappings ( cls , index_name , doc_type , mappings ) : cache . set ( cls . get_cache_item_name ( index_name , doc_type ) , mappings )

set new mapped - items structure into cache


def log_indexing_error(cls, indexing_errors):
        """ Logs indexing errors and raises a general ElasticSearch Exception"""
        indexing_errors_log = []
        for indexing_error in indexing_errors:
            indexing_errors_log.append(str(indexing_error))
        raise exceptions.ElasticsearchException(', '.join(indexing_errors_log))

def
log_indexing_error
(
cls
,
indexing_errors
)
:
indexing_errors_log
=
[
]
for
indexing_error
in
indexing_errors
:
indexing_errors_log
.
append
(
str
(
indexing_error
)
)
raise
exceptions
.
ElasticsearchException
(
', '
.
join
(
indexing_errors_log
)
)


def _get_mappings(self, doc_type):
        """
        Interfaces with the elasticsearch mappings for the index
        prevents multiple loading of the same mappings from ES when called more than once

        Mappings format in elasticsearch is as follows:
        {
           "doc_type": {
              "properties": {
                 "nested_property": {
                    "properties": {
                       "an_analysed_property": {
                          "type": "string"
                       },
                       "another_analysed_property": {
                          "type": "string"
                       }
                    }
                 },
                 "a_not_analysed_property": {
                    "type": "string",
                    "index": "not_analyzed"
                 },
                 "a_date_property": {
                    "type": "date"
                 }
              }
           }
        }

        We cache the properties of each doc_type, if they are not available, we'll load them again from Elasticsearch
        """
        # Try loading the mapping from the cache.
        mapping = ElasticSearchEngine.get_mappings(self.index_name, doc_type)

        # Fall back to Elasticsearch
        if not mapping:
            mapping = self._es.indices.get_mapping(
                index=self.index_name,
                doc_type=doc_type,
            ).get(self.index_name, {}).get('mappings', {}).get(doc_type, {})

            # Cache the mapping, if one was retrieved
            if mapping:
                ElasticSearchEngine.set_mappings(
                    self.index_name,
                    doc_type,
                    mapping
                )

        return mapping

def _get_mappings ( self , doc_type ) : mapping = ElasticSearchEngine . get_mappings ( self . index_name , doc_type ) if not mapping : mapping = self . _es . indices . get_mapping ( index = self . index_name , doc_type = doc_type , ) . get ( self . index_name , { } ) . get ( 'mappings' , { } ) . get ( doc_type , { } ) if mapping : ElasticSearchEngine . set_mappings ( self . index_name , doc_type , mapping ) return mapping


def index(self, doc_type, sources, **kwargs):
        """
        Implements call to add documents to the ES index
        Note the call to _check_mappings which will setup fields with the desired mappings
        """

        try:
            actions = []
            for source in sources:
                self._check_mappings(doc_type, source)
                id_ = source['id'] if 'id' in source else None
                log.debug("indexing %s object with id %s", doc_type, id_)
                action = {
                    "_index": self.index_name,
                    "_type": doc_type,
                    "_id": id_,
                    "_source": source
                }
                actions.append(action)
            # bulk() returns a tuple with summary information
            # number of successfully executed actions and number of errors if stats_only is set to True.
            _, indexing_errors = bulk(
                self._es,
                actions,
                **kwargs
            )
            if indexing_errors:
                ElasticSearchEngine.log_indexing_error(indexing_errors)
        # Broad exception handler to protect around bulk call
        except Exception as ex:
            # log information and re-raise
            log.exception("error while indexing - %s", str(ex))
            raise

def
index
(
self
,
doc_type
,
sources
,
**
kwargs
)
:
try
:
actions
=
[
]
for
source
in
sources
:
self
.
_check_mappings
(
doc_type
,
source
)
id_
=
source
[
'id'
]
if
'id'
in
source
else
log
.
debug
(
"indexing %s object with id %s"
,
doc_type
,
id_
)
action
=
{
"_index"
:
self
.
index_name
,
"_type"
:
doc_type
,
"_id"
:
id_
,
"_source"
:
source
}
actions
.
append
(
action
)
_
,
indexing_errors
=
bulk
(
self
.
_es
,
actions
,
**
kwargs
)
if
indexing_errors
:
ElasticSearchEngine
.
log_indexing_error
(
indexing_errors
)
except
Exception
as
ex
:
log
.
exception
(
"error while indexing - %s"
,
str
(
ex
)
)
raise


def remove(self, doc_type, doc_ids, **kwargs):
        """ Implements call to remove the documents from the index """

        try:
            # ignore is flagged as an unexpected-keyword-arg; ES python client documents that it can be used
            # pylint: disable=unexpected-keyword-arg
            actions = []
            for doc_id in doc_ids:
                log.debug("Removing document of type %s and index %s", doc_type, doc_id)
                action = {
                    '_op_type': 'delete',
                    "_index": self.index_name,
                    "_type": doc_type,
                    "_id": doc_id
                }
                actions.append(action)
            bulk(self._es, actions, **kwargs)
        except BulkIndexError as ex:
            valid_errors = [error for error in ex.errors if error['delete']['status'] != 404]

            if valid_errors:
                log.exception("An error occurred while removing documents from the index.")
                raise


def search(self,
               query_string=None,
               field_dictionary=None,
               filter_dictionary=None,
               exclude_dictionary=None,
               facet_terms=None,
               exclude_ids=None,
               use_field_match=False,
               **kwargs):  # pylint: disable=too-many-arguments, too-many-locals, too-many-branches, arguments-differ
        """
        Implements call to search the index for the desired content.

        Args:
            query_string (str): the string of values upon which to search within the
            content of the objects within the index

            field_dictionary (dict): dictionary of values which _must_ exist and
            _must_ match in order for the documents to be included in the results

            filter_dictionary (dict): dictionary of values which _must_ match if the
            field exists in order for the documents to be included in the results;
            documents for which the field does not exist may be included in the
            results if they are not otherwise filtered out

            exclude_dictionary(dict): dictionary of values all of which which must
            not match in order for the documents to be included in the results;
            documents which have any of these fields and for which the value matches
            one of the specified values shall be filtered out of the result set

            facet_terms (dict): dictionary of terms to include within search
            facets list - key is the term desired to facet upon, and the value is a
            dictionary of extended information to include. Supported right now is a
            size specification for a cap upon how many facet results to return (can
            be an empty dictionary to use default size for underlying engine):

            e.g.
            {
                "org": {"size": 10},  # only show top 10 organizations
                "modes": {}
            }

            use_field_match (bool): flag to indicate whether to use elastic
            filtering or elastic matching for field matches - this is nothing but a
            potential performance tune for certain queries

            (deprecated) exclude_ids (list): list of id values to exclude from the results -
            useful for finding maches that aren't "one of these"

        Returns:
            dict object with results in the desired format
            {
                "took": 3,
                "total": 4,
                "max_score": 2.0123,
                "results": [
                    {
                        "score": 2.0123,
                        "data": {
                            ...
                        }
                    },
                    {
                        "score": 0.0983,
                        "data": {
                            ...
                        }
                    }
                ],
                "facets": {
                    "org": {
                        "total": total_count,
                        "other": 1,
                        "terms": {
                            "MITx": 25,
                            "HarvardX": 18
                        }
                    },
                    "modes": {
                        "total": modes_count,
                        "other": 15,
                        "terms": {
                            "honor": 58,
                            "verified": 44,
                        }
                    }
                }
            }

        Raises:
            ElasticsearchException when there is a problem with the response from elasticsearch

        Example usage:
            .search(
                "find the words within this string",
                {
                    "must_have_field": "mast_have_value for must_have_field"
                },
                {

                }
            )
        """

        log.debug("searching index with %s", query_string)

        elastic_queries = []
        elastic_filters = []

        # We have a query string, search all fields for matching text within the "content" node
        if query_string:
            if six.PY2:
                query_string = query_string.encode('utf-8').translate(None, RESERVED_CHARACTERS)
            else:
                query_string = query_string.translate(query_string.maketrans('', '', RESERVED_CHARACTERS))
            elastic_queries.append({
                "query_string": {
                    "fields": ["content.*"],
                    "query": query_string
                }
            })

        if field_dictionary:
            if use_field_match:
                elastic_queries.extend(_process_field_queries(field_dictionary))
            else:
                elastic_filters.extend(_process_field_filters(field_dictionary))

        if filter_dictionary:
            elastic_filters.extend(_process_filters(filter_dictionary))

        # Support deprecated argument of exclude_ids
        if exclude_ids:
            if not exclude_dictionary:
                exclude_dictionary = {}
            if "_id" not in exclude_dictionary:
                exclude_dictionary["_id"] = []
            exclude_dictionary["_id"].extend(exclude_ids)

        if exclude_dictionary:
            elastic_filters.append(_process_exclude_dictionary(exclude_dictionary))

        query_segment = {
            "match_all": {}
        }
        if elastic_queries:
            query_segment = {
                "bool": {
                    "must": elastic_queries
                }
            }

        query = query_segment
        if elastic_filters:
            filter_segment = {
                "bool": {
                    "must": elastic_filters
                }
            }
            query = {
                "filtered": {
                    "query": query_segment,
                    "filter": filter_segment,
                }
            }

        body = {"query": query}
        if facet_terms:
            facet_query = _process_facet_terms(facet_terms)
            if facet_query:
                body["facets"] = facet_query

        try:
            es_response = self._es.search(
                index=self.index_name,
                body=body,
                **kwargs
            )
        except exceptions.ElasticsearchException as ex:
            message = six.text_type(ex)
            if 'QueryParsingException' in message:
                log.exception("Malformed search query: %s", message)
                raise QueryParseError('Malformed search query.')
            else:
                # log information and re-raise
                log.exception("error while searching index - %s", str(message))
                raise

        return _translate_hits(es_response)

def
search
(
self
,
query_string
=
,
field_dictionary
=
,
filter_dictionary
=
,
exclude_dictionary
=
,
facet_terms
=
,
exclude_ids
=
,
use_field_match
=
False
,
**
kwargs
)
:
log
.
debug
(
"searching index with %s"
,
query_string
)
elastic_queries
=
[
]
elastic_filters
=
[
]
if
query_string
:
if
six
.
PY2
:
query_string
=
query_string
.
encode
(
'utf-8'
)
.
translate
(
,
RESERVED_CHARACTERS
)
else
:
query_string
=
query_string
.
translate
(
query_string
.
maketrans
(
''
,
''
,
RESERVED_CHARACTERS
)
)
elastic_queries
.
append
(
{
"query_string"
:
{
"fields"
:
[
"content.*"
]
,
"query"
:
query_string
}
}
)
if
field_dictionary
:
if
use_field_match
:
elastic_queries
.
extend
(
_process_field_queries
(
field_dictionary
)
)
else
:
elastic_filters
.
extend
(
_process_field_filters
(
field_dictionary
)
)
if
filter_dictionary
:
elastic_filters
.
extend
(
_process_filters
(
filter_dictionary
)
)
if
exclude_ids
:
if
not
exclude_dictionary
:
exclude_dictionary
=
{
}
if
"_id"
not
in
exclude_dictionary
:
exclude_dictionary
[
"_id"
]
=
[
]
exclude_dictionary
[
"_id"
]
.
extend
(
exclude_ids
)
if
exclude_dictionary
:
elastic_filters
.
append
(
_process_exclude_dictionary
(
exclude_dictionary
)
)
query_segment
=
{
"match_all"
:
{
}
}
if
elastic_queries
:
query_segment
=
{
"bool"
:
{
"must"
:
elastic_queries
}
}
query
=
query_segment
if
elastic_filters
:
filter_segment
=
{
"bool"
:
{
"must"
:
elastic_filters
}
}
query
=
{
"filtered"
:
{
"query"
:
query_segment
,
"filter"
:
filter_segment
,
}
}
body
=
{
"query"
:
query
}
if
facet_terms
:
facet_query
=
_process_facet_terms
(
facet_terms
)
if
facet_query
:
body
[
"facets"
]
=
facet_query
try
:
es_response
=
self
.
_es
.
search
(
index
=
self
.
index_name
,
body
=
body
,
**
kwargs
)
except
exceptions
.
ElasticsearchException
as
ex
:
message
=
six
.
text_type
(
ex
)
if
'QueryParsingException'
in
message
:
log
.
exception
(
"Malformed search query: %s"
,
message
)
raise
QueryParseError
(
'Malformed search query.'
)
else
:
log
.
exception
(
"error while searching index - %s"
,
str
(
message
)
)
raise
return
_translate_hits
(
es_response
)

Implements call to search the index for the desired content .


def perform_search(
        search_term,
        user=None,
        size=10,
        from_=0,
        course_id=None):
    """ Call the search engine with the appropriate parameters """
    # field_, filter_ and exclude_dictionary(s) can be overridden by calling application
    # field_dictionary includes course if course_id provided
    (field_dictionary, filter_dictionary, exclude_dictionary) = SearchFilterGenerator.generate_field_filters(
        user=user,
        course_id=course_id
    )

    searcher = SearchEngine.get_search_engine(getattr(settings, "COURSEWARE_INDEX_NAME", "courseware_index"))
    if not searcher:
        raise NoSearchEngineError("No search engine specified in settings.SEARCH_ENGINE")

    results = searcher.search_string(
        search_term,
        field_dictionary=field_dictionary,
        filter_dictionary=filter_dictionary,
        exclude_dictionary=exclude_dictionary,
        size=size,
        from_=from_,
        doc_type="courseware_content",
    )

    # post-process the result
    for result in results["results"]:
        result["data"] = SearchResultProcessor.process_result(result["data"], search_term, user)

    results["access_denied_count"] = len([r for r in results["results"] if r["data"] is None])
    results["results"] = [r for r in results["results"] if r["data"] is not None]

    return results


def course_discovery_search(search_term=None, size=20, from_=0, field_dictionary=None):
    """
    Course Discovery activities against the search engine index of course details
    """
    # We'll ignore the course-enrollemnt informaiton in field and filter
    # dictionary, and use our own logic upon enrollment dates for these
    use_search_fields = ["org"]
    (search_fields, _, exclude_dictionary) = SearchFilterGenerator.generate_field_filters()
    use_field_dictionary = {}
    use_field_dictionary.update({field: search_fields[field] for field in search_fields if field in use_search_fields})
    if field_dictionary:
        use_field_dictionary.update(field_dictionary)
    if not getattr(settings, "SEARCH_SKIP_ENROLLMENT_START_DATE_FILTERING", False):
        use_field_dictionary["enrollment_start"] = DateRange(None, datetime.utcnow())

    searcher = SearchEngine.get_search_engine(getattr(settings, "COURSEWARE_INDEX_NAME", "courseware_index"))
    if not searcher:
        raise NoSearchEngineError("No search engine specified in settings.SEARCH_ENGINE")

    results = searcher.search(
        query_string=search_term,
        doc_type="course_info",
        size=size,
        from_=from_,
        # only show when enrollment start IS provided and is before now
        field_dictionary=use_field_dictionary,
        # show if no enrollment end is provided and has not yet been reached
        filter_dictionary={"enrollment_end": DateRange(datetime.utcnow(), None)},
        exclude_dictionary=exclude_dictionary,
        facet_terms=course_discovery_facets(),
    )

    return results


def strings_in_dictionary(dictionary):
        """ Used by default implementation for finding excerpt """
        strings = [value for value in six.itervalues(dictionary) if not isinstance(value, dict)]
        for child_dict in [dv for dv in six.itervalues(dictionary) if isinstance(dv, dict)]:
            strings.extend(SearchResultProcessor.strings_in_dictionary(child_dict))
        return strings

def strings_in_dictionary ( dictionary ) : strings = [ value for value in six . itervalues ( dictionary ) if not isinstance ( value , dict ) ] for child_dict in [ dv for dv in six . itervalues ( dictionary ) if isinstance ( dv , dict ) ] : strings . extend ( SearchResultProcessor . strings_in_dictionary ( child_dict ) ) return strings


def find_matches(strings, words, length_hoped):
        """ Used by default property excerpt """
        lower_words = [w.lower() for w in words]

        def has_match(string):
            """ Do any of the words match within the string """
            lower_string = string.lower()
            for test_word in lower_words:
                if test_word in lower_string:
                    return True
            return False

        shortened_strings = [textwrap.wrap(s) for s in strings]
        short_string_list = list(chain.from_iterable(shortened_strings))
        matches = [ms for ms in short_string_list if has_match(ms)]

        cumulative_len = 0
        break_at = None
        for idx, match in enumerate(matches):
            cumulative_len += len(match)
            if cumulative_len >= length_hoped:
                break_at = idx
                break

        return matches[0:break_at]

def find_matches ( strings , words , length_hoped ) : lower_words = [ w . lower ( ) for w in words ] def has_match ( string ) : lower_string = string . lower ( ) for test_word in lower_words : if test_word in lower_string : return True return False shortened_strings = [ textwrap . wrap ( s ) for s in strings ] short_string_list = list ( chain . from_iterable ( shortened_strings ) ) matches = [ ms for ms in short_string_list if has_match ( ms ) ] cumulative_len = 0 break_at = None for idx , match in enumerate ( matches ) : cumulative_len += len ( match ) if cumulative_len >= length_hoped : break_at = idx break return matches [ 0 : break_at ]


def decorate_matches(match_in, match_word):
        """ decorate the matches within the excerpt """
        matches = re.finditer(match_word, match_in, re.IGNORECASE)
        for matched_string in set([match.group() for match in matches]):
            match_in = match_in.replace(
                matched_string,
                getattr(settings, "SEARCH_MATCH_DECORATION", u"<b>{}</b>").format(matched_string)
            )
        return match_in

def decorate_matches ( match_in , match_word ) : matches = re . finditer ( match_word , match_in , re . IGNORECASE ) for matched_string in set ( [ match . group ( ) for match in matches ] ) : match_in = match_in . replace ( matched_string , getattr ( settings , "SEARCH_MATCH_DECORATION" , u"<b>{}</b>" ) . format ( matched_string ) ) return match_in


def add_properties(self):
        """
        Called during post processing of result
        Any properties defined in your subclass will get exposed as members of the result json from the search
        """
        for property_name in [p[0] for p in inspect.getmembers(self.__class__) if isinstance(p[1], property)]:
            self._results_fields[property_name] = getattr(self, property_name, None)

def add_properties ( self ) : for property_name in [ p [ 0 ] for p in inspect . getmembers ( self . __class__ ) if isinstance ( p [ 1 ] , property ) ] : self . _results_fields [ property_name ] = getattr ( self , property_name , None )


def process_result(cls, dictionary, match_phrase, user):
        """
        Called from within search handler. Finds desired subclass and decides if the
        result should be removed and adds properties derived from the result information
        """
        result_processor = _load_class(getattr(settings, "SEARCH_RESULT_PROCESSOR", None), cls)
        srp = result_processor(dictionary, match_phrase)
        if srp.should_remove(user):
            return None
        try:
            srp.add_properties()
        # protect around any problems introduced by subclasses within their properties
        except Exception as ex:  # pylint: disable=broad-except
            log.exception("error processing properties for %s - %s: will remove from results",
                          json.dumps(dictionary, cls=DjangoJSONEncoder), str(ex))
            return None
        return dictionary

Called from within search handler . Finds desired subclass and decides if the result should be removed and adds properties derived from the result information


def excerpt(self):
        """
        Property to display a useful excerpt representing the matches within the results
        """
        if "content" not in self._results_fields:
            return None

        match_phrases = [self._match_phrase]
        if six.PY2:
            separate_phrases = [
                phrase.decode('utf-8')
                for phrase in shlex.split(self._match_phrase.encode('utf-8'))
            ]
        else:
            separate_phrases = [
                phrase
                for phrase in shlex.split(self._match_phrase)
            ]
        if len(separate_phrases) > 1:
            match_phrases.extend(separate_phrases)
        else:
            match_phrases = separate_phrases

        matches = SearchResultProcessor.find_matches(
            SearchResultProcessor.strings_in_dictionary(self._results_fields["content"]),
            match_phrases,
            DESIRED_EXCERPT_LENGTH
        )
        excerpt_text = ELLIPSIS.join(matches)

        for match_word in match_phrases:
            excerpt_text = SearchResultProcessor.decorate_matches(excerpt_text, match_word)

        return excerpt_text

def excerpt ( self ) : if "content" not in self . _results_fields : return None match_phrases = [ self . _match_phrase ] if six . PY2 : separate_phrases = [ phrase . decode ( 'utf-8' ) for phrase in shlex . split ( self . _match_phrase . encode ( 'utf-8' ) ) ] else : separate_phrases = [ phrase for phrase in shlex . split ( self . _match_phrase ) ] if len ( separate_phrases ) > 1 : match_phrases . extend ( separate_phrases ) else : match_phrases = separate_phrases matches = SearchResultProcessor . find_matches ( SearchResultProcessor . strings_in_dictionary ( self . _results_fields [ "content" ] ) , match_phrases , DESIRED_EXCERPT_LENGTH ) excerpt_text = ELLIPSIS . join ( matches ) for match_word in match_phrases : excerpt_text = SearchResultProcessor . decorate_matches ( excerpt_text , match_word ) return excerpt_text


def generate_field_filters(cls, **kwargs):
        """
        Called from within search handler
        Finds desired subclass and adds filter information based upon user information
        """
        generator = _load_class(getattr(settings, "SEARCH_FILTER_GENERATOR", None), cls)()
        return (
            generator.field_dictionary(**kwargs),
            generator.filter_dictionary(**kwargs),
            generator.exclude_dictionary(**kwargs),
        )

def generate_field_filters ( cls , ** kwargs ) : generator = _load_class ( getattr ( settings , "SEARCH_FILTER_GENERATOR" , None ) , cls ) ( ) return ( generator . field_dictionary ( ** kwargs ) , generator . filter_dictionary ( ** kwargs ) , generator . exclude_dictionary ( ** kwargs ) , )


def set_search_enviroment(cls, **kwargs):
        """
        Called from within search handler
        Finds desired subclass and calls initialize method
        """
        initializer = _load_class(getattr(settings, "SEARCH_INITIALIZER", None), cls)()
        return initializer.initialize(**kwargs)

def set_search_enviroment ( cls , ** kwargs ) : initializer = _load_class ( getattr ( settings , "SEARCH_INITIALIZER" , None ) , cls ) ( ) return initializer . initialize ( ** kwargs )


def _parse(self, filename):
        """Opens data file and for each line, calls _eat_name_line"""
        self.names = {}
        with codecs.open(filename, encoding="iso8859-1") as f:
            for line in f:
                if any(map(lambda c: 128 < ord(c) < 160, line)):
                    line = line.encode("iso8859-1").decode("windows-1252")
                self._eat_name_line(line.strip())

def _parse ( self , filename ) : self . names = { } with codecs . open ( filename , encoding = "iso8859-1" ) as f : for line in f : if any ( map ( lambda c : 128 < ord ( c ) < 160 , line ) ) : line = line . encode ( "iso8859-1" ) . decode ( "windows-1252" ) self . _eat_name_line ( line . strip ( ) )

Opens data file and for each line calls _eat_name_line


def _eat_name_line(self, line):
        """Parses one line of data file"""
        if line[0] not in "#=":
            parts = line.split()
            country_values = line[30:-1]
            name = map_name(parts[1])
            if not self.case_sensitive:
                name = name.lower()

            if parts[0] == "M":
                self._set(name, u"male", country_values)
            elif parts[0] == "1M" or parts[0] == "?M":
                self._set(name, u"mostly_male", country_values)
            elif parts[0] == "F":
                self._set(name, u"female", country_values)
            elif parts[0] == "1F" or parts[0] == "?F":
                self._set(name, u"mostly_female", country_values)
            elif parts[0] == "?":
                self._set(name, self.unknown_value, country_values)
            else:
                raise "Not sure what to do with a sex of %s" % parts[0]

def
_eat_name_line
(
self
,
line
)
:
if
line
[
0
]
not
in
"#="
:
parts
=
line
.
split
(
)
country_values
=
line
[
30
:
1
]
name
=
map_name
(
parts
[
1
]
)
if
not
self
.
case_sensitive
:
name
=
name
.
lower
(
)
if
parts
[
0
]
==
"M"
:
self
.
_set
(
name
,
u"male"
,
country_values
)
elif
parts
[
0
]
==
"1M"
or
parts
[
0
]
==
"?M"
:
self
.
_set
(
name
,
u"mostly_male"
,
country_values
)
elif
parts
[
0
]
==
"F"
:
self
.
_set
(
name
,
u"female"
,
country_values
)
elif
parts
[
0
]
==
"1F"
or
parts
[
0
]
==
"?F"
:
self
.
_set
(
name
,
u"mostly_female"
,
country_values
)
elif
parts
[
0
]
==
"?"
:
self
.
_set
(
name
,
self
.
unknown_value
,
country_values
)
else
:
raise
"Not sure what to do with a sex of %s"
%
parts
[
0
]


def _set(self, name, gender, country_values):
        """Sets gender and relevant country values for names dictionary of detector"""
        if '+' in name:
            for replacement in ['', ' ', '-']:
                self._set(name.replace('+', replacement), gender, country_values)
        else:
            if name not in self.names:
                self.names[name] = {}
            self.names[name][gender] = country_values

def
_set
(
self
,
name
,
gender
,
country_values
)
:
if
'+'
in
name
:
for
replacement
in
[
''
,
' '
,
'-'
]
:
self
.
_set
(
name
.
replace
(
'+'
,
replacement
)
,
gender
,
country_values
)
else
:
if
name
not
in
self
.
names
:
self
.
names
[
name
]
=
{
}
self
.
names
[
name
]
[
gender
]
=
country_values


def _most_popular_gender(self, name, counter):
        """Finds the most popular gender for the given name counting by given counter"""
        if name not in self.names:
            return self.unknown_value

        max_count, max_tie = (0, 0)
        best = self.names[name].keys()[0]
        for gender, country_values in self.names[name].items():
            count, tie = counter(country_values)
            if count > max_count or (count == max_count and tie > max_tie):
                max_count, max_tie, best = count, tie, gender

        return best if max_count > 0 else self.unknown_value

def _most_popular_gender ( self , name , counter ) : if name not in self . names : return self . unknown_value max_count , max_tie = ( 0 , 0 ) best = self . names [ name ] . keys ( ) [ 0 ] for gender , country_values in self . names [ name ] . items ( ) : count , tie = counter ( country_values ) if count > max_count or ( count == max_count and tie > max_tie ) : max_count , max_tie , best = count , tie , gender return best if max_count > 0 else self . unknown_value


def get_gender(self, name, country=None):
        """Returns best gender for the given name and country pair"""
        if not self.case_sensitive:
            name = name.lower()

        if name not in self.names:
            return self.unknown_value
        elif not country:
            def counter(country_values):
                country_values = map(ord, country_values.replace(" ", ""))
                return (len(country_values),
                        sum(map(lambda c: c > 64 and c-55 or c-48, country_values)))
            return self._most_popular_gender(name, counter)
        elif country in self.__class__.COUNTRIES:
            index = self.__class__.COUNTRIES.index(country)
            counter = lambda e: (ord(e[index])-32, 0)
            return self._most_popular_gender(name, counter)
        else:
            raise NoCountryError("No such country: %s" % country)

def
get_gender
(
self
,
name
,
country
=
)
:
if
not
self
.
case_sensitive
:
name
=
name
.
lower
(
)
if
name
not
in
self
.
names
:
return
self
.
unknown_value
elif
not
country
:
def
counter
(
country_values
)
:
country_values
=
map
(
ord
,
country_values
.
replace
(
" "
,
""
)
)
return
(
len
(
country_values
)
,
sum
(
map
(
lambda
c
:
c
>
64
and
c
55
or
c
48
,
country_values
)
)
)
return
self
.
_most_popular_gender
(
name
,
counter
)
elif
country
in
self
.
__class__
.
COUNTRIES
:
index
=
self
.
__class__
.
COUNTRIES
.
index
(
country
)
counter
=
lambda
e
:
(
ord
(
e
[
index
]
)
32
,
0
)
return
self
.
_most_popular_gender
(
name
,
counter
)
else
:
raise
NoCountryError
(
"No such country: %s"
%
country
)


def output(self, msg, newline=True):
        """
        Writes the specified string to the output target of the report.

        :param msg: the message to output.
        :type msg: str
        :param newline:
            whether or not to append a newline to the end of the message
        :type newline: str
        """

        click.echo(text_type(msg), nl=newline, file=self.output_file)

def output ( self , msg , newline = True ) : click . echo ( text_type ( msg ) , nl = newline , file = self . output_file )

Writes the specified string to the output target of the report .


def execute_tools(config, path, progress=None):
    """
    Executes the suite of TidyPy tools upon the project and returns the
    issues that are found.

    :param config: the TidyPy configuration to use
    :type config: dict
    :param path: that path to the project to analyze
    :type path: str
    :param progress:
        the progress reporter object that will receive callbacks during the
        execution of the tool suite. If not specified, not progress
        notifications will occur.
    :type progress: tidypy.Progress
    :rtype: tidypy.Collector
    """

    progress = progress or QuietProgress()
    progress.on_start()

    manager = SyncManager()
    manager.start()

    num_tools = 0
    tools = manager.Queue()
    for name, cls in iteritems(get_tools()):
        if config[name]['use'] and cls.can_be_used():
            num_tools += 1
            tools.put({
                'name': name,
                'config': config[name],
            })

    collector = Collector(config)
    if not num_tools:
        progress.on_finish()
        return collector

    notifications = manager.Queue()
    environment = manager.dict({
        'finder': Finder(path, config),
    })

    workers = []
    for _ in range(config['workers']):
        worker = Worker(
            args=(
                tools,
                notifications,
                environment,
            ),
        )
        worker.start()
        workers.append(worker)

    while num_tools:
        try:
            notification = notifications.get(True, 0.25)
        except Empty:
            pass
        else:
            if notification['type'] == 'start':
                progress.on_tool_start(notification['tool'])
            elif notification['type'] == 'complete':
                collector.add_issues(notification['issues'])
                progress.on_tool_finish(notification['tool'])
                num_tools -= 1

    progress.on_finish()

    return collector

def execute_tools ( config , path , progress = None ) : progress = progress or QuietProgress ( ) progress . on_start ( ) manager = SyncManager ( ) manager . start ( ) num_tools = 0 tools = manager . Queue ( ) for name , cls in iteritems ( get_tools ( ) ) : if config [ name ] [ 'use' ] and cls . can_be_used ( ) : num_tools += 1 tools . put ( { 'name' : name , 'config' : config [ name ] , } ) collector = Collector ( config ) if not num_tools : progress . on_finish ( ) return collector notifications = manager . Queue ( ) environment = manager . dict ( { 'finder' : Finder ( path , config ) , } ) workers = [ ] for _ in range ( config [ 'workers' ] ) : worker = Worker ( args = ( tools , notifications , environment , ) , ) worker . start ( ) workers . append ( worker ) while num_tools : try : notification = notifications . get ( True , 0.25 ) except Empty : pass else : if notification [ 'type' ] == 'start' : progress . on_tool_start ( notification [ 'tool' ] ) elif notification [ 'type' ] == 'complete' : collector . add_issues ( notification [ 'issues' ] ) progress . on_tool_finish ( notification [ 'tool' ] ) num_tools -= 1 progress . on_finish ( ) return collector

Executes the suite of TidyPy tools upon the project and returns the issues that are found .


def execute_reports(
        config,
        path,
        collector,
        on_report_finish=None,
        output_file=None):
    """
    Executes the configured suite of issue reports.

    :param config: the TidyPy configuration to use
    :type config: dict
    :param path: that path to the project that was analyzed
    :type path: str
    :param collector: the issues to report
    :type collector: tidypy.Collector
    """

    reports = get_reports()
    for report in config.get('requested_reports', []):
        if report.get('type') and report['type'] in reports:
            cfg = config.get('report', {}).get(report['type'], {})
            cfg.update(report)
            reporter = reports[report['type']](
                cfg,
                path,
                output_file=output_file,
            )
            reporter.produce(collector)
            if on_report_finish:
                on_report_finish(report)

def execute_reports ( config , path , collector , on_report_finish = None , output_file = None ) : reports = get_reports ( ) for report in config . get ( 'requested_reports' , [ ] ) : if report . get ( 'type' ) and report [ 'type' ] in reports : cfg = config . get ( 'report' , { } ) . get ( report [ 'type' ] , { } ) cfg . update ( report ) reporter = reports [ report [ 'type' ] ] ( cfg , path , output_file = output_file , ) reporter . produce ( collector ) if on_report_finish : on_report_finish ( report )

Executes the configured suite of issue reports .


def is_excluded(self, path):
        """
        Determines whether or not the specified file is excluded by the
        project's configuration.

        :param path: the path to check
        :type path: pathlib.Path
        :rtype: bool
        """

        relpath = path.relative_to(self.base_path).as_posix()
        return matches_masks(relpath, self.excludes)

def is_excluded ( self , path ) : relpath = path . relative_to ( self . base_path ) . as_posix ( ) return matches_masks ( relpath , self . excludes )

Determines whether or not the specified file is excluded by the project s configuration .


def is_excluded_dir(self, path):
        """
        Determines whether or not the specified directory is excluded by the
        project's configuration.

        :param path: the path to check
        :type path: pathlib.Path
        :rtype: bool
        """

        if self.is_excluded(path):
            return True
        return matches_masks(path.name, ALWAYS_EXCLUDED_DIRS)

def is_excluded_dir ( self , path ) : if self . is_excluded ( path ) : return True return matches_masks ( path . name , ALWAYS_EXCLUDED_DIRS )

Determines whether or not the specified directory is excluded by the project s configuration .


def files(self, filters=None):
        """
        A generator that produces a sequence of paths to files in the project
        that matches the specified filters.

        :param filters:
            the regular expressions to use when finding files in the project.
            If not specified, all files are returned.
        :type filters: list(str)
        """

        filters = compile_masks(filters or [r'.*'])

        for files in itervalues(self._found):
            for file_ in files:
                relpath = text_type(Path(file_).relative_to(self.base_path))
                if matches_masks(relpath, filters):
                    yield file_

def files ( self , filters = None ) : filters = compile_masks ( filters or [ r'.*' ] ) for files in itervalues ( self . _found ) : for file_ in files : relpath = text_type ( Path ( file_ ) . relative_to ( self . base_path ) ) if matches_masks ( relpath , filters ) : yield file_

A generator that produces a sequence of paths to files in the project that matches the specified filters .


def directories(self, filters=None, containing=None):
        """
        A generator that produces a sequence of paths to directories in the
        project that matches the specified filters.

        :param filters:
            the regular expressions to use when finding directories in the
            project. If not specified, all directories are returned.
        :type filters: list(str)
        :param containing:
            if a directory passes through the specified filters, it is checked
            for the presence of a file that matches one of the regular
            expressions in this parameter.
        :type containing: list(str)
        """

        filters = compile_masks(filters or [r'.*'])
        contains = compile_masks(containing)

        for dirname, files in iteritems(self._found):
            relpath = text_type(Path(dirname).relative_to(self.base_path))
            if matches_masks(relpath, filters):
                if not contains or self._contains(files, contains):
                    yield dirname

def directories ( self , filters = None , containing = None ) : filters = compile_masks ( filters or [ r'.*' ] ) contains = compile_masks ( containing ) for dirname , files in iteritems ( self . _found ) : relpath = text_type ( Path ( dirname ) . relative_to ( self . base_path ) ) if matches_masks ( relpath , filters ) : if not contains or self . _contains ( files , contains ) : yield dirname

A generator that produces a sequence of paths to directories in the project that matches the specified filters .


def add_issues(self, issues):
        """
        Adds an issue to the collection.

        :param issues: the issue(s) to add
        :type issues: tidypy.Issue or list(tidypy.Issue)
        """

        if not isinstance(issues, (list, tuple)):
            issues = [issues]
        with self._lock:
            self._all_issues.extend(issues)
            self._cleaned_issues = None

def add_issues ( self , issues ) : if not isinstance ( issues , ( list , tuple ) ) : issues = [ issues ] with self . _lock : self . _all_issues . extend ( issues ) self . _cleaned_issues = None


def issue_count(self, include_unclean=False):
        """
        Returns the number of issues in the collection.

        :param include_unclean:
            whether or not to include issues that are being ignored due to
            being a duplicate, excluded, etc.
        :type include_unclean: bool
        :rtype: int
        """

        if include_unclean:
            return len(self._all_issues)
        self._ensure_cleaned_issues()
        return len(self._cleaned_issues)

def issue_count ( self , include_unclean = False ) : if include_unclean : return len ( self . _all_issues ) self . _ensure_cleaned_issues ( ) return len ( self . _cleaned_issues )

Returns the number of issues in the collection .


def get_issues(self, sortby=None):
        """
        Retrieves the issues in the collection.

        :param sortby: the properties to sort the issues by
        :type sortby: list(str)
        :rtype: list(tidypy.Issue)
        """

        self._ensure_cleaned_issues()
        return self._sort_issues(self._cleaned_issues, sortby)

def get_issues ( self , sortby = None ) : self . _ensure_cleaned_issues ( ) return self . _sort_issues ( self . _cleaned_issues , sortby )

Retrieves the issues in the collection .


def get_grouped_issues(self, keyfunc=None, sortby=None):
        """
        Retrieves the issues in the collection grouped into buckets according
        to the key generated by the keyfunc.

        :param keyfunc:
            a function that will be used to generate the key that identifies
            the group that an issue will be assigned to. This function receives
            a single tidypy.Issue argument and must return a string. If not
            specified, the filename of the issue will be used.
        :type keyfunc: func
        :param sortby: the properties to sort the issues by
        :type sortby: list(str)
        :rtype: OrderedDict
        """

        if not keyfunc:
            keyfunc = default_group
        if not sortby:
            sortby = self.DEFAULT_SORT
        self._ensure_cleaned_issues()
        return self._group_issues(self._cleaned_issues, keyfunc, sortby)

def get_grouped_issues ( self , keyfunc = None , sortby = None ) : if not keyfunc : keyfunc = default_group if not sortby : sortby = self . DEFAULT_SORT self . _ensure_cleaned_issues ( ) return self . _group_issues ( self . _cleaned_issues , keyfunc , sortby )

Retrieves the issues in the collection grouped into buckets according to the key generated by the keyfunc .


def parse(cls, content, is_pyproject=False):
        """
        A convenience method for parsing a TOML-serialized configuration.

        :param content: a TOML string containing a TidyPy configuration
        :type content: str
        :param is_pyproject:
            whether or not the content is (or resembles) a ``pyproject.toml``
            file, where the TidyPy configuration is located within a key named
            ``tool``.
        :type is_pyproject: bool
        :rtype: dict
        """

        parsed = pytoml.loads(content)

        if is_pyproject:
            parsed = parsed.get('tool', {})
        parsed = parsed.get('tidypy', {})

        return parsed

def parse ( cls , content , is_pyproject = False ) : parsed = pytoml . loads ( content ) if is_pyproject : parsed = parsed . get ( 'tool' , { } ) parsed = parsed . get ( 'tidypy' , { } ) return parsed

A convenience method for parsing a TOML - serialized configuration .


def get_tools():
    """
    Retrieves the TidyPy tools that are available in the current Python
    environment.

    The returned dictionary has keys that are the tool names and values are the
    tool classes.

    :rtype: dict
    """

    # pylint: disable=protected-access

    if not hasattr(get_tools, '_CACHE'):
        get_tools._CACHE = dict()
        for entry in pkg_resources.iter_entry_points('tidypy.tools'):
            try:
                get_tools._CACHE[entry.name] = entry.load()
            except ImportError as exc:  # pragma: no cover
                output_error(
                    'Could not load tool "%s" defined by "%s": %s' % (
                        entry,
                        entry.dist,
                        exc,
                    ),
                )
    return get_tools._CACHE

Retrieves the TidyPy tools that are available in the current Python environment .


def get_reports():
    """
    Retrieves the TidyPy issue reports that are available in the current Python
    environment.

    The returned dictionary has keys are the report names and values are the
    report classes.

    :rtype: dict
    """

    # pylint: disable=protected-access

    if not hasattr(get_reports, '_CACHE'):
        get_reports._CACHE = dict()
        for entry in pkg_resources.iter_entry_points('tidypy.reports'):
            try:
                get_reports._CACHE[entry.name] = entry.load()
            except ImportError as exc:  # pragma: no cover
                output_error(
                    'Could not load report "%s" defined by "%s": %s' % (
                        entry,
                        entry.dist,
                        exc,
                    ),
                )
    return get_reports._CACHE

Retrieves the TidyPy issue reports that are available in the current Python environment .


def get_extenders():
    """
    Retrieves the TidyPy configuration extenders that are available in the
    current Python environment.

    The returned dictionary has keys are the extender names and values are the
    extender classes.

    :rtype: dict
    """

    # pylint: disable=protected-access

    if not hasattr(get_extenders, '_CACHE'):
        get_extenders._CACHE = dict()
        for entry in pkg_resources.iter_entry_points('tidypy.extenders'):
            try:
                get_extenders._CACHE[entry.name] = entry.load()
            except ImportError as exc:  # pragma: no cover
                output_error(
                    'Could not load extender "%s" defined by "%s": %s' % (
                        entry,
                        entry.dist,
                        exc,
                    ),
                )
    return get_extenders._CACHE

Retrieves the TidyPy configuration extenders that are available in the current Python environment .


def purge_config_cache(location=None):
    """
    Clears out the cache of TidyPy configurations that were retrieved from
    outside the normal locations.
    """

    cache_path = get_cache_path(location)

    if location:
        os.remove(cache_path)
    else:
        shutil.rmtree(cache_path)

def purge_config_cache ( location = None ) : cache_path = get_cache_path ( location ) if location : os . remove ( cache_path ) else : shutil . rmtree ( cache_path )

Clears out the cache of TidyPy configurations that were retrieved from outside the normal locations .


def get_user_config(project_path, use_cache=True):
    """
    Produces a TidyPy configuration that incorporates the configuration files
    stored in the current user's home directory.

    :param project_path: the path to the project that is going to be analyzed
    :type project_path: str
    :param use_cache:
        whether or not to use cached versions of any remote/referenced TidyPy
        configurations. If not specified, defaults to ``True``.
    :type use_cache: bool
    :rtype: dict
    """

    if sys.platform == 'win32':
        user_config = os.path.expanduser(r'~\\tidypy')
    else:
        user_config = os.path.join(
            os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'),
            'tidypy'
        )

    if os.path.exists(user_config):
        with open(user_config, 'r') as config_file:
            config = pytoml.load(config_file).get('tidypy', {})

        config = merge_dict(get_default_config(), config)
        config = process_extensions(config, project_path, use_cache=use_cache)
        return config

    return None

def get_user_config ( project_path , use_cache = True ) : if sys . platform == 'win32' : user_config = os . path . expanduser ( r'~\\tidypy' ) else : user_config = os . path . join ( os . getenv ( 'XDG_CONFIG_HOME' ) or os . path . expanduser ( '~/.config' ) , 'tidypy' ) if os . path . exists ( user_config ) : with open ( user_config , 'r' ) as config_file : config = pytoml . load ( config_file ) . get ( 'tidypy' , { } ) config = merge_dict ( get_default_config ( ) , config ) config = process_extensions ( config , project_path , use_cache = use_cache ) return config return None

Produces a TidyPy configuration that incorporates the configuration files stored in the current user s home directory .


def get_local_config(project_path, use_cache=True):
    """
    Produces a TidyPy configuration using the ``pyproject.toml`` in the
    project's directory.

    :param project_path: the path to the project that is going to be analyzed
    :type project_path: str
    :param use_cache:
        whether or not to use cached versions of any remote/referenced TidyPy
        configurations. If not specified, defaults to ``True``.
    :type use_cache: bool
    :rtype: dict
    """

    pyproject_path = os.path.join(project_path, 'pyproject.toml')

    if os.path.exists(pyproject_path):
        with open(pyproject_path, 'r') as config_file:
            config = pytoml.load(config_file)

        config = config.get('tool', {}).get('tidypy', {})
        config = merge_dict(get_default_config(), config)
        config = process_extensions(config, project_path, use_cache=use_cache)
        return config

    return None

def get_local_config ( project_path , use_cache = True ) : pyproject_path = os . path . join ( project_path , 'pyproject.toml' ) if os . path . exists ( pyproject_path ) : with open ( pyproject_path , 'r' ) as config_file : config = pytoml . load ( config_file ) config = config . get ( 'tool' , { } ) . get ( 'tidypy' , { } ) config = merge_dict ( get_default_config ( ) , config ) config = process_extensions ( config , project_path , use_cache = use_cache ) return config return None

Produces a TidyPy configuration using the pyproject . toml in the project s directory .


def get_project_config(project_path, use_cache=True):
    """
    Produces the Tidypy configuration to use for the specified project.

    If a ``pyproject.toml`` exists, the configuration will be based on that. If
    not, the TidyPy configuration in the user's home directory will be used. If
    one does not exist, the default configuration will be used.

    :param project_path: the path to the project that is going to be analyzed
    :type project_path: str
    :param use_cache:
        whether or not to use cached versions of any remote/referenced TidyPy
        configurations. If not specified, defaults to ``True``.
    :type use_cache: bool
    :rtype: dict
    """

    return get_local_config(project_path, use_cache=use_cache) \
        or get_user_config(project_path, use_cache=use_cache) \
        or get_default_config()

Produces the Tidypy configuration to use for the specified project .


def merge_list(list1, list2):
    """
    Merges the contents of two lists into a new list.

    :param list1: the first list
    :type list1: list
    :param list2: the second list
    :type list2: list
    :returns: list
    """

    merged = list(list1)

    for value in list2:
        if value not in merged:
            merged.append(value)

    return merged

def merge_list ( list1 , list2 ) : merged = list ( list1 ) for value in list2 : if value not in merged : merged . append ( value ) return merged

Merges the contents of two lists into a new list .


def merge_dict(dict1, dict2, merge_lists=False):
    """
    Recursively merges the contents of two dictionaries into a new dictionary.

    When both input dictionaries share a key, the value from ``dict2`` is
    kept.

    :param dict1: the first dictionary
    :type dict1: dict
    :param dict2: the second dictionary
    :type dict2: dict
    :param merge_lists:
        when this function encounters a key that contains lists in both input
        dictionaries, this parameter dictates whether or not those lists should
        be merged. If not specified, defaults to ``False``.
    :type merge_lists: bool
    :returns: dict
    """

    merged = dict(dict1)

    for key, value in iteritems(dict2):
        if isinstance(merged.get(key), dict):
            merged[key] = merge_dict(merged[key], value)
        elif merge_lists and isinstance(merged.get(key), list):
            merged[key] = merge_list(merged[key], value)
        else:
            merged[key] = value

    return merged

def merge_dict ( dict1 , dict2 , merge_lists = False ) : merged = dict ( dict1 ) for key , value in iteritems ( dict2 ) : if isinstance ( merged . get ( key ) , dict ) : merged [ key ] = merge_dict ( merged [ key ] , value ) elif merge_lists and isinstance ( merged . get ( key ) , list ) : merged [ key ] = merge_list ( merged [ key ] , value ) else : merged [ key ] = value return merged

Recursively merges the contents of two dictionaries into a new dictionary .


def output_error(msg):
    """
    Prints the specified string to ``stderr``.

    :param msg: the message to print
    :type msg: str
    """

    click.echo(click.style(msg, fg='red'), err=True)

def output_error ( msg ) : click . echo ( click . style ( msg , fg = 'red' ) , err = True )


def mod_sys_path(paths):
    """
    A context manager that will append the specified paths to Python's
    ``sys.path`` during the execution of the block.

    :param paths: the paths to append
    :type paths: list(str)
    """

    old_path = sys.path
    sys.path = paths + sys.path
    try:
        yield
    finally:
        sys.path = old_path

def mod_sys_path ( paths ) : old_path = sys . path sys . path = paths + sys . path try : yield finally : sys . path = old_path

A context manager that will append the specified paths to Python s sys . path during the execution of the block .


def compile_masks(masks):
    """
    Compiles a list of regular expressions.

    :param masks: the regular expressions to compile
    :type masks: list(str) or str
    :returns: list(regular expression object)
    """

    if not masks:
        masks = []
    elif not isinstance(masks, (list, tuple)):
        masks = [masks]

    return [
        re.compile(mask)
        for mask in masks
    ]

def compile_masks ( masks ) : if not masks : masks = [ ] elif not isinstance ( masks , ( list , tuple ) ) : masks = [ masks ] return [ re . compile ( mask ) for mask in masks ]

Compiles a list of regular expressions .


def matches_masks(target, masks):
    """
    Determines whether or not the target string matches any of the regular
    expressions specified.

    :param target: the string to check
    :type target: str
    :param masks: the regular expressions to check against
    :type masks: list(regular expression object)
    :returns: bool
    """

    for mask in masks:
        if mask.search(target):
            return True
    return False

def matches_masks ( target , masks ) : for mask in masks : if mask . search ( target ) : return True return False

Determines whether or not the target string matches any of the regular expressions specified .


def read_file(filepath):
    """
    Retrieves the contents of the specified file.

    This function performs simple caching so that the same file isn't read more
    than once per process.

    :param filepath: the file to read
    :type filepath: str
    :returns: str
    """

    with _FILE_CACHE_LOCK:
        if filepath not in _FILE_CACHE:
            _FILE_CACHE[filepath] = _read_file(filepath)
    return _FILE_CACHE[filepath]

def read_file ( filepath ) : with _FILE_CACHE_LOCK : if filepath not in _FILE_CACHE : _FILE_CACHE [ filepath ] = _read_file ( filepath ) return _FILE_CACHE [ filepath ]

Retrieves the contents of the specified file .


def parse_python_file(filepath):
    """
    Retrieves the AST of the specified file.

    This function performs simple caching so that the same file isn't read or
    parsed more than once per process.

    :param filepath: the file to parse
    :type filepath: str
    :returns: ast.AST
    """

    with _AST_CACHE_LOCK:
        if filepath not in _AST_CACHE:
            source = read_file(filepath)
            _AST_CACHE[filepath] = ast.parse(source, filename=filepath)
    return _AST_CACHE[filepath]

def parse_python_file ( filepath ) : with _AST_CACHE_LOCK : if filepath not in _AST_CACHE : source = read_file ( filepath ) _AST_CACHE [ filepath ] = ast . parse ( source , filename = filepath ) return _AST_CACHE [ filepath ]

Retrieves the AST of the specified file .


def on_tool_finish(self, tool):
        """
        Called when an individual tool completes execution.

        :param tool: the name of the tool that completed
        :type tool: str
        """

        with self._lock:
            if tool in self.current_tools:
                self.current_tools.remove(tool)
                self.completed_tools.append(tool)

def on_tool_finish ( self , tool ) : with self . _lock : if tool in self . current_tools : self . current_tools . remove ( tool ) self . completed_tools . append ( tool )

Called when an individual tool completes execution .


def exec_command(self, cmdstr):
        """
            Execute an x3270 command

            `cmdstr` gets sent directly to the x3270 subprocess on it's stdin.
        """
        if self.is_terminated:
            raise TerminatedError("this TerminalClient instance has been terminated")

        log.debug("sending command: %s", cmdstr)
        c = Command(self.app, cmdstr)
        start = time.time()
        c.execute()
        elapsed = time.time() - start
        log.debug("elapsed execution: {0}".format(elapsed))
        self.status = Status(c.status_line)

        return c


def terminate(self):
        """
            terminates the underlying x3270 subprocess. Once called, this
            Emulator instance must no longer be used.
        """
        if not self.is_terminated:
            log.debug("terminal client terminated")
            try:
                self.exec_command(b"Quit")
            except BrokenPipeError:  # noqa
                # x3270 was terminated, since we are just quitting anyway, ignore it.
                pass
            except socket.error as e:
                if e.errno != errno.ECONNRESET:
                    raise
                # this can happen because wc3270 closes the socket before
                # the read() can happen, causing a socket error

            self.app.close()

            self.is_terminated = True

def
terminate
(
self
)
:
if
not
self
.
is_terminated
:
log
.
debug
(
"terminal client terminated"
)
try
:
self
.
exec_command
(
b"Quit"
)
except
BrokenPipeError
:
pass
except
socket
.
error
as
e
:
if
e
.
errno
!=
errno
.
ECONNRESET
:
raise
self
.
app
.
close
(
)
self
.
is_terminated
=
True

terminates the underlying x3270 subprocess . Once called this Emulator instance must no longer be used .