def imap_send(func, gen):
    """Apply a function to all ``send`` values of a generator

    Parameters
    ----------
    func: ~typing.Callable[[T_send], T_mapped]
        the function to apply
    gen: Generable[T_yield, T_mapped, T_return]
        the generator iterable.

    Returns
    -------
    ~typing.Generator[T_yield, T_send, T_return]
        the mapped generator
    """
    gen = iter(gen)
    assert _is_just_started(gen)
    yielder = yield_from(gen)
    for item in yielder:
        with yielder:
            yielder.send(func((yield item)))
    return_(yielder.result)

def imap_send ( func , gen ) : gen = iter ( gen ) assert _is_just_started ( gen ) yielder = yield_from ( gen ) for item in yielder : with yielder : yielder . send ( func ( ( yield item ) ) ) return_ ( yielder . result )

Apply a function to all send values of a generator


def bug_info(exc_type, exc_value, exc_trace):
    """Prints the traceback and invokes the ipython debugger on any exception

    Only invokes ipydb if you are outside ipython or python interactive session.
    So scripts must be called from OS shell in order for exceptions to ipy-shell-out.

    Dependencies:
      Needs `pip install ipdb`

    Arguments:
      exc_type (type): The exception type/class (e.g. RuntimeError)
      exc_value (Exception): The exception instance (e.g. the error message passed to the Exception constructor)
      exc_trace (Traceback): The traceback instance

    References:
      http://stackoverflow.com/a/242531/623735

    Example Usage:
      $  python -c 'from pug import debug;x=[];x[0]'
      Traceback (most recent call last):
        File "<string>", line 1, in <module>
      IndexError: list index out of range

      > <string>(1)<module>()

      ipdb> x
      []
      ipdb> locals()
      {'__builtins__': <module '__builtin__' (built-in)>, '__package__': None, 'x': [], 'debug': <module 'pug.debug' from 'pug/debug.py'>, '__name__': '__main__', '__doc__': None}
      ipdb>
    """
    if hasattr(sys, 'ps1') or not sys.stderr.isatty():
        # We are in interactive mode or don't have a tty-like device, so we call the default hook
        sys.__excepthook__(exc_type, exc_value, exc_trace)
    else:
        # Need to import non-built-ins here, so if dependencies haven't been installed, both tracebacks will print
        # (e.g. the ImportError and the Exception that got you here)
        import ipdb
        # We are NOT in interactive mode, print the exception
        traceback.print_exception(exc_type, exc_value, exc_trace)
        print
        # Start the debugger in post-mortem mode.
        ipdb.post_mortem(exc_trace)

def bug_info ( exc_type , exc_value , exc_trace ) : if hasattr ( sys , 'ps1' ) or not sys . stderr . isatty ( ) : sys . __excepthook__ ( exc_type , exc_value , exc_trace ) else : import ipdb traceback . print_exception ( exc_type , exc_value , exc_trace ) print ipdb . post_mortem ( exc_trace )


def copy_web_file_to_local(file_path, target_path):
    """Copies a file from its location on the web to a designated
    place on the local machine.

    Args:
        file_path: Complete url of the file to copy, string (e.g. http://fool.com/input.css).

        target_path: Path and name of file on the local machine, string. (e.g. /directory/output.css)

    Returns:
        None.

    """
    response = urllib.request.urlopen(file_path)
    f = open(target_path, 'w')
    f.write(response.read())
    f.close()

def copy_web_file_to_local ( file_path , target_path ) : response = urllib . request . urlopen ( file_path ) f = open ( target_path , 'w' ) f . write ( response . read ( ) ) f . close ( )

Copies a file from its location on the web to a designated place on the local machine .


def get_line_count(fname):
    """Counts the number of lines in a file.

    Args:
        fname: string, name of the file.

    Returns:
        integer, the number of lines in the file.

    """
    i = 0
    with open(fname) as f:
        for i, l in enumerate(f):
            pass
    return i + 1

def get_line_count ( fname ) : i = 0 with open ( fname ) as f : for i , l in enumerate ( f ) : pass return i + 1


def indent_css(f, output):
    """Indentes css that has not been indented and saves it to a new file.
    A new file is created if the output destination does not already exist.

    Args:
        f: string, path to file.

        output: string, path/name of the output file (e.g. /directory/output.css).
    print type(response.read())

    Returns:
        None.
    """
    line_count = get_line_count(f)
    f = open(f, 'r+')
    output = open(output, 'r+')
    for line in range(line_count):
        string = f.readline().rstrip()
        if len(string) > 0:
            if string[-1] == ";":
                output.write("    " + string + "\n")
            else:
                output.write(string + "\n")
    output.close()
    f.close()

def
indent_css
(
f
,
output
)
:
line_count
=
get_line_count
(
f
)
f
=
open
(
f
,
'r+'
)
output
=
open
(
output
,
'r+'
)
for
line
in
range
(
line_count
)
:
string
=
f
.
readline
(
)
.
rstrip
(
)
if
len
(
string
)
>
0
:
if
string
[
1
]
==
";"
:
output
.
write
(
"    "
+
string
+
"\n"
)
else
:
output
.
write
(
string
+
"\n"
)
output
.
close
(
)
f
.
close
(
)

Indentes css that has not been indented and saves it to a new file . A new file is created if the output destination does not already exist .


def add_newlines(f, output, char):
    """Adds line breaks after every occurance of a given character in a file.

    Args:
        f: string, path to input file.

        output: string, path to output file.

    Returns:
        None.
    """
    line_count = get_line_count(f)
    f = open(f, 'r+')
    output = open(output, 'r+')
    for line in range(line_count):
        string = f.readline()
        string = re.sub(char, char + '\n', string)
        output.write(string)

def add_newlines ( f , output , char ) : line_count = get_line_count ( f ) f = open ( f , 'r+' ) output = open ( output , 'r+' ) for line in range ( line_count ) : string = f . readline ( ) string = re . sub ( char , char + '\n' , string ) output . write ( string )

Adds line breaks after every occurance of a given character in a file .


def reformat_css(input_file, output_file):
    """Reformats poorly written css. This function does not validate or fix errors in the code.
    It only gives code the proper indentation.

    Args:
        input_file: string, path to the input file.

        output_file: string, path to where the reformatted css should be saved. If the target file
        doesn't exist, a new file is created.

    Returns:
        None.
    """
    # Number of lines in the file.
    line_count = get_line_count(input_file)

    # Open source and target files.
    f = open(input_file, 'r+')
    output = open(output_file, 'w')

    # Loop over every line in the file.
    for line in range(line_count):
        # Eliminate whitespace at the beginning and end of lines.
        string = f.readline().strip()
        # New lines after {
        string = re.sub('\{', '{\n', string)
        # New lines after ;
        string = re.sub('; ', ';', string)
        string = re.sub(';', ';\n', string)
        # Eliminate whitespace before comments
        string = re.sub('} /*', '}/*', string)
        # New lines after }
        string = re.sub('\}', '}\n', string)
        # New lines at the end of comments
        string = re.sub('\*/', '*/\n', string)
        # Write to the output file.
        output.write(string)

    # Close the files.
    output.close()
    f.close()

    # Indent the css.
    indent_css(output_file, output_file)

    # Make sure there's a space before every {
    add_whitespace_before("{", output_file, output_file)

def
reformat_css
(
input_file
,
output_file
)
:
line_count
=
get_line_count
(
input_file
)
f
=
open
(
input_file
,
'r+'
)
output
=
open
(
output_file
,
'w'
)
for
line
in
range
(
line_count
)
:
string
=
f
.
readline
(
)
.
strip
(
)
string
=
re
.
sub
(
'\{'
,
'{\n'
,
string
)
string
=
re
.
sub
(
'; '
,
';'
,
string
)
string
=
re
.
sub
(
';'
,
';\n'
,
string
)
string
=
re
.
sub
(
'} /*'
,
'}/*'
,
string
)
string
=
re
.
sub
(
'\}'
,
'}\n'
,
string
)
string
=
re
.
sub
(
'\*/'
,
'*/\n'
,
string
)
output
.
write
(
string
)
output
.
close
(
)
f
.
close
(
)
indent_css
(
output_file
,
output_file
)
add_whitespace_before
(
"{"
,
output_file
,
output_file
)

Reformats poorly written css . This function does not validate or fix errors in the code . It only gives code the proper indentation .


def clean_strings(iterable):
    """
    Take a list of strings and clear whitespace
    on each one. If a value in the list is not a
    string pass it through untouched.

    Args:
        iterable: mixed list

    Returns:
        mixed list
    """
    retval = []
    for val in iterable:
        try:
            retval.append(val.strip())
        except(AttributeError):
            retval.append(val)
    return retval

def clean_strings ( iterable ) : retval = [ ] for val in iterable : try : retval . append ( val . strip ( ) ) except ( AttributeError ) : retval . append ( val ) return retval

Take a list of strings and clear whitespace on each one . If a value in the list is not a string pass it through untouched .


def future_value(present_value, annual_rate, periods_per_year, years):
    """
    Calculates the future value of money invested at an anual interest rate,
    x times per year, for a given number of years.

    Args:
        present_value: int or float, the current value of the money (principal).

        annual_rate: float 0 to 1 e.g., .5 = 50%), the interest rate paid out.

        periods_per_year: int, the number of times money is invested per year.

        years: int, the number of years invested.

    Returns:
        Float, the future value of the money invested with compound interest.
    """

    # The nominal interest rate per period (rate) is how much interest you earn during a
    # particular length of time, before accounting for compounding. This is typically
    # expressed as a percentage.
    rate_per_period = annual_rate / float(periods_per_year)

    # How many periods in the future the calculation is for.
    periods = periods_per_year * years

    return present_value * (1 + rate_per_period) ** periods

def future_value ( present_value , annual_rate , periods_per_year , years ) : rate_per_period = annual_rate / float ( periods_per_year ) periods = periods_per_year * years return present_value * ( 1 + rate_per_period ) ** periods

Calculates the future value of money invested at an anual interest rate x times per year for a given number of years .


def triangle_area(point1, point2, point3):
    """
    Uses Heron's formula to find the area of a triangle
    based on the coordinates of three points.

    Args:
        point1: list or tuple, the x y coordinate of point one.

        point2: list or tuple, the x y coordinate of point two.

        point3: list or tuple, the x y coordinate of point three.

    Returns:
        The area of a triangle as a floating point number.

    Requires:
        The math module, point_distance().
    """

    """Lengths of the three sides of the triangle"""
    a = point_distance(point1, point2)
    b = point_distance(point1, point3)
    c = point_distance(point2, point3)

    """Where s is the semiperimeter"""
    s = (a + b + c) / 2.0

    """Return the area of the triangle (using Heron's formula)"""
    return math.sqrt(s * (s - a) * (s - b) * (s - c))

def triangle_area ( point1 , point2 , point3 ) : a = point_distance ( point1 , point2 ) b = point_distance ( point1 , point3 ) c = point_distance ( point2 , point3 ) s = ( a + b + c ) / 2.0 return math . sqrt ( s * ( s - a ) * ( s - b ) * ( s - c ) )

Uses Heron s formula to find the area of a triangle based on the coordinates of three points .


def median(data):
    """
    Calculates  the median of a list of integers or floating point numbers.

    Args:
        data: A list of integers or floating point numbers

    Returns:
        Sorts the list numerically and returns the middle number if the list has an odd number
        of items. If the list contains an even number of items the mean of the two middle numbers
        is returned.
    """
    ordered = sorted(data)
    length = len(ordered)
    if length % 2 == 0:
        return (
            ordered[math.floor(length / 2) - 1] + ordered[math.floor(length / 2)]
        ) / 2.0

    elif length % 2 != 0:
        return ordered[math.floor(length / 2)]

def median ( data ) : ordered = sorted ( data ) length = len ( ordered ) if length % 2 == 0 : return ( ordered [ math . floor ( length / 2 ) - 1 ] + ordered [ math . floor ( length / 2 ) ] ) / 2.0 elif length % 2 != 0 : return ordered [ math . floor ( length / 2 ) ]

Calculates the median of a list of integers or floating point numbers .


def average(numbers, numtype='float'):
    """
    Calculates the average or mean of a list of numbers

    Args:
        numbers: a list of integers or floating point numbers.

        numtype: string, 'decimal' or 'float'; the type of number to return.

    Returns:
        The average (mean) of the numbers as a floating point number
        or a Decimal object.

    Requires:
        The math module
    """
    if type == 'decimal':
        return Decimal(sum(numbers)) / len(numbers)
    else:
        return float(sum(numbers)) / len(numbers)

def average ( numbers , numtype = 'float' ) : if type == 'decimal' : return Decimal ( sum ( numbers ) ) / len ( numbers ) else : return float ( sum ( numbers ) ) / len ( numbers )


def variance(numbers, type='population'):
    """
    Calculates the population or sample variance of a list of numbers.
    A large number means the results are all over the place, while a
    small number means the results are comparatively close to the average.

    Args:
        numbers: a list  of integers or floating point numbers to compare.

        type: string, 'population' or 'sample', the kind of variance to be computed.

    Returns:
        The computed population or sample variance.
        Defaults to population variance.

    Requires:
        The math module, average()
    """
    mean = average(numbers)
    variance = 0
    for number in numbers:
        variance += (mean - number) ** 2

    if type == 'population':
        return variance / len(numbers)
    else:
        return variance / (len(numbers) - 1)

def variance ( numbers , type = 'population' ) : mean = average ( numbers ) variance = 0 for number in numbers : variance += ( mean - number ) ** 2 if type == 'population' : return variance / len ( numbers ) else : return variance / ( len ( numbers ) - 1 )

Calculates the population or sample variance of a list of numbers . A large number means the results are all over the place while a small number means the results are comparatively close to the average .


def get_percentage(a, b, i=False, r=False):
    """
    Finds the percentage of one number over another.

    Args:
        a: The number that is a percent, int or float.

        b: The base number that a is a percent of, int or float.

        i: Optional boolean integer. True if the user wants the result returned as
        a whole number. Assumes False.

        r: Optional boolean round. True if the user wants the result rounded.
        Rounds to the second decimal point on floating point numbers. Assumes False.

    Returns:
        The argument a as a percentage of b. Throws a warning if integer is set to True
        and round is set to False.
    """
    # Round to the second decimal
    if i is False and r is True:
        percentage = round(100.0 * (float(a) / b), 2)

    # Round to the nearest whole number
    elif (i is True and r is True) or (i is True and r is False):
        percentage = int(round(100 * (float(a) / b)))

        # A rounded number and an integer were requested
        if r is False:
            warnings.warn(
                "If integer is set to True and Round is set to False, you will still get a rounded number if you pass floating point numbers as arguments."
            )

    # A precise unrounded decimal
    else:
        percentage = 100.0 * (float(a) / b)

    return percentage

Finds the percentage of one number over another .


def get_datetime_string(datetime_obj):
        '''
        Get datetime string from datetime object

        :param datetime datetime_obj: datetime object
        :return: datetime string
        :rtype: str
        '''

        if isinstance(datetime_obj, datetime):
            dft = DTFormat()
            return datetime_obj.strftime(dft.datetime_format)

        return None

def get_datetime_string ( datetime_obj ) : if isinstance ( datetime_obj , datetime ) : dft = DTFormat ( ) return datetime_obj . strftime ( dft . datetime_format ) return None


def attr(prev, attr_name):
    """attr pipe can extract attribute value of object.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param attr_name: The name of attribute
    :type attr_name: str
    :returns: generator
    """
    for obj in prev:
        if hasattr(obj, attr_name):
            yield getattr(obj, attr_name)

def attr ( prev , attr_name ) : for obj in prev : if hasattr ( obj , attr_name ) : yield getattr ( obj , attr_name )

attr pipe can extract attribute value of object .


def attrs(prev, attr_names):
    """attrs pipe can extract attribute values of object.

    If attr_names is a list and its item is not a valid attribute of
    prev's object. It will be excluded from yielded dict.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param attr_names: The list of attribute names
    :type attr_names: str of list
    :returns: generator
    """
    for obj in prev:
        attr_values = []
        for name in attr_names:
            if hasattr(obj, name):
                attr_values.append(getattr(obj, name))
        yield attr_values

def attrs ( prev , attr_names ) : for obj in prev : attr_values = [ ] for name in attr_names : if hasattr ( obj , name ) : attr_values . append ( getattr ( obj , name ) ) yield attr_values

attrs pipe can extract attribute values of object .


def attrdict(prev, attr_names):
    """attrdict pipe can extract attribute values of object into a dict.

    The argument attr_names can be a list or a dict.

    If attr_names is a list and its item is not a valid attribute of
    prev's object. It will be excluded from yielded dict.

    If attr_names is dict and the key doesn't exist in prev's object.
    the value of corresponding attr_names key will be copy to yielded dict.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param attr_names: The list or dict of attribute names
    :type attr_names: str of list or dict
    :returns: generator
    """
    if isinstance(attr_names, dict):
        for obj in prev:
            attr_values = dict()
            for name in attr_names.keys():
                if hasattr(obj, name):
                    attr_values[name] = getattr(obj, name)
                else:
                    attr_values[name] = attr_names[name]
            yield attr_values
    else:
        for obj in prev:
            attr_values = dict()
            for name in attr_names:
                if hasattr(obj, name):
                    attr_values[name] = getattr(obj, name)
            yield attr_values

def attrdict ( prev , attr_names ) : if isinstance ( attr_names , dict ) : for obj in prev : attr_values = dict ( ) for name in attr_names . keys ( ) : if hasattr ( obj , name ) : attr_values [ name ] = getattr ( obj , name ) else : attr_values [ name ] = attr_names [ name ] yield attr_values else : for obj in prev : attr_values = dict ( ) for name in attr_names : if hasattr ( obj , name ) : attr_values [ name ] = getattr ( obj , name ) yield attr_values

attrdict pipe can extract attribute values of object into a dict .


def flatten(prev, depth=sys.maxsize):
    """flatten pipe extracts nested item from previous pipe.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param depth: The deepest nested level to be extracted. 0 means no extraction.
    :type depth: integer
    :returns: generator
    """
    def inner_flatten(iterable, curr_level, max_levels):
        for i in iterable:
            if hasattr(i, '__iter__') and curr_level < max_levels:
                for j in inner_flatten(i, curr_level + 1, max_levels):
                    yield j
            else:
                yield i

    for d in prev:
        if hasattr(d, '__iter__') and depth > 0:
            for inner_d in inner_flatten(d, 1, depth):
                yield inner_d
        else:
            yield d

def flatten ( prev , depth = sys . maxsize ) : def inner_flatten ( iterable , curr_level , max_levels ) : for i in iterable : if hasattr ( i , '__iter__' ) and curr_level < max_levels : for j in inner_flatten ( i , curr_level + 1 , max_levels ) : yield j else : yield i for d in prev : if hasattr ( d , '__iter__' ) and depth > 0 : for inner_d in inner_flatten ( d , 1 , depth ) : yield inner_d else : yield d

flatten pipe extracts nested item from previous pipe .


def values(prev, *keys, **kw):
    """values pipe extract value from previous pipe.

    If previous pipe send a dictionary to values pipe, keys should contains
    the key of dictionary which you want to get. If previous pipe send list or
    tuple,

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :returns: generator
    """
    d = next(prev)
    if isinstance(d, dict):
        yield [d[k] for k in keys if k in d]
        for d in prev:
            yield [d[k] for k in keys if k in d]
    else:
        yield [d[i] for i in keys if 0 <= i < len(d)]
        for d in prev:
            yield [d[i] for i in keys if 0 <= i < len(d)]

def values ( prev , * keys , ** kw ) : d = next ( prev ) if isinstance ( d , dict ) : yield [ d [ k ] for k in keys if k in d ] for d in prev : yield [ d [ k ] for k in keys if k in d ] else : yield [ d [ i ] for i in keys if 0 <= i < len ( d ) ] for d in prev : yield [ d [ i ] for i in keys if 0 <= i < len ( d ) ]

values pipe extract value from previous pipe .


def pack(prev, n, rest=False, **kw):
    """pack pipe takes n elements from previous generator and yield one
    list to next.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param rest: Set True to allow to output the rest part of last elements.
    :type prev: boolean
    :param padding: Specify the padding element for the rest part of last elements.
    :type prev: boolean
    :returns: generator

    :Example:
    >>> result([1,2,3,4,5,6,7] | pack(3))
    [[1, 2, 3], [4, 5, 6]]

    >>> result([1,2,3,4,5,6,7] | pack(3, rest=True))
    [[1, 2, 3], [4, 5, 6], [7,]]

    >>> result([1,2,3,4,5,6,7] | pack(3, padding=None))
    [[1, 2, 3], [4, 5, 6], [7, None, None]]
    """

    if 'padding' in kw:
        use_padding = True
        padding = kw['padding']
    else:
        use_padding = False
        padding = None

    items = []
    for i, data in enumerate(prev, 1):
        items.append(data)
        if (i % n) == 0:
            yield items
            items = []
    if len(items) != 0 and rest:
        if use_padding:
            items.extend([padding, ] * (n - (i % n)))
        yield items

def pack ( prev , n , rest = False , ** kw ) : if 'padding' in kw : use_padding = True padding = kw [ 'padding' ] else : use_padding = False padding = None items = [ ] for i , data in enumerate ( prev , 1 ) : items . append ( data ) if ( i % n ) == 0 : yield items items = [ ] if len ( items ) != 0 and rest : if use_padding : items . extend ( [ padding , ] * ( n - ( i % n ) ) ) yield items

pack pipe takes n elements from previous generator and yield one list to next .


def grep(prev, pattern, *args, **kw):
    """The pipe greps the data passed from previous generator according to
    given regular expression.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param pattern: The pattern which used to filter out data.
    :type pattern: str|unicode|re pattern object
    :param inv: If true, invert the match condition.
    :type inv: boolean
    :param kw:
    :type kw: dict
    :returns: generator
    """
    inv = False if 'inv' not in kw else kw.pop('inv')
    pattern_obj = re.compile(pattern, *args, **kw)

    for data in prev:
        if bool(inv) ^ bool(pattern_obj.match(data)):
            yield data

def grep ( prev , pattern , * args , ** kw ) : inv = False if 'inv' not in kw else kw . pop ( 'inv' ) pattern_obj = re . compile ( pattern , * args , ** kw ) for data in prev : if bool ( inv ) ^ bool ( pattern_obj . match ( data ) ) : yield data

The pipe greps the data passed from previous generator according to given regular expression .


def match(prev, pattern, *args, **kw):
    """The pipe greps the data passed from previous generator according to
    given regular expression. The data passed to next pipe is MatchObject
    , dict or tuple which determined by 'to' in keyword argument.

    By default, match pipe yields MatchObject. Use 'to' in keyword argument
    to change the type of match result.

    If 'to' is dict, yield MatchObject.groupdict().
    If 'to' is tuple, yield MatchObject.groups().
    If 'to' is list, yield list(MatchObject.groups()).

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param pattern: The pattern which used to filter data.
    :type pattern: str|unicode
    :param to: What data type the result should be stored. dict|tuple|list
    :type to: type
    :returns: generator
    """
    to = 'to' in kw and kw.pop('to')
    pattern_obj = re.compile(pattern, *args, **kw)

    if to is dict:
        for data in prev:
            match = pattern_obj.match(data)
            if match is not None:
                yield match.groupdict()
    elif to is tuple:
        for data in prev:
            match = pattern_obj.match(data)
            if match is not None:
                yield match.groups()
    elif to is list:
        for data in prev:
            match = pattern_obj.match(data)
            if match is not None:
                yield list(match.groups())
    else:
        for data in prev:
            match = pattern_obj.match(data)
            if match is not None:
                yield match

def match ( prev , pattern , * args , ** kw ) : to = 'to' in kw and kw . pop ( 'to' ) pattern_obj = re . compile ( pattern , * args , ** kw ) if to is dict : for data in prev : match = pattern_obj . match ( data ) if match is not None : yield match . groupdict ( ) elif to is tuple : for data in prev : match = pattern_obj . match ( data ) if match is not None : yield match . groups ( ) elif to is list : for data in prev : match = pattern_obj . match ( data ) if match is not None : yield list ( match . groups ( ) ) else : for data in prev : match = pattern_obj . match ( data ) if match is not None : yield match

The pipe greps the data passed from previous generator according to given regular expression . The data passed to next pipe is MatchObject dict or tuple which determined by to in keyword argument .


def resplit(prev, pattern, *args, **kw):
    """The resplit pipe split previous pipe input by regular expression.

    Use 'maxsplit' keyword argument to limit the number of split.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param pattern: The pattern which used to split string.
    :type pattern: str|unicode
    """
    maxsplit = 0 if 'maxsplit' not in kw else kw.pop('maxsplit')
    pattern_obj = re.compile(pattern, *args, **kw)
    for s in prev:
        yield pattern_obj.split(s, maxsplit=maxsplit)

def resplit ( prev , pattern , * args , ** kw ) : maxsplit = 0 if 'maxsplit' not in kw else kw . pop ( 'maxsplit' ) pattern_obj = re . compile ( pattern , * args , ** kw ) for s in prev : yield pattern_obj . split ( s , maxsplit = maxsplit )

The resplit pipe split previous pipe input by regular expression .


def sub(prev, pattern, repl, *args, **kw):
    """sub pipe is a wrapper of re.sub method.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param pattern: The pattern string.
    :type pattern: str|unicode
    :param repl: Check repl argument in re.sub method.
    :type repl: str|unicode|callable
    """
    count = 0 if 'count' not in kw else kw.pop('count')
    pattern_obj = re.compile(pattern, *args, **kw)
    for s in prev:
        yield pattern_obj.sub(repl, s, count=count)

def sub ( prev , pattern , repl , * args , ** kw ) : count = 0 if 'count' not in kw else kw . pop ( 'count' ) pattern_obj = re . compile ( pattern , * args , ** kw ) for s in prev : yield pattern_obj . sub ( repl , s , count = count )

sub pipe is a wrapper of re . sub method .


def wildcard(prev, pattern, *args, **kw):
    """wildcard pipe greps data passed from previous generator
    according to given regular expression.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param pattern: The wildcard string which used to filter data.
    :type pattern: str|unicode|re pattern object
    :param inv: If true, invert the match condition.
    :type inv: boolean
    :returns: generator
    """
    import fnmatch

    inv = 'inv' in kw and kw.pop('inv')
    pattern_obj = re.compile(fnmatch.translate(pattern), *args, **kw)

    if not inv:
        for data in prev:
            if pattern_obj.match(data):
                yield data
    else:
        for data in prev:
            if not pattern_obj.match(data):
                yield data

def wildcard ( prev , pattern , * args , ** kw ) : import fnmatch inv = 'inv' in kw and kw . pop ( 'inv' ) pattern_obj = re . compile ( fnmatch . translate ( pattern ) , * args , ** kw ) if not inv : for data in prev : if pattern_obj . match ( data ) : yield data else : for data in prev : if not pattern_obj . match ( data ) : yield data

wildcard pipe greps data passed from previous generator according to given regular expression .


def stdout(prev, endl='\n', thru=False):
    """This pipe read data from previous iterator and write it to stdout.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param endl: The end-of-line symbol for each output.
    :type endl: str
    :param thru: If true, data will passed to next generator. If false, data
                 will be dropped.
    :type thru: bool
    :returns: generator
    """
    for i in prev:
        sys.stdout.write(str(i) + endl)
        if thru:
            yield i

def stdout ( prev , endl = '\n' , thru = False ) : for i in prev : sys . stdout . write ( str ( i ) + endl ) if thru : yield i

This pipe read data from previous iterator and write it to stdout .


def readline(prev, filename=None, mode='r', trim=str.rstrip, start=1, end=sys.maxsize):
    """This pipe get filenames or file object from previous pipe and read the
    content of file. Then, send the content of file line by line to next pipe.

    The start and end parameters are used to limit the range of reading from file.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param filename: The files to be read. If None, use previous pipe input as filenames.
    :type filename: None|str|unicode|list|tuple
    :param mode: The mode to open file. default is 'r'
    :type mode: str
    :param trim: The function to trim the line before send to next pipe.
    :type trim: function object.
    :param start: if star is specified, only line number larger or equal to start will be sent.
    :type start: integer
    :param end: The last line number to read.
    :type end: integer
    :returns: generator
    """
    if prev is None:
        if filename is None:
            raise Exception('No input available for readline.')
        elif is_str_type(filename):
            file_list = [filename, ]
        else:
            file_list = filename
    else:
        file_list = prev

    for fn in file_list:
        if isinstance(fn, file_type):
            fd = fn
        else:
            fd = open(fn, mode)

        try:
            if start <= 1 and end == sys.maxsize:
                for line in fd:
                    yield trim(line)
            else:
                for line_no, line in enumerate(fd, 1):
                    if line_no < start:
                        continue
                    yield trim(line)
                    if line_no >= end:
                        break
        finally:
            if fd != fn:
                fd.close()

def
readline
(
prev
,
filename
=
,
mode
=
'r'
,
trim
=
str
.
rstrip
,
start
=
1
,
end
=
sys
.
maxsize
)
:
if
prev
is
:
if
filename
is
:
raise
Exception
(
'No input available for readline.'
)
elif
is_str_type
(
filename
)
:
file_list
=
[
filename
,
]
else
:
file_list
=
filename
else
:
file_list
=
prev
for
fn
in
file_list
:
if
isinstance
(
fn
,
file_type
)
:
fd
=
fn
else
:
fd
=
open
(
fn
,
mode
)
try
:
if
start
<=
1
and
end
==
sys
.
maxsize
:
for
line
in
fd
:
yield
trim
(
line
)
else
:
for
line_no
,
line
in
enumerate
(
fd
,
1
)
:
if
line_no
<
start
:
continue
yield
trim
(
line
)
if
line_no
>=
end
:
break
finally
:
if
fd
!=
fn
:
fd
.
close
(
)

This pipe get filenames or file object from previous pipe and read the content of file . Then send the content of file line by line to next pipe .


def sh(prev, *args, **kw):
    """sh pipe execute shell command specified by args. If previous pipe exists,
    read data from it and write it to stdin of shell process. The stdout of
    shell process will be passed to next pipe object line by line.

    A optional keyword argument 'trim' can pass a function into sh pipe. It is
    used to trim the output from shell process. The default trim function is
    str.rstrip. Therefore, any space characters in tail of
    shell process output line will be removed.

    For example:

    py_files = result(sh('ls') | strip | wildcard('*.py'))

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param args: The command line arguments. It will be joined by space character.
    :type args: list of string.
    :param kw: arguments for subprocess.Popen.
    :type kw: dictionary of options.
    :returns: generator
    """
    endl = '\n' if 'endl' not in kw else kw.pop('endl')
    trim = None if 'trim' not in kw else kw.pop('trim')
    if trim is None:
        trim = bytes.rstrip if is_py3 else str.rstrip

    cmdline = ' '.join(args)
    if not cmdline:
        if prev is not None:
            for i in prev:
                yield i
        else:
            while True:
                yield None

    process = subprocess.Popen(cmdline, shell=True,
        stdin=subprocess.PIPE, stdout=subprocess.PIPE,
        **kw)
    if prev is not None:
        stdin_buffer = StringIO()
        for i in prev:
            stdin_buffer.write(i)
            if endl:
                stdin_buffer.write(endl)
        if is_py3:
            process.stdin.write(stdin_buffer.getvalue().encode('utf-8'))
        else:
            process.stdin.write(stdin_buffer.getvalue())
        process.stdin.flush()
        process.stdin.close()
        stdin_buffer.close()

    for line in process.stdout:
        yield trim(line)

    process.wait()

def
sh
(
prev
,
*
args
,
**
kw
)
:
endl
=
'\n'
if
'endl'
not
in
kw
else
kw
.
pop
(
'endl'
)
trim
=
if
'trim'
not
in
kw
else
kw
.
pop
(
'trim'
)
if
trim
is
:
trim
=
bytes
.
rstrip
if
is_py3
else
str
.
rstrip
cmdline
=
' '
.
join
(
args
)
if
not
cmdline
:
if
prev
is
not
:
for
i
in
prev
:
yield
i
else
:
while
True
:
yield
process
=
subprocess
.
Popen
(
cmdline
,
shell
=
True
,
stdin
=
subprocess
.
PIPE
,
stdout
=
subprocess
.
PIPE
,
**
kw
)
if
prev
is
not
:
stdin_buffer
=
StringIO
(
)
for
i
in
prev
:
stdin_buffer
.
write
(
i
)
if
endl
:
stdin_buffer
.
write
(
endl
)
if
is_py3
:
process
.
stdin
.
write
(
stdin_buffer
.
getvalue
(
)
.
encode
(
'utf-8'
)
)
else
:
process
.
stdin
.
write
(
stdin_buffer
.
getvalue
(
)
)
process
.
stdin
.
flush
(
)
process
.
stdin
.
close
(
)
stdin_buffer
.
close
(
)
for
line
in
process
.
stdout
:
yield
trim
(
line
)
process
.
wait
(
)

sh pipe execute shell command specified by args . If previous pipe exists read data from it and write it to stdin of shell process . The stdout of shell process will be passed to next pipe object line by line .


def walk(prev, inital_path, *args, **kw):
    """This pipe wrap os.walk and yield absolute path one by one.

    :param prev: The previous iterator of pipe.
    :type prev: Pipe
    :param args: The end-of-line symbol for each output.
    :type args: list of string.
    :param kw: The end-of-line symbol for each output.
    :type kw: dictionary of options. Add 'endl' in kw to specify end-of-line symbol.
    :returns: generator
    """
    for dir_path, dir_names, filenames in os.walk(inital_path):
        for filename in filenames:
            yield os.path.join(dir_path, filename)

def walk ( prev , inital_path , * args , ** kw ) : for dir_path , dir_names , filenames in os . walk ( inital_path ) : for filename in filenames : yield os . path . join ( dir_path , filename )

This pipe wrap os . walk and yield absolute path one by one .


def join(prev, sep, *args, **kw):
    '''alias of str.join'''
    yield sep.join(prev, *args, **kw)

def join ( prev , sep , * args , ** kw ) : yield sep . join ( prev , * args , ** kw )


def substitute(prev, *args, **kw):
    '''alias of string.Template.substitute'''
    template_obj = string.Template(*args, **kw)
    for data in prev:
        yield template_obj.substitute(data)

def substitute ( prev , * args , ** kw ) : template_obj = string . Template ( * args , ** kw ) for data in prev : yield template_obj . substitute ( data )


def safe_substitute(prev, *args, **kw):
    '''alias of string.Template.safe_substitute'''
    template_obj = string.Template(*args, **kw)
    for data in prev:
        yield template_obj.safe_substitute(data)

def safe_substitute ( prev , * args , ** kw ) : template_obj = string . Template ( * args , ** kw ) for data in prev : yield template_obj . safe_substitute ( data )

alias of string . Template . safe_substitute


def to_str(prev, encoding=None):
    """Convert data from previous pipe with specified encoding."""
    first = next(prev)
    if isinstance(first, str):
        if encoding is None:
            yield first
            for s in prev:
                yield s
        else:
            yield first.encode(encoding)
            for s in prev:
                yield s.encode(encoding)
    else:
        if encoding is None:
            encoding = sys.stdout.encoding or 'utf-8'
        yield first.decode(encoding)
        for s in prev:
            yield s.decode(encoding)

def to_str ( prev , encoding = None ) : first = next ( prev ) if isinstance ( first , str ) : if encoding is None : yield first for s in prev : yield s else : yield first . encode ( encoding ) for s in prev : yield s . encode ( encoding ) else : if encoding is None : encoding = sys . stdout . encoding or 'utf-8' yield first . decode ( encoding ) for s in prev : yield s . decode ( encoding )

Convert data from previous pipe with specified encoding .


def register_default_types():
    """Regiser all default type-to-pipe convertors."""
    register_type(type, pipe.map)
    register_type(types.FunctionType, pipe.map)
    register_type(types.MethodType, pipe.map)
    register_type(tuple, seq)
    register_type(list, seq)
    register_type(types.GeneratorType, seq)
    register_type(string_type, sh)
    register_type(unicode_type, sh)
    register_type(file_type, fileobj)

    if is_py3:
        register_type(range, seq)
        register_type(map, seq)

def register_default_types ( ) : register_type ( type , pipe . map ) register_type ( types . FunctionType , pipe . map ) register_type ( types . MethodType , pipe . map ) register_type ( tuple , seq ) register_type ( list , seq ) register_type ( types . GeneratorType , seq ) register_type ( string_type , sh ) register_type ( unicode_type , sh ) register_type ( file_type , fileobj ) if is_py3 : register_type ( range , seq ) register_type ( map , seq )

Regiser all default type - to - pipe convertors .


def get_dict(self):
        '''
        Convert Paginator instance to dict

        :return: Paging data
        :rtype: dict
        '''

        return dict(
            current_page=self.current_page,
            total_page_count=self.total_page_count,
            items=self.items,
            total_item_count=self.total_item_count,
            page_size=self.page_size
        )

def get_dict ( self ) : return dict ( current_page = self . current_page , total_page_count = self . total_page_count , items = self . items , total_item_count = self . total_item_count , page_size = self . page_size )


def check_pidfile(pidfile, debug):
    """Check that a process is not running more than once, using PIDFILE"""
    # Check PID exists and see if the PID is running
    if os.path.isfile(pidfile):
        pidfile_handle = open(pidfile, 'r')
        # try and read the PID file. If no luck, remove it
        try:
            pid = int(pidfile_handle.read())
            pidfile_handle.close()
            if check_pid(pid, debug):
                return True
        except:
            pass

        # PID is not active, remove the PID file
        os.unlink(pidfile)

    # Create a PID file, to ensure this is script is only run once (at a time)
    pid = str(os.getpid())
    open(pidfile, 'w').write(pid)
    return False

def check_pidfile ( pidfile , debug ) : if os . path . isfile ( pidfile ) : pidfile_handle = open ( pidfile , 'r' ) try : pid = int ( pidfile_handle . read ( ) ) pidfile_handle . close ( ) if check_pid ( pid , debug ) : return True except : pass os . unlink ( pidfile ) pid = str ( os . getpid ( ) ) open ( pidfile , 'w' ) . write ( pid ) return False

Check that a process is not running more than once using PIDFILE


def check_pid(pid, debug):
    """This function will check whether a PID is currently running"""
    try:
        # A Kill of 0 is to check if the PID is active. It won't kill the process
        os.kill(pid, 0)
        if debug > 1:
            print("Script has a PIDFILE where the process is still running")
        return True
    except OSError:
        if debug > 1:
            print("Script does not appear to be running")
        return False


def disown(debug):
    """This function will disown, so the Ardexa service can be restarted"""
    # Get the current PID
    pid = os.getpid()
    cgroup_file = "/proc/" + str(pid) + "/cgroup"
    try:
        infile = open(cgroup_file, "r")
    except IOError:
        print("Could not open cgroup file: ", cgroup_file)
        return False

    # Read each line
    for line in infile:
        # Check if the line contains "ardexa.service"
        if line.find("ardexa.service") == -1:
            continue

        # if the lines contains "name=", replace it with nothing
        line = line.replace("name=", "")
        # Split  the line by commas
        items_list = line.split(':')
        accounts = items_list[1]
        dir_str = accounts + "/ardexa.disown"
        # If accounts is empty, continue
        if not accounts:
            continue

        # Create the dir and all subdirs
        full_dir = "/sys/fs/cgroup/" + dir_str
        if not os.path.exists(full_dir):
            os.makedirs(full_dir)
            if debug >= 1:
                print("Making directory: ", full_dir)
        else:
            if debug >= 1:
                print("Directory already exists: ", full_dir)

        # Add the PID to the file
        full_path = full_dir + "/cgroup.procs"
        prog_list = ["echo", str(pid), ">", full_path]
        run_program(prog_list, debug, True)

        # If this item contains a comma, then separate it, and reverse
        # some OSes will need cpuacct,cpu reversed to actually work
        if accounts.find(",") != -1:
            acct_list = accounts.split(',')
            accounts = acct_list[1] + "," + acct_list[0]
            dir_str = accounts + "/ardexa.disown"
            # Create the dir and all subdirs. But it may not work. So use a TRY
            full_dir = "/sys/fs/cgroup/" + dir_str
            try:
                if not os.path.exists(full_dir):
                    os.makedirs(full_dir)
            except:
                continue

            # Add the PID to the file
            full_path = full_dir + "/cgroup.procs"
            prog_list = ["echo", str(pid), ">", full_path]
            run_program(prog_list, debug, True)

    infile.close()

    # For debug purposes only
    if debug >= 1:
        prog_list = ["cat", cgroup_file]
        run_program(prog_list, debug, False)

    # If there are any "ardexa.service" in the proc file. If so, exit with error
    prog_list = ["grep", "-q", "ardexa.service", cgroup_file]
    if run_program(prog_list, debug, False):
        # There are entries still left in the file
        return False

    return True

def
disown
(
debug
)
:
pid
=
os
.
getpid
(
)
cgroup_file
=
"/proc/"
+
str
(
pid
)
+
"/cgroup"
try
:
infile
=
open
(
cgroup_file
,
"r"
)
except
IOError
:
print
(
"Could not open cgroup file: "
,
cgroup_file
)
return
False
for
line
in
infile
:
if
line
.
find
(
"ardexa.service"
)
==
1
:
continue
line
=
line
.
replace
(
"name="
,
""
)
items_list
=
line
.
split
(
':'
)
accounts
=
items_list
[
1
]
dir_str
=
accounts
+
"/ardexa.disown"
if
not
accounts
:
continue
full_dir
=
"/sys/fs/cgroup/"
+
dir_str
if
not
os
.
path
.
exists
(
full_dir
)
:
os
.
makedirs
(
full_dir
)
if
debug
>=
1
:
print
(
"Making directory: "
,
full_dir
)
else
:
if
debug
>=
1
:
print
(
"Directory already exists: "
,
full_dir
)
full_path
=
full_dir
+
"/cgroup.procs"
prog_list
=
[
"echo"
,
str
(
pid
)
,
">"
,
full_path
]
run_program
(
prog_list
,
debug
,
True
)
if
accounts
.
find
(
","
)
!=
1
:
acct_list
=
accounts
.
split
(
','
)
accounts
=
acct_list
[
1
]
+
","
+
acct_list
[
0
]
dir_str
=
accounts
+
"/ardexa.disown"
full_dir
=
"/sys/fs/cgroup/"
+
dir_str
try
:
if
not
os
.
path
.
exists
(
full_dir
)
:
os
.
makedirs
(
full_dir
)
except
:
continue
full_path
=
full_dir
+
"/cgroup.procs"
prog_list
=
[
"echo"
,
str
(
pid
)
,
">"
,
full_path
]
run_program
(
prog_list
,
debug
,
True
)
infile
.
close
(
)
if
debug
>=
1
:
prog_list
=
[
"cat"
,
cgroup_file
]
run_program
(
prog_list
,
debug
,
False
)
prog_list
=
[
"grep"
,
"-q"
,
"ardexa.service"
,
cgroup_file
]
if
run_program
(
prog_list
,
debug
,
False
)
:
return
False
return
True

This function will disown so the Ardexa service can be restarted


def run_program(prog_list, debug, shell):
    """Run a  program and check program return code Note that some commands don't work
    well with Popen.  So if this function is specifically called with 'shell=True',
    then it will run the old 'os.system'. In which case, there is no program output
    """
    try:
        if not shell:
            process = Popen(prog_list, stdout=PIPE, stderr=PIPE)
            stdout, stderr = process.communicate()
            retcode = process.returncode
            if debug >= 1:
                print("Program : ", " ".join(prog_list))
                print("Return Code: ", retcode)
                print("Stdout: ", stdout)
                print("Stderr: ", stderr)
            return bool(retcode)
        else:
            command = " ".join(prog_list)
            os.system(command)
            return True
    except:
        return False

def
run_program
(
prog_list
,
debug
,
shell
)
:
try
:
if
not
shell
:
process
=
Popen
(
prog_list
,
stdout
=
PIPE
,
stderr
=
PIPE
)
stdout
,
stderr
=
process
.
communicate
(
)
retcode
=
process
.
returncode
if
debug
>=
1
:
print
(
"Program : "
,
" "
.
join
(
prog_list
)
)
print
(
"Return Code: "
,
retcode
)
print
(
"Stdout: "
,
stdout
)
print
(
"Stderr: "
,
stderr
)
return
bool
(
retcode
)
else
:
command
=
" "
.
join
(
prog_list
)
os
.
system
(
command
)
return
True
except
:
return
False

Run a program and check program return code Note that some commands don t work well with Popen . So if this function is specifically called with shell = True then it will run the old os . system . In which case there is no program output


def parse_address_list(addrs):
    """Yield each integer from a complex range string like "1-9,12,15-20,23"

    >>> list(parse_address_list('1-9,12,15-20,23'))
    [1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 15, 16, 17, 18, 19, 20, 23]

    >>> list(parse_address_list('1-9,12,15-20,2-3-4'))
    Traceback (most recent call last):
        ...
    ValueError: format error in 2-3-4
    """
    for addr in addrs.split(','):
        elem = addr.split('-')
        if len(elem) == 1: # a number
            yield int(elem[0])
        elif len(elem) == 2: # a range inclusive
            start, end = list(map(int, elem))
            for i in range(start, end+1):
                yield i
        else: # more than one hyphen
            raise ValueError('format error in %s' % addr)

def
parse_address_list
(
addrs
)
:
for
addr
in
addrs
.
split
(
','
)
:
elem
=
addr
.
split
(
'-'
)
if
len
(
elem
)
==
1
:
yield
int
(
elem
[
0
]
)
elif
len
(
elem
)
==
2
:
start
,
end
=
list
(
map
(
int
,
elem
)
)
for
i
in
range
(
start
,
end
+
1
)
:
yield
i
else
:
raise
ValueError
(
'format error in %s'
%
addr
)


def _encode_ids(*args):
    """
    Do url-encode resource ids
    """

    ids = []
    for v in args:
        if isinstance(v, basestring):
            qv = v.encode('utf-8') if isinstance(v, unicode) else v
            ids.append(urllib.quote(qv))
        else:
            qv = str(v)
            ids.append(urllib.quote(qv))

    return ';'.join(ids)

def _encode_ids ( * args ) : ids = [ ] for v in args : if isinstance ( v , basestring ) : qv = v . encode ( 'utf-8' ) if isinstance ( v , unicode ) else v ids . append ( urllib . quote ( qv ) ) else : qv = str ( v ) ids . append ( urllib . quote ( qv ) ) return ';' . join ( ids )


def get_item_creator(item_type):
    """Get item creator according registered item type.

    :param item_type: The type of item to be checed.
    :type item_type: types.TypeType.
    :returns: Creator function. None if type not found.
    """
    if item_type not in Pipe.pipe_item_types:
        for registered_type in Pipe.pipe_item_types:
            if issubclass(item_type, registered_type):
                return Pipe.pipe_item_types[registered_type]
        return None
    else:
        return Pipe.pipe_item_types[item_type]

def get_item_creator ( item_type ) : if item_type not in Pipe . pipe_item_types : for registered_type in Pipe . pipe_item_types : if issubclass ( item_type , registered_type ) : return Pipe . pipe_item_types [ registered_type ] return None else : return Pipe . pipe_item_types [ item_type ]

Get item creator according registered item type .


def clone(self):
        """Self-cloning. All its next Pipe objects are cloned too.

        :returns: cloned object
        """
        new_object = copy.copy(self)
        if new_object.next:
            new_object.next = new_object.next.clone()
        return new_object

def clone ( self ) : new_object = copy . copy ( self ) if new_object . next : new_object . next = new_object . next . clone ( ) return new_object

Self - cloning . All its next Pipe objects are cloned too .


def append(self, next):
        """Append next object to pipe tail.

        :param next: The Pipe object to be appended to tail.
        :type next: Pipe object.
        """
        next.chained = True
        if self.next:
            self.next.append(next)
        else:
            self.next = next

def append ( self , next ) : next . chained = True if self . next : self . next . append ( next ) else : self . next = next


def iter(self, prev=None):
        """Return an generator as iterator object.

        :param prev: Previous Pipe object which used for data input.
        :returns: A generator for iteration.
        """

        if self.next:
            generator = self.next.iter(self.func(prev, *self.args, **self.kw))
        else:
            generator = self.func(prev, *self.args, **self.kw)
        return generator

def iter ( self , prev = None ) : if self . next : generator = self . next . iter ( self . func ( prev , * self . args , ** self . kw ) ) else : generator = self . func ( prev , * self . args , ** self . kw ) return generator

Return an generator as iterator object .


def reduce(func):
        """Wrap a reduce function to Pipe object. Reduce function is a function
        with at least two arguments. It works like built-in reduce function.
        It takes first argument for accumulated result, second argument for
        the new data to process. A keyword-based argument named 'init' is
        optional. If init is provided, it is used for the initial value of
        accumulated result. Or, the initial value is None.

        The first argument is the data to be converted. The return data from
        filter function should be a boolean value. If true, data can pass.
        Otherwise, data is omitted.

        :param func: The filter function to be wrapped.
        :type func: function object
        :param args: The default arguments to be used for filter function.
        :param kw: The default keyword arguments to be used for filter function.
        :returns: Pipe object
        """
        def wrapper(prev, *argv, **kw):
            accum_value = None if 'init' not in kw else kw.pop('init')
            if prev is None:
                raise TypeError('A reducer must have input.')
            for i in prev:
                accum_value = func(accum_value, i, *argv, **kw)
            yield accum_value
        return Pipe(wrapper)

def
reduce
(
func
)
:
def
wrapper
(
prev
,
*
argv
,
**
kw
)
:
accum_value
=
if
'init'
not
in
kw
else
kw
.
pop
(
'init'
)
if
prev
is
:
raise
TypeError
(
'A reducer must have input.'
)
for
i
in
prev
:
accum_value
=
func
(
accum_value
,
i
,
*
argv
,
**
kw
)
yield
accum_value
return
Pipe
(
wrapper
)

Wrap a reduce function to Pipe object . Reduce function is a function with at least two arguments . It works like built - in reduce function . It takes first argument for accumulated result second argument for the new data to process . A keyword - based argument named init is optional . If init is provided it is used for the initial value of accumulated result . Or the initial value is None .


def _list_networks():
    """Return a dictionary of network name to active status bools.

        Sample virsh net-list output::

    Name                 State      Autostart
    -----------------------------------------
    default              active     yes
    juju-test            inactive   no
    foobar               inactive   no

    Parsing the above would return::
    {"default": True, "juju-test": False, "foobar": False}

    See: http://goo.gl/kXwfC
    """
    output = core.run("virsh net-list --all")
    networks = {}

    # Take the header off and normalize whitespace.
    net_lines = [n.strip() for n in output.splitlines()[2:]]
    for line in net_lines:
        if not line:
            continue
        name, state, auto = line.split()
        networks[name] = state == "active"
    return networks

def
_list_networks
(
)
:
output
=
core
.
run
(
"virsh net-list --all"
)
networks
=
{
}
net_lines
=
[
n
.
strip
(
)
for
n
in
output
.
splitlines
(
)
[
2
:
]
]
for
line
in
net_lines
:
if
not
line
:
continue
name
,
state
,
auto
=
line
.
split
(
)
networks
[
name
]
=
state
==
"active"
return
networks

Return a dictionary of network name to active status bools .


def flush(self, line):
        """flush the line to stdout"""
        # TODO -- maybe use echo?
        sys.stdout.write(line)
        sys.stdout.flush()

def flush ( self , line ) : sys . stdout . write ( line ) sys . stdout . flush ( )


def execute(self, arg_str='', **kwargs):
        """runs the passed in arguments and returns an iterator on the output of
        running command"""
        cmd = "{} {} {}".format(self.cmd_prefix, self.script, arg_str)
        expected_ret_code = kwargs.pop('code', 0)

        # any kwargs with all capital letters should be considered environment
        # variables
        environ = self.environ
        for k in list(kwargs.keys()):
            if k.isupper():
                environ[k] = kwargs.pop(k)

        # we will allow overriding of these values
        kwargs.setdefault("stderr", subprocess.STDOUT)

        # we will not allow these to be overridden via kwargs
        kwargs["shell"] = True
        kwargs["stdout"] = subprocess.PIPE
        kwargs["cwd"] = self.cwd
        kwargs["env"] = environ

        process = None
        self.buf = deque(maxlen=self.bufsize)

        try:
            process = subprocess.Popen(
                cmd,
                **kwargs
            )

            # another round of links
            # http://stackoverflow.com/a/17413045/5006 (what I used)
            # http://stackoverflow.com/questions/2715847/
            for line in iter(process.stdout.readline, b""):
                line = line.decode(self.encoding)
                self.buf.append(line.rstrip())
                yield line

            process.wait()
            if process.returncode != expected_ret_code:
                if process.returncode > 0:
                    raise RuntimeError("{} returned {} with output: {}".format(
                        cmd,
                        process.returncode,
                        self.output
                    ))

        except subprocess.CalledProcessError as e:
            if e.returncode != expected_ret_code:
                raise RuntimeError("{} returned {} with output: {}".format(
                    cmd,
                    e.returncode,
                    self.output
                ))

        finally:
            if process:
                process.stdout.close()

def
execute
(
self
,
arg_str
=
''
,
**
kwargs
)
:
cmd
=
"{} {} {}"
.
format
(
self
.
cmd_prefix
,
self
.
script
,
arg_str
)
expected_ret_code
=
kwargs
.
pop
(
'code'
,
0
)
environ
=
self
.
environ
for
k
in
list
(
kwargs
.
keys
(
)
)
:
if
k
.
isupper
(
)
:
environ
[
k
]
=
kwargs
.
pop
(
k
)
kwargs
.
setdefault
(
"stderr"
,
subprocess
.
STDOUT
)
kwargs
[
"shell"
]
=
True
kwargs
[
"stdout"
]
=
subprocess
.
PIPE
kwargs
[
"cwd"
]
=
self
.
cwd
kwargs
[
"env"
]
=
environ
process
=
self
.
buf
=
deque
(
maxlen
=
self
.
bufsize
)
try
:
process
=
subprocess
.
Popen
(
cmd
,
**
kwargs
)
for
line
in
iter
(
process
.
stdout
.
readline
,
b""
)
:
line
=
line
.
decode
(
self
.
encoding
)
self
.
buf
.
append
(
line
.
rstrip
(
)
)
yield
line
process
.
wait
(
)
if
process
.
returncode
!=
expected_ret_code
:
if
process
.
returncode
>
0
:
raise
RuntimeError
(
"{} returned {} with output: {}"
.
format
(
cmd
,
process
.
returncode
,
self
.
output
)
)
except
subprocess
.
CalledProcessError
as
e
:
if
e
.
returncode
!=
expected_ret_code
:
raise
RuntimeError
(
"{} returned {} with output: {}"
.
format
(
cmd
,
e
.
returncode
,
self
.
output
)
)
finally
:
if
process
:
process
.
stdout
.
close
(
)


def get_request_subfields(root):
    """Build a basic 035 subfield with basic information from the OAI-PMH request.

    :param root: ElementTree root node

    :return: list of subfield tuples [(..),(..)]
    """
    request = root.find('request')
    responsedate = root.find('responseDate')

    subs = [("9", request.text),
            ("h", responsedate.text),
            ("m", request.attrib["metadataPrefix"])]
    return subs

def get_request_subfields ( root ) : request = root . find ( 'request' ) responsedate = root . find ( 'responseDate' ) subs = [ ( "9" , request . text ) , ( "h" , responsedate . text ) , ( "m" , request . attrib [ "metadataPrefix" ] ) ] return subs

Build a basic 035 subfield with basic information from the OAI - PMH request .


def strip_xml_namespace(root):
    """Strip out namespace data from an ElementTree.

    This function is recursive and will traverse all
    subnodes to the root element

    @param root: the root element

    @return: the same root element, minus namespace
    """
    try:
        root.tag = root.tag.split('}')[1]
    except IndexError:
        pass

    for element in root.getchildren():
        strip_xml_namespace(element)

def strip_xml_namespace ( root ) : try : root . tag = root . tag . split ( '}' ) [ 1 ] except IndexError : pass for element in root . getchildren ( ) : strip_xml_namespace ( element )

Strip out namespace data from an ElementTree .


def load_dict(self, source, namespace=''):
        """ Load values from a dictionary structure. Nesting can be used to
            represent namespaces.

            >>> c = ConfigDict()
            >>> c.load_dict({'some': {'namespace': {'key': 'value'} } })
            {'some.namespace.key': 'value'}
        """
        for key, value in source.items():
            if isinstance(key, str):
                nskey = (namespace + '.' + key).strip('.')
                if isinstance(value, dict):
                    self.load_dict(value, namespace=nskey)
                else:
                    self[nskey] = value
            else:
                raise TypeError('Key has type %r (not a string)' % type(key))
        return self

def
load_dict
(
self
,
source
,
namespace
=
''
)
:
for
key
,
value
in
source
.
items
(
)
:
if
isinstance
(
key
,
str
)
:
nskey
=
(
namespace
+
'.'
+
key
)
.
strip
(
'.'
)
if
isinstance
(
value
,
dict
)
:
self
.
load_dict
(
value
,
namespace
=
nskey
)
else
:
self
[
nskey
]
=
value
else
:
raise
TypeError
(
'Key has type %r (not a string)'
%
type
(
key
)
)
return
self

Load values from a dictionary structure . Nesting can be used to represent namespaces .


def json(request, *args, **kwargs):
    """
    The oembed endpoint, or the url to which requests for metadata are passed.
    Third parties will want to access this view with URLs for your site's
    content and be returned OEmbed metadata.
    """
    # coerce to dictionary
    params = dict(request.GET.items())

    callback = params.pop('callback', None)
    url = params.pop('url', None)

    if not url:
        return HttpResponseBadRequest('Required parameter missing: URL')

    try:
        provider = oembed.site.provider_for_url(url)
        if not provider.provides:
            raise OEmbedMissingEndpoint()
    except OEmbedMissingEndpoint:
        raise Http404('No provider found for %s' % url)

    query = dict([(smart_str(k), smart_str(v)) for k, v in params.items() if v])

    try:
        resource = oembed.site.embed(url, **query)
    except OEmbedException, e:
        raise Http404('Error embedding %s: %s' % (url, str(e)))

    response = HttpResponse(mimetype='application/json')
    json = resource.json

    if callback:
        response.write('%s(%s)' % (defaultfilters.force_escape(callback), json))
    else:
        response.write(json)

    return response

def
json
(
request
,
*
args
,
**
kwargs
)
:
params
=
dict
(
request
.
GET
.
items
(
)
)
callback
=
params
.
pop
(
'callback'
,
)
url
=
params
.
pop
(
'url'
,
)
if
not
url
:
return
HttpResponseBadRequest
(
'Required parameter missing: URL'
)
try
:
provider
=
oembed
.
site
.
provider_for_url
(
url
)
if
not
provider
.
provides
:
raise
OEmbedMissingEndpoint
(
)
except
OEmbedMissingEndpoint
:
raise
Http404
(
'No provider found for %s'
%
url
)
query
=
dict
(
[
(
smart_str
(
k
)
,
smart_str
(
v
)
)
for
k
,
v
in
params
.
items
(
)
if
v
]
)
try
:
resource
=
oembed
.
site
.
embed
(
url
,
**
query
)
except
OEmbedException
,
e
:
raise
Http404
(
'Error embedding %s: %s'
%
(
url
,
str
(
e
)
)
)
response
=
HttpResponse
(
mimetype
=
'application/json'
)
json
=
resource
.
json
if
callback
:
response
.
write
(
'%s(%s)'
%
(
defaultfilters
.
force_escape
(
callback
)
,
json
)
)
else
:
response
.
write
(
json
)
return
response

The oembed endpoint or the url to which requests for metadata are passed . Third parties will want to access this view with URLs for your site s content and be returned OEmbed metadata .


def consume_json(request):
    """
    Extract and return oembed content for given urls.

    Required GET params:
        urls - list of urls to consume

    Optional GET params:
        width - maxwidth attribute for oembed content
        height - maxheight attribute for oembed content
        template_dir - template_dir to use when rendering oembed

    Returns:
        list of dictionaries with oembed metadata and renderings, json encoded
    """
    client = OEmbedConsumer()

    urls = request.GET.getlist('urls')
    width = request.GET.get('width')
    height = request.GET.get('height')
    template_dir = request.GET.get('template_dir')

    output = {}
    ctx = RequestContext(request)

    for url in urls:
        try:
            provider = oembed.site.provider_for_url(url)
        except OEmbedMissingEndpoint:
            oembeds = None
            rendered = None
        else:
            oembeds = url
            rendered = client.parse_text(url, width, height, context=ctx, template_dir=template_dir)

        output[url] = {
            'oembeds': oembeds,
            'rendered': rendered,
        }

    return HttpResponse(simplejson.dumps(output), mimetype='application/json')

def consume_json ( request ) : client = OEmbedConsumer ( ) urls = request . GET . getlist ( 'urls' ) width = request . GET . get ( 'width' ) height = request . GET . get ( 'height' ) template_dir = request . GET . get ( 'template_dir' ) output = { } ctx = RequestContext ( request ) for url in urls : try : provider = oembed . site . provider_for_url ( url ) except OEmbedMissingEndpoint : oembeds = None rendered = None else : oembeds = url rendered = client . parse_text ( url , width , height , context = ctx , template_dir = template_dir ) output [ url ] = { 'oembeds' : oembeds , 'rendered' : rendered , } return HttpResponse ( simplejson . dumps ( output ) , mimetype = 'application/json' )

Extract and return oembed content for given urls .


def oembed_schema(request):
    """
    A site profile detailing valid endpoints for a given domain.  Allows for
    better auto-discovery of embeddable content.

    OEmbed-able content lives at a URL that maps to a provider.
    """
    current_domain = Site.objects.get_current().domain
    url_schemes = [] # a list of dictionaries for all the urls we can match
    endpoint = reverse('oembed_json') # the public endpoint for our oembeds
    providers = oembed.site.get_providers()

    for provider in providers:
        # first make sure this provider class is exposed at the public endpoint
        if not provider.provides:
            continue

        match = None
        if isinstance(provider, DjangoProvider):
            # django providers define their regex_list by using urlreversing
            url_pattern = resolver.reverse_dict.get(provider._meta.named_view)

            # this regex replacement is set to be non-greedy, which results
            # in things like /news/*/*/*/*/ -- this is more explicit
            if url_pattern:
                regex = re.sub(r'%\(.+?\)s', '*', url_pattern[0][0][0])
                match = 'http://%s/%s' % (current_domain, regex)
        elif isinstance(provider, HTTPProvider):
            match = provider.url_scheme
        else:
            match = provider.regex

        if match:
            url_schemes.append({
                'type': provider.resource_type,
                'matches': match,
                'endpoint': endpoint
            })

    url_schemes.sort(key=lambda item: item['matches'])

    response = HttpResponse(mimetype='application/json')
    response.write(simplejson.dumps(url_schemes))
    return response

def oembed_schema ( request ) : current_domain = Site . objects . get_current ( ) . domain url_schemes = [ ] endpoint = reverse ( 'oembed_json' ) providers = oembed . site . get_providers ( ) for provider in providers : if not provider . provides : continue match = None if isinstance ( provider , DjangoProvider ) : url_pattern = resolver . reverse_dict . get ( provider . _meta . named_view ) if url_pattern : regex = re . sub ( r'%\(.+?\)s' , '*' , url_pattern [ 0 ] [ 0 ] [ 0 ] ) match = 'http://%s/%s' % ( current_domain , regex ) elif isinstance ( provider , HTTPProvider ) : match = provider . url_scheme else : match = provider . regex if match : url_schemes . append ( { 'type' : provider . resource_type , 'matches' : match , 'endpoint' : endpoint } ) url_schemes . sort ( key = lambda item : item [ 'matches' ] ) response = HttpResponse ( mimetype = 'application/json' ) response . write ( simplejson . dumps ( url_schemes ) ) return response

A site profile detailing valid endpoints for a given domain . Allows for better auto - discovery of embeddable content .


def main(path):
    '''scan path directory and any subdirectories for valid captain scripts'''
    basepath = os.path.abspath(os.path.expanduser(str(path)))

    echo.h2("Available scripts in {}".format(basepath))
    echo.br()
    for root_dir, dirs, files in os.walk(basepath, topdown=True):
        for f in fnmatch.filter(files, '*.py'):
            try:
                filepath = os.path.join(root_dir, f)

                # super edge case, this makes sure the python script won't start
                # an interactive console session which would cause the session
                # to start and not allow the for loop to complete
                with open(filepath, encoding="UTF-8") as fp:
                    body = fp.read()
                    is_console = "InteractiveConsole" in body
                    is_console = is_console or "code" in body
                    is_console = is_console and "interact(" in body
                    if is_console:
                        continue

                s = captain.Script(filepath)
                if s.can_run_from_cli():
                    rel_filepath = s.call_path(basepath)
                    p = s.parser

                    echo.h3(rel_filepath)

                    desc = p.description
                    if desc:
                        echo.indent(desc, indent=(" " * 4))

                    subcommands = s.subcommands
                    if subcommands:
                        echo.br()
                        echo.indent("Subcommands:", indent=(" " * 4))
                        for sc in subcommands.keys():
                            echo.indent(sc, indent=(" " * 6))

                    echo.br()

            except captain.ParseError:
                pass

            except Exception as e:
                #echo.exception(e)
                #echo.err("Failed to parse {} because {}", f, e.message)
                echo.err("Failed to parse {}", f)
                echo.verbose(e.message)
                echo.br()

def
main
(
path
)
:
basepath
=
os
.
path
.
abspath
(
os
.
path
.
expanduser
(
str
(
path
)
)
)
echo
.
h2
(
"Available scripts in {}"
.
format
(
basepath
)
)
echo
.
br
(
)
for
root_dir
,
dirs
,
files
in
os
.
walk
(
basepath
,
topdown
=
True
)
:
for
f
in
fnmatch
.
filter
(
files
,
'*.py'
)
:
try
:
filepath
=
os
.
path
.
join
(
root_dir
,
f
)
with
open
(
filepath
,
encoding
=
"UTF-8"
)
as
fp
:
body
=
fp
.
read
(
)
is_console
=
"InteractiveConsole"
in
body
is_console
=
is_console
or
"code"
in
body
is_console
=
is_console
and
"interact("
in
body
if
is_console
:
continue
s
=
captain
.
Script
(
filepath
)
if
s
.
can_run_from_cli
(
)
:
rel_filepath
=
s
.
call_path
(
basepath
)
p
=
s
.
parser
echo
.
h3
(
rel_filepath
)
desc
=
p
.
description
if
desc
:
echo
.
indent
(
desc
,
indent
=
(
" "
*
4
)
)
subcommands
=
s
.
subcommands
if
subcommands
:
echo
.
br
(
)
echo
.
indent
(
"Subcommands:"
,
indent
=
(
" "
*
4
)
)
for
sc
in
subcommands
.
keys
(
)
:
echo
.
indent
(
sc
,
indent
=
(
" "
*
6
)
)
echo
.
br
(
)
except
captain
.
ParseError
:
pass
except
Exception
as
e
:
echo
.
err
(
"Failed to parse {}"
,
f
)
echo
.
verbose
(
e
.
message
)
echo
.
br
(
)


def make_request_data(self, zipcode, city, state):
        """ Make the request params given location data """
        data = {'key': self.api_key,
                'postalcode': str(zipcode),
                'city': city,
                'state': state
        }
        data = ZipTaxClient._clean_request_data(data)
        return data

def make_request_data ( self , zipcode , city , state ) : data = { 'key' : self . api_key , 'postalcode' : str ( zipcode ) , 'city' : city , 'state' : state } data = ZipTaxClient . _clean_request_data ( data ) return data


def process_response(self, resp, multiple_rates):
        """ Get the tax rate from the ZipTax response """
        self._check_for_exceptions(resp, multiple_rates)

        rates = {}
        for result in resp['results']:
            rate = ZipTaxClient._cast_tax_rate(result['taxSales'])
            rates[result['geoCity']] = rate
        if not multiple_rates:
            return rates[list(rates.keys())[0]]
        return rates

def process_response ( self , resp , multiple_rates ) : self . _check_for_exceptions ( resp , multiple_rates ) rates = { } for result in resp [ 'results' ] : rate = ZipTaxClient . _cast_tax_rate ( result [ 'taxSales' ] ) rates [ result [ 'geoCity' ] ] = rate if not multiple_rates : return rates [ list ( rates . keys ( ) ) [ 0 ] ] return rates


def _check_for_exceptions(self, resp, multiple_rates):
        """ Check if there are exceptions that should be raised """
        if resp['rCode'] != 100:
            raise exceptions.get_exception_for_code(resp['rCode'])(resp)

        results = resp['results']
        if len(results) == 0:
            raise exceptions.ZipTaxNoResults('No results found')
        if len(results) > 1 and not multiple_rates:
            # It's fine if all the taxes are the same
            rates = [result['taxSales'] for result in results]
            if len(set(rates)) != 1:
                raise exceptions.ZipTaxMultipleResults('Multiple results found but requested only one')


def get_all_text(node):
    """Recursively extract all text from node."""
    if node.nodeType == node.TEXT_NODE:
        return node.data
    else:
        text_string = ""
        for child_node in node.childNodes:
            text_string += get_all_text(child_node)
        return text_string

def get_all_text ( node ) : if node . nodeType == node . TEXT_NODE : return node . data else : text_string = "" for child_node in node . childNodes : text_string += get_all_text ( child_node ) return text_string

Recursively extract all text from node .


def register(self, provider_class):
        """
        Registers a provider with the site.
        """
        if not issubclass(provider_class, BaseProvider):
            raise TypeError('%s is not a subclass of BaseProvider' % provider_class.__name__)

        if provider_class in self._registered_providers:
            raise AlreadyRegistered('%s is already registered' % provider_class.__name__)

        if issubclass(provider_class, DjangoProvider):
            # set up signal handler for cache invalidation
            signals.post_save.connect(
                self.invalidate_stored_oembeds,
                sender=provider_class._meta.model
            )

        # don't build the regex yet - if not all urlconfs have been loaded
        # and processed at this point, the DjangoProvider instances will fail
        # when attempting to reverse urlpatterns that haven't been created.
        # Rather, the regex-list will be populated once, on-demand.
        self._registered_providers.append(provider_class)

        # flag for re-population
        self.invalidate_providers()

def
register
(
self
,
provider_class
)
:
if
not
issubclass
(
provider_class
,
BaseProvider
)
:
raise
TypeError
(
'%s is not a subclass of BaseProvider'
%
provider_class
.
__name__
)
if
provider_class
in
self
.
_registered_providers
:
raise
AlreadyRegistered
(
'%s is already registered'
%
provider_class
.
__name__
)
if
issubclass
(
provider_class
,
DjangoProvider
)
:
signals
.
post_save
.
connect
(
self
.
invalidate_stored_oembeds
,
sender
=
provider_class
.
_meta
.
model
)
self
.
_registered_providers
.
append
(
provider_class
)
self
.
invalidate_providers
(
)


def unregister(self, provider_class):
        """
        Unregisters a provider from the site.
        """
        if not issubclass(provider_class, BaseProvider):
            raise TypeError('%s must be a subclass of BaseProvider' % provider_class.__name__)

        if provider_class not in self._registered_providers:
            raise NotRegistered('%s is not registered' % provider_class.__name__)

        self._registered_providers.remove(provider_class)

        # flag for repopulation
        self.invalidate_providers()

def
unregister
(
self
,
provider_class
)
:
if
not
issubclass
(
provider_class
,
BaseProvider
)
:
raise
TypeError
(
'%s must be a subclass of BaseProvider'
%
provider_class
.
__name__
)
if
provider_class
not
in
self
.
_registered_providers
:
raise
NotRegistered
(
'%s is not registered'
%
provider_class
.
__name__
)
self
.
_registered_providers
.
remove
(
provider_class
)
self
.
invalidate_providers
(
)


def populate(self):
        """
        Populate the internal registry's dictionary with the regexes for each
        provider instance
        """
        self._registry = {}

        for provider_class in self._registered_providers:
            instance = provider_class()
            self._registry[instance] = instance.regex

        for stored_provider in StoredProvider.objects.active():
            self._registry[stored_provider] = stored_provider.regex

        self._populated = True

def populate ( self ) : self . _registry = { } for provider_class in self . _registered_providers : instance = provider_class ( ) self . _registry [ instance ] = instance . regex for stored_provider in StoredProvider . objects . active ( ) : self . _registry [ stored_provider ] = stored_provider . regex self . _populated = True

Populate the internal registry s dictionary with the regexes for each provider instance


def provider_for_url(self, url):
        """
        Find the right provider for a URL
        """
        for provider, regex in self.get_registry().items():
            if re.match(regex, url) is not None:
                return provider

        raise OEmbedMissingEndpoint('No endpoint matches URL: %s' % url)

def
provider_for_url
(
self
,
url
)
:
for
provider
,
regex
in
self
.
get_registry
(
)
.
items
(
)
:
if
re
.
match
(
regex
,
url
)
is
not
:
return
provider
raise
OEmbedMissingEndpoint
(
'No endpoint matches URL: %s'
%
url
)


def invalidate_stored_oembeds(self, sender, instance, created, **kwargs):
        """
        A hook for django-based oembed providers to delete any stored oembeds
        """
        ctype = ContentType.objects.get_for_model(instance)
        StoredOEmbed.objects.filter(
            object_id=instance.pk,
            content_type=ctype).delete()

def invalidate_stored_oembeds ( self , sender , instance , created , ** kwargs ) : ctype = ContentType . objects . get_for_model ( instance ) StoredOEmbed . objects . filter ( object_id = instance . pk , content_type = ctype ) . delete ( )

A hook for django - based oembed providers to delete any stored oembeds


def embed(self, url, **kwargs):
        """
        The heart of the matter
        """
        try:
            # first figure out the provider
            provider = self.provider_for_url(url)
        except OEmbedMissingEndpoint:
            raise
        else:
            try:
                # check the database for a cached response, because of certain
                # race conditions that exist with get_or_create(), do a filter
                # lookup and just grab the first item
                stored_match = StoredOEmbed.objects.filter(
                    match=url,
                    maxwidth=kwargs.get('maxwidth', None),
                    maxheight=kwargs.get('maxheight', None),
                    date_expires__gte=datetime.datetime.now())[0]
                return OEmbedResource.create_json(stored_match.response_json)
            except IndexError:
                # query the endpoint and cache response in db
                # prevent None from being passed in as a GET param
                params = dict([(k, v) for k, v in kwargs.items() if v])

                # request an oembed resource for the url
                resource = provider.request_resource(url, **params)

                try:
                    cache_age = int(resource.cache_age)
                    if cache_age < MIN_OEMBED_TTL:
                        cache_age = MIN_OEMBED_TTL
                except:
                    cache_age = DEFAULT_OEMBED_TTL

                date_expires = datetime.datetime.now() + datetime.timedelta(seconds=cache_age)

                stored_oembed, created = StoredOEmbed.objects.get_or_create(
                    match=url,
                    maxwidth=kwargs.get('maxwidth', None),
                    maxheight=kwargs.get('maxheight', None))

                stored_oembed.response_json = resource.json
                stored_oembed.resource_type = resource.type
                stored_oembed.date_expires = date_expires

                if resource.content_object:
                    stored_oembed.content_object = resource.content_object

                stored_oembed.save()
                return resource

def embed ( self , url , ** kwargs ) : try : provider = self . provider_for_url ( url ) except OEmbedMissingEndpoint : raise else : try : stored_match = StoredOEmbed . objects . filter ( match = url , maxwidth = kwargs . get ( 'maxwidth' , None ) , maxheight = kwargs . get ( 'maxheight' , None ) , date_expires__gte = datetime . datetime . now ( ) ) [ 0 ] return OEmbedResource . create_json ( stored_match . response_json ) except IndexError : params = dict ( [ ( k , v ) for k , v in kwargs . items ( ) if v ] ) resource = provider . request_resource ( url , ** params ) try : cache_age = int ( resource . cache_age ) if cache_age < MIN_OEMBED_TTL : cache_age = MIN_OEMBED_TTL except : cache_age = DEFAULT_OEMBED_TTL date_expires = datetime . datetime . now ( ) + datetime . timedelta ( seconds = cache_age ) stored_oembed , created = StoredOEmbed . objects . get_or_create ( match = url , maxwidth = kwargs . get ( 'maxwidth' , None ) , maxheight = kwargs . get ( 'maxheight' , None ) ) stored_oembed . response_json = resource . json stored_oembed . resource_type = resource . type stored_oembed . date_expires = date_expires if resource . content_object : stored_oembed . content_object = resource . content_object stored_oembed . save ( ) return resource


def autodiscover(self, url):
        """
        Load up StoredProviders from url if it is an oembed scheme
        """
        headers, response = fetch_url(url)
        if headers['content-type'].split(';')[0] in ('application/json', 'text/javascript'):
            provider_data = json.loads(response)
            return self.store_providers(provider_data)

def autodiscover ( self , url ) : headers , response = fetch_url ( url ) if headers [ 'content-type' ] . split ( ';' ) [ 0 ] in ( 'application/json' , 'text/javascript' ) : provider_data = json . loads ( response ) return self . store_providers ( provider_data )


def store_providers(self, provider_data):
        """
        Iterate over the returned json and try to sort out any new providers
        """
        if not hasattr(provider_data, '__iter__'):
            raise OEmbedException('Autodiscovered response not iterable')

        provider_pks = []

        for provider in provider_data:
            if 'endpoint' not in provider or \
               'matches' not in provider:
                continue

            resource_type = provider.get('type')
            if resource_type not in RESOURCE_TYPES:
                continue

            stored_provider, created = StoredProvider.objects.get_or_create(
                wildcard_regex=provider['matches']
            )

            if created:
                stored_provider.endpoint_url = relative_to_full(
                    provider['endpoint'],
                    provider['matches']
                )
                stored_provider.resource_type = resource_type
                stored_provider.save()

            provider_pks.append(stored_provider.pk)

        return StoredProvider.objects.filter(pk__in=provider_pks)

def
store_providers
(
self
,
provider_data
)
:
if
not
hasattr
(
provider_data
,
'__iter__'
)
:
raise
OEmbedException
(
'Autodiscovered response not iterable'
)
provider_pks
=
[
]
for
provider
in
provider_data
:
if
'endpoint'
not
in
provider
or
'matches'
not
in
provider
:
continue
resource_type
=
provider
.
get
(
'type'
)
if
resource_type
not
in
RESOURCE_TYPES
:
continue
stored_provider
,
created
=
StoredProvider
.
objects
.
get_or_create
(
wildcard_regex
=
provider
[
'matches'
]
)
if
created
:
stored_provider
.
endpoint_url
=
relative_to_full
(
provider
[
'endpoint'
]
,
provider
[
'matches'
]
)
stored_provider
.
resource_type
=
resource_type
stored_provider
.
save
(
)
provider_pks
.
append
(
stored_provider
.
pk
)
return
StoredProvider
.
objects
.
filter
(
pk__in
=
provider_pks
)


def map_attr(self, mapping, attr, obj):
        """
        A kind of cheesy method that allows for callables or attributes to
        be used interchangably
        """
        if attr not in mapping and hasattr(self, attr):
            if not callable(getattr(self, attr)):
                mapping[attr] = getattr(self, attr)
            else:
                mapping[attr] = getattr(self, attr)(obj)

def map_attr ( self , mapping , attr , obj ) : if attr not in mapping and hasattr ( self , attr ) : if not callable ( getattr ( self , attr ) ) : mapping [ attr ] = getattr ( self , attr ) else : mapping [ attr ] = getattr ( self , attr ) ( obj )


def get_image(self, obj):
        """
        Return an ImageFileField instance
        """
        if self._meta.image_field:
            return getattr(obj, self._meta.image_field)

def get_image ( self , obj ) : if self . _meta . image_field : return getattr ( obj , self . _meta . image_field )


def map_to_dictionary(self, url, obj, **kwargs):
        """
        Build a dictionary of metadata for the requested object.
        """
        maxwidth = kwargs.get('maxwidth', None)
        maxheight = kwargs.get('maxheight', None)

        provider_url, provider_name = self.provider_from_url(url)

        mapping = {
            'version': '1.0',
            'url': url,
            'provider_name': provider_name,
            'provider_url': provider_url,
            'type': self.resource_type
        }

        # a hook
        self.preprocess(obj, mapping, **kwargs)

        # resize image if we have a photo, otherwise use the given maximums
        if self.resource_type == 'photo' and self.get_image(obj):
            self.resize_photo(obj, mapping, maxwidth, maxheight)
        elif self.resource_type in ('video', 'rich', 'photo'):
            width, height = size_to_nearest(
                maxwidth,
                maxheight,
                self._meta.valid_sizes,
                self._meta.force_fit
            )
            mapping.update(width=width, height=height)

        # create a thumbnail
        if self.get_image(obj):
            self.thumbnail(obj, mapping)

        # map attributes to the mapping dictionary.  if the attribute is
        # a callable, it must have an argument signature of
        # (self, obj)
        for attr in ('title', 'author_name', 'author_url', 'html'):
            self.map_attr(mapping, attr, obj)

        # fix any urls
        if 'url' in mapping:
            mapping['url'] = relative_to_full(mapping['url'], url)

        if 'thumbnail_url' in mapping:
            mapping['thumbnail_url'] = relative_to_full(mapping['thumbnail_url'], url)

        if 'html' not in mapping and mapping['type'] in ('video', 'rich'):
            mapping['html'] = self.render_html(obj, context=Context(mapping))

        # a hook
        self.postprocess(obj, mapping, **kwargs)

        return mapping

def map_to_dictionary ( self , url , obj , ** kwargs ) : maxwidth = kwargs . get ( 'maxwidth' , None ) maxheight = kwargs . get ( 'maxheight' , None ) provider_url , provider_name = self . provider_from_url ( url ) mapping = { 'version' : '1.0' , 'url' : url , 'provider_name' : provider_name , 'provider_url' : provider_url , 'type' : self . resource_type } self . preprocess ( obj , mapping , ** kwargs ) if self . resource_type == 'photo' and self . get_image ( obj ) : self . resize_photo ( obj , mapping , maxwidth , maxheight ) elif self . resource_type in ( 'video' , 'rich' , 'photo' ) : width , height = size_to_nearest ( maxwidth , maxheight , self . _meta . valid_sizes , self . _meta . force_fit ) mapping . update ( width = width , height = height ) if self . get_image ( obj ) : self . thumbnail ( obj , mapping ) for attr in ( 'title' , 'author_name' , 'author_url' , 'html' ) : self . map_attr ( mapping , attr , obj ) if 'url' in mapping : mapping [ 'url' ] = relative_to_full ( mapping [ 'url' ] , url ) if 'thumbnail_url' in mapping : mapping [ 'thumbnail_url' ] = relative_to_full ( mapping [ 'thumbnail_url' ] , url ) if 'html' not in mapping and mapping [ 'type' ] in ( 'video' , 'rich' ) : mapping [ 'html' ] = self . render_html ( obj , context = Context ( mapping ) ) self . postprocess ( obj , mapping , ** kwargs ) return mapping

Build a dictionary of metadata for the requested object .


def get_object(self, url, month_format='%b', day_format='%d'):
        """
        Parses the date from a url and uses it in the query.  For objects which
        are unique for date.
        """
        params = self.get_params(url)
        try:
            year = params[self._meta.year_part]
            month = params[self._meta.month_part]
            day = params[self._meta.day_part]
        except KeyError:
            try:
                # named lookups failed, so try to get the date using the first
                # three parameters
                year, month, day = params['_0'], params['_1'], params['_2']
            except KeyError:
                raise OEmbedException('Error extracting date from url parameters')

        try:
            tt = time.strptime('%s-%s-%s' % (year, month, day),
                               '%s-%s-%s' % ('%Y', month_format, day_format))
            date = datetime.date(*tt[:3])
        except ValueError:
            raise OEmbedException('Error parsing date from: %s' % url)

        # apply the date-specific lookups
        if isinstance(self._meta.model._meta.get_field(self._meta.date_field), DateTimeField):
            min_date = datetime.datetime.combine(date, datetime.time.min)
            max_date = datetime.datetime.combine(date, datetime.time.max)
            query = {'%s__range' % self._meta.date_field: (min_date, max_date)}
        else:
            query = {self._meta.date_field: date}

        # apply the regular search lookups
        for key, value in self._meta.fields_to_match.iteritems():
            try:
                query[value] = params[key]
            except KeyError:
                raise OEmbedException('%s was not found in the urlpattern parameters.  Valid names are: %s' % (key, ', '.join(params.keys())))

        try:
            obj = self.get_queryset().get(**query)
        except self._meta.model.DoesNotExist:
            raise OEmbedException('Requested object not found')

        return obj

Parses the date from a url and uses it in the query . For objects which are unique for date .


def get_record(self):
        """Override the base."""
        self.recid = self.get_recid()
        self.remove_controlfields()
        self.update_system_numbers()
        self.add_systemnumber("Inspire", recid=self.recid)
        self.add_control_number("003", "SzGeCERN")
        self.update_collections()
        self.update_languages()
        self.update_reportnumbers()
        self.update_authors()
        self.update_journals()
        self.update_subject_categories("INSPIRE", "SzGeCERN", "categories_cds")
        self.update_pagenumber()
        self.update_notes()
        self.update_experiments()
        self.update_isbn()
        self.update_dois()
        self.update_links_and_ffts()
        self.update_date()
        self.update_date_year()
        self.update_hidden_notes()
        self.update_oai_info()
        self.update_cnum()
        self.update_conference_info()

        self.fields_list = [
            "909", "541", "961",
            "970", "690", "695",
            "981",
        ]
        self.strip_fields()

        if "ANNOUNCEMENT" in self.collections:
            self.update_conference_111()
            self.update_conference_links()
            record_add_field(self.record, "690", ind1="C", subfields=[("a", "CONFERENCE")])

        if "THESIS" in self.collections:
            self.update_thesis_information()
            self.update_thesis_supervisors()

        if "PROCEEDINGS" in self.collections:
            # Special proceeding syntax
            self.update_title_to_proceeding()
            self.update_author_to_proceeding()
            record_add_field(self.record, "690", ind1="C", subfields=[("a", "CONFERENCE")])

        # 690 tags
        if self.tag_as_cern:
            record_add_field(self.record, "690", ind1="C", subfields=[("a", "CERN")])

        return self.record

def get_record ( self ) : self . recid = self . get_recid ( ) self . remove_controlfields ( ) self . update_system_numbers ( ) self . add_systemnumber ( "Inspire" , recid = self . recid ) self . add_control_number ( "003" , "SzGeCERN" ) self . update_collections ( ) self . update_languages ( ) self . update_reportnumbers ( ) self . update_authors ( ) self . update_journals ( ) self . update_subject_categories ( "INSPIRE" , "SzGeCERN" , "categories_cds" ) self . update_pagenumber ( ) self . update_notes ( ) self . update_experiments ( ) self . update_isbn ( ) self . update_dois ( ) self . update_links_and_ffts ( ) self . update_date ( ) self . update_date_year ( ) self . update_hidden_notes ( ) self . update_oai_info ( ) self . update_cnum ( ) self . update_conference_info ( ) self . fields_list = [ "909" , "541" , "961" , "970" , "690" , "695" , "981" , ] self . strip_fields ( ) if "ANNOUNCEMENT" in self . collections : self . update_conference_111 ( ) self . update_conference_links ( ) record_add_field ( self . record , "690" , ind1 = "C" , subfields = [ ( "a" , "CONFERENCE" ) ] ) if "THESIS" in self . collections : self . update_thesis_information ( ) self . update_thesis_supervisors ( ) if "PROCEEDINGS" in self . collections : self . update_title_to_proceeding ( ) self . update_author_to_proceeding ( ) record_add_field ( self . record , "690" , ind1 = "C" , subfields = [ ( "a" , "CONFERENCE" ) ] ) if self . tag_as_cern : record_add_field ( self . record , "690" , ind1 = "C" , subfields = [ ( "a" , "CERN" ) ] ) return self . record


def update_oai_info(self):
        """Add the 909 OAI info to 035."""
        for field in record_get_field_instances(self.record, '909', ind1="C", ind2="O"):
            new_subs = []
            for tag, value in field[0]:
                if tag == "o":
                    new_subs.append(("a", value))
                else:
                    new_subs.append((tag, value))
                if value in ["CERN", "CDS", "ForCDS"]:
                    self.tag_as_cern = True
            record_add_field(self.record, '024', ind1="8", subfields=new_subs)
        record_delete_fields(self.record, '909')

def update_oai_info ( self ) : for field in record_get_field_instances ( self . record , '909' , ind1 = "C" , ind2 = "O" ) : new_subs = [ ] for tag , value in field [ 0 ] : if tag == "o" : new_subs . append ( ( "a" , value ) ) else : new_subs . append ( ( tag , value ) ) if value in [ "CERN" , "CDS" , "ForCDS" ] : self . tag_as_cern = True record_add_field ( self . record , '024' , ind1 = "8" , subfields = new_subs ) record_delete_fields ( self . record , '909' )


def update_cnum(self):
        """Check if we shall add cnum in 035."""
        if "ConferencePaper" not in self.collections:
            cnums = record_get_field_values(self.record, '773', code="w")
            for cnum in cnums:
                cnum_subs = [
                    ("9", "INSPIRE-CNUM"),
                    ("a", cnum)
                ]
                record_add_field(self.record, "035", subfields=cnum_subs)

def update_cnum ( self ) : if "ConferencePaper" not in self . collections : cnums = record_get_field_values ( self . record , '773' , code = "w" ) for cnum in cnums : cnum_subs = [ ( "9" , "INSPIRE-CNUM" ) , ( "a" , cnum ) ] record_add_field ( self . record , "035" , subfields = cnum_subs )


def update_hidden_notes(self):
        """Remove hidden notes and tag a CERN if detected."""
        if not self.tag_as_cern:
            notes = record_get_field_instances(self.record,
                                               tag="595")
            for field in notes:
                for dummy, value in field[0]:
                    if value == "CDS":
                        self.tag_as_cern = True
        record_delete_fields(self.record, tag="595")

def update_hidden_notes ( self ) : if not self . tag_as_cern : notes = record_get_field_instances ( self . record , tag = "595" ) for field in notes : for dummy , value in field [ 0 ] : if value == "CDS" : self . tag_as_cern = True record_delete_fields ( self . record , tag = "595" )

Remove hidden notes and tag a CERN if detected .


def update_notes(self):
        """Remove INSPIRE specific notes."""
        fields = record_get_field_instances(self.record, '500')
        for field in fields:
            subs = field_get_subfields(field)
            for sub in subs.get('a', []):
                sub = sub.strip()  # remove any spaces before/after
                if sub.startswith("*") and sub.endswith("*"):
                    record_delete_field(self.record, tag="500",
                                        field_position_global=field[4])

def update_notes ( self ) : fields = record_get_field_instances ( self . record , '500' ) for field in fields : subs = field_get_subfields ( field ) for sub in subs . get ( 'a' , [ ] ) : sub = sub . strip ( ) if sub . startswith ( "*" ) and sub . endswith ( "*" ) : record_delete_field ( self . record , tag = "500" , field_position_global = field [ 4 ] )


def update_title_to_proceeding(self):
        """Move title info from 245 to 111 proceeding style."""
        titles = record_get_field_instances(self.record,
                                            tag="245")
        for title in titles:
            subs = field_get_subfields(title)
            new_subs = []
            if "a" in subs:
                new_subs.append(("a", subs['a'][0]))
            if "b" in subs:
                new_subs.append(("c", subs['b'][0]))
            record_add_field(self.record,
                             tag="111",
                             subfields=new_subs)
        record_delete_fields(self.record, tag="245")
        record_delete_fields(self.record, tag="246")

def update_title_to_proceeding ( self ) : titles = record_get_field_instances ( self . record , tag = "245" ) for title in titles : subs = field_get_subfields ( title ) new_subs = [ ] if "a" in subs : new_subs . append ( ( "a" , subs [ 'a' ] [ 0 ] ) ) if "b" in subs : new_subs . append ( ( "c" , subs [ 'b' ] [ 0 ] ) ) record_add_field ( self . record , tag = "111" , subfields = new_subs ) record_delete_fields ( self . record , tag = "245" ) record_delete_fields ( self . record , tag = "246" )

Move title info from 245 to 111 proceeding style .