def parse_commits(data):
    '''Accept a string and parse it into many commits.
    Parse and yield each commit-dictionary.
    This function is a generator.
    '''
    raw_commits = RE_COMMIT.finditer(data)
    for rc in raw_commits:
        full_commit = rc.groups()[0]
        parts = RE_COMMIT.match(full_commit).groupdict()
        parsed_commit = parse_commit(parts)
        yield parsed_commit

def parse_commits ( data ) : raw_commits = RE_COMMIT . finditer ( data ) for rc in raw_commits : full_commit = rc . groups ( ) [ 0 ] parts = RE_COMMIT . match ( full_commit ) . groupdict ( ) parsed_commit = parse_commit ( parts ) yield parsed_commit

Accept a string and parse it into many commits . Parse and yield each commit - dictionary . This function is a generator .


def parse_commit(parts):
    '''Accept a parsed single commit. Some of the named groups
    require further processing, so parse those groups.
    Return a dictionary representing the completely parsed
    commit.
    '''
    commit = {}
    commit['commit'] = parts['commit']
    commit['tree'] = parts['tree']
    parent_block = parts['parents']
    commit['parents'] = [
        parse_parent_line(parentline)
        for parentline in
        parent_block.splitlines()
    ]
    commit['author'] = parse_author_line(parts['author'])
    commit['committer'] = parse_committer_line(parts['committer'])
    message_lines = [
        parse_message_line(msgline)
        for msgline in
        parts['message'].split("\n")
    ]
    commit['message'] = "\n".join(
        msgline
        for msgline in
        message_lines
        if msgline is not None
    )
    commit['changes'] = [
        parse_numstat_line(numstat)
        for numstat in
        parts['numstats'].splitlines()
    ]
    return commit

def parse_commit ( parts ) : commit = { } commit [ 'commit' ] = parts [ 'commit' ] commit [ 'tree' ] = parts [ 'tree' ] parent_block = parts [ 'parents' ] commit [ 'parents' ] = [ parse_parent_line ( parentline ) for parentline in parent_block . splitlines ( ) ] commit [ 'author' ] = parse_author_line ( parts [ 'author' ] ) commit [ 'committer' ] = parse_committer_line ( parts [ 'committer' ] ) message_lines = [ parse_message_line ( msgline ) for msgline in parts [ 'message' ] . split ( "\n" ) ] commit [ 'message' ] = "\n" . join ( msgline for msgline in message_lines if msgline is not None ) commit [ 'changes' ] = [ parse_numstat_line ( numstat ) for numstat in parts [ 'numstats' ] . splitlines ( ) ] return commit

Accept a parsed single commit . Some of the named groups require further processing so parse those groups . Return a dictionary representing the completely parsed commit .


def load_config_from_cli(config: GoodConf, argv: List[str]) -> List[str]:
    """Loads config, checking CLI arguments for a config file"""

    # Monkey patch Django's command parser
    from django.core.management.base import BaseCommand
    original_parser = BaseCommand.create_parser

    def patched_parser(self, prog_name, subcommand):
        parser = original_parser(self, prog_name, subcommand)
        argparser_add_argument(parser, config)
        return parser

    BaseCommand.create_parser = patched_parser

    try:
        parser = argparse.ArgumentParser(add_help=False)
        argparser_add_argument(parser, config)

        config_arg, default_args = parser.parse_known_args(argv)
        config.load(config_arg.config)
        yield default_args
    finally:
        # Put that create_parser back where it came from or so help me!
        BaseCommand.create_parser = original_parser

def load_config_from_cli ( config : GoodConf , argv : List [ str ] ) -> List [ str ] : from django . core . management . base import BaseCommand original_parser = BaseCommand . create_parser def patched_parser ( self , prog_name , subcommand ) : parser = original_parser ( self , prog_name , subcommand ) argparser_add_argument ( parser , config ) return parser BaseCommand . create_parser = patched_parser try : parser = argparse . ArgumentParser ( add_help = False ) argparser_add_argument ( parser , config ) config_arg , default_args = parser . parse_known_args ( argv ) config . load ( config_arg . config ) yield default_args finally : BaseCommand . create_parser = original_parser

Loads config checking CLI arguments for a config file


def execute_from_command_line_with_config(config: GoodConf, argv: List[str]):
    """Load's config then runs Django's execute_from_command_line"""
    with load_config_from_cli(config, argv) as args:
        from django.core.management import execute_from_command_line
        execute_from_command_line(args)

def execute_from_command_line_with_config ( config : GoodConf , argv : List [ str ] ) : with load_config_from_cli ( config , argv ) as args : from django . core . management import execute_from_command_line execute_from_command_line ( args )

Load s config then runs Django s execute_from_command_line


def argparser_add_argument(parser: argparse.ArgumentParser, config: GoodConf):
    """Adds argument for config to existing argparser"""
    help = "Config file."
    if config.file_env_var:
        help += (" Can also be configured via the "
                 "environment variable: {}".format(config.file_env_var))
    if config.default_files:
        help += (" Defaults to the first file that exists from "
                 "[{}].".format(', '.join(config.default_files)))
    parser.add_argument('-C', '--config', metavar='FILE', help=help)


def load(self, filename: str = None):
        """Find config file and set values"""
        if filename:
            self.config_file = _find_file(filename)
        else:
            if self.file_env_var and self.file_env_var in os.environ:
                self.config_file = _find_file(os.environ[self.file_env_var])
            if not self.config_file:
                for filename in self.default_files:
                    self.config_file = _find_file(filename, require=False)
                    if self.config_file:
                        break
        if self.config_file:
            config = _load_config(self.config_file)
            log.info("Loading config from %s", self.config_file)
        else:
            config = {}
            log.info("No config file specified. "
                     "Loading with environment variables.")
        self.set_values(config)

def
load
(
self
,
filename
:
str
=
)
:
if
filename
:
self
.
config_file
=
_find_file
(
filename
)
else
:
if
self
.
file_env_var
and
self
.
file_env_var
in
os
.
environ
:
self
.
config_file
=
_find_file
(
os
.
environ
[
self
.
file_env_var
]
)
if
not
self
.
config_file
:
for
filename
in
self
.
default_files
:
self
.
config_file
=
_find_file
(
filename
,
require
=
False
)
if
self
.
config_file
:
break
if
self
.
config_file
:
config
=
_load_config
(
self
.
config_file
)
log
.
info
(
"Loading config from %s"
,
self
.
config_file
)
else
:
config
=
{
}
log
.
info
(
"No config file specified. "
"Loading with environment variables."
)
self
.
set_values
(
config
)


def generate_yaml(cls, **override):
        """
        Dumps initial config in YAML
        """
        import ruamel.yaml
        yaml = ruamel.yaml.YAML()
        yaml_str = StringIO()
        yaml.dump(cls.get_initial(**override), stream=yaml_str)
        yaml_str.seek(0)
        dict_from_yaml = yaml.load(yaml_str)
        if cls.__doc__:
            dict_from_yaml.yaml_set_start_comment(
                '\n' + cls.__doc__ + '\n\n')
        for k in dict_from_yaml.keys():
            if cls._values[k].help:
                dict_from_yaml.yaml_set_comment_before_after_key(
                    k, before='\n' + cls._values[k].help)
        yaml_str = StringIO()
        yaml.dump(dict_from_yaml, yaml_str)
        yaml_str.seek(0)
        return yaml_str.read()

def generate_yaml ( cls , ** override ) : import ruamel . yaml yaml = ruamel . yaml . YAML ( ) yaml_str = StringIO ( ) yaml . dump ( cls . get_initial ( ** override ) , stream = yaml_str ) yaml_str . seek ( 0 ) dict_from_yaml = yaml . load ( yaml_str ) if cls . __doc__ : dict_from_yaml . yaml_set_start_comment ( '\n' + cls . __doc__ + '\n\n' ) for k in dict_from_yaml . keys ( ) : if cls . _values [ k ] . help : dict_from_yaml . yaml_set_comment_before_after_key ( k , before = '\n' + cls . _values [ k ] . help ) yaml_str = StringIO ( ) yaml . dump ( dict_from_yaml , yaml_str ) yaml_str . seek ( 0 ) return yaml_str . read ( )


def generate_markdown(cls):
        """
        Documents values in markdown
        """
        lines = []
        if cls.__doc__:
            lines.extend(['# {}'.format(cls.__doc__), ''])
        for k, v in cls._values.items():
            lines.append('* **{}**  '.format(k))
            if v.required:
                lines[-1] = lines[-1] + '_REQUIRED_  '
            if v.help:
                lines.append('  {}  '.format(v.help))
            lines.append('  type: `{}`  '.format(v.cast_as.__name__))
            if v.default is not None:
                lines.append('  default: `{}`  '.format(v.default))
        return '\n'.join(lines)

def
generate_markdown
(
cls
)
:
lines
=
[
]
if
cls
.
__doc__
:
lines
.
extend
(
[
'# {}'
.
format
(
cls
.
__doc__
)
,
''
]
)
for
k
,
v
in
cls
.
_values
.
items
(
)
:
lines
.
append
(
'* **{}**  '
.
format
(
k
)
)
if
v
.
required
:
lines
[
1
]
=
lines
[
1
]
+
'_REQUIRED_  '
if
v
.
help
:
lines
.
append
(
'  {}  '
.
format
(
v
.
help
)
)
lines
.
append
(
'  type: `{}`  '
.
format
(
v
.
cast_as
.
__name__
)
)
if
v
.
default
is
not
:
lines
.
append
(
'  default: `{}`  '
.
format
(
v
.
default
)
)
return
'\n'
.
join
(
lines
)


def cast(self, val: str):
        """converts string to type requested by `cast_as`"""
        try:
            return getattr(self, 'cast_as_{}'.format(
                self.cast_as.__name__.lower()))(val)
        except AttributeError:
            return self.cast_as(val)

def cast ( self , val : str ) : try : return getattr ( self , 'cast_as_{}' . format ( self . cast_as . __name__ . lower ( ) ) ) ( val ) except AttributeError : return self . cast_as ( val )

converts string to type requested by cast_as


def list_dates_between(first_date, last_date):
    """Returns all dates from first to last included."""
    return [first_date + timedelta(days=n)
            for n in range(1 + (last_date - first_date).days)]

def list_dates_between ( first_date , last_date ) : return [ first_date + timedelta ( days = n ) for n in range ( 1 + ( last_date - first_date ) . days ) ]

Returns all dates from first to last included .


def parse_date(s):
    """Fast %Y-%m-%d parsing."""
    try:
        return datetime.date(int(s[:4]), int(s[5:7]), int(s[8:10]))
    except ValueError:  # other accepted format used in one-day data set
        return datetime.datetime.strptime(s, '%d %B %Y').date()

def
parse_date
(
s
)
:
try
:
return
datetime
.
date
(
int
(
s
[
:
4
]
)
,
int
(
s
[
5
:
7
]
)
,
int
(
s
[
8
:
10
]
)
)
except
ValueError
:
return
datetime
.
datetime
.
strptime
(
s
,
'%d %B %Y'
)
.
date
(
)


def load_file(self, currency_file):
        """To be subclassed if alternate methods of loading data.
        """
        if currency_file.startswith(('http://', 'https://')):
            content = urlopen(currency_file).read()
        else:
            with open(currency_file, 'rb') as f:
                content = f.read()

        if currency_file.endswith('.zip'):
            self.load_lines(get_lines_from_zip(content))
        else:
            self.load_lines(content.decode('utf-8').splitlines())

def load_file ( self , currency_file ) : if currency_file . startswith ( ( 'http://' , 'https://' ) ) : content = urlopen ( currency_file ) . read ( ) else : with open ( currency_file , 'rb' ) as f : content = f . read ( ) if currency_file . endswith ( '.zip' ) : self . load_lines ( get_lines_from_zip ( content ) ) else : self . load_lines ( content . decode ( 'utf-8' ) . splitlines ( ) )

To be subclassed if alternate methods of loading data .


def _set_missing_to_none(self, currency):
        """Fill missing rates of a currency with the closest available ones."""
        rates = self._rates[currency]
        first_date, last_date = self.bounds[currency]

        for date in list_dates_between(first_date, last_date):
            if date not in rates:
                rates[date] = None

        if self.verbose:
            missing = len([r for r in itervalues(rates) if r is None])
            if missing:
                print('{0}: {1} missing rates from {2} to {3} ({4} days)'.format(
                    currency, missing, first_date, last_date,
                    1 + (last_date - first_date).days))

Fill missing rates of a currency with the closest available ones .


def _compute_missing_rates(self, currency):
        """Fill missing rates of a currency.

        This is done by linear interpolation of the two closest available rates.

        :param str currency: The currency to fill missing rates for.
        """
        rates = self._rates[currency]

        # tmp will store the closest rates forward and backward
        tmp = defaultdict(lambda: [None, None])

        for date in sorted(rates):
            rate = rates[date]
            if rate is not None:
                closest_rate = rate
                dist = 0
            else:
                dist += 1
                tmp[date][0] = closest_rate, dist

        for date in sorted(rates, reverse=True):
            rate = rates[date]
            if rate is not None:
                closest_rate = rate
                dist = 0
            else:
                dist += 1
                tmp[date][1] = closest_rate, dist

        for date in sorted(tmp):
            (r0, d0), (r1, d1) = tmp[date]
            rates[date] = (r0 * d1 + r1 * d0) / (d0 + d1)
            if self.verbose:
                print(('{0}: filling {1} missing rate using {2} ({3}d old) and '
                       '{4} ({5}d later)').format(currency, date, r0, d0, r1, d1))


def _get_rate(self, currency, date):
        """Get a rate for a given currency and date.

        :type date: datetime.date

        >>> from datetime import date
        >>> c = CurrencyConverter()
        >>> c._get_rate('USD', date=date(2014, 3, 28))
        1.375...
        >>> c._get_rate('BGN', date=date(2010, 11, 21))
        Traceback (most recent call last):
        RateNotFoundError: BGN has no rate for 2010-11-21
        """
        if currency == self.ref_currency:
            return 1.0

        if date not in self._rates[currency]:
            first_date, last_date = self.bounds[currency]

            if not self.fallback_on_wrong_date:
                raise RateNotFoundError('{0} not in {1} bounds {2}/{3}'.format(
                    date, currency, first_date, last_date))

            if date < first_date:
                fallback_date = first_date
            elif date > last_date:
                fallback_date = last_date
            else:
                raise AssertionError('Should never happen, bug in the code!')

            if self.verbose:
                print(r'/!\ {0} not in {1} bounds {2}/{3}, falling back to {4}'.format(
                    date, currency, first_date, last_date, fallback_date))

            date = fallback_date

        rate = self._rates[currency][date]
        if rate is None:
            raise RateNotFoundError('{0} has no rate for {1}'.format(currency, date))
        return rate

Get a rate for a given currency and date .


def convert(self, amount, currency, new_currency='EUR', date=None):
        """Convert amount from a currency to another one.

        :param float amount: The amount of `currency` to convert.
        :param str currency: The currency to convert from.
        :param str new_currency: The currency to convert to.
        :param datetime.date date: Use the conversion rate of this date. If this
            is not given, the most recent rate is used.

        :return: The value of `amount` in `new_currency`.
        :rtype: float

        >>> from datetime import date
        >>> c = CurrencyConverter()
        >>> c.convert(100, 'EUR', 'USD', date=date(2014, 3, 28))
        137.5...
        >>> c.convert(100, 'USD', date=date(2014, 3, 28))
        72.67...
        >>> c.convert(100, 'BGN', date=date(2010, 11, 21))
        Traceback (most recent call last):
        RateNotFoundError: BGN has no rate for 2010-11-21
        """
        for c in currency, new_currency:
            if c not in self.currencies:
                raise ValueError('{0} is not a supported currency'.format(c))

        if date is None:
            date = self.bounds[currency].last_date
        else:
            try:
                date = date.date()  # fallback if input was a datetime object
            except AttributeError:
                pass

        r0 = self._get_rate(currency, date)
        r1 = self._get_rate(new_currency, date)

        return float(amount) / r0 * r1

def
convert
(
self
,
amount
,
currency
,
new_currency
=
'EUR'
,
date
=
)
:
for
c
in
currency
,
new_currency
:
if
c
not
in
self
.
currencies
:
raise
ValueError
(
'{0} is not a supported currency'
.
format
(
c
)
)
if
date
is
:
date
=
self
.
bounds
[
currency
]
.
last_date
else
:
try
:
date
=
date
.
date
(
)
except
AttributeError
:
pass
r0
=
self
.
_get_rate
(
currency
,
date
)
r1
=
self
.
_get_rate
(
new_currency
,
date
)
return
float
(
amount
)
/
r0
*
r1

Convert amount from a currency to another one .


def grouper(iterable, n, fillvalue=None):
    """Group iterable by n elements.

    >>> for t in grouper('abcdefg', 3, fillvalue='x'):
    ...     print(''.join(t))
    abc
    def
    gxx
    """
    return list(zip_longest(*[iter(iterable)] * n, fillvalue=fillvalue))

def grouper ( iterable , n , fillvalue = None ) : return list ( zip_longest ( * [ iter ( iterable ) ] * n , fillvalue = fillvalue ) )


def animate(frames, interval, name, iterations=2):
    """Animate given frame for set number of iterations.

    Parameters
    ----------
    frames : list
        Frames for animating
    interval : float
        Interval between two frames
    name : str
        Name of animation
    iterations : int, optional
        Number of loops for animations
    """
    for i in range(iterations):
        for frame in frames:
            frame = get_coded_text(frame)
            output = "\r{0} {1}".format(frame, name)
            sys.stdout.write(output)
            sys.stdout.write(CLEAR_LINE)
            sys.stdout.flush()
            time.sleep(0.001 * interval)

def
animate
(
frames
,
interval
,
name
,
iterations
=
2
)
:
for
i
in
range
(
iterations
)
:
for
frame
in
frames
:
frame
=
get_coded_text
(
frame
)
output
=
"\r{0} {1}"
.
format
(
frame
,
name
)
sys
.
stdout
.
write
(
output
)
sys
.
stdout
.
write
(
CLEAR_LINE
)
sys
.
stdout
.
flush
(
)
time
.
sleep
(
0.001
*
interval
)

Animate given frame for set number of iterations .


def read_record(self, n):
        """Return record `n` as 1,024 bytes; records are indexed from 1."""
        self.file.seek(n * K - K)
        return self.file.read(K)

def read_record ( self , n ) : self . file . seek ( n * K - K ) return self . file . read ( K )

Return record n as 1 024 bytes ; records are indexed from 1 .


def write_record(self, n, data):
        """Write `data` to file record `n`; records are indexed from 1."""
        self.file.seek(n * K - K)
        return self.file.write(data)

def write_record ( self , n , data ) : self . file . seek ( n * K - K ) return self . file . write ( data )

Write data to file record n ; records are indexed from 1 .


def map_words(self, start, end):
        """Return a memory-map of the elements `start` through `end`.

        The memory map will offer the 8-byte double-precision floats
        ("elements") in the file from index `start` through to the index
        `end`, inclusive, both counting the first float as element 1.
        Memory maps must begin on a page boundary, so `skip` returns the
        number of extra bytes at the beginning of the return value.

        """
        i, j = 8 * start - 8, 8 * end
        try:
            fileno = self.file.fileno()
        except (AttributeError, io.UnsupportedOperation):
            fileno = None
        if fileno is None:
            skip = 0
            self.file.seek(i)
            m = self.file.read(j - i)
        else:
            skip = i % mmap.ALLOCATIONGRANULARITY
            r = mmap.ACCESS_READ
            m = mmap.mmap(fileno, length=j-i+skip, access=r, offset=i-skip)
        if sys.version_info > (3,):
            m = memoryview(m)  # so further slicing can return views
        return m, skip

def map_words ( self , start , end ) : i , j = 8 * start - 8 , 8 * end try : fileno = self . file . fileno ( ) except ( AttributeError , io . UnsupportedOperation ) : fileno = None if fileno is None : skip = 0 self . file . seek ( i ) m = self . file . read ( j - i ) else : skip = i % mmap . ALLOCATIONGRANULARITY r = mmap . ACCESS_READ m = mmap . mmap ( fileno , length = j - i + skip , access = r , offset = i - skip ) if sys . version_info > ( 3 , ) : m = memoryview ( m ) return m , skip

Return a memory - map of the elements start through end .


def comments(self):
        """Return the text inside the comment area of the file."""
        record_numbers = range(2, self.fward)
        if not record_numbers:
            return ''
        data = b''.join(self.read_record(n)[0:1000] for n in record_numbers)
        try:
            return data[:data.find(b'\4')].decode('ascii').replace('\0', '\n')
        except IndexError:
            raise ValueError('DAF file comment area is missing its EOT byte')
        except UnicodeDecodeError:
            raise ValueError('DAF file comment area is not ASCII text')

Return the text inside the comment area of the file .


def add_array(self, name, values, array):
        """Add a new array to the DAF file.

        The summary will be initialized with the `name` and `values`,
        and will have its start word and end word fields set to point to
        where the `array` of floats has been appended to the file.

        """
        f = self.file
        scs = self.summary_control_struct

        record_number = self.bward
        data = bytearray(self.read_record(record_number))
        next_record, previous_record, n_summaries = scs.unpack(data[:24])

        if n_summaries < self.summaries_per_record:
            summary_record = record_number
            name_record = summary_record + 1
            data[:24] = scs.pack(next_record, previous_record, n_summaries + 1)
            self.write_record(summary_record, data)
        else:
            summary_record = ((self.free - 1) * 8 + 1023) // 1024 + 1
            name_record = summary_record + 1
            free_record = summary_record + 2

            n_summaries = 0
            data[:24] = scs.pack(summary_record, previous_record, n_summaries)
            self.write_record(record_number, data)

            summaries = scs.pack(0, record_number, 1).ljust(1024, b'\0')
            names = b'\0' * 1024
            self.write_record(summary_record, summaries)
            self.write_record(name_record, names)

            self.bward = summary_record
            self.free = (free_record - 1) * 1024 // 8 + 1

        start_word = self.free
        f.seek((start_word - 1) * 8)
        array = numpy_array(array)  # TODO: force correct endian
        f.write(array.view())
        end_word = f.tell() // 8

        self.free = end_word + 1
        self.write_file_record()

        values = values[:self.nd + self.ni - 2] + (start_word, end_word)

        base = 1024 * (summary_record - 1)
        offset = int(n_summaries) * self.summary_step
        f.seek(base + scs.size + offset)
        f.write(self.summary_struct.pack(*values))
        f.seek(base + 1024 + offset)
        f.write(name[:self.summary_length].ljust(self.summary_step, b' '))

def
add_array
(
self
,
name
,
values
,
array
)
:
f
=
self
.
file
scs
=
self
.
summary_control_struct
record_number
=
self
.
bward
data
=
bytearray
(
self
.
read_record
(
record_number
)
)
next_record
,
previous_record
,
n_summaries
=
scs
.
unpack
(
data
[
:
24
]
)
if
n_summaries
<
self
.
summaries_per_record
:
summary_record
=
record_number
name_record
=
summary_record
+
1
data
[
:
24
]
=
scs
.
pack
(
next_record
,
previous_record
,
n_summaries
+
1
)
self
.
write_record
(
summary_record
,
data
)
else
:
summary_record
=
(
(
self
.
free
1
)
*
8
+
1023
)
//
1024
+
1
name_record
=
summary_record
+
1
free_record
=
summary_record
+
2
n_summaries
=
0
data
[
:
24
]
=
scs
.
pack
(
summary_record
,
previous_record
,
n_summaries
)
self
.
write_record
(
record_number
,
data
)
summaries
=
scs
.
pack
(
0
,
record_number
,
1
)
.
ljust
(
1024
,
b'\0'
)
names
=
b'\0'
*
1024
self
.
write_record
(
summary_record
,
summaries
)
self
.
write_record
(
name_record
,
names
)
self
.
bward
=
summary_record
self
.
free
=
(
free_record
1
)
*
1024
//
8
+
1
start_word
=
self
.
free
f
.
seek
(
(
start_word
1
)
*
8
)
array
=
numpy_array
(
array
)
f
.
write
(
array
.
view
(
)
)
end_word
=
f
.
tell
(
)
//
8
self
.
free
=
end_word
+
1
self
.
write_file_record
(
)
values
=
values
[
:
self
.
nd
+
self
.
ni
2
]
+
(
start_word
,
end_word
)
base
=
1024
*
(
summary_record
1
)
offset
=
int
(
n_summaries
)
*
self
.
summary_step
f
.
seek
(
base
+
scs
.
size
+
offset
)
f
.
write
(
self
.
summary_struct
.
pack
(
*
values
)
)
f
.
seek
(
base
+
1024
+
offset
)
f
.
write
(
name
[
:
self
.
summary_length
]
.
ljust
(
self
.
summary_step
,
b' '
)
)


def close(self):
        """Close this SPK file."""
        self.daf.file.close()
        for segment in self.segments:
            if hasattr(segment, '_data'):
                del segment._data
        self.daf._array = None
        self.daf._map = None

def close ( self ) : self . daf . file . close ( ) for segment in self . segments : if hasattr ( segment , '_data' ) : del segment . _data self . daf . _array = None self . daf . _map = None


def compute(self, tdb, tdb2=0.0):
        """Compute the component values for the time `tdb` plus `tdb2`."""
        for position in self.generate(tdb, tdb2):
            return position

def compute ( self , tdb , tdb2 = 0.0 ) : for position in self . generate ( tdb , tdb2 ) : return position

Compute the component values for the time tdb plus tdb2 .


def close(self):
        """Close this file."""
        self.daf.file.close()
        for segment in self.segments:
            if hasattr(segment, '_data'):
                del segment._data

def close ( self ) : self . daf . file . close ( ) for segment in self . segments : if hasattr ( segment , '_data' ) : del segment . _data


def _load(self):
        """Map the coefficients into memory using a NumPy array.

        """
        if self.data_type == 2:
            component_count = 3
        else:
            raise ValueError('only binary PCK data type 2 is supported')

        init, intlen, rsize, n = self.daf.read_array(self.end_i - 3, self.end_i)
        initial_epoch = jd(init)
        interval_length = intlen / S_PER_DAY
        coefficient_count = int(rsize - 2) // component_count
        coefficients = self.daf.map_array(self.start_i, self.end_i - 4)

        coefficients.shape = (int(n), int(rsize))
        coefficients = coefficients[:,2:]  # ignore MID and RADIUS elements
        coefficients.shape = (int(n), component_count, coefficient_count)
        coefficients = rollaxis(coefficients, 1)
        return initial_epoch, interval_length, coefficients

Map the coefficients into memory using a NumPy array .


def compute(self, tdb, tdb2, derivative=True):
        """Generate angles and derivatives for time `tdb` plus `tdb2`.

        If ``derivative`` is true, return a tuple containing both the
        angle and its derivative; otherwise simply return the angles.

        """
        scalar = not getattr(tdb, 'shape', 0) and not getattr(tdb2, 'shape', 0)
        if scalar:
            tdb = array((tdb,))

        data = self._data
        if data is None:
            self._data = data = self._load()

        initial_epoch, interval_length, coefficients = data
        component_count, n, coefficient_count = coefficients.shape

        # Subtracting tdb before adding tdb2 affords greater precision.
        index, offset = divmod((tdb - initial_epoch) + tdb2, interval_length)
        index = index.astype(int)

        if (index < 0).any() or (index > n).any():
            final_epoch = initial_epoch + interval_length * n
            raise ValueError('segment only covers dates %.1f through %.1f'
                            % (initial_epoch, final_epoch))

        omegas = (index == n)
        index[omegas] -= 1
        offset[omegas] += interval_length

        coefficients = coefficients[:,index]

        # Chebyshev polynomial.

        T = empty((coefficient_count, len(index)))
        T[0] = 1.0
        T[1] = t1 = 2.0 * offset / interval_length - 1.0
        twot1 = t1 + t1
        for i in range(2, coefficient_count):
            T[i] = twot1 * T[i-1] - T[i-2]

        components = (T.T * coefficients).sum(axis=2)
        if scalar:
            components = components[:,0]

        if not derivative:
            return components

        # Chebyshev differentiation.

        dT = empty_like(T)
        dT[0] = 0.0
        dT[1] = 1.0
        if coefficient_count > 2:
            dT[2] = twot1 + twot1
            for i in range(3, coefficient_count):
                dT[i] = twot1 * dT[i-1] - dT[i-2] + T[i-1] + T[i-1]
        dT *= 2.0
        dT /= interval_length

        rates = (dT.T * coefficients).sum(axis=2)
        if scalar:
            rates = rates[:,0]

        return components, rates

Generate angles and derivatives for time tdb plus tdb2 .


def visit_Call(self, node):
        """
        Visit a function call.

        We expect every logging statement and string format to be a function call.

        """
        # CASE 1: We're in a logging statement
        if self.within_logging_statement():
            if self.within_logging_argument() and self.is_format_call(node):
                self.violations.append((node, STRING_FORMAT_VIOLATION))
                super(LoggingVisitor, self).generic_visit(node)
                return

        logging_level = self.detect_logging_level(node)

        if logging_level and self.current_logging_level is None:
            self.current_logging_level = logging_level

        # CASE 2: We're in some other statement
        if logging_level is None:
            super(LoggingVisitor, self).generic_visit(node)
            return

        # CASE 3: We're entering a new logging statement
        self.current_logging_call = node

        if logging_level == "warn":
            self.violations.append((node, WARN_VIOLATION))

        self.check_exc_info(node)

        for index, child in enumerate(iter_child_nodes(node)):
            if index == 1:
                self.current_logging_argument = child
            if index >= 1:
                self.check_exception_arg(child)
            if index > 1 and isinstance(child, keyword) and child.arg == "extra":
                self.current_extra_keyword = child

            super(LoggingVisitor, self).visit(child)

            self.current_logging_argument = None
            self.current_extra_keyword = None

        self.current_logging_call = None
        self.current_logging_level = None

def visit_Call ( self , node ) : if self . within_logging_statement ( ) : if self . within_logging_argument ( ) and self . is_format_call ( node ) : self . violations . append ( ( node , STRING_FORMAT_VIOLATION ) ) super ( LoggingVisitor , self ) . generic_visit ( node ) return logging_level = self . detect_logging_level ( node ) if logging_level and self . current_logging_level is None : self . current_logging_level = logging_level if logging_level is None : super ( LoggingVisitor , self ) . generic_visit ( node ) return self . current_logging_call = node if logging_level == "warn" : self . violations . append ( ( node , WARN_VIOLATION ) ) self . check_exc_info ( node ) for index , child in enumerate ( iter_child_nodes ( node ) ) : if index == 1 : self . current_logging_argument = child if index >= 1 : self . check_exception_arg ( child ) if index > 1 and isinstance ( child , keyword ) and child . arg == "extra" : self . current_extra_keyword = child super ( LoggingVisitor , self ) . visit ( child ) self . current_logging_argument = None self . current_extra_keyword = None self . current_logging_call = None self . current_logging_level = None


def visit_BinOp(self, node):
        """
        Process binary operations while processing the first logging argument.

        """
        if self.within_logging_statement() and self.within_logging_argument():
            # handle percent format
            if isinstance(node.op, Mod):
                self.violations.append((node, PERCENT_FORMAT_VIOLATION))
            # handle string concat
            if isinstance(node.op, Add):
                self.violations.append((node, STRING_CONCAT_VIOLATION))
        super(LoggingVisitor, self).generic_visit(node)

def visit_BinOp ( self , node ) : if self . within_logging_statement ( ) and self . within_logging_argument ( ) : if isinstance ( node . op , Mod ) : self . violations . append ( ( node , PERCENT_FORMAT_VIOLATION ) ) if isinstance ( node . op , Add ) : self . violations . append ( ( node , STRING_CONCAT_VIOLATION ) ) super ( LoggingVisitor , self ) . generic_visit ( node )

Process binary operations while processing the first logging argument .


def visit_Dict(self, node):
        """
        Process dict arguments.

        """
        if self.should_check_whitelist(node):
            for key in node.keys:
                if key.s in self.whitelist or key.s.startswith("debug_"):
                    continue
                self.violations.append((self.current_logging_call, WHITELIST_VIOLATION.format(key.s)))

        if self.should_check_extra_exception(node):
            for value in node.values:
                self.check_exception_arg(value)

        super(LoggingVisitor, self).generic_visit(node)

def visit_Dict ( self , node ) : if self . should_check_whitelist ( node ) : for key in node . keys : if key . s in self . whitelist or key . s . startswith ( "debug_" ) : continue self . violations . append ( ( self . current_logging_call , WHITELIST_VIOLATION . format ( key . s ) ) ) if self . should_check_extra_exception ( node ) : for value in node . values : self . check_exception_arg ( value ) super ( LoggingVisitor , self ) . generic_visit ( node )


def visit_JoinedStr(self, node):
        """
        Process f-string arguments.

        """
        if version_info >= (3, 6):
            if self.within_logging_statement():
                if any(isinstance(i, FormattedValue) for i in node.values):
                    if self.within_logging_argument():
                        self.violations.append((node, FSTRING_VIOLATION))
                        super(LoggingVisitor, self).generic_visit(node)

def visit_JoinedStr ( self , node ) : if version_info >= ( 3 , 6 ) : if self . within_logging_statement ( ) : if any ( isinstance ( i , FormattedValue ) for i in node . values ) : if self . within_logging_argument ( ) : self . violations . append ( ( node , FSTRING_VIOLATION ) ) super ( LoggingVisitor , self ) . generic_visit ( node )


def visit_keyword(self, node):
        """
        Process keyword arguments.

        """
        if self.should_check_whitelist(node):
            if node.arg not in self.whitelist and not node.arg.startswith("debug_"):
                self.violations.append((self.current_logging_call, WHITELIST_VIOLATION.format(node.arg)))

        if self.should_check_extra_exception(node):
            self.check_exception_arg(node.value)

        super(LoggingVisitor, self).generic_visit(node)

def visit_keyword ( self , node ) : if self . should_check_whitelist ( node ) : if node . arg not in self . whitelist and not node . arg . startswith ( "debug_" ) : self . violations . append ( ( self . current_logging_call , WHITELIST_VIOLATION . format ( node . arg ) ) ) if self . should_check_extra_exception ( node ) : self . check_exception_arg ( node . value ) super ( LoggingVisitor , self ) . generic_visit ( node )


def visit_ExceptHandler(self, node):
        """
        Process except blocks.

        """
        name = self.get_except_handler_name(node)
        if not name:
            super(LoggingVisitor, self).generic_visit(node)
            return

        self.current_except_names.append(name)
        super(LoggingVisitor, self).generic_visit(node)
        self.current_except_names.pop()

def visit_ExceptHandler ( self , node ) : name = self . get_except_handler_name ( node ) if not name : super ( LoggingVisitor , self ) . generic_visit ( node ) return self . current_except_names . append ( name ) super ( LoggingVisitor , self ) . generic_visit ( node ) self . current_except_names . pop ( )


def detect_logging_level(self, node):
        """
        Heuristic to decide whether an AST Call is a logging call.

        """
        try:
            if self.get_id_attr(node.func.value) == "warnings":
                return None
            # NB: We could also look at the argument signature or the target attribute
            if node.func.attr in LOGGING_LEVELS:
                return node.func.attr
        except AttributeError:
            pass
        return None

def detect_logging_level ( self , node ) : try : if self . get_id_attr ( node . func . value ) == "warnings" : return None if node . func . attr in LOGGING_LEVELS : return node . func . attr except AttributeError : pass return None

Heuristic to decide whether an AST Call is a logging call .


def get_except_handler_name(self, node):
        """
        Helper to get the exception name from an ExceptHandler node in both py2 and py3.

        """
        name = node.name
        if not name:
            return None

        if version_info < (3,):
            return name.id
        return name

def get_except_handler_name ( self , node ) : name = node . name if not name : return None if version_info < ( 3 , ) : return name . id return name

Helper to get the exception name from an ExceptHandler node in both py2 and py3 .


def get_id_attr(self, value):
        """Check if value has id attribute and return it.

        :param value: The value to get id from.
        :return: The value.id.
        """
        if not hasattr(value, "id") and hasattr(value, "value"):
            value = value.value
        return value.id

def get_id_attr ( self , value ) : if not hasattr ( value , "id" ) and hasattr ( value , "value" ) : value = value . value return value . id

Check if value has id attribute and return it .


def is_bare_exception(self, node):
        """
        Checks if the node is a bare exception name from an except block.

        """
        return isinstance(node, Name) and node.id in self.current_except_names

def is_bare_exception ( self , node ) : return isinstance ( node , Name ) and node . id in self . current_except_names

Checks if the node is a bare exception name from an except block .


def check_exc_info(self, node):
        """
        Reports a violation if exc_info keyword is used with logging.error or logging.exception.

        """
        if self.current_logging_level not in ('error', 'exception'):
            return

        for kw in node.keywords:
            if kw.arg == 'exc_info':
                if self.current_logging_level == 'error':
                    violation = ERROR_EXC_INFO_VIOLATION
                else:
                    violation = REDUNDANT_EXC_INFO_VIOLATION
                self.violations.append((node, violation))

def check_exc_info ( self , node ) : if self . current_logging_level not in ( 'error' , 'exception' ) : return for kw in node . keywords : if kw . arg == 'exc_info' : if self . current_logging_level == 'error' : violation = ERROR_EXC_INFO_VIOLATION else : violation = REDUNDANT_EXC_INFO_VIOLATION self . violations . append ( ( node , violation ) )

Reports a violation if exc_info keyword is used with logging . error or logging . exception .


def delete_file_if_needed(instance, filefield_name):
    """Delete file from database only if needed.

    When editing and the filefield is a new file,
    deletes the previous file (if any) from the database.
    Call this function immediately BEFORE saving the instance.
    """
    if instance.pk:
        model_class = type(instance)

        # Check if there is a file for the instance in the database
        if model_class.objects.filter(pk=instance.pk).exclude(
            **{'%s__isnull' % filefield_name: True}
        ).exclude(
            **{'%s__exact' % filefield_name: ''}
        ).exists():
            old_file = getattr(
                model_class.objects.only(filefield_name).get(pk=instance.pk),
                filefield_name
            )
        else:
            old_file = None

        # If there is a file, delete it if needed
        if old_file:
            # When editing and NOT changing the file,
            #   old_file.name == getattr(instance, filefield_name)
            #   returns True. In this case, the file must NOT be deleted.
            # If the file IS being changed, the comparison returns False.
            #   In this case, the old file MUST be deleted.
            if (old_file.name == getattr(instance, filefield_name)) is False:
                DatabaseFileStorage().delete(old_file.name)

def delete_file_if_needed ( instance , filefield_name ) : if instance . pk : model_class = type ( instance ) if model_class . objects . filter ( pk = instance . pk ) . exclude ( ** { '%s__isnull' % filefield_name : True } ) . exclude ( ** { '%s__exact' % filefield_name : '' } ) . exists ( ) : old_file = getattr ( model_class . objects . only ( filefield_name ) . get ( pk = instance . pk ) , filefield_name ) else : old_file = None if old_file : if ( old_file . name == getattr ( instance , filefield_name ) ) is False : DatabaseFileStorage ( ) . delete ( old_file . name )

Delete file from database only if needed .


def db_file_widget(cls):
    """Edit the download-link inner text."""

    def get_link_display(url):
        unquoted = unquote(url.split('%2F')[-1])
        if sys.version_info.major == 2:  # python 2
            from django.utils.encoding import force_unicode
            unquoted = force_unicode(unquoted)
        return escape(unquoted)

    def get_template_substitution_values(self, value):
        # Used by Django < 1.11
        subst = super(cls, self).get_template_substitution_values(value)
        subst['initial'] = get_link_display(value.url)
        return subst
    setattr(cls,
            'get_template_substitution_values',
            get_template_substitution_values)

    def get_context(self, name, value, attrs):
        context = super(cls, self).get_context(name, value, attrs)
        if value and hasattr(value, 'url'):
            context['widget']['display'] = get_link_display(value.url)
        return context
    setattr(cls, 'get_context', get_context)

    return cls

def db_file_widget ( cls ) : def get_link_display ( url ) : unquoted = unquote ( url . split ( '%2F' ) [ - 1 ] ) if sys . version_info . major == 2 : from django . utils . encoding import force_unicode unquoted = force_unicode ( unquoted ) return escape ( unquoted ) def get_template_substitution_values ( self , value ) : subst = super ( cls , self ) . get_template_substitution_values ( value ) subst [ 'initial' ] = get_link_display ( value . url ) return subst setattr ( cls , 'get_template_substitution_values' , get_template_substitution_values ) def get_context ( self , name , value , attrs ) : context = super ( cls , self ) . get_context ( name , value , attrs ) if value and hasattr ( value , 'url' ) : context [ 'widget' ] [ 'display' ] = get_link_display ( value . url ) return context setattr ( cls , 'get_context' , get_context ) return cls


def rendered_content(self):
        """Returns the freshly rendered content for the template and context
        described by the PDFResponse.

        This *does not* set the final content of the response. To set the
        response content, you must either call render(), or set the
        content explicitly using the value of this property.
        """
        cmd_options = self.cmd_options.copy()
        return render_pdf_from_template(
            self.resolve_template(self.template_name),
            self.resolve_template(self.header_template),
            self.resolve_template(self.footer_template),
            context=self.resolve_context(self.context_data),
            request=self._request,
            cmd_options=cmd_options,
            cover_template=self.resolve_template(self.cover_template)

        )

def rendered_content ( self ) : cmd_options = self . cmd_options . copy ( ) return render_pdf_from_template ( self . resolve_template ( self . template_name ) , self . resolve_template ( self . header_template ) , self . resolve_template ( self . footer_template ) , context = self . resolve_context ( self . context_data ) , request = self . _request , cmd_options = cmd_options , cover_template = self . resolve_template ( self . cover_template ) )

Returns the freshly rendered content for the template and context described by the PDFResponse .


def render_to_response(self, context, **response_kwargs):
        """
        Returns a PDF response with a template rendered with the given context.
        """
        filename = response_kwargs.pop('filename', None)
        cmd_options = response_kwargs.pop('cmd_options', None)

        if issubclass(self.response_class, PDFTemplateResponse):
            if filename is None:
                filename = self.get_filename()

            if cmd_options is None:
                cmd_options = self.get_cmd_options()

            return super(PDFTemplateView, self).render_to_response(
                context=context, filename=filename,
                show_content_in_browser=self.show_content_in_browser,
                header_template=self.header_template,
                footer_template=self.footer_template,
                cmd_options=cmd_options,
                cover_template=self.cover_template,
                **response_kwargs
            )
        else:
            return super(PDFTemplateView, self).render_to_response(
                context=context,
                **response_kwargs
            )

def render_to_response ( self , context , ** response_kwargs ) : filename = response_kwargs . pop ( 'filename' , None ) cmd_options = response_kwargs . pop ( 'cmd_options' , None ) if issubclass ( self . response_class , PDFTemplateResponse ) : if filename is None : filename = self . get_filename ( ) if cmd_options is None : cmd_options = self . get_cmd_options ( ) return super ( PDFTemplateView , self ) . render_to_response ( context = context , filename = filename , show_content_in_browser = self . show_content_in_browser , header_template = self . header_template , footer_template = self . footer_template , cmd_options = cmd_options , cover_template = self . cover_template , ** response_kwargs ) else : return super ( PDFTemplateView , self ) . render_to_response ( context = context , ** response_kwargs )

Returns a PDF response with a template rendered with the given context .


def http_quote(string):
    """
    Given a unicode string, will do its dandiest to give you back a
    valid ascii charset string you can use in, say, http headers and the
    like.
    """
    if isinstance(string, six.text_type):
        try:
            import unidecode
        except ImportError:
            pass
        else:
            string = unidecode.unidecode(string)
        string = string.encode('ascii', 'replace')
    # Wrap in double-quotes for ; , and the like
    string = string.replace(b'\\', b'\\\\').replace(b'"', b'\\"')
    return '"{0!s}"'.format(string.decode())

def http_quote ( string ) : if isinstance ( string , six . text_type ) : try : import unidecode except ImportError : pass else : string = unidecode . unidecode ( string ) string = string . encode ( 'ascii' , 'replace' ) string = string . replace ( b'\\' , b'\\\\' ) . replace ( b'"' , b'\\"' ) return '"{0!s}"' . format ( string . decode ( ) )

Given a unicode string will do its dandiest to give you back a valid ascii charset string you can use in say http headers and the like .


def configure(module=None, prefix='MONGODB_', **kwargs):
    """Sets defaults for ``class Meta`` declarations.

    Arguments can either be extracted from a `module` (in that case
    all attributes starting from `prefix` are used):

    >>> import foo
    >>> configure(foo)

    or passed explicictly as keyword arguments:

    >>> configure(database='foo')

    .. warning:: Current implementation is by no means thread-safe --
                 use it wisely.
    """
    if module is not None and isinstance(module, types.ModuleType):
        # Search module for MONGODB_* attributes and converting them
        # to _Options' values, ex: MONGODB_PORT ==> port.
        attrs = ((attr.replace(prefix, '').lower(), value)
                 for attr, value in vars(module).items()
                 if attr.startswith(prefix))

        _Options._configure(**dict(attrs))
    elif kwargs:
        _Options._configure(**kwargs)

def configure ( module = None , prefix = 'MONGODB_' , ** kwargs ) : if module is not None and isinstance ( module , types . ModuleType ) : attrs = ( ( attr . replace ( prefix , '' ) . lower ( ) , value ) for attr , value in vars ( module ) . items ( ) if attr . startswith ( prefix ) ) _Options . _configure ( ** dict ( attrs ) ) elif kwargs : _Options . _configure ( ** kwargs )

Sets defaults for class Meta declarations .


def to_underscore(string):
    """Converts a given string from CamelCase to under_score.

    >>> to_underscore('FooBar')
    'foo_bar'
    """
    new_string = re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', string)
    new_string = re.sub(r'([a-z\d])([A-Z])', r'\1_\2', new_string)
    return new_string.lower()

def to_underscore ( string ) : new_string = re . sub ( r'([A-Z]+)([A-Z][a-z])' , r'\1_\2' , string ) new_string = re . sub ( r'([a-z\d])([A-Z])' , r'\1_\2' , new_string ) return new_string . lower ( )

Converts a given string from CamelCase to under_score .


def auto_index(mcs):
        """Builds all indices, listed in model's Meta class.

           >>> class SomeModel(Model)
           ...     class Meta:
           ...         indices = (
           ...             Index('foo'),
           ...         )

        .. note:: this will result in calls to
                  :meth:`pymongo.collection.Collection.ensure_index`
                  method at import time, so import all your models up
                  front.
        """
        for index in mcs._meta.indices:
            index.ensure(mcs.collection)

def auto_index ( mcs ) : for index in mcs . _meta . indices : index . ensure ( mcs . collection )

Builds all indices listed in model s Meta class .


def parse_file(self, file_path, currency) -> List[PriceModel]:
        """ Load and parse a .csv file """
        # load file
                # read csv into memory?
        contents = self.load_file(file_path)
        prices = []

        # parse price elements
        for line in contents:
            price = self.parse_line(line)
            assert isinstance(price, PriceModel)
            price.currency = currency
            prices.append(price)

        return prices

def parse_file ( self , file_path , currency ) -> List [ PriceModel ] : contents = self . load_file ( file_path ) prices = [ ] for line in contents : price = self . parse_line ( line ) assert isinstance ( price , PriceModel ) price . currency = currency prices . append ( price ) return prices


def load_file(self, file_path) -> List[str]:
        """ Loads the content of the text file """
        content = []
        content = read_lines_from_file(file_path)
        return content

def load_file ( self , file_path ) -> List [ str ] : content = [ ] content = read_lines_from_file ( file_path ) return content


def parse_line(self, line: str) -> PriceModel:
        """ Parse a CSV line into a price element """
        line = line.rstrip()
        parts = line.split(',')

        result = PriceModel()

        # symbol
        result.symbol = self.translate_symbol(parts[0])

        # value
        result.value = Decimal(parts[1])

        # date
        date_str = parts[2]
        date_str = date_str.replace('"', '')
        date_parts = date_str.split('/')

        year_str = date_parts[2]
        month_str = date_parts[1]
        day_str = date_parts[0]

        logging.debug(f"parsing {date_parts} into date")
        result.datetime = datetime(int(year_str), int(month_str), int(day_str))

        return result

def
parse_line
(
self
,
line
:
str
)
->
PriceModel
:
line
=
line
.
rstrip
(
)
parts
=
line
.
split
(
','
)
result
=
PriceModel
(
)
result
.
symbol
=
self
.
translate_symbol
(
parts
[
0
]
)
result
.
value
=
Decimal
(
parts
[
1
]
)
date_str
=
parts
[
2
]
date_str
=
date_str
.
replace
(
'"'
,
''
)
date_parts
=
date_str
.
split
(
'/'
)
year_str
=
date_parts
[
2
]
month_str
=
date_parts
[
1
]
day_str
=
date_parts
[
0
]
logging
.
debug
(
f"parsing {date_parts} into date"
)
result
.
datetime
=
datetime
(
int
(
year_str
)
,
int
(
month_str
)
,
int
(
day_str
)
)
return
result


def translate_symbol(self, in_symbol: str) -> str:
        """ translate the incoming symbol into locally-used """
        # read all mappings from the db
        if not self.symbol_maps:
            self.__load_symbol_maps()
        # translate the incoming symbol
        result = self.symbol_maps[in_symbol] if in_symbol in self.symbol_maps else in_symbol

        return result

def translate_symbol ( self , in_symbol : str ) -> str : if not self . symbol_maps : self . __load_symbol_maps ( ) result = self . symbol_maps [ in_symbol ] if in_symbol in self . symbol_maps else in_symbol return result

translate the incoming symbol into locally - used


def __load_symbol_maps(self):
        """ Loads all symbol maps from db """
        repo = SymbolMapRepository(self.__get_session())
        all_maps = repo.get_all()
        self.symbol_maps = {}
        for item in all_maps:
            self.symbol_maps[item.in_symbol] = item.out_symbol

def __load_symbol_maps ( self ) : repo = SymbolMapRepository ( self . __get_session ( ) ) all_maps = repo . get_all ( ) self . symbol_maps = { } for item in all_maps : self . symbol_maps [ item . in_symbol ] = item . out_symbol


def __get_session(self):
        """ Reuses the same db session """
        if not self.session:
            self.session = dal.get_default_session()
        return self.session

def __get_session ( self ) : if not self . session : self . session = dal . get_default_session ( ) return self . session


def add(symbol: str, date, value, currency: str):
    """ Add individual price """
    symbol = symbol.upper()
    currency = currency.upper()

    app = PriceDbApplication()
    price = PriceModel()

    # security = SecuritySymbol("", "")
    price.symbol.parse(symbol)
    # price.symbol.mnemonic = price.symbol.mnemonic.upper()

    # date_str = f"{date}"
    # date_format = "%Y-%m-%d"
    # if time:
    #     date_str = f"{date_str}T{time}"
    #     date_format += "T%H:%M:%S"
    # datum.from_iso_date_string(date)
    # price.datetime = datetime.strptime(date_str, date_format)
    price.datum.from_iso_date_string(date)

    price.value = Decimal(value)
    price.currency = currency
    app.add_price(price)
    app.save()

    click.echo("Price added.")

def
add
(
symbol
:
str
,
date
,
value
,
currency
:
str
)
:
symbol
=
symbol
.
upper
(
)
currency
=
currency
.
upper
(
)
app
=
PriceDbApplication
(
)
price
=
PriceModel
(
)
price
.
symbol
.
parse
(
symbol
)
price
.
datum
.
from_iso_date_string
(
date
)
price
.
value
=
Decimal
(
value
)
price
.
currency
=
currency
app
.
add_price
(
price
)
app
.
save
(
)
click
.
echo
(
"Price added."
)


def import_csv(filepath: str, currency: str):
    """ Import prices from CSV file """
    logger.debug(f"currency = {currency}")
    # auto-convert to uppercase.
    currency = currency.upper()

    app = PriceDbApplication()
    app.logger = logger
    app.import_prices(filepath, currency)

def
import_csv
(
filepath
:
str
,
currency
:
str
)
:
logger
.
debug
(
f"currency = {currency}"
)
currency
=
currency
.
upper
(
)
app
=
PriceDbApplication
(
)
app
.
logger
=
logger
app
.
import_prices
(
filepath
,
currency
)


def last(symbol: str):
    """ displays last price, for symbol if provided """
    app = PriceDbApplication()

    # convert to uppercase
    if symbol:
        symbol = symbol.upper()
        # extract namespace
        sec_symbol = SecuritySymbol("", "")
        sec_symbol.parse(symbol)

        latest = app.get_latest_price(sec_symbol)
        assert isinstance(latest, PriceModel)
        print(f"{latest}")
    else:
        # Show the latest prices available for all securities.
        latest = app.get_latest_prices()
        for price in latest:
            print(f"{price}")

def last ( symbol : str ) : app = PriceDbApplication ( ) if symbol : symbol = symbol . upper ( ) sec_symbol = SecuritySymbol ( "" , "" ) sec_symbol . parse ( symbol ) latest = app . get_latest_price ( sec_symbol ) assert isinstance ( latest , PriceModel ) print ( f"{latest}" ) else : latest = app . get_latest_prices ( ) for price in latest : print ( f"{price}" )

displays last price for symbol if provided


def list_prices(date, currency, last):
    """ Display all prices """
    app = PriceDbApplication()
    app.logger = logger

    if last:
        # fetch only the last prices
        prices = app.get_latest_prices()
    else:
        prices = app.get_prices(date, currency)
    for price in prices:
        print(price)

    print(f"{len(prices)} records found.")

def
list_prices
(
date
,
currency
,
last
)
:
app
=
PriceDbApplication
(
)
app
.
logger
=
logger
if
last
:
prices
=
app
.
get_latest_prices
(
)
else
:
prices
=
app
.
get_prices
(
date
,
currency
)
for
price
in
prices
:
print
(
price
)
print
(
f"{len(prices)} records found."
)


def download(ctx, help: bool, symbol: str, namespace: str, agent: str, currency: str):
    """ Download the latest prices """
    if help:
        click.echo(ctx.get_help())
        ctx.exit()

    app = PriceDbApplication()
    app.logger = logger

    if currency:
        currency = currency.strip()
        currency = currency.upper()

    # Otherwise download the prices for securities listed in the database.
    app.download_prices(currency=currency, agent=agent, symbol=symbol, namespace=namespace)

def download ( ctx , help : bool , symbol : str , namespace : str , agent : str , currency : str ) : if help : click . echo ( ctx . get_help ( ) ) ctx . exit ( ) app = PriceDbApplication ( ) app . logger = logger if currency : currency = currency . strip ( ) currency = currency . upper ( ) app . download_prices ( currency = currency , agent = agent , symbol = symbol , namespace = namespace )


def prune(symbol: str, all: str):
    """ Delete old prices, leaving just the last. """
    app = PriceDbApplication()
    app.logger = logger
    count = 0

    if symbol is not None:
        sec_symbol = SecuritySymbol("", "")
        sec_symbol.parse(symbol)

        deleted = app.prune(sec_symbol)
        if deleted:
            count = 1
    else:
        count = app.prune_all()

    print(f"Removed {count} old price entries.")

def
prune
(
symbol
:
str
,
all
:
str
)
:
app
=
PriceDbApplication
(
)
app
.
logger
=
logger
count
=
0
if
symbol
is
not
:
sec_symbol
=
SecuritySymbol
(
""
,
""
)
sec_symbol
.
parse
(
symbol
)
deleted
=
app
.
prune
(
sec_symbol
)
if
deleted
:
count
=
1
else
:
count
=
app
.
prune_all
(
)
print
(
f"Removed {count} old price entries."
)

Delete old prices leaving just the last .


def get_default_session():
    """ Return the default session. The path is read from the default config. """
    from .config import Config, ConfigKeys

    db_path = Config().get(ConfigKeys.price_database)
    if not db_path:
        raise ValueError("Price database not set in the configuration file!")
    return get_session(db_path)

Return the default session . The path is read from the default config .


def add_map(incoming, outgoing):
    """ Creates a symbol mapping """
    db_path = Config().get(ConfigKeys.pricedb_path)
    session = get_session(db_path)

    new_map = SymbolMap()
    new_map.in_symbol = incoming
    new_map.out_symbol = outgoing

    session.add(new_map)
    session.commit()
    click.echo("Record saved.")

def
add_map
(
incoming
,
outgoing
)
:
db_path
=
Config
(
)
.
get
(
ConfigKeys
.
pricedb_path
)
session
=
get_session
(
db_path
)
new_map
=
SymbolMap
(
)
new_map
.
in_symbol
=
incoming
new_map
.
out_symbol
=
outgoing
session
.
add
(
new_map
)
session
.
commit
(
)
click
.
echo
(
"Record saved."
)


def list_maps():
    """ Displays all symbol maps """
    db_path = Config().get(ConfigKeys.price_database)
    session = get_session(db_path)

    maps = session.query(SymbolMap).all()
    for item in maps:
        click.echo(item)

def list_maps ( ) : db_path = Config ( ) . get ( ConfigKeys . price_database ) session = get_session ( db_path ) maps = session . query ( SymbolMap ) . all ( ) for item in maps : click . echo ( item )


def get_by_id(self, symbol: str) -> SymbolMap:
        """ Finds the map by in-symbol """
        return self.query.filter(SymbolMap.in_symbol == symbol).first()

def get_by_id ( self , symbol : str ) -> SymbolMap : return self . query . filter ( SymbolMap . in_symbol == symbol ) . first ( )


def read_lines_from_file(file_path: str) -> List[str]:
    """ Read text lines from a file """
    # check if the file exists?
    with open(file_path) as csv_file:
        content = csv_file.readlines()
    return content

def read_lines_from_file ( file_path : str ) -> List [ str ] : with open ( file_path ) as csv_file : content = csv_file . readlines ( ) return content


def map_entity(self, entity: dal.Price) -> PriceModel:
        """ Map the price entity """
        if not entity:
            return None

        result = PriceModel()
        result.currency = entity.currency

        # date/time
        dt_string = entity.date
        format_string = "%Y-%m-%d"
        if entity.time:
            dt_string += f"T{entity.time}"
            format_string += "T%H:%M:%S"
        price_datetime = datetime.strptime(dt_string, format_string)
        result.datum = Datum()
        result.datum.from_datetime(price_datetime)
        assert isinstance(result.datum, Datum)

        #result.namespace = entity.namespace
        #result.symbol = entity.symbol
        result.symbol = SecuritySymbol(entity.namespace, entity.symbol)

        # Value
        value = Decimal(entity.value) / Decimal(entity.denom)
        result.value = Decimal(value)

        return result

def map_entity ( self , entity : dal . Price ) -> PriceModel : if not entity : return None result = PriceModel ( ) result . currency = entity . currency dt_string = entity . date format_string = "%Y-%m-%d" if entity . time : dt_string += f"T{entity.time}" format_string += "T%H:%M:%S" price_datetime = datetime . strptime ( dt_string , format_string ) result . datum = Datum ( ) result . datum . from_datetime ( price_datetime ) assert isinstance ( result . datum , Datum ) result . symbol = SecuritySymbol ( entity . namespace , entity . symbol ) value = Decimal ( entity . value ) / Decimal ( entity . denom ) result . value = Decimal ( value ) return result


def map_model(self, model: PriceModel) -> Price:
        """ Parse into the Price entity, ready for saving """
        # assert isinstance(model, PriceModel)
        assert isinstance(model.symbol, SecuritySymbol)
        assert isinstance(model.datum, Datum)

        entity = Price()

        # Format date as ISO string
        date_iso = f"{model.datum.value.year}-{model.datum.value.month:02d}-{model.datum.value.day:02d}"
        entity.date = date_iso

        entity.time = f"{model.datum.value.hour:02d}:{model.datum.value.minute:02d}:{model.datum.value.second:02d}"

        # Symbol
        # properly mapped symbols have a namespace, except for the US markets
        # TODO check this with .csv import
        if model.symbol.namespace:
            entity.namespace = model.symbol.namespace.upper()
        entity.symbol = model.symbol.mnemonic.upper()

        assert isinstance(model.value, Decimal)
        # Find number of decimal places
        dec_places = abs(model.value.as_tuple().exponent)
        entity.denom = 10 ** dec_places
        # Price value
        entity.value = int(model.value * entity.denom)

        # Currency
        entity.currency = model.currency.upper()

        # self.logger.debug(f"{entity}")
        return entity

Parse into the Price entity ready for saving


def __read_config(self, file_path: str):
        """ Read the config file """
        if not os.path.exists(file_path):
            raise FileNotFoundError(f"File path not found: {file_path}")
        # check if file exists
        if not os.path.isfile(file_path):
            self.logger.error(f"file not found: {file_path}")
            raise FileNotFoundError(f"configuration file not found {file_path}")

        self.config.read(file_path)


def __get_config_template_path(self) -> str:
        """ gets the default config path from resources """
        filename = resource_filename(
            Requirement.parse(package_name),
            template_path + config_filename)
        return filename

def __get_config_template_path ( self ) -> str : filename = resource_filename ( Requirement . parse ( package_name ) , template_path + config_filename ) return filename


def __create_user_config(self):
        """ Copy the config template into user's directory """
        src_path = self.__get_config_template_path()
        src = os.path.abspath(src_path)

        if not os.path.exists(src):
            message = f"Config template not found {src}"
            self.logger.error(message)
            raise FileNotFoundError(message)

        dst = os.path.abspath(self.get_config_path())

        shutil.copyfile(src, dst)

        if not os.path.exists(dst):
            raise FileNotFoundError("Config file could not be copied to user dir!")

Copy the config template into user s directory


def get_config_path(self) -> str:
        """
        Returns the path where the active config file is expected.
        This is the user's profile folder.
        """
        dst_dir = self.__get_user_path()
        dst = dst_dir + "/" + config_filename
        return dst

def get_config_path ( self ) -> str : dst_dir = self . __get_user_path ( ) dst = dst_dir + "/" + config_filename return dst

Returns the path where the active config file is expected . This is the user s profile folder .


def get_contents(self) -> str:
        """ Reads the contents of the config file """
        content = None
        # with open(file_path) as cfg_file:
        #     contents = cfg_file.read()

        # Dump the current contents into an in-memory file.
        in_memory = io.StringIO("")
        self.config.write(in_memory)
        in_memory.seek(0)
        content = in_memory.read()
        #     log(DEBUG, "config content: %s", content)
        in_memory.close()
        return content

def get_contents ( self ) -> str : content = None in_memory = io . StringIO ( "" ) self . config . write ( in_memory ) in_memory . seek ( 0 ) content = in_memory . read ( ) in_memory . close ( ) return content


def set(self, option: ConfigKeys, value):
        """ Sets a value in config """
        assert isinstance(option, ConfigKeys)

        # As currently we only have 1 section.
        section = SECTION
        self.config.set(section, option.name, value)
        self.save()

def set ( self , option : ConfigKeys , value ) : assert isinstance ( option , ConfigKeys ) section = SECTION self . config . set ( section , option . name , value ) self . save ( )


def get(self, option: ConfigKeys):
        """ Retrieves a config value """
        assert isinstance(option, ConfigKeys)

        # Currently only one section is used
        section = SECTION
        return self.config.get(section, option.name)

def get ( self , option : ConfigKeys ) : assert isinstance ( option , ConfigKeys ) section = SECTION return self . config . get ( section , option . name )


def save(self):
        """ Save the config file """
        file_path = self.get_config_path()
        contents = self.get_contents()
        with open(file_path, mode='w') as cfg_file:
            cfg_file.write(contents)

def save ( self ) : file_path = self . get_config_path ( ) contents = self . get_contents ( ) with open ( file_path , mode = 'w' ) as cfg_file : cfg_file . write ( contents )


def parse(self, symbol: str) -> (str, str):
        """ Splits the symbol into namespace, symbol tuple """
        symbol_parts = symbol.split(":")
        namespace = None
        mnemonic = symbol

        if len(symbol_parts) > 1:
            namespace = symbol_parts[0]
            mnemonic = symbol_parts[1]

        self.namespace = namespace
        self.mnemonic = mnemonic

        return namespace, mnemonic

def parse ( self , symbol : str ) -> ( str , str ) : symbol_parts = symbol . split ( ":" ) namespace = None mnemonic = symbol if len ( symbol_parts ) > 1 : namespace = symbol_parts [ 0 ] mnemonic = symbol_parts [ 1 ] self . namespace = namespace self . mnemonic = mnemonic return namespace , mnemonic

Splits the symbol into namespace symbol tuple


def add_price(self, price: PriceModel):
        """ Creates a new price record """
        # assert isinstance(price, PriceModel)

        if not price:
            raise ValueError("Cannot add price. The received model is null!")

        mapper = mappers.PriceMapper()
        entity = mapper.map_model(price)

        self.add_price_entity(entity)


def add_price_entity(self, price: dal.Price):
        """ Adds the price """
        from decimal import Decimal

        # check if the price already exists in db.
        repo = self.get_price_repository()
        existing = (
            repo.query
            .filter(dal.Price.namespace == price.namespace)
            .filter(dal.Price.symbol == price.symbol)
            .filter(dal.Price.date == price.date)
            .filter(dal.Price.time == price.time)
            .first()
        )
        if existing:
            # Update existing price.
            new_value = Decimal(price.value) / Decimal(price.denom)
            self.logger.info(f"Exists: {price}")
            if price.currency != existing.currency:
                raise ValueError(
                    f"The currency is different for price {price}!")
            if existing.value != price.value:
                existing.value = price.value
                self.logger.info(f"Updating to {new_value}.")
            if existing.denom != price.denom:
                existing.denom = price.denom
        else:
            # Insert new price
            self.session.add(price)
            self.logger.info(f"Added {price}")


def download_price(self, symbol: str, currency: str, agent: str) -> PriceModel:
        """ Download and save price online """
        price = self.__download_price(symbol, currency, agent)
        self.save()
        return price

def download_price ( self , symbol : str , currency : str , agent : str ) -> PriceModel : price = self . __download_price ( symbol , currency , agent ) self . save ( ) return price


def session(self):
        """ Returns the current db session """
        if not self.__session:
            self.__session = dal.get_default_session()
        return self.__session

def session ( self ) : if not self . __session : self . __session = dal . get_default_session ( ) return self . __session


def get_prices(self, date: str, currency: str) -> List[PriceModel]:
        """ Fetches all the prices for the given arguments """
        from .repositories import PriceRepository

        session = self.session
        repo = PriceRepository(session)
        query = repo.query
        if date:
            query = query.filter(dal.Price.date == date)
        if currency:
            query = query.filter(dal.Price.currency == currency)
        # Sort by symbol.
        query = query.order_by(dal.Price.namespace, dal.Price.symbol)
        price_entities = query.all()

        mapper = mappers.PriceMapper()
        result = []
        for entity in price_entities:
            model = mapper.map_entity(entity)
            result.append(model)
        return result

def get_prices ( self , date : str , currency : str ) -> List [ PriceModel ] : from . repositories import PriceRepository session = self . session repo = PriceRepository ( session ) query = repo . query if date : query = query . filter ( dal . Price . date == date ) if currency : query = query . filter ( dal . Price . currency == currency ) query = query . order_by ( dal . Price . namespace , dal . Price . symbol ) price_entities = query . all ( ) mapper = mappers . PriceMapper ( ) result = [ ] for entity in price_entities : model = mapper . map_entity ( entity ) result . append ( model ) return result


def get_prices_on(self, on_date: str, namespace: str, symbol: str):
        """ Returns the latest price on the date """
        repo = self.get_price_repository()
        query = (
            repo.query.filter(dal.Price.namespace == namespace)
            .filter(dal.Price.symbol == symbol)
            .filter(dal.Price.date == on_date)
            .order_by(dal.Price.time.desc())
        )
        result = query.first()
        # logging.debug(result)
        return result

def get_prices_on ( self , on_date : str , namespace : str , symbol : str ) : repo = self . get_price_repository ( ) query = ( repo . query . filter ( dal . Price . namespace == namespace ) . filter ( dal . Price . symbol == symbol ) . filter ( dal . Price . date == on_date ) . order_by ( dal . Price . time . desc ( ) ) ) result = query . first ( ) return result


def prune_all(self) -> int:
        """
        Prune historical prices for all symbols, leaving only the latest.
        Returns the number of items removed.
        """
        from .repositories import PriceRepository

        # get all symbols that have prices
        repo = PriceRepository()
        items = repo.query.distinct(dal.Price.namespace, dal.Price.symbol).all()
        # self.logger.debug(items)
        count = 0

        for item in items:
            symbol = SecuritySymbol(item.namespace, item.symbol)
            deleted = self.prune(symbol)
            if deleted:
                count += 1

        return count

def prune_all ( self ) -> int : from . repositories import PriceRepository repo = PriceRepository ( ) items = repo . query . distinct ( dal . Price . namespace , dal . Price . symbol ) . all ( ) count = 0 for item in items : symbol = SecuritySymbol ( item . namespace , item . symbol ) deleted = self . prune ( symbol ) if deleted : count += 1 return count

Prune historical prices for all symbols leaving only the latest . Returns the number of items removed .


def prune(self, symbol: SecuritySymbol):
        """
        Delete all but the latest available price for the given symbol.
        Returns the number of items removed.
        """
        from .repositories import PriceRepository

        assert isinstance(symbol, SecuritySymbol)

        self.logger.debug(f"pruning prices for {symbol}")

        repo = PriceRepository()
        query = (
            repo.query.filter(dal.Price.namespace == symbol.namespace)
            .filter(dal.Price.symbol == symbol.mnemonic)
            .order_by(dal.Price.date.desc())
            .order_by(dal.Price.time.desc())
        )
        all_prices = query.all()
        # self.logger.debug(f"fetched {all_prices}")

        deleted = False
        first = True
        for single in all_prices:
            if not first:
                repo.query.filter(dal.Price.id == single.id).delete()
                deleted = True
                self.logger.debug(f"deleting {single.id}")
            else:
                first = False

        repo.save()

        return deleted

def
prune
(
self
,
symbol
:
SecuritySymbol
)
:
from
.
repositories
import
PriceRepository
assert
isinstance
(
symbol
,
SecuritySymbol
)
self
.
logger
.
debug
(
f"pruning prices for {symbol}"
)
repo
=
PriceRepository
(
)
query
=
(
repo
.
query
.
filter
(
dal
.
Price
.
namespace
==
symbol
.
namespace
)
.
filter
(
dal
.
Price
.
symbol
==
symbol
.
mnemonic
)
.
order_by
(
dal
.
Price
.
date
.
desc
(
)
)
.
order_by
(
dal
.
Price
.
time
.
desc
(
)
)
)
all_prices
=
query
.
all
(
)
deleted
=
False
first
=
True
for
single
in
all_prices
:
if
not
first
:
repo
.
query
.
filter
(
dal
.
Price
.
id
==
single
.
id
)
.
delete
(
)
deleted
=
True
self
.
logger
.
debug
(
f"deleting {single.id}"
)
else
:
first
=
False
repo
.
save
(
)
return
deleted

Delete all but the latest available price for the given symbol . Returns the number of items removed .


def __download_price(self, symbol: str, currency: str, agent: str):
        """ Downloads and parses the price """
        from finance_quote_python import Quote

        assert isinstance(symbol, str)
        assert isinstance(currency, str)
        assert isinstance(agent, str)

        if not symbol:
            return None

        #self.logger.info(f"Downloading {symbol}... ")

        dl = Quote()
        dl.logger = self.logger

        dl.set_source(agent)
        dl.set_currency(currency)

        result = dl.fetch(agent, [symbol])

        if not result:
            raise ValueError(f"Did not receive a response for {symbol}.")

        price = result[0]

        if not price:
            raise ValueError(f"Price not downloaded/parsed for {symbol}.")
        else:
            # Create price data entity, to be inserted.
            self.add_price(price)

        return price


def __get_securities(self, currency: str, agent: str, symbol: str,
                         namespace: str) -> List[dal.Security]:
        """ Fetches the securities that match the given filters """
        repo = self.get_security_repository()
        query = repo.query

        if currency is not None:
            query = query.filter(dal.Security.currency == currency)

        if agent is not None:
            query = query.filter(dal.Security.updater == agent)

        if symbol is not None:
            query = query.filter(dal.Security.symbol == symbol)

        if namespace is not None:
            query = query.filter(dal.Security.namespace == namespace)

        # Sorting
        query = query.order_by(dal.Security.namespace, dal.Security.symbol)

        securities = query.all()
        return securities

def __get_securities ( self , currency : str , agent : str , symbol : str , namespace : str ) -> List [ dal . Security ] : repo = self . get_security_repository ( ) query = repo . query if currency is not None : query = query . filter ( dal . Security . currency == currency ) if agent is not None : query = query . filter ( dal . Security . updater == agent ) if symbol is not None : query = query . filter ( dal . Security . symbol == symbol ) if namespace is not None : query = query . filter ( dal . Security . namespace == namespace ) query = query . order_by ( dal . Security . namespace , dal . Security . symbol ) securities = query . all ( ) return securities


def partial(self):
        """Return partial of original function call"""
        ba = self.data["bound_args"]
        return state_partial(self.data["func"], *ba.args[1:], **ba.kwargs)

def partial ( self ) : ba = self . data [ "bound_args" ] return state_partial ( self . data [ "func" ] , * ba . args [ 1 : ] , ** ba . kwargs )


def update_child_calls(self):
        """Replace child nodes on original function call with their partials"""

        for node in filter(lambda n: len(n.arg_name), self.child_list):
            self.data["bound_args"].arguments[node.arg_name] = node.partial()
        self.updated = True

def update_child_calls ( self ) : for node in filter ( lambda n : len ( n . arg_name ) , self . child_list ) : self . data [ "bound_args" ] . arguments [ node . arg_name ] = node . partial ( ) self . updated = True


def descend(self, include_me=True):
        """Descend depth first into all child nodes"""
        if include_me:
            yield self

        for child in self.child_list:
            yield child
            yield from child.descend()

def descend ( self , include_me = True ) : if include_me : yield self for child in self . child_list : yield child yield from child . descend ( )


def multi_dec(f):
    """Decorator for multi to remove nodes for original test functions from root node"""

    @wraps(f)
    def wrapper(*args, **kwargs):
        args = (
            args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args
        )
        for arg in args:
            if isinstance(arg, Node) and arg.parent.name is "root":
                arg.parent.remove_child(arg)
                arg.update_child_calls()
        return f(*args, **kwargs)

    return wrapper

def multi_dec ( f ) : @ wraps ( f ) def wrapper ( * args , ** kwargs ) : args = ( args [ 0 ] if len ( args ) == 1 and isinstance ( args [ 0 ] , ( list , tuple ) ) else args ) for arg in args : if isinstance ( arg , Node ) and arg . parent . name is "root" : arg . parent . remove_child ( arg ) arg . update_child_calls ( ) return f ( * args , ** kwargs ) return wrapper


def has_equal_part_len(state, name, unequal_msg):
    """Verify that a part that is zoomed in on has equal length.

    Typically used in the context of ``check_function_def()``

    Arguments:
        name (str): name of the part for which to check the length to the corresponding part in the solution.
        unequal_msg (str): Message in case the lengths do not match.
        state (State): state as passed by the SCT chain. Don't specify this explicitly.

    :Examples:

        Student and solution code::

            def shout(word):
                return word + '!!!'

        SCT that checks number of arguments::

            Ex().check_function_def('shout').has_equal_part_len('args', 'not enough args!')
    """
    d = dict(
        stu_len=len(state.student_parts[name]), sol_len=len(state.solution_parts[name])
    )

    if d["stu_len"] != d["sol_len"]:
        _msg = state.build_message(unequal_msg, d)
        state.report(Feedback(_msg, state))

    return state

def has_equal_part_len ( state , name , unequal_msg ) : d = dict ( stu_len = len ( state . student_parts [ name ] ) , sol_len = len ( state . solution_parts [ name ] ) ) if d [ "stu_len" ] != d [ "sol_len" ] : _msg = state . build_message ( unequal_msg , d ) state . report ( Feedback ( _msg , state ) ) return state

Verify that a part that is zoomed in on has equal length .


def has_equal_ast(state, incorrect_msg=None, code=None, exact=True, append=None):
    """Test whether abstract syntax trees match between the student and solution code.

    ``has_equal_ast()`` can be used in two ways:

    * As a robust version of ``has_code()``. By setting ``code``, you can look for the AST representation of ``code`` in the student's submission.
      But be aware that ``a`` and ``a = 1`` won't match, as reading and assigning are not the same in an AST.
      Use ``ast.dump(ast.parse(code))`` to see an AST representation of ``code``.
    * As an expression-based check when using more advanced SCT chain, e.g. to compare the equality of expressions to set function arguments.

    Args:
        incorrect_msg: message displayed when ASTs mismatch. When you specify ``code`` yourself, you have to specify this.
        code: optional code to use instead of the solution AST.
        exact: whether the representations must match exactly. If false, the solution AST
               only needs to be contained within the student AST (similar to using test student typed).
               Defaults to ``True``, unless the ``code`` argument has been specified.

    :Example:

        Student and Solution Code::

            dict(a = 'value').keys()

        SCT::

            # all pass
            Ex().has_equal_ast()
            Ex().has_equal_ast(code = "dict(a = 'value').keys()")
            Ex().has_equal_ast(code = "dict(a = 'value')", exact = False)

        Student and Solution Code::

            import numpy as np
            arr = np.array([1, 2, 3, 4, 5])
            np.mean(arr)

        SCT::

            # Check underlying value of arugment a of np.mean:
            Ex().check_function('numpy.mean').check_args('a').has_equal_ast()

            # Only check AST equality of expression used to specify argument a:
            Ex().check_function('numpy.mean').check_args('a').has_equal_ast()

    """
    if utils.v2_only():
        state.assert_is_not(["object_assignments"], "has_equal_ast", ["check_object"])
        state.assert_is_not(["function_calls"], "has_equal_ast", ["check_function"])

    if code and incorrect_msg is None:
        raise InstructorError(
            "If you manually specify the code to match inside has_equal_ast(), "
            "you have to explicitly set the `incorrect_msg` argument."
        )

    if (
        append is None
    ):  # if not specified, set to False if incorrect_msg was manually specified
        append = incorrect_msg is None
    if incorrect_msg is None:
        incorrect_msg = "Expected `{{sol_str}}`, but got `{{stu_str}}`."

    def parse_tree(tree):
        # get contents of module.body if only 1 element
        crnt = (
            tree.body[0]
            if isinstance(tree, ast.Module) and len(tree.body) == 1
            else tree
        )

        # remove Expr if it exists
        return ast.dump(crnt.value if isinstance(crnt, ast.Expr) else crnt)

    stu_rep = parse_tree(state.student_ast)
    sol_rep = parse_tree(state.solution_ast if not code else ast.parse(code))

    fmt_kwargs = {
        "sol_str": state.solution_code if not code else code,
        "stu_str": state.student_code,
    }

    _msg = state.build_message(incorrect_msg, fmt_kwargs, append=append)

    if exact and not code:
        state.do_test(EqualTest(stu_rep, sol_rep, Feedback(_msg, state)))
    elif not sol_rep in stu_rep:
        state.report(Feedback(_msg, state))

    return state

Test whether abstract syntax trees match between the student and solution code .


def has_code(state, text, pattern=True, not_typed_msg=None):
    """Test the student code.

    Tests if the student typed a (pattern of) text. It is advised to use ``has_equal_ast()`` instead of ``has_code()``,
    as it is more robust to small syntactical differences that don't change the code's behavior.

    Args:
        text (str): the text that is searched for
        pattern (bool): if True (the default), the text is treated as a pattern. If False, it is treated as plain text.
        not_typed_msg (str): feedback message to be displayed if the student did not type the text.

    :Example:

        Student code and solution code::

            y = 1 + 2 + 3

        SCT::

            # Verify that student code contains pattern (not robust!!):
            Ex().has_code(r"1\\s*\\+2\\s*\\+3")

    """
    if not not_typed_msg:
        if pattern:
            not_typed_msg = "Could not find the correct pattern in your code."
        else:
            not_typed_msg = "Could not find the following text in your code: %r" % text

    student_code = state.student_code

    _msg = state.build_message(not_typed_msg)
    state.do_test(
        StringContainsTest(student_code, text, pattern, Feedback(_msg, state))
    )

    return state


def has_import(
    state,
    name,
    same_as=False,
    not_imported_msg="Did you import `{{pkg}}`?",
    incorrect_as_msg="Did you import `{{pkg}}` as `{{alias}}`?",
):
    """Checks whether student imported a package or function correctly.

    Python features many ways to import packages.
    All of these different methods revolve around the ``import``, ``from`` and ``as`` keywords.
    ``has_import()`` provides a robust way to check whether a student correctly imported a certain package.

    By default, ``has_import()`` allows for different ways of aliasing the imported package or function.
    If you want to make sure the correct alias was used to refer to the package or function that was imported,
    set ``same_as=True``.

    Args:
        name (str): the name of the package that has to be checked.
        same_as (bool): if True, the alias of the package or function has to be the same. Defaults to False.
        not_imported_msg (str): feedback message when the package is not imported.
        incorrect_as_msg (str): feedback message if the alias is wrong.

    :Example:

        Example 1, where aliases don't matter (defaut): ::

            # solution
            import matplotlib.pyplot as plt

            # sct
            Ex().has_import("matplotlib.pyplot")

            # passing submissions
            import matplotlib.pyplot as plt
            from matplotlib import pyplot as plt
            import matplotlib.pyplot as pltttt

            # failing submissions
            import matplotlib as mpl

        Example 2, where the SCT is coded so aliases do matter: ::

            # solution
            import matplotlib.pyplot as plt

            # sct
            Ex().has_import("matplotlib.pyplot", same_as=True)

            # passing submissions
            import matplotlib.pyplot as plt
            from matplotlib import pyplot as plt

            # failing submissions
            import matplotlib.pyplot as pltttt

    """
    student_imports = state.ast_dispatcher("imports", state.student_ast)
    solution_imports = state.ast_dispatcher("imports", state.solution_ast)

    if name not in solution_imports:
        raise InstructorError(
            "`has_import()` couldn't find an import of the package %s in your solution code."
            % name
        )

    fmt_kwargs = {"pkg": name, "alias": solution_imports[name]}

    _msg = state.build_message(not_imported_msg, fmt_kwargs)
    state.do_test(DefinedCollTest(name, student_imports, _msg))

    if same_as:
        _msg = state.build_message(incorrect_as_msg, fmt_kwargs)
        state.do_test(EqualTest(solution_imports[name], student_imports[name], _msg))

    return state

Checks whether student imported a package or function correctly .


def has_output(state, text, pattern=True, no_output_msg=None):
    """Search student output for a pattern.

    Among the student and solution process, the student submission and solution code as a string,
    the ``Ex()`` state also contains the output that a student generated with his or her submission.

    With ``has_output()``, you can access this output and match it against a regular or fixed expression.

    Args:
        text (str): the text that is searched for
        pattern (bool): if True (default), the text is treated as a pattern. If False, it is treated as plain text.
        no_output_msg (str): feedback message to be displayed if the output is not found.

    :Example:

        As an example, suppose we want a student to print out a sentence: ::

            # Print the "This is some ... stuff"
            print("This is some weird stuff")

        The following SCT tests whether the student prints out ``This is some weird stuff``: ::

            # Using exact string matching
            Ex().has_output("This is some weird stuff", pattern = False)

            # Using a regular expression (more robust)
            # pattern = True is the default
            msg = "Print out ``This is some ... stuff`` to the output, " + \\
                  "fill in ``...`` with a word you like."
            Ex().has_output(r"This is some \w* stuff", no_output_msg = msg)

    """
    if not no_output_msg:
        no_output_msg = "You did not output the correct things."

    _msg = state.build_message(no_output_msg)
    state.do_test(StringContainsTest(state.raw_student_output, text, pattern, _msg))

    return state


def has_printout(
    state, index, not_printed_msg=None, pre_code=None, name=None, copy=False
):
    """Check if the right printouts happened.

    ``has_printout()`` will look for the printout in the solution code that you specified with ``index`` (0 in this case), rerun the ``print()`` call in
    the solution process, capture its output, and verify whether the output is present in the output of the student.

    This is more robust as ``Ex().check_function('print')`` initiated chains as students can use as many
    printouts as they want, as long as they do the correct one somewhere.

    Args:
        index (int): index of the ``print()`` call in the solution whose output you want to search for in the student output.
        not_printed_msg (str): if specified, this overrides the default message that is generated when the output
          is not found in the student output.
        pre_code (str): Python code as a string that is executed before running the targeted student call.
          This is the ideal place to set a random seed, for example.
        copy (bool): whether to try to deep copy objects in the environment, such as lists, that could
          accidentally be mutated. Disabled by default, which speeds up SCTs.
        state (State): state as passed by the SCT chain. Don't specify this explicitly.

    :Example:

        Suppose you want somebody to print out 4: ::

            print(1, 2, 3, 4)

        The following SCT would check that: ::

            Ex().has_printout(0)

        All of the following SCTs would pass: ::

            print(1, 2, 3, 4)
            print('1 2 3 4')
            print(1, 2, '3 4')
            print("random"); print(1, 2, 3, 4)

    :Example:

        Watch out: ``has_printout()`` will effectively **rerun** the ``print()`` call in the solution process after the entire solution script was executed.
        If your solution script updates the value of `x` after executing it, ``has_printout()`` will not work.

        Suppose you have the following solution: ::

            x = 4
            print(x)
            x = 6

        The following SCT will not work: ::

            Ex().has_printout(0)

        Why? When the ``print(x)`` call is executed, the value of ``x`` will be 6, and pythonwhat will look for the output `'6`' in the output the student generated.
        In cases like these, ``has_printout()`` cannot be used.

    :Example:

        Inside a for loop ``has_printout()``

        Suppose you have the following solution: ::

            for i in range(5):
                print(i)

        The following SCT will not work: ::

            Ex().check_for_loop().check_body().has_printout(0)

        The reason is that ``has_printout()`` can only be called from the root state. ``Ex()``.
        If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead: ::

            Ex().check_for_loop().check_body().\\
                set_context(0).check_function('print').\\
                check_args(0).has_equal_value()

    """

    extra_msg = "If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead."
    state.assert_root("has_printout", extra_msg=extra_msg)

    if not_printed_msg is None:
        not_printed_msg = (
            "Have you used `{{sol_call}}` to do the appropriate printouts?"
        )

    try:
        sol_call_ast = state.ast_dispatcher("function_calls", state.solution_ast)[
            "print"
        ][index]["node"]
    except (KeyError, IndexError):
        raise InstructorError(
            "`has_printout({})` couldn't find the {} print call in your solution.".format(
                index, utils.get_ord(index + 1)
            )
        )

    out_sol, str_sol = getOutputInProcess(
        tree=sol_call_ast,
        process=state.solution_process,
        context=state.solution_context,
        env=state.solution_env,
        pre_code=pre_code,
        copy=copy,
    )

    sol_call_str = state.solution_ast_tokens.get_text(sol_call_ast)

    if isinstance(str_sol, Exception):
        raise InstructorError(
            "Evaluating the solution expression {} raised error in solution process."
            "Error: {} - {}".format(sol_call_str, type(out_sol), str_sol)
        )

    _msg = state.build_message(not_printed_msg, {"sol_call": sol_call_str})

    has_output(state, out_sol.strip(), pattern=False, no_output_msg=_msg)

    return state


def has_no_error(
    state,
    incorrect_msg="Have a look at the console: your code contains an error. Fix it and try again!",
):
    """Check whether the submission did not generate a runtime error.

    If all SCTs for an exercise pass, before marking the submission as correct pythonwhat will automatically check whether
    the student submission generated an error. This means it is not needed to use ``has_no_error()`` explicitly.

    However, in some cases, using ``has_no_error()`` explicitly somewhere throughout your SCT execution can be helpful:

    - If you want to make sure people didn't write typos when writing a long function name.
    - If you want to first verify whether a function actually runs, before checking whether the arguments were specified correctly.
    - More generally, if, because of the content, it's instrumental that the script runs without
      errors before doing any other verifications.

    Args:
        incorrect_msg: if specified, this overrides the default message if the student code generated an error.

    :Example:

        Suppose you're verifying an exercise about model training and validation: ::

            # pre exercise code
            import numpy as np
            from sklearn.model_selection import train_test_split
            from sklearn import datasets
            from sklearn import svm

            iris = datasets.load_iris()
            iris.data.shape, iris.target.shape

            # solution
            X_train, X_test, y_train, y_test = train_test_split(
                iris.data, iris.target, test_size=0.4, random_state=0)

        If you want to make sure that ``train_test_split()`` ran without errors,
        which would check if the student typed the function without typos and used
        sensical arguments, you could use the following SCT: ::

            Ex().has_no_error()
            Ex().check_function('sklearn.model_selection.train_test_split').multi(
                check_args(['arrays', 0]).has_equal_value(),
                check_args(['arrays', 0]).has_equal_value(),
                check_args(['options', 'test_size']).has_equal_value(),
                check_args(['options', 'random_state']).has_equal_value()
            )

        If, on the other hand, you want to fall back onto pythonwhat's built in behavior,
        that checks for an error before marking the exercise as correct, you can simply
        leave of the ``has_no_error()`` step.

    """
    state.assert_root("has_no_error")

    if state.reporter.errors:
        _msg = state.build_message(
            incorrect_msg, {"error": str(state.reporter.errors[0])}
        )
        state.report(Feedback(_msg, state))

    return state

Check whether the submission did not generate a runtime error .