def build_minimal_runs(events):
    """Compile list of minimal runs for given events."""
    # Eliminate multiples
    events = [e for i, e in enumerate(events) if events.index(e) == i]

    # Build list of runs per register group
    scheduled_runs = {}
    scheduled_events = []
    cur_run = 0
    while len(scheduled_events) != len(events):
        for event_tpl in events:
            event, registers, parameters = event_tpl
            # Skip allready scheduled events
            if event_tpl in scheduled_events:
                continue
            # Compile explicit list of possible register locations
            for possible_reg in register_options(registers):
                # Schedule in current run, if register is not yet in use
                s = scheduled_runs.setdefault(cur_run, {})
                if possible_reg not in s:
                    s[possible_reg] = (event, possible_reg, parameters)
                    # ban from further scheduling attempts
                    scheduled_events.append(event_tpl)
                    break
        cur_run += 1

    # Collaps all register dicts to single runs
    runs = [list(v.values()) for v in scheduled_runs.values()]

    return runs

def build_minimal_runs ( events ) : events = [ e for i , e in enumerate ( events ) if events . index ( e ) == i ] scheduled_runs = { } scheduled_events = [ ] cur_run = 0 while len ( scheduled_events ) != len ( events ) : for event_tpl in events : event , registers , parameters = event_tpl if event_tpl in scheduled_events : continue for possible_reg in register_options ( registers ) : s = scheduled_runs . setdefault ( cur_run , { } ) if possible_reg not in s : s [ possible_reg ] = ( event , possible_reg , parameters ) scheduled_events . append ( event_tpl ) break cur_run += 1 runs = [ list ( v . values ( ) ) for v in scheduled_runs . values ( ) ] return runs

Compile list of minimal runs for given events .


def report(self, output_file=sys.stdout):
        """Report analysis outcome in human readable form."""
        max_perf = self.results['max_perf']

        if self._args and self._args.verbose >= 3:
            print('{}'.format(pformat(self.results)), file=output_file)

        if self._args and self._args.verbose >= 1:
            print('{}'.format(pformat(self.results['verbose infos'])), file=output_file)
            print('Bottlenecks:', file=output_file)
            print('  level | a. intensity |   performance   |   peak bandwidth  | peak bandwidth kernel',
                  file=output_file)
            print('--------+--------------+-----------------+-------------------+----------------------',
                  file=output_file)
            print('    CPU |              | {!s:>15} |                   |'.format(
                max_perf[self._args.unit]),
                  file=output_file)
            for b in self.results['mem bottlenecks']:
                print('{level:>7} | {arithmetic intensity:>5.2} FLOP/B | {0!s:>15} |'
                      ' {bandwidth!s:>17} | {bw kernel:<8}'.format(
                          b['performance'][self._args.unit], **b),
                      file=output_file)
            print('', file=output_file)

        if self.results['min performance']['FLOP/s'] > max_perf['FLOP/s']:
            # CPU bound
            print('CPU bound. {!s} due to CPU max. FLOP/s'.format(max_perf), file=output_file)
        else:
            # Cache or mem bound
            print('Cache or mem bound.', file=output_file)

            bottleneck = self.results['mem bottlenecks'][self.results['bottleneck level']]
            print('{!s} due to {} transfer bottleneck (with bw from {} benchmark)'.format(
                    bottleneck['performance'][self._args.unit],
                    bottleneck['level'],
                    bottleneck['bw kernel']),
                  file=output_file)
            print('Arithmetic Intensity: {:.2f} FLOP/B'.format(bottleneck['arithmetic intensity']),
                  file=output_file)

Report analysis outcome in human readable form .


def report(self, output_file=sys.stdout):
        """Print human readable report of model."""
        cpu_perf = self.results['cpu bottleneck']['performance throughput']

        if self.verbose >= 3:
            print('{}'.format(pformat(self.results)), file=output_file)

        if self.verbose >= 1:
            print('Bottlenecks:', file=output_file)
            print('  level | a. intensity |   performance   |   peak bandwidth  | peak bandwidth kernel',
                  file=output_file)
            print('--------+--------------+-----------------+-------------------+----------------------',
                  file=output_file)
            print('    CPU |              | {!s:>15} |                   |'.format(
                cpu_perf[self._args.unit]),
                  file=output_file)
            for b in self.results['mem bottlenecks']:
                # Skip CPU-L1 from Roofline model
                if b is None:
                    continue
                print('{level:>7} | {arithmetic intensity:>5.2} FLOP/B | {0!s:>15} |'
                      ' {bandwidth!s:>17} | {bw kernel:<8}'.format(
                          b['performance'][self._args.unit], **b),
                      file=output_file)
            print('', file=output_file)
            print('IACA analisys:', file=output_file)
            print('{!s}'.format(
                {k: v
                 for k, v in list(self.results['cpu bottleneck'].items())
                 if k not in['IACA output']}),
                file=output_file)

        if self.results['min performance']['FLOP/s'] > cpu_perf['FLOP/s']:
            # CPU bound
            print('CPU bound. {!s} due to CPU bottleneck'.format(cpu_perf[self._args.unit]),
                  file=output_file)
        else:
            # Cache or mem bound
            print('Cache or mem bound.', file=output_file)

            bottleneck = self.results['mem bottlenecks'][self.results['bottleneck level']]
            print('{!s} due to {} transfer bottleneck (with bw from {} benchmark)'.format(
                      bottleneck['performance'][self._args.unit],
                      bottleneck['level'],
                      bottleneck['bw kernel']),
                  file=output_file)
            print('Arithmetic Intensity: {:.2f} FLOP/B'.format(bottleneck['arithmetic intensity']),
                  file=output_file)


def report(self, output_file=sys.stdout):
        """Report generated model in human readable form."""
        if self._args and self._args.verbose > 2:
            pprint(self.results)

        for dimension, lc_info in self.results['dimensions'].items():
            print("{}D layer condition:".format(dimension), file=output_file)
            for cache, lc_solution in sorted(lc_info['caches'].items()):
                print(cache+": ", end='', file=output_file)
                if lc_solution['lt'] is sympy.true:
                    print("unconditionally fulfilled", file=output_file)
                else:
                    if lc_solution['eq'] is None:
                        print("{}".format(lc_solution['lt']), file=output_file)
                    elif type(lc_solution['eq']) is not list:
                        print("{}".format(lc_solution['eq']), file=output_file)
                    else:
                        for solu in lc_solution['eq']:
                            for s, v in solu.items():
                                print("{} <= {}".format(s, v), file=output_file)

def
report
(
self
,
output_file
=
sys
.
stdout
)
:
if
self
.
_args
and
self
.
_args
.
verbose
>
2
:
pprint
(
self
.
results
)
for
dimension
,
lc_info
in
self
.
results
[
'dimensions'
]
.
items
(
)
:
print
(
"{}D layer condition:"
.
format
(
dimension
)
,
file
=
output_file
)
for
cache
,
lc_solution
in
sorted
(
lc_info
[
'caches'
]
.
items
(
)
)
:
print
(
cache
+
": "
,
end
=
''
,
file
=
output_file
)
if
lc_solution
[
'lt'
]
is
sympy
.
true
:
print
(
"unconditionally fulfilled"
,
file
=
output_file
)
else
:
if
lc_solution
[
'eq'
]
is
:
print
(
"{}"
.
format
(
lc_solution
[
'lt'
]
)
,
file
=
output_file
)
elif
type
(
lc_solution
[
'eq'
]
)
is
not
list
:
print
(
"{}"
.
format
(
lc_solution
[
'eq'
]
)
,
file
=
output_file
)
else
:
for
solu
in
lc_solution
[
'eq'
]
:
for
s
,
v
in
solu
.
items
(
)
:
print
(
"{} <= {}"
.
format
(
s
,
v
)
,
file
=
output_file
)

Report generated model in human readable form .


def clean_code(code, comments=True, macros=False, pragmas=False):
    """
    Naive comment and macro striping from source code

    :param comments: If True, all comments are stripped from code
    :param macros: If True, all macros are stripped from code
    :param pragmas: If True, all pragmas are stripped from code

    :return: cleaned code. Line numbers are preserved with blank lines,
    and multiline comments and macros are supported. BUT comment-like
    strings are (wrongfully) treated as comments.
    """
    if macros or pragmas:
        lines = code.split('\n')
        in_macro = False
        in_pragma = False
        for i in range(len(lines)):
            l = lines[i].strip()

            if macros and (l.startswith('#') and not l.startswith('#pragma') or in_macro):
                lines[i] = ''
                in_macro = l.endswith('\\')
            if pragmas and (l.startswith('#pragma') or in_pragma):
                lines[i] = ''
                in_pragma = l.endswith('\\')
        code = '\n'.join(lines)

    if comments:
        idx = 0
        comment_start = None
        while idx < len(code) - 1:
            if comment_start is None and code[idx:idx + 2] == '//':
                end_idx = code.find('\n', idx)
                code = code[:idx] + code[end_idx:]
                idx -= end_idx - idx
            elif comment_start is None and code[idx:idx + 2] == '/*':
                comment_start = idx
            elif comment_start is not None and code[idx:idx + 2] == '*/':
                code = (code[:comment_start] +
                        '\n' * code[comment_start:idx].count('\n') +
                        code[idx + 2:])
                idx -= idx - comment_start
                comment_start = None
            idx += 1

    return code

def clean_code ( code , comments = True , macros = False , pragmas = False ) : if macros or pragmas : lines = code . split ( '\n' ) in_macro = False in_pragma = False for i in range ( len ( lines ) ) : l = lines [ i ] . strip ( ) if macros and ( l . startswith ( '#' ) and not l . startswith ( '#pragma' ) or in_macro ) : lines [ i ] = '' in_macro = l . endswith ( '\\' ) if pragmas and ( l . startswith ( '#pragma' ) or in_pragma ) : lines [ i ] = '' in_pragma = l . endswith ( '\\' ) code = '\n' . join ( lines ) if comments : idx = 0 comment_start = None while idx < len ( code ) - 1 : if comment_start is None and code [ idx : idx + 2 ] == '//' : end_idx = code . find ( '\n' , idx ) code = code [ : idx ] + code [ end_idx : ] idx -= end_idx - idx elif comment_start is None and code [ idx : idx + 2 ] == '/*' : comment_start = idx elif comment_start is not None and code [ idx : idx + 2 ] == '*/' : code = ( code [ : comment_start ] + '\n' * code [ comment_start : idx ] . count ( '\n' ) + code [ idx + 2 : ] ) idx -= idx - comment_start comment_start = None idx += 1 return code


def round_to_next(x, base):
    """Round float to next multiple of base."""
    # Based on: http://stackoverflow.com/a/2272174
    return int(base * math.ceil(float(x)/base))

def round_to_next ( x , base ) : return int ( base * math . ceil ( float ( x ) / base ) )


def blocking(indices, block_size, initial_boundary=0):
    """
    Split list of integers into blocks of block_size and return block indices.

    First block element will be located at initial_boundary (default 0).

    >>> blocking([0, -1, -2, -3, -4, -5, -6, -7, -8, -9], 8)
    [0,-1]
    >>> blocking([0], 8)
    [0]
    >>> blocking([0], 8, initial_boundary=32)
    [-4]
    """
    blocks = []

    for idx in indices:
        bl_idx = (idx-initial_boundary)//float(block_size)
        if bl_idx not in blocks:
            blocks.append(bl_idx)
    blocks.sort()

    return blocks

def blocking ( indices , block_size , initial_boundary = 0 ) : blocks = [ ] for idx in indices : bl_idx = ( idx - initial_boundary ) // float ( block_size ) if bl_idx not in blocks : blocks . append ( bl_idx ) blocks . sort ( ) return blocks

Split list of integers into blocks of block_size and return block indices .


def calculate_cache_access(self):
        """Dispatch to cache predictor to get cache stats."""
        self.results.update({
                        'cycles': [],  # will be filled by caclculate_cycles()
                        'misses': self.predictor.get_misses(),
                        'hits': self.predictor.get_hits(),
                        'evicts': self.predictor.get_evicts(),
                        'verbose infos': self.predictor.get_infos()})

def
calculate_cache_access
(
self
)
:
self
.
results
.
update
(
{
'cycles'
:
[
]
,
'misses'
:
self
.
predictor
.
get_misses
(
)
,
'hits'
:
self
.
predictor
.
get_hits
(
)
,
'evicts'
:
self
.
predictor
.
get_evicts
(
)
,
'verbose infos'
:
self
.
predictor
.
get_infos
(
)
}
)

Dispatch to cache predictor to get cache stats .


def calculate_cycles(self):
        """
        Calculate performance model cycles from cache stats.

        calculate_cache_access() needs to have been execute before.
        """
        element_size = self.kernel.datatypes_size[self.kernel.datatype]
        elements_per_cacheline = float(self.machine['cacheline size']) // element_size
        iterations_per_cacheline = (sympy.Integer(self.machine['cacheline size']) /
                                    sympy.Integer(self.kernel.bytes_per_iteration))
        self.results['iterations per cacheline'] = iterations_per_cacheline
        cacheline_size = float(self.machine['cacheline size'])

        loads, stores = (self.predictor.get_loads(), self.predictor.get_stores())

        for cache_level, cache_info in list(enumerate(self.machine['memory hierarchy']))[1:]:
            throughput, duplexness = cache_info['non-overlap upstream throughput']

            if type(throughput) is str and throughput == 'full socket memory bandwidth':
                # Memory transfer
                # we use bandwidth to calculate cycles and then add panalty cycles (if given)

                # choose bw according to cache level and problem
                # first, compile stream counts at current cache level
                # write-allocate is allready resolved in cache predictor
                read_streams = loads[cache_level]
                write_streams = stores[cache_level]
                # second, try to find best fitting kernel (closest to stream seen stream counts):
                threads_per_core = 1
                bw, measurement_kernel = self.machine.get_bandwidth(
                    cache_level, read_streams, write_streams, threads_per_core)

                # calculate cycles
                if duplexness == 'half-duplex':
                    cycles = float(loads[cache_level] + stores[cache_level]) * \
                             float(elements_per_cacheline) * float(element_size) * \
                             float(self.machine['clock']) / float(bw)
                else:  # full-duplex
                    raise NotImplementedError(
                        "full-duplex mode is not (yet) supported for memory transfers.")
                # add penalty cycles for each read stream
                if 'penalty cycles per read stream' in cache_info:
                    cycles += stores[cache_level] * \
                              cache_info['penalty cycles per read stream']

                self.results.update({
                    'memory bandwidth kernel': measurement_kernel,
                    'memory bandwidth': bw})
            else:
                # since throughput is given in B/cy, and we need CL/cy:
                throughput = float(throughput) / cacheline_size
                # only cache cycles count
                if duplexness == 'half-duplex':
                    cycles = (loads[cache_level] + stores[cache_level]) / float(throughput)
                elif duplexness == 'full-duplex':
                    cycles = max(loads[cache_level] / float(throughput),
                                 stores[cache_level] / float(throughput))
                else:
                    raise ValueError("Duplexness of cache throughput may only be 'half-duplex'"
                                     "or 'full-duplex', found {} in {}.".format(
                        duplexness, cache_info['name']))

            self.results['cycles'].append((cache_info['level'], cycles))

            self.results[cache_info['level']] = cycles

        return self.results

Calculate performance model cycles from cache stats .


def analyze(self):
        """Run complete anaylysis and return results."""
        self.calculate_cache_access()
        self.calculate_cycles()
        self.results['flops per iteration'] = sum(self.kernel._flops.values())

        return self.results

def
analyze
(
self
)
:
self
.
calculate_cache_access
(
)
self
.
calculate_cycles
(
)
self
.
results
[
'flops per iteration'
]
=
sum
(
self
.
kernel
.
_flops
.
values
(
)
)
return
self
.
results

Run complete anaylysis and return results .


def analyze(self):
        """
        Run complete analysis and return results.
        """
        try:
            incore_analysis, asm_block =  self.kernel.iaca_analysis(
                micro_architecture=self.machine['micro-architecture'],
                asm_block=self.asm_block,
                pointer_increment=self.pointer_increment,
                verbose=self.verbose > 2)
        except RuntimeError as e:
            print("IACA analysis failed: " + str(e))
            sys.exit(1)

        block_throughput = incore_analysis['throughput']
        port_cycles = incore_analysis['port cycles']
        uops = incore_analysis['uops']

        # Normalize to cycles per cacheline
        elements_per_block = abs(asm_block['pointer_increment']
                                 // self.kernel.datatypes_size[self.kernel.datatype])
        block_size = elements_per_block*self.kernel.datatypes_size[self.kernel.datatype]
        try:
            block_to_cl_ratio = float(self.machine['cacheline size'])/block_size
        except ZeroDivisionError as e:
            print("Too small block_size / pointer_increment:", e, file=sys.stderr)
            sys.exit(1)

        port_cycles = dict([(i[0], i[1]*block_to_cl_ratio) for i in list(port_cycles.items())])
        uops = uops*block_to_cl_ratio
        cl_throughput = block_throughput*block_to_cl_ratio

        # Compile most relevant information
        T_OL = max([v for k, v in list(port_cycles.items())
                    if k in self.machine['overlapping model']['ports']])
        T_nOL = max([v for k, v in list(port_cycles.items())
                     if k in self.machine['non-overlapping model']['ports']])

        # Use IACA throughput prediction if it is slower then T_nOL
        if T_nOL < cl_throughput:
            T_OL = cl_throughput

        # Create result dictionary
        self.results = {
            'port cycles': port_cycles,
            'cl throughput': self.conv_cy(cl_throughput),
            'uops': uops,
            'T_nOL': T_nOL,
            'T_OL': T_OL,
            'IACA output': incore_analysis['output'],
            'elements_per_block': elements_per_block,
            'pointer_increment': asm_block['pointer_increment'],
            'flops per iteration': sum(self.kernel._flops.values())}
        return self.results

Run complete analysis and return results .


def strip_and_uncomment(asm_lines):
    """Strip whitespaces and comments from asm lines."""
    asm_stripped = []
    for line in asm_lines:
        # Strip comments and whitespaces
        asm_stripped.append(line.split('#')[0].strip())
    return asm_stripped

def strip_and_uncomment ( asm_lines ) : asm_stripped = [ ] for line in asm_lines : asm_stripped . append ( line . split ( '#' ) [ 0 ] . strip ( ) ) return asm_stripped

Strip whitespaces and comments from asm lines .


def strip_unreferenced_labels(asm_lines):
    """Strip all labels, which are never referenced."""
    asm_stripped = []
    for line in asm_lines:
        if re.match(r'^\S+:', line):
            # Found label
            label = line[0:line.find(':')]
            # Search for references to current label
            if not any([re.match(r'^[^#]*\s' + re.escape(label) + '[\s,]?.*$', l) for l in asm_lines]):
                # Skip labels without seen reference
                line = ''
        asm_stripped.append(line)
    return asm_stripped

def strip_unreferenced_labels ( asm_lines ) : asm_stripped = [ ] for line in asm_lines : if re . match ( r'^\S+:' , line ) : label = line [ 0 : line . find ( ':' ) ] if not any ( [ re . match ( r'^[^#]*\s' + re . escape ( label ) + '[\s,]?.*$' , l ) for l in asm_lines ] ) : line = '' asm_stripped . append ( line ) return asm_stripped

Strip all labels which are never referenced .


def select_best_block(blocks):
    """Return best block selected based on simple heuristic."""
    # TODO make this cleverer with more stats
    if not blocks:
        raise ValueError("No suitable blocks were found in assembly.")
    best_block = max(blocks, key=lambda b: b[1]['packed_instr'])
    if best_block[1]['packed_instr'] == 0:
        best_block = max(blocks,
                         key=lambda b: (b[1]['ops'] + b[1]['packed_instr'] + b[1]['avx_instr'],
                                        b[1]['ZMM'], b[1]['YMM'], b[1]['XMM']))
    return best_block[0]

Return best block selected based on simple heuristic .


def userselect_increment(block):
    """Let user interactively select byte increment."""
    print("Selected block:")
    print('\n    ' + ('\n    '.join(block['lines'])))
    print()

    increment = None
    while increment is None:
        increment = input("Choose store pointer increment (number of bytes): ")
        try:
            increment = int(increment)
        except ValueError:
            increment = None

    block['pointer_increment'] = increment
    return increment

Let user interactively select byte increment .


def userselect_block(blocks, default=None, debug=False):
    """Let user interactively select block."""
    print("Blocks found in assembly file:")
    print("      block     | OPs | pck. | AVX || Registers |    ZMM   |    YMM   |    XMM   |    GP   ||ptr.inc|\n"
          "----------------+-----+------+-----++-----------+----------+----------+----------+---------++-------|")
    for idx, b in blocks:
        print('{:>2} {b[labels]!r:>12} | {b[ops]:>3} | {b[packed_instr]:>4} | {b[avx_instr]:>3} |'
              '| {b[regs][0]:>3} ({b[regs][1]:>3}) | {b[ZMM][0]:>3} ({b[ZMM][1]:>2}) | '
              '{b[YMM][0]:>3} ({b[YMM][1]:>2}) | '
              '{b[XMM][0]:>3} ({b[XMM][1]:>2}) | {b[GP][0]:>2} ({b[GP][1]:>2}) || '
              '{b[pointer_increment]!s:>5} |'.format(idx, b=b))

        if debug:
            ln = b['first_line']
            print(' '*4 + 'Code:')
            for l in b['lines']:
                print(' '*8 + '{:>5} | {}'.format(ln, l))
                ln += 1
            print(' '*4 + 'Metadata:')
            print(textwrap.indent(
                pformat({k: v for k,v in b.items() if k not in ['lines']}),
                ' '*8))

    # Let user select block:
    block_idx = -1
    while not (0 <= block_idx < len(blocks)):
        block_idx = input("Choose block to be marked [" + str(default) + "]: ") or default
        try:
            block_idx = int(block_idx)
        except ValueError:
            block_idx = -1
    # block = blocks[block_idx][1]

    return block_idx


def insert_markers(asm_lines, start_line, end_line):
    """Insert IACA marker into list of ASM instructions at given indices."""
    asm_lines = (asm_lines[:start_line] + START_MARKER +
                 asm_lines[start_line:end_line + 1] + END_MARKER +
                 asm_lines[end_line + 1:])
    return asm_lines

Insert IACA marker into list of ASM instructions at given indices .


def iaca_instrumentation(input_file, output_file,
                         block_selection='auto',
                         pointer_increment='auto_with_manual_fallback',
                         debug=False):
    """
    Add IACA markers to an assembly file.

    If instrumentation fails because loop increment could not determined automatically, a ValueError
    is raised.

    :param input_file: file-like object to read from
    :param output_file: file-like object to write to
    :param block_selection: index of the assembly block to instrument, or 'auto' for automatically
                            using block with the
                            most vector instructions, or 'manual' to read index to prompt user
    :param pointer_increment: number of bytes the pointer is incremented after the loop or
                              - 'auto': automatic detection, otherwise RuntimeError is raised
                              - 'auto_with_manual_fallback': like auto with fallback to manual input
                              - 'manual': prompt user
    :param debug: output additional internal analysis information. Only works with manual selection.
    :return: the instrumented assembly block
    """
    assembly_orig = input_file.readlines()

    # If input and output files are the same, overwrite with output
    if input_file is output_file:
        output_file.seek(0)
        output_file.truncate()

    if debug:
        block_selection = 'manual'

    assembly = strip_and_uncomment(copy(assembly_orig))
    assembly = strip_unreferenced_labels(assembly)
    blocks = find_asm_blocks(assembly)
    if block_selection == 'auto':
        block_idx = select_best_block(blocks)
    elif block_selection == 'manual':
        block_idx = userselect_block(blocks, default=select_best_block(blocks), debug=debug)
    elif isinstance(block_selection, int):
        block_idx = block_selection
    else:
        raise ValueError("block_selection has to be an integer, 'auto' or 'manual' ")

    block = blocks[block_idx][1]

    if pointer_increment == 'auto':
        if block['pointer_increment'] is None:
            raise RuntimeError("pointer_increment could not be detected automatically. Use "
                               "--pointer-increment to set manually to byte offset of store "
                               "pointer address between consecutive assembly block iterations.")
    elif pointer_increment == 'auto_with_manual_fallback':
        if block['pointer_increment'] is None:
            block['pointer_increment'] = userselect_increment(block)
    elif pointer_increment == 'manual':
        block['pointer_increment'] = userselect_increment(block)
    elif isinstance(pointer_increment, int):
        block['pointer_increment'] = pointer_increment
    else:
        raise ValueError("pointer_increment has to be an integer, 'auto', 'manual' or  "
                         "'auto_with_manual_fallback' ")

    instrumented_asm = insert_markers(assembly_orig, block['first_line'], block['last_line'])
    output_file.writelines(instrumented_asm)

    return block


def main():
    """Execute command line interface."""
    parser = argparse.ArgumentParser(
        description='Find and analyze basic loop blocks and mark for IACA.',
        epilog='For help, examples, documentation and bug reports go to:\nhttps://github.com'
               '/RRZE-HPC/kerncraft\nLicense: AGPLv3')
    parser.add_argument('--version', action='version', version='%(prog)s {}'.format(__version__))
    parser.add_argument('source', type=argparse.FileType(), nargs='?', default=sys.stdin,
                        help='assembly file to analyze (default: stdin)')
    parser.add_argument('--outfile', '-o', type=argparse.FileType('w'), nargs='?',
                        default=sys.stdout, help='output file location (default: stdout)')
    parser.add_argument('--debug', action='store_true',
                        help='Output nternal analysis information for debugging.')
    args = parser.parse_args()

    # pointer_increment is given, since it makes no difference on the command lien and requires
    # less user input
    iaca_instrumentation(input_file=args.source, output_file=args.outfile,
                         block_selection='manual', pointer_increment=1, debug=args.debug)


def simulate(kernel, model, define_dict, blocking_constant, blocking_length):
    """Setup and execute model with given blocking length"""
    kernel.clear_state()

    # Add constants from define arguments
    for k, v in define_dict.items():
        kernel.set_constant(k, v)

    kernel.set_constant(blocking_constant, blocking_length)

    model.analyze()
    return sum([cy for dscr, cy in model.results['cycles']])

def simulate ( kernel , model , define_dict , blocking_constant , blocking_length ) : kernel . clear_state ( ) for k , v in define_dict . items ( ) : kernel . set_constant ( k , v ) kernel . set_constant ( blocking_constant , blocking_length ) model . analyze ( ) return sum ( [ cy for dscr , cy in model . results [ 'cycles' ] ] )


def space(start, stop, num, endpoint=True, log=False, base=10):
    """
    Return list of evenly spaced integers over an interval.

    Numbers can either be evenly distributed in a linear space (if *log* is False) or in a log
    space (if *log* is True). If *log* is True, base is used to define the log space basis.

    If *endpoint* is True, *stop* will be the last retruned value, as long as *num* >= 2.
    """
    assert type(start) is int and type(stop) is int and type(num) is int, \
        "start, stop and num need to be intergers"
    assert num >= 2, "num has to be atleast 2"

    if log:
        start = math.log(start, base)
        stop = math.log(stop, base)

    if endpoint:
        step_length = float((stop - start)) / float(num - 1)
    else:
        step_length = float((stop - start)) / float(num)

    i = 0
    while i < num:
        if log:
            yield int(round(base ** (start + i * step_length)))
        else:
            yield int(round(start + i * step_length))
        i += 1

Return list of evenly spaced integers over an interval .


def get_last_modified_datetime(dir_path=os.path.dirname(__file__)):
    """Return datetime object of latest change in kerncraft module directory."""
    max_mtime = 0
    for root, dirs, files in os.walk(dir_path):
        for f in files:
            p = os.path.join(root, f)
            try:
                max_mtime = max(max_mtime, os.stat(p).st_mtime)
            except FileNotFoundError:
                pass
    return datetime.utcfromtimestamp(max_mtime)

def get_last_modified_datetime ( dir_path = os . path . dirname ( __file__ ) ) : max_mtime = 0 for root , dirs , files in os . walk ( dir_path ) : for f in files : p = os . path . join ( root , f ) try : max_mtime = max ( max_mtime , os . stat ( p ) . st_mtime ) except FileNotFoundError : pass return datetime . utcfromtimestamp ( max_mtime )

Return datetime object of latest change in kerncraft module directory .


def check_arguments(args, parser):
    """Check arguments passed by user that are not checked by argparse itself."""
    if args.asm_block not in ['auto', 'manual']:
        try:
            args.asm_block = int(args.asm_block)
        except ValueError:
            parser.error('--asm-block can only be "auto", "manual" or an integer')

    # Set default unit depending on performance model requested
    if not args.unit:
        if 'Roofline' in args.pmodel or 'RooflineIACA' in args.pmodel:
            args.unit = 'FLOP/s'
        else:
            args.unit = 'cy/CL'

Check arguments passed by user that are not checked by argparse itself .


def main():
    """Initialize and run command line interface."""
    # Create and populate parser
    parser = create_parser()

    # Parse given arguments
    args = parser.parse_args()

    # Checking arguments
    check_arguments(args, parser)

    # BUSINESS LOGIC IS FOLLOWING
    run(parser, args)

def main ( ) : parser = create_parser ( ) args = parser . parse_args ( ) check_arguments ( args , parser ) run ( parser , args )

Initialize and run command line interface .


def main():
    """Comand line interface of picklemerge."""
    parser = argparse.ArgumentParser(
        description='Recursively merges two or more pickle files. Only supports pickles consisting '
        'of a single dictionary object.')
    parser.add_argument('destination', type=argparse.FileType('r+b'),
                        help='File to write to and include in resulting pickle. (WILL BE CHANGED)')
    parser.add_argument('source', type=argparse.FileType('rb'), nargs='+',
                        help='File to include in resulting pickle.')

    args = parser.parse_args()

    result = pickle.load(args.destination)
    assert isinstance(result, collections.Mapping), "only Mapping types can be handled."

    for s in args.source:
        data = pickle.load(s)
        assert isinstance(data, collections.Mapping), "only Mapping types can be handled."

        update(result, data)

    args.destination.seek(0)
    args.destination.truncate()
    pickle.dump(result, args.destination)


def symbol_pos_int(*args, **kwargs):
    """Create a sympy.Symbol with positive and integer assumptions."""
    kwargs.update({'positive': True,
                   'integer': True})
    return sympy.Symbol(*args, **kwargs)

def symbol_pos_int ( * args , ** kwargs ) : kwargs . update ( { 'positive' : True , 'integer' : True } ) return sympy . Symbol ( * args , ** kwargs )

Create a sympy . Symbol with positive and integer assumptions .


def transform_multidim_to_1d_decl(decl):
    """
    Transform ast of multidimensional declaration to a single dimension declaration.

    In-place operation!

    Returns name and dimensions of array (to be used with transform_multidim_to_1d_ref())
    """
    dims = []
    type_ = decl.type
    while type(type_) is c_ast.ArrayDecl:
        dims.append(type_.dim)
        type_ = type_.type

    if dims:
        # Multidimensional array
        decl.type.dim = reduce(lambda l, r: c_ast.BinaryOp('*', l, r), dims)
        decl.type.type = type_

    return decl.name, dims

def transform_multidim_to_1d_decl ( decl ) : dims = [ ] type_ = decl . type while type ( type_ ) is c_ast . ArrayDecl : dims . append ( type_ . dim ) type_ = type_ . type if dims : decl . type . dim = reduce ( lambda l , r : c_ast . BinaryOp ( '*' , l , r ) , dims ) decl . type . type = type_ return decl . name , dims

Transform ast of multidimensional declaration to a single dimension declaration .


def transform_multidim_to_1d_ref(aref, dimension_dict):
    """
    Transform ast of multidimensional reference to a single dimension reference.

    In-place operation!
    """
    dims = []
    name = aref
    while type(name) is c_ast.ArrayRef:
        dims.append(name.subscript)
        name = name.name

    subscript_list = []
    for i, d in enumerate(dims):
        if i == 0:
            subscript_list.append(d)
        else:
            subscript_list.append(c_ast.BinaryOp('*', d, reduce(
                lambda l, r: c_ast.BinaryOp('*', l, r),
                dimension_dict[name.name][-1:-i-1:-1])))

    aref.subscript = reduce(
        lambda l, r: c_ast.BinaryOp('+', l, r), subscript_list)
    aref.name = name

def transform_multidim_to_1d_ref ( aref , dimension_dict ) : dims = [ ] name = aref while type ( name ) is c_ast . ArrayRef : dims . append ( name . subscript ) name = name . name subscript_list = [ ] for i , d in enumerate ( dims ) : if i == 0 : subscript_list . append ( d ) else : subscript_list . append ( c_ast . BinaryOp ( '*' , d , reduce ( lambda l , r : c_ast . BinaryOp ( '*' , l , r ) , dimension_dict [ name . name ] [ - 1 : - i - 1 : - 1 ] ) ) ) aref . subscript = reduce ( lambda l , r : c_ast . BinaryOp ( '+' , l , r ) , subscript_list ) aref . name = name

Transform ast of multidimensional reference to a single dimension reference .


def find_node_type(ast, node_type):
    """Return list of array references in AST."""
    if type(ast) is node_type:
        return [ast]
    elif type(ast) is list:
        return reduce(operator.add, list(map(lambda a: find_node_type(a, node_type), ast)), [])
    elif ast is None:
        return []
    else:
        return reduce(operator.add,
                      [find_node_type(o[1], node_type) for o in ast.children()], [])

def find_node_type ( ast , node_type ) : if type ( ast ) is node_type : return [ ast ] elif type ( ast ) is list : return reduce ( operator . add , list ( map ( lambda a : find_node_type ( a , node_type ) , ast ) ) , [ ] ) elif ast is None : return [ ] else : return reduce ( operator . add , [ find_node_type ( o [ 1 ] , node_type ) for o in ast . children ( ) ] , [ ] )

Return list of array references in AST .


def force_iterable(f):
    """Will make any functions return an iterable objects by wrapping its result in a list."""
    def wrapper(*args, **kwargs):
        r = f(*args, **kwargs)
        if hasattr(r, '__iter__'):
            return r
        else:
            return [r]
    return wrapper

def force_iterable ( f ) : def wrapper ( * args , ** kwargs ) : r = f ( * args , ** kwargs ) if hasattr ( r , '__iter__' ) : return r else : return [ r ] return wrapper

Will make any functions return an iterable objects by wrapping its result in a list .


def check(self):
        """Check that information about kernel makes sens and is valid."""
        datatypes = [v[0] for v in self.variables.values()]
        assert len(set(datatypes)) <= 1, 'mixing of datatypes within a kernel is not supported.'

Check that information about kernel makes sens and is valid .


def set_constant(self, name, value):
        """
        Set constant of name to value.

        :param name: may be a str or a sympy.Symbol
        :param value: must be an int
        """
        assert isinstance(name, str) or isinstance(name, sympy.Symbol), \
            "constant name needs to be of type str, unicode or a sympy.Symbol"
        assert type(value) is int, "constant value needs to be of type int"
        if isinstance(name, sympy.Symbol):
            self.constants[name] = value
        else:
            self.constants[symbol_pos_int(name)] = value


def subs_consts(self, expr):
        """Substitute constants in expression unless it is already a number."""
        if isinstance(expr, numbers.Number):
            return expr
        else:
            return expr.subs(self.constants)

def subs_consts ( self , expr ) : if isinstance ( expr , numbers . Number ) : return expr else : return expr . subs ( self . constants )

Substitute constants in expression unless it is already a number .


def array_sizes(self, in_bytes=False, subs_consts=False):
        """
        Return a dictionary with all arrays sizes.

        :param in_bytes: If True, output will be in bytes, not element counts.
        :param subs_consts: If True, output will be numbers and not symbolic.

        Scalar variables are ignored.
        """
        var_sizes = {}

        for var_name, var_info in self.variables.items():
            var_type, var_size = var_info

            # Skiping sclars
            if var_size is None:
                continue

            var_sizes[var_name] = reduce(operator.mul, var_size, 1)

            # Multiply by bytes per element if requested
            if in_bytes:
                element_size = self.datatypes_size[var_type]
                var_sizes[var_name] *= element_size

        if subs_consts:
            return {k: self.subs_consts(v) for k, v in var_sizes.items()}
        else:
            return var_sizes

def array_sizes ( self , in_bytes = False , subs_consts = False ) : var_sizes = { } for var_name , var_info in self . variables . items ( ) : var_type , var_size = var_info if var_size is None : continue var_sizes [ var_name ] = reduce ( operator . mul , var_size , 1 ) if in_bytes : element_size = self . datatypes_size [ var_type ] var_sizes [ var_name ] *= element_size if subs_consts : return { k : self . subs_consts ( v ) for k , v in var_sizes . items ( ) } else : return var_sizes

Return a dictionary with all arrays sizes .


def _calculate_relative_offset(self, name, access_dimensions):
        """
        Return the offset from the iteration center in number of elements.

        The order of indices used in access is preserved.
        """
        # TODO to be replaced with compile_global_offsets
        offset = 0
        base_dims = self.variables[name][1]

        for dim, offset_info in enumerate(access_dimensions):
            offset_type, idx_name, dim_offset = offset_info
            assert offset_type == 'rel', 'Only relative access to arrays is supported at the moment'

            if offset_type == 'rel':
                offset += self.subs_consts(
                    dim_offset*reduce(operator.mul, base_dims[dim+1:], sympy.Integer(1)))
            else:
                # should not happen
                pass

        return offset

Return the offset from the iteration center in number of elements .


def _remove_duplicate_accesses(self):
        """
        Remove duplicate source and destination accesses
        """
        self.destinations = {var_name: set(acs) for var_name, acs in self.destinations.items()}
        self.sources = {var_name: set(acs) for var_name, acs in self.sources.items()}

def _remove_duplicate_accesses ( self ) : self . destinations = { var_name : set ( acs ) for var_name , acs in self . destinations . items ( ) } self . sources = { var_name : set ( acs ) for var_name , acs in self . sources . items ( ) }


def iteration_length(self, dimension=None):
        """
        Return the number of global loop iterations that are performed.

        If dimension is not None, it is the loop dimension that is returned
        (-1 is the inner most loop and 0 the outermost)
        """
        total_length = 1

        if dimension is not None:
            loops = [self._loop_stack[dimension]]
        else:
            loops = reversed(self._loop_stack)

        for var_name, start, end, incr in loops:
            # This unspools the iterations:
            length = end-start
            total_length = total_length*length
        return self.subs_consts(total_length)

def iteration_length ( self , dimension = None ) : total_length = 1 if dimension is not None : loops = [ self . _loop_stack [ dimension ] ] else : loops = reversed ( self . _loop_stack ) for var_name , start , end , incr in loops : length = end - start total_length = total_length * length return self . subs_consts ( total_length )

Return the number of global loop iterations that are performed .


def get_loop_stack(self, subs_consts=False):
        """Yield loop stack dictionaries in order from outer to inner."""
        for l in self._loop_stack:
            if subs_consts:
                yield {'index': l[0],
                       'start': self.subs_consts(l[1]),
                       'stop': self.subs_consts(l[2]),
                       'increment': self.subs_consts(l[3])}
            else:
                yield {'index': l[0], 'start': l[1], 'stop': l[2], 'increment': l[3]}

def get_loop_stack ( self , subs_consts = False ) : for l in self . _loop_stack : if subs_consts : yield { 'index' : l [ 0 ] , 'start' : self . subs_consts ( l [ 1 ] ) , 'stop' : self . subs_consts ( l [ 2 ] ) , 'increment' : self . subs_consts ( l [ 3 ] ) } else : yield { 'index' : l [ 0 ] , 'start' : l [ 1 ] , 'stop' : l [ 2 ] , 'increment' : l [ 3 ] }

Yield loop stack dictionaries in order from outer to inner .


def index_order(self, sources=True, destinations=True):
        """
        Return the order of indices as they appear in array references.

        Use *source* and *destination* to filter output
        """
        if sources:
            arefs = chain(*self.sources.values())
        else:
            arefs = []
        if destinations:
            arefs = chain(arefs, *self.destinations.values())

        ret = []
        for a in [aref for aref in arefs if aref is not None]:
            ref = []
            for expr in a:
                ref.append(expr.free_symbols)
            ret.append(ref)

        return ret

def index_order ( self , sources = True , destinations = True ) : if sources : arefs = chain ( * self . sources . values ( ) ) else : arefs = [ ] if destinations : arefs = chain ( arefs , * self . destinations . values ( ) ) ret = [ ] for a in [ aref for aref in arefs if aref is not None ] : ref = [ ] for expr in a : ref . append ( expr . free_symbols ) ret . append ( ref ) return ret

Return the order of indices as they appear in array references .


def compile_sympy_accesses(self, sources=True, destinations=True):
        """
        Return a dictionary of lists of sympy accesses, for each variable.

        Use *source* and *destination* to filter output
        """
        sympy_accesses = defaultdict(list)
        # Compile sympy accesses
        for var_name in self.variables:
            if sources:
                for r in self.sources.get(var_name, []):
                    if r is None:
                        continue
                    sympy_accesses[var_name].append(self.access_to_sympy(var_name, r))
            if destinations:
                for w in self.destinations.get(var_name, []):
                    if w is None:
                        continue
                    sympy_accesses[var_name].append(self.access_to_sympy(var_name, w))

        return sympy_accesses

def compile_sympy_accesses ( self , sources = True , destinations = True ) : sympy_accesses = defaultdict ( list ) for var_name in self . variables : if sources : for r in self . sources . get ( var_name , [ ] ) : if r is None : continue sympy_accesses [ var_name ] . append ( self . access_to_sympy ( var_name , r ) ) if destinations : for w in self . destinations . get ( var_name , [ ] ) : if w is None : continue sympy_accesses [ var_name ] . append ( self . access_to_sympy ( var_name , w ) ) return sympy_accesses

Return a dictionary of lists of sympy accesses for each variable .


def compile_relative_distances(self, sympy_accesses=None):
        """
        Return load and store distances between accesses.

        :param sympy_accesses: optionally restrict accesses, default from compile_sympy_accesses()

        e.g. if accesses are to [+N, +1, -1, -N], relative distances are [N-1, 2, N-1]

        returned is a dict of list of sympy expressions, for each variable
        """
        if sympy_accesses is None:
            sympy_accesses = self.compile_sympy_accesses()

        sympy_distances = defaultdict(list)
        for var_name, accesses in sympy_accesses.items():
            for i in range(1, len(accesses)):
                sympy_distances[var_name].append((accesses[i-1]-accesses[i]).simplify())

        return sympy_distances

def compile_relative_distances ( self , sympy_accesses = None ) : if sympy_accesses is None : sympy_accesses = self . compile_sympy_accesses ( ) sympy_distances = defaultdict ( list ) for var_name , accesses in sympy_accesses . items ( ) : for i in range ( 1 , len ( accesses ) ) : sympy_distances [ var_name ] . append ( ( accesses [ i - 1 ] - accesses [ i ] ) . simplify ( ) ) return sympy_distances

Return load and store distances between accesses .


def global_iterator_to_indices(self, git=None):
        """
        Return sympy expressions translating global_iterator to loop indices.

        If global_iterator is given, an integer is returned
        """
        # unwind global iteration count into loop counters:
        base_loop_counters = {}
        global_iterator = symbol_pos_int('global_iterator')
        idiv = implemented_function(sympy.Function(str('idiv')), lambda x, y: x//y)
        total_length = 1
        last_incr = 1
        for var_name, start, end, incr in reversed(self._loop_stack):
            loop_var = symbol_pos_int(var_name)

            # This unspools the iterations:
            length = end-start  # FIXME is incr handled correct here?
            counter = start+(idiv(global_iterator*last_incr, total_length)*incr) % length
            total_length = total_length*length
            last_incr = incr

            base_loop_counters[loop_var] = sympy.lambdify(
                global_iterator,
                self.subs_consts(counter), modules=[numpy, {'Mod': numpy.mod}])

            if git is not None:
                try:  # Try to resolve to integer if global_iterator was given
                    base_loop_counters[loop_var] = sympy.Integer(self.subs_consts(counter))
                    continue
                except (ValueError, TypeError):
                    base_loop_counters[loop_var] = base_loop_counters[loop_var](git)

        return base_loop_counters

def global_iterator_to_indices ( self , git = None ) : base_loop_counters = { } global_iterator = symbol_pos_int ( 'global_iterator' ) idiv = implemented_function ( sympy . Function ( str ( 'idiv' ) ) , lambda x , y : x // y ) total_length = 1 last_incr = 1 for var_name , start , end , incr in reversed ( self . _loop_stack ) : loop_var = symbol_pos_int ( var_name ) length = end - start counter = start + ( idiv ( global_iterator * last_incr , total_length ) * incr ) % length total_length = total_length * length last_incr = incr base_loop_counters [ loop_var ] = sympy . lambdify ( global_iterator , self . subs_consts ( counter ) , modules = [ numpy , { 'Mod' : numpy . mod } ] ) if git is not None : try : base_loop_counters [ loop_var ] = sympy . Integer ( self . subs_consts ( counter ) ) continue except ( ValueError , TypeError ) : base_loop_counters [ loop_var ] = base_loop_counters [ loop_var ] ( git ) return base_loop_counters

Return sympy expressions translating global_iterator to loop indices .


def global_iterator(self):
        """
        Return global iterator sympy expression
        """
        global_iterator = sympy.Integer(0)
        total_length = sympy.Integer(1)
        for var_name, start, end, incr in reversed(self._loop_stack):
            loop_var = symbol_pos_int(var_name)
            length = end - start  # FIXME is incr handled correct here?
            global_iterator += (loop_var - start) * total_length
            total_length *= length
        return global_iterator

def global_iterator ( self ) : global_iterator = sympy . Integer ( 0 ) total_length = sympy . Integer ( 1 ) for var_name , start , end , incr in reversed ( self . _loop_stack ) : loop_var = symbol_pos_int ( var_name ) length = end - start global_iterator += ( loop_var - start ) * total_length total_length *= length return global_iterator


def indices_to_global_iterator(self, indices):
        """
        Transform a dictionary of indices to a global iterator integer.

        Inverse of global_iterator_to_indices().
        """
        global_iterator = self.subs_consts(self.global_iterator().subs(indices))
        return global_iterator

def indices_to_global_iterator ( self , indices ) : global_iterator = self . subs_consts ( self . global_iterator ( ) . subs ( indices ) ) return global_iterator

Transform a dictionary of indices to a global iterator integer .


def max_global_iteration(self):
        """Return global iterator with last iteration number"""
        return self.indices_to_global_iterator({
            symbol_pos_int(var_name): end-1 for var_name, start, end, incr in self._loop_stack
        })

def max_global_iteration ( self ) : return self . indices_to_global_iterator ( { symbol_pos_int ( var_name ) : end - 1 for var_name , start , end , incr in self . _loop_stack } )


def print_kernel_info(self, output_file=sys.stdout):
        """Print kernel information in human readble format."""
        table = ('     idx |        min        max       step\n' +
                 '---------+---------------------------------\n')
        for l in self._loop_stack:
            table += '{:>8} | {!r:>10} {!r:>10} {!r:>10}\n'.format(*l)
        print(prefix_indent('loop stack:        ', table), file=output_file)

        table = ('    name |  offsets   ...\n' +
                 '---------+------------...\n')
        for name, offsets in list(self.sources.items()):
            prefix = '{:>8} | '.format(name)
            right_side = '\n'.join(['{!r:}'.format(o) for o in offsets])
            table += prefix_indent(prefix, right_side, later_prefix='         | ')
        print(prefix_indent('data sources:      ', table), file=output_file)

        table = ('    name |  offsets   ...\n' +
                 '---------+------------...\n')
        for name, offsets in list(self.destinations.items()):
            prefix = '{:>8} | '.format(name)
            right_side = '\n'.join(['{!r:}'.format(o) for o in offsets])
            table += prefix_indent(prefix, right_side, later_prefix='         | ')
        print(prefix_indent('data destinations: ', table), file=output_file)

        table = (' op | count \n' +
                 '----+-------\n')
        for op, count in list(self._flops.items()):
            table += '{:>3} | {:>4}\n'.format(op, count)
        table += '     =======\n'
        table += '      {:>4}'.format(sum(self._flops.values()))
        print(prefix_indent('FLOPs:     ', table), file=output_file)

'     idx |        min        max       step\n'

Print kernel information in human readble format .


def print_variables_info(self, output_file=sys.stdout):
        """Print variables information in human readble format."""
        table = ('    name |   type size             \n' +
                 '---------+-------------------------\n')
        for name, var_info in list(self.variables.items()):
            table += '{:>8} | {:>6} {!s:<10}\n'.format(name, var_info[0], var_info[1])
        print(prefix_indent('variables: ', table), file=output_file)

def
print_variables_info
(
self
,
output_file
=
sys
.
stdout
)
:
table
=
(
'    name |   type size             \n'
+
'---------+-------------------------\n'
)
for
name
,
var_info
in
list
(
self
.
variables
.
items
(
)
)
:
table
+=
'{:>8} | {:>6} {!s:<10}\n'
.
format
(
name
,
var_info
[
0
]
,
var_info
[
1
]
)
print
(
prefix_indent
(
'variables: '
,
table
)
,
file
=
output_file
)

Print variables information in human readble format .


def print_constants_info(self, output_file=sys.stdout):
        """Print constants information in human readble format."""
        table = ('    name | value     \n' +
                 '---------+-----------\n')
        for name, value in list(self.constants.items()):
            table += '{!s:>8} | {:<10}\n'.format(name, value)
        print(prefix_indent('constants: ', table), file=output_file)

def
print_constants_info
(
self
,
output_file
=
sys
.
stdout
)
:
table
=
(
'    name | value     \n'
+
'---------+-----------\n'
)
for
name
,
value
in
list
(
self
.
constants
.
items
(
)
)
:
table
+=
'{!s:>8} | {:<10}\n'
.
format
(
name
,
value
)
print
(
prefix_indent
(
'constants: '
,
table
)
,
file
=
output_file
)

Print constants information in human readble format .


def print_kernel_code(self, output_file=sys.stdout):
        """Print source code of kernel."""
        print(self.kernel_code, file=output_file)

def print_kernel_code ( self , output_file = sys . stdout ) : print ( self . kernel_code , file = output_file )


def conv_ast_to_sym(self, math_ast):
        """
        Convert mathematical expressions to a sympy representation.

        May only contain paranthesis, addition, subtraction and multiplication from AST.
        """
        if type(math_ast) is c_ast.ID:
            return symbol_pos_int(math_ast.name)
        elif type(math_ast) is c_ast.Constant:
            return sympy.Integer(math_ast.value)
        else:  # elif type(dim) is c_ast.BinaryOp:
            op = {
                '*': operator.mul,
                '+': operator.add,
                '-': operator.sub
            }

            return op[math_ast.op](
                self.conv_ast_to_sym(math_ast.left),
                self.conv_ast_to_sym(math_ast.right))

def conv_ast_to_sym ( self , math_ast ) : if type ( math_ast ) is c_ast . ID : return symbol_pos_int ( math_ast . name ) elif type ( math_ast ) is c_ast . Constant : return sympy . Integer ( math_ast . value ) else : op = { '*' : operator . mul , '+' : operator . add , '-' : operator . sub } return op [ math_ast . op ] ( self . conv_ast_to_sym ( math_ast . left ) , self . conv_ast_to_sym ( math_ast . right ) )

Convert mathematical expressions to a sympy representation .


def _get_offsets(self, aref, dim=0):
        """
        Return a tuple of offsets of an ArrayRef object in all dimensions.

        The index order is right to left (c-code order).
        e.g. c[i+1][j-2] -> (-2, +1)

        If aref is actually a c_ast.ID, None will be returned.
        """
        if isinstance(aref, c_ast.ID):
            return None

        # Check for restrictions
        assert type(aref.name) in [c_ast.ArrayRef, c_ast.ID], \
            "array references must only be used with variables or other array references"
        assert type(aref.subscript) in [c_ast.ID, c_ast.Constant, c_ast.BinaryOp], \
            'array subscript must only contain variables or binary operations'

        # Convert subscript to sympy and append
        idxs = [self.conv_ast_to_sym(aref.subscript)]

        # Check for more indices (multi-dimensional access)
        if type(aref.name) is c_ast.ArrayRef:
            idxs += self._get_offsets(aref.name, dim=dim+1)

        # Reverse to preserver order (the subscripts in the AST are traversed backwards)
        if dim == 0:
            idxs.reverse()

        return tuple(idxs)

Return a tuple of offsets of an ArrayRef object in all dimensions .


def _get_basename(cls, aref):
        """
        Return base name of ArrayRef object.

        e.g. c[i+1][j-2] -> 'c'
        """
        if isinstance(aref.name, c_ast.ArrayRef):
            return cls._get_basename(aref.name)
        elif isinstance(aref.name, str):
            return aref.name
        else:
            return aref.name.name

def _get_basename ( cls , aref ) : if isinstance ( aref . name , c_ast . ArrayRef ) : return cls . _get_basename ( aref . name ) elif isinstance ( aref . name , str ) : return aref . name else : return aref . name . name


def get_index_type(self, loop_nest=None):
        """
        Return index type used in loop nest.

        If index type between loops differ, an exception is raised.
        """
        if loop_nest is None:
            loop_nest = self.get_kernel_loop_nest()
        if type(loop_nest) is c_ast.For:
            loop_nest = [loop_nest]
        index_types = (None, None)
        for s in loop_nest:
            if type(s) is c_ast.For:
                if type(s.stmt) in [c_ast.For, c_ast.Compound]:
                    other = self.get_index_type(loop_nest=s.stmt)
                else:
                    other = None
                index_types = (s.init.decls[0].type.type.names, other)
                break
        if index_types[0] == index_types[1] or index_types[1] is None:
            return index_types[0]
        else:
            raise ValueError("Loop indices must have same type, found {}.".format(index_types))


def _build_const_declartions(self, with_init=True):
        """
        Generate constants declarations

        :return: list of declarations
        """
        decls = []

        # Use type as provided by user in loop indices
        index_type = self.get_index_type()

        i = 2  # subscript for cli input, 1 is reserved for repeat
        for k in self.constants:
            # const long long N = strtoul(argv[2])
            # with increasing N and 1
            # TODO change subscript of argv depending on constant count
            type_decl = c_ast.TypeDecl(k.name, ['const'], c_ast.IdentifierType(index_type))
            init = None
            if with_init:
                init = c_ast.FuncCall(
                    c_ast.ID('atoi'),
                    c_ast.ExprList([c_ast.ArrayRef(c_ast.ID('argv'),
                                                   c_ast.Constant('int', str(i)))]))
            i += 1
            decls.append(c_ast.Decl(
                k.name, ['const'], [], [],
                type_decl, init, None))

        return decls

def _build_const_declartions ( self , with_init = True ) : decls = [ ] index_type = self . get_index_type ( ) i = 2 for k in self . constants : type_decl = c_ast . TypeDecl ( k . name , [ 'const' ] , c_ast . IdentifierType ( index_type ) ) init = None if with_init : init = c_ast . FuncCall ( c_ast . ID ( 'atoi' ) , c_ast . ExprList ( [ c_ast . ArrayRef ( c_ast . ID ( 'argv' ) , c_ast . Constant ( 'int' , str ( i ) ) ) ] ) ) i += 1 decls . append ( c_ast . Decl ( k . name , [ 'const' ] , [ ] , [ ] , type_decl , init , None ) ) return decls


def get_array_declarations(self):
        """Return array declarations."""
        return [d for d in self.kernel_ast.block_items
                if type(d) is c_ast.Decl and type(d.type) is c_ast.ArrayDecl]

def get_array_declarations ( self ) : return [ d for d in self . kernel_ast . block_items if type ( d ) is c_ast . Decl and type ( d . type ) is c_ast . ArrayDecl ]


def get_kernel_loop_nest(self):
        """Return kernel loop nest including any preceding pragmas and following swaps."""
        loop_nest = [s for s in self.kernel_ast.block_items
                     if type(s) in [c_ast.For, c_ast.Pragma, c_ast.FuncCall]]
        assert len(loop_nest) >= 1, "Found to few for statements in kernel"
        return loop_nest

def
get_kernel_loop_nest
(
self
)
:
loop_nest
=
[
s
for
s
in
self
.
kernel_ast
.
block_items
if
type
(
s
)
in
[
c_ast
.
For
,
c_ast
.
Pragma
,
c_ast
.
FuncCall
]
]
assert
len
(
loop_nest
)
>=
1
,
"Found to few for statements in kernel"
return
loop_nest

Return kernel loop nest including any preceding pragmas and following swaps .


def _build_array_declarations(self, with_init=True):
        """
        Generate declaration statements for arrays.

        Also transforming multi-dim to 1d arrays and initializing with malloc.

        :param with_init: ommit malloc initialization

        :return: list of declarations nodes, dictionary of array names and original dimensions
        """
        # copy array declarations from from kernel ast
        array_declarations = deepcopy(self.get_array_declarations())
        array_dict = []
        for d in array_declarations:
            # We need to transform
            array_dict.append(transform_multidim_to_1d_decl(d))
            transform_array_decl_to_malloc(d, with_init=with_init)
        return array_declarations, dict(array_dict)

def _build_array_declarations ( self , with_init = True ) : array_declarations = deepcopy ( self . get_array_declarations ( ) ) array_dict = [ ] for d in array_declarations : array_dict . append ( transform_multidim_to_1d_decl ( d ) ) transform_array_decl_to_malloc ( d , with_init = with_init ) return array_declarations , dict ( array_dict )

Generate declaration statements for arrays .


def _find_inner_most_loop(self, loop_nest):
        """Return inner most for loop in loop nest"""
        r = None
        for s in loop_nest:
            if type(s) is c_ast.For:
                return self._find_inner_most_loop(s) or s
            else:
                r = r or self._find_inner_most_loop(s)
        return r

def _find_inner_most_loop ( self , loop_nest ) : r = None for s in loop_nest : if type ( s ) is c_ast . For : return self . _find_inner_most_loop ( s ) or s else : r = r or self . _find_inner_most_loop ( s ) return r


def _build_array_initializations(self, array_dimensions):
        """
        Generate initialization statements for arrays.

        :param array_dimensions: dictionary of array dimensions

        :return: list of nodes
        """
        kernel = deepcopy(deepcopy(self.get_kernel_loop_nest()))
        # traverse to the inner most for loop:
        inner_most = self._find_inner_most_loop(kernel)
        orig_inner_stmt = inner_most.stmt
        inner_most.stmt = c_ast.Compound([])

        rand_float_str = str(random.uniform(1.0, 0.1))

        # find all array references in original orig_inner_stmt
        for aref in find_node_type(orig_inner_stmt, c_ast.ArrayRef):
            # transform to 1d references
            transform_multidim_to_1d_ref(aref, array_dimensions)
            # build static assignments and inject into inner_most.stmt
            inner_most.stmt.block_items.append(c_ast.Assignment(
                '=', aref, c_ast.Constant('float', rand_float_str)))

        return kernel

def _build_array_initializations ( self , array_dimensions ) : kernel = deepcopy ( deepcopy ( self . get_kernel_loop_nest ( ) ) ) inner_most = self . _find_inner_most_loop ( kernel ) orig_inner_stmt = inner_most . stmt inner_most . stmt = c_ast . Compound ( [ ] ) rand_float_str = str ( random . uniform ( 1.0 , 0.1 ) ) for aref in find_node_type ( orig_inner_stmt , c_ast . ArrayRef ) : transform_multidim_to_1d_ref ( aref , array_dimensions ) inner_most . stmt . block_items . append ( c_ast . Assignment ( '=' , aref , c_ast . Constant ( 'float' , rand_float_str ) ) ) return kernel

Generate initialization statements for arrays .


def _build_dummy_calls(self):
        """
        Generate false if branch with dummy calls

        Requires kerncraft.h to be included, which defines dummy(...) and var_false.

        :return: dummy statement
        """
        # Make sure nothing gets removed by inserting dummy calls
        dummy_calls = []
        for d in self.kernel_ast.block_items:
            # Only consider toplevel declarations from kernel ast
            if type(d) is not c_ast.Decl: continue
            if type(d.type) is c_ast.ArrayDecl:
                dummy_calls.append(c_ast.FuncCall(
                    c_ast.ID('dummy'),
                    c_ast.ExprList([c_ast.ID(d.name)])))
            else:
                dummy_calls.append(c_ast.FuncCall(
                    c_ast.ID('dummy'),
                    c_ast.ExprList([c_ast.UnaryOp('&', c_ast.ID(d.name))])))
        dummy_stmt = c_ast.If(
            cond=c_ast.ID('var_false'),
            iftrue=c_ast.Compound(dummy_calls),
            iffalse=None)
        return dummy_stmt

def _build_dummy_calls ( self ) : dummy_calls = [ ] for d in self . kernel_ast . block_items : if type ( d ) is not c_ast . Decl : continue if type ( d . type ) is c_ast . ArrayDecl : dummy_calls . append ( c_ast . FuncCall ( c_ast . ID ( 'dummy' ) , c_ast . ExprList ( [ c_ast . ID ( d . name ) ] ) ) ) else : dummy_calls . append ( c_ast . FuncCall ( c_ast . ID ( 'dummy' ) , c_ast . ExprList ( [ c_ast . UnaryOp ( '&' , c_ast . ID ( d . name ) ) ] ) ) ) dummy_stmt = c_ast . If ( cond = c_ast . ID ( 'var_false' ) , iftrue = c_ast . Compound ( dummy_calls ) , iffalse = None ) return dummy_stmt


def _build_kernel_function_declaration(self, name='kernel'):
        """Build and return kernel function declaration"""
        array_declarations, array_dimensions = self._build_array_declarations(with_init=False)
        scalar_declarations = self._build_scalar_declarations(with_init=False)
        const_declarations = self._build_const_declartions(with_init=False)
        return c_ast.FuncDecl(args=c_ast.ParamList(params=array_declarations + scalar_declarations +
                                                          const_declarations),
                              type=c_ast.TypeDecl(declname=name,
                                                  quals=[],
                                                  type=c_ast.IdentifierType(names=['void'])))

def _build_kernel_function_declaration ( self , name = 'kernel' ) : array_declarations , array_dimensions = self . _build_array_declarations ( with_init = False ) scalar_declarations = self . _build_scalar_declarations ( with_init = False ) const_declarations = self . _build_const_declartions ( with_init = False ) return c_ast . FuncDecl ( args = c_ast . ParamList ( params = array_declarations + scalar_declarations + const_declarations ) , type = c_ast . TypeDecl ( declname = name , quals = [ ] , type = c_ast . IdentifierType ( names = [ 'void' ] ) ) )


def _build_scalar_declarations(self, with_init=True):
        """Build and return scalar variable declarations"""
        # copy scalar declarations from from kernel ast
        scalar_declarations = [deepcopy(d) for d in self.kernel_ast.block_items
                               if type(d) is c_ast.Decl and type(d.type) is c_ast.TypeDecl]
        # add init values to declarations
        if with_init:
            random.seed(2342)  # we want reproducible random numbers
            for d in scalar_declarations:
                if d.type.type.names[0] in ['double', 'float']:
                    d.init = c_ast.Constant('float', str(random.uniform(1.0, 0.1)))
                elif d.type.type.names[0] in ['int', 'long', 'long long',
                                              'unsigned int', 'unsigned long', 'unsigned long long']:
                    d.init = c_ast.Constant('int', 2)

        return scalar_declarations

def
_build_scalar_declarations
(
self
,
with_init
=
True
)
:
scalar_declarations
=
[
deepcopy
(
d
)
for
d
in
self
.
kernel_ast
.
block_items
if
type
(
d
)
is
c_ast
.
Decl
and
type
(
d
.
type
)
is
c_ast
.
TypeDecl
]
if
with_init
:
random
.
seed
(
2342
)
for
d
in
scalar_declarations
:
if
d
.
type
.
type
.
names
[
0
]
in
[
'double'
,
'float'
]
:
d
.
init
=
c_ast
.
Constant
(
'float'
,
str
(
random
.
uniform
(
1.0
,
0.1
)
)
)
elif
d
.
type
.
type
.
names
[
0
]
in
[
'int'
,
'long'
,
'long long'
,
'unsigned int'
,
'unsigned long'
,
'unsigned long long'
]
:
d
.
init
=
c_ast
.
Constant
(
'int'
,
2
)
return
scalar_declarations


def get_kernel_code(self, openmp=False, as_filename=False, name='kernel'):
        """
        Generate and return compilable source code with kernel function from AST.

        :param openmp: if true, OpenMP code will be generated
        :param as_filename: if true, will save to file and return filename
        :param name: name of kernel function
        """
        assert self.kernel_ast is not None, "AST does not exist, this could be due to running " \
                                            "based on a kernel description rather than code."
        file_name = 'kernel'
        if openmp:
            file_name += '-omp'
        file_name += '.c'

        fp, already_available = self._get_intermediate_file(
            file_name, machine_and_compiler_dependent=False)

        # Use already cached version
        if already_available:
            code = fp.read()
        else:
            array_declarations, array_dimensions = self._build_array_declarations()

            # Prepare actual kernel loop nest
            if openmp:
                # with OpenMP code
                kernel = deepcopy(self.get_kernel_loop_nest())
                # find all array references in kernel
                for aref in find_node_type(kernel, c_ast.ArrayRef):
                    # transform to 1d references
                    transform_multidim_to_1d_ref(aref, array_dimensions)
                omp_pragmas = [p for p in find_node_type(kernel, c_ast.Pragma)
                               if 'omp' in p.string]
                # TODO if omp parallel was found, remove it (also replace "parallel for" -> "for")
                # if no omp for pragmas are present, insert suitable ones
                if not omp_pragmas:
                    kernel.insert(0, c_ast.Pragma("omp for"))
                # otherwise do not change anything
            else:
                # with original code
                kernel = deepcopy(self.get_kernel_loop_nest())
                # find all array references in kernel
                for aref in find_node_type(kernel, c_ast.ArrayRef):
                    # transform to 1d references
                    transform_multidim_to_1d_ref(aref, array_dimensions)

            function_ast = c_ast.FuncDef(decl=c_ast.Decl(
                name=name, type=self._build_kernel_function_declaration(name=name), quals=[],
                storage=[], funcspec=[], init=None, bitsize=None),
                body=c_ast.Compound(block_items=kernel),
                param_decls=None)

            # Generate code
            code = CGenerator().visit(function_ast)

            # Insert missing #includes from template to top of code
            code = '#include "kerncraft.h"\n\n' + code

            # Store to file
            fp.write(code)
        fp.close()

        if as_filename:
            return fp.name
        else:
            return code

Generate and return compilable source code with kernel function from AST .


def _build_kernel_call(self, name='kernel'):
        """Generate and return kernel call ast."""
        return c_ast.FuncCall(name=c_ast.ID(name=name), args=c_ast.ExprList(exprs=[
            c_ast.ID(name=d.name) for d in (
                    self._build_array_declarations()[0] +
                    self._build_scalar_declarations() +
                    self._build_const_declartions())]))

def _build_kernel_call ( self , name = 'kernel' ) : return c_ast . FuncCall ( name = c_ast . ID ( name = name ) , args = c_ast . ExprList ( exprs = [ c_ast . ID ( name = d . name ) for d in ( self . _build_array_declarations ( ) [ 0 ] + self . _build_scalar_declarations ( ) + self . _build_const_declartions ( ) ) ] ) )


def get_main_code(self, as_filename=False, kernel_function_name='kernel'):
        """
        Generate and return compilable source code from AST.
        """
        # TODO produce nicer code, including help text and other "comfort features".
        assert self.kernel_ast is not None, "AST does not exist, this could be due to running " \
                                            "based on a kernel description rather than code."

        fp, already_available = self._get_intermediate_file('main.c',
                                                            machine_and_compiler_dependent=False)

        # Use already cached version
        if already_available:
            code = fp.read()
        else:
            parser = CParser()
            template_code = self.CODE_TEMPLATE
            template_ast = parser.parse(clean_code(template_code,
                                                   macros=True, comments=True, pragmas=False))
            ast = deepcopy(template_ast)

            # Define and replace DECLARE_CONSTS
            replace_id(ast, "DECLARE_CONSTS", self._build_const_declartions(with_init=True))

            # Define and replace DECLARE_ARRAYS
            array_declarations, array_dimensions = self._build_array_declarations()
            replace_id(ast, "DECLARE_ARRAYS", array_declarations)

            # Define and replace DECLARE_INIT_SCALARS
            replace_id(ast, "DECLARE_INIT_SCALARS", self._build_scalar_declarations())

            # Define and replace DUMMY_CALLS
            replace_id(ast, "DUMMY_CALLS", self._build_dummy_calls())

            # Define and replace KERNEL_DECL
            ast.ext.insert(0, self._build_kernel_function_declaration(
                name=kernel_function_name))

            # Define and replace KERNEL_CALL
            replace_id(ast, "KERNEL_CALL", self._build_kernel_call())

            # Define and replace INIT_ARRAYS based on previously generated kernel
            replace_id(ast, "INIT_ARRAYS", self._build_array_initializations(array_dimensions))

            # Generate code
            code = CGenerator().visit(ast)

            # Insert missing #includes from template to top of code
            code = '\n'.join([l for l in template_code.split('\n') if l.startswith("#include")]) + \
                   '\n\n' + code

            # Store to file
            fp.write(code)
        fp.close()

        if as_filename:
            return fp.name
        else:
            return code

Generate and return compilable source code from AST .


def iaca_analysis(self, micro_architecture, asm_block='auto',
                      pointer_increment='auto_with_manual_fallback', verbose=False):
        """
        Run an IACA analysis and return its outcome.

        *asm_block* controls how the to-be-marked block is chosen. "auto" (default) results in
        the largest block, "manual" results in interactive and a number in the according block.

        *pointer_increment* is the number of bytes the pointer is incremented after the loop or
           - 'auto': automatic detection, RuntimeError is raised in case of failure
           - 'auto_with_manual_fallback': automatic detection, fallback to manual input
           - 'manual': prompt user
        """
        asm_filename = self.compile_kernel(assembly=True, verbose=verbose)
        asm_marked_filename = os.path.splitext(asm_filename)[0]+'-iaca.s'
        with open(asm_filename, 'r') as in_file, open(asm_marked_filename, 'w') as out_file:
            self.asm_block = iaca.iaca_instrumentation(
                in_file, out_file,
                block_selection=asm_block,
                pointer_increment=pointer_increment)
        obj_name = self.assemble_to_object(asm_marked_filename, verbose=verbose)
        return iaca.iaca_analyse_instrumented_binary(obj_name, micro_architecture), self.asm_block

def iaca_analysis ( self , micro_architecture , asm_block = 'auto' , pointer_increment = 'auto_with_manual_fallback' , verbose = False ) : asm_filename = self . compile_kernel ( assembly = True , verbose = verbose ) asm_marked_filename = os . path . splitext ( asm_filename ) [ 0 ] + '-iaca.s' with open ( asm_filename , 'r' ) as in_file , open ( asm_marked_filename , 'w' ) as out_file : self . asm_block = iaca . iaca_instrumentation ( in_file , out_file , block_selection = asm_block , pointer_increment = pointer_increment ) obj_name = self . assemble_to_object ( asm_marked_filename , verbose = verbose ) return iaca . iaca_analyse_instrumented_binary ( obj_name , micro_architecture ) , self . asm_block

Run an IACA analysis and return its outcome .


def build_executable(self, lflags=None, verbose=False, openmp=False):
        """Compile source to executable with likwid capabilities and return the executable name."""
        compiler, compiler_args = self._machine.get_compiler()

        kernel_obj_filename = self.compile_kernel(openmp=openmp, verbose=verbose)
        out_filename, already_exists = self._get_intermediate_file(
            os.path.splitext(os.path.basename(kernel_obj_filename))[0], binary=True, fp=False)

        if not already_exists:
            main_source_filename = self.get_main_code(as_filename=True)

            if not (('LIKWID_INCLUDE' in os.environ or 'LIKWID_INC' in os.environ) and
                    'LIKWID_LIB' in os.environ):
                print('Could not find LIKWID_INCLUDE (e.g., "-I/app/likwid/4.1.2/include") and '
                      'LIKWID_LIB (e.g., "-L/apps/likwid/4.1.2/lib") environment variables',
                      file=sys.stderr)
                sys.exit(1)

            compiler_args += [
                '-std=c99',
                '-I'+reduce_path(os.path.abspath(os.path.dirname(
                    os.path.realpath(__file__)))+'/headers/'),
                os.environ.get('LIKWID_INCLUDE', ''),
                os.environ.get('LIKWID_INC', ''),
                '-llikwid']

            # This is a special case for unittesting
            if os.environ.get('LIKWID_LIB') == '':
                compiler_args = compiler_args[:-1]

            if lflags is None:
                lflags = []
            lflags += os.environ['LIKWID_LIB'].split(' ') + ['-pthread']
            compiler_args += os.environ['LIKWID_LIB'].split(' ') + ['-pthread']

            infiles = [reduce_path(os.path.abspath(os.path.dirname(
                os.path.realpath(__file__)))+'/headers/dummy.c'),
                       kernel_obj_filename, main_source_filename]

            cmd = [compiler] + infiles + compiler_args + ['-o', out_filename]
            # remove empty arguments
            cmd = list(filter(bool, cmd))
            if verbose:
                print('Executing (build_executable): ', ' '.join(cmd))
            try:
                subprocess.check_output(cmd)
            except subprocess.CalledProcessError as e:
                print("Build failed:", e, file=sys.stderr)
                sys.exit(1)
        else:
            if verbose:
                print('Executing (build_executable): ', 'using cached', out_filename)

        return out_filename

Compile source to executable with likwid capabilities and return the executable name .


def string_to_sympy(cls, s):
        """Convert any string to a sympy object or None."""
        if isinstance(s, int):
            return sympy.Integer(s)
        elif isinstance(s, list):
            return tuple([cls.string_to_sympy(e) for e in s])
        elif s is None:
            return None
        else:
            # Step 1 build expression with the whole alphabet redefined:
            local_dict = {c: symbol_pos_int(c) for c in s if c in string.ascii_letters}
            # TODO find nicer solution for N and other pre-mapped letters
            preliminary_expr = parse_expr(s, local_dict=local_dict)
            # Replace all free symbols with positive integer versions:
            local_dict.update(
                {s.name: symbol_pos_int(s.name) for s in preliminary_expr.free_symbols})
            return parse_expr(s, local_dict=local_dict)

def string_to_sympy ( cls , s ) : if isinstance ( s , int ) : return sympy . Integer ( s ) elif isinstance ( s , list ) : return tuple ( [ cls . string_to_sympy ( e ) for e in s ] ) elif s is None : return None else : local_dict = { c : symbol_pos_int ( c ) for c in s if c in string . ascii_letters } preliminary_expr = parse_expr ( s , local_dict = local_dict ) local_dict . update ( { s . name : symbol_pos_int ( s . name ) for s in preliminary_expr . free_symbols } ) return parse_expr ( s , local_dict = local_dict )

Convert any string to a sympy object or None .


def get_identifier(self):
        """Return identifier which is either the machine file name or sha256 checksum of data."""
        if self._path:
            return os.path.basename(self._path)
        else:
            return hashlib.sha256(hashlib.sha256(repr(self._data).encode())).hexdigest()

def get_identifier ( self ) : if self . _path : return os . path . basename ( self . _path ) else : return hashlib . sha256 ( hashlib . sha256 ( repr ( self . _data ) . encode ( ) ) ) . hexdigest ( )

Return identifier which is either the machine file name or sha256 checksum of data .


def get_last_modified_datetime(self):
        """Return datetime object of modified time of machine file. Return now if not a file."""
        if self._path:
            statbuf = os.stat(self._path)
            return datetime.utcfromtimestamp(statbuf.st_mtime)
        else:
            return datetime.now()

def get_last_modified_datetime ( self ) : if self . _path : statbuf = os . stat ( self . _path ) return datetime . utcfromtimestamp ( statbuf . st_mtime ) else : return datetime . now ( )

Return datetime object of modified time of machine file . Return now if not a file .


def get_cachesim(self, cores=1):
        """
        Return a cachesim.CacheSimulator object based on the machine description.

        :param cores: core count (default: 1)
        """
        cache_dict = {}
        for c in self['memory hierarchy']:
            # Skip main memory
            if 'cache per group' not in c:
                continue
            cache_dict[c['level']] = deepcopy(c['cache per group'])
            # Scale size of shared caches according to cores
            if c['cores per group'] > 1:
                cache_dict[c['level']]['sets'] //= cores

        cs, caches, mem = cachesim.CacheSimulator.from_dict(cache_dict)

        return cs

def
get_cachesim
(
self
,
cores
=
1
)
:
cache_dict
=
{
}
for
c
in
self
[
'memory hierarchy'
]
:
if
'cache per group'
not
in
c
:
continue
cache_dict
[
c
[
'level'
]
]
=
deepcopy
(
c
[
'cache per group'
]
)
if
c
[
'cores per group'
]
>
1
:
cache_dict
[
c
[
'level'
]
]
[
'sets'
]
//=
cores
cs
,
caches
,
mem
=
cachesim
.
CacheSimulator
.
from_dict
(
cache_dict
)
return
cs

Return a cachesim . CacheSimulator object based on the machine description .


def get_bandwidth(self, cache_level, read_streams, write_streams, threads_per_core, cores=None):
        """
        Return best fitting bandwidth according to number of threads, read and write streams.

        :param cache_level: integer of cache (0 is L1, 1 is L2 ...)
        :param read_streams: number of read streams expected
        :param write_streams: number of write streams expected
        :param threads_per_core: number of threads that are run on each core
        :param cores: if not given, will choose maximum bandwidth for single NUMA domain
        """
        # try to find best fitting kernel (closest to read/write ratio):
        # write allocate has to be handled in kernel information (all writes are also reads)
        # TODO support for non-write-allocate architectures
        try:
            target_ratio = read_streams/write_streams
        except ZeroDivisionError:
            target_ratio = float('inf')
        measurement_kernel = 'load'
        measurement_kernel_info = self['benchmarks']['kernels'][measurement_kernel]
        measurement_kernel_ratio = float('inf')
        for kernel_name, kernel_info in sorted(self['benchmarks']['kernels'].items()):
            try:
                kernel_ratio = ((kernel_info['read streams']['streams'] +
                                 kernel_info['write streams']['streams'] -
                                 kernel_info['read+write streams']['streams']) /
                                kernel_info['write streams']['streams'])
            except ZeroDivisionError:
                kernel_ratio = float('inf')

            if abs(kernel_ratio - target_ratio) < abs(measurement_kernel_ratio - target_ratio):
                measurement_kernel = kernel_name
                measurement_kernel_info = kernel_info
                measurement_kernel_ratio = kernel_ratio

        # choose smt, and then use max/saturation bw
        bw_level = self['memory hierarchy'][cache_level]['level']
        bw_measurements = \
            self['benchmarks']['measurements'][bw_level][threads_per_core]
        assert threads_per_core == bw_measurements['threads per core'], \
            'malformed measurement dictionary in machine file.'
        if cores is not None:
            # Used by Roofline model
            run_index = bw_measurements['cores'].index(cores)
            bw = bw_measurements['results'][measurement_kernel][run_index]
        else:
            # Used by ECM model
            # Choose maximum number of cores to get bandwidth for
            max_cores = min(self['memory hierarchy'][cache_level]['cores per group'],
                            self['cores per NUMA domain'])
            bw = max(bw_measurements['results'][measurement_kernel][:max_cores])

        # Correct bandwidth due to miss-measurement of write allocation
        # TODO support non-temporal stores and non-write-allocate architectures
        if cache_level == 0:
            # L1 does not have write-allocate, so everything is measured correctly
            factor = 1.0
        else:
            factor = (float(measurement_kernel_info['read streams']['bytes']) +
                      2.0*float(measurement_kernel_info['write streams']['bytes']) -
                      float(measurement_kernel_info['read+write streams']['bytes'])) / \
                     (float(measurement_kernel_info['read streams']['bytes']) +
                      float(measurement_kernel_info['write streams']['bytes']))
        bw = bw * factor

        return bw, measurement_kernel

Return best fitting bandwidth according to number of threads read and write streams .


def get_compiler(self, compiler=None, flags=None):
        """
        Return tuple of compiler and compiler flags.

        Selects compiler and flags from machine description file, commandline arguments or call
        arguements.
        """
        if self._args:
            compiler = compiler or self._args.compiler
            flags = flags or self._args.compiler_flags
        if compiler is None:
            # Select first available compiler in machine description file's compiler dict
            for c in self['compiler'].keys():
                # Making sure compiler is available:
                if find_executable(c) is not None:
                    compiler = c
                    break
            else:
                raise RuntimeError("No compiler ({}) was found. Add different one in machine file, "
                                   "via --compiler argument or make sure it will be found in "
                                   "$PATH.".format(list(self['compiler'].keys())))
        if flags is None:
            # Select from machine description file
            flags = self['compiler'].get(compiler, '')

        return compiler, flags.split(' ')

Return tuple of compiler and compiler flags .


def parse_perfctr_event(perfctr):
        """
        Parse events in machine description to tuple representation used in Benchmark module.

        Examples:
        >>> parse_perfctr_event('PERF_EVENT:REG[0-3]')
        ('PERF_EVENT', 'REG[0-3]')
        >>> parse_perfctr_event('PERF_EVENT:REG[0-3]:STAY:FOO=23:BAR=0x23')
        ('PERF_EVENT', 'REG[0-3]', {'STAY': None, 'FOO': 23, 'BAR': 35})

        """
        split_perfctr = perfctr.split(':')
        assert len(split_perfctr) >= 2, "Atleast one colon (:) is required in the event name"
        event_tuple = split_perfctr[:2]
        parameters = {}
        for p in split_perfctr[2:]:
            if '=' in p:
                k, v = p.split('=')
                if v.startswith('0x'):
                    parameters[k] = int(v, 16)
                else:
                    parameters[k] = int(v)
            else:
                parameters[p] = None
        event_tuple.append(parameters)
        return tuple(event_tuple)

Parse events in machine description to tuple representation used in Benchmark module .


def _enforce_no_overlap(self, start_at=0):
        """Enforce that no ranges overlap in internal storage."""
        i = start_at
        while i+1 < len(self.data):
            if self.data[i][1] >= self.data[i+1][0]:
                # beginning of i+1-th range is contained in i-th range
                if self.data[i][1] < self.data[i+1][1]:
                    # i+1-th range is longer, thus enlarge i-th range
                    self.data[i][1] = self.data[i+1][1]

                # removed contained range
                del self.data[i+1]
            i += 1

def _enforce_no_overlap ( self , start_at = 0 ) : i = start_at while i + 1 < len ( self . data ) : if self . data [ i ] [ 1 ] >= self . data [ i + 1 ] [ 0 ] : if self . data [ i ] [ 1 ] < self . data [ i + 1 ] [ 1 ] : self . data [ i ] [ 1 ] = self . data [ i + 1 ] [ 1 ] del self . data [ i + 1 ] i += 1

Enforce that no ranges overlap in internal storage .


def get_header_path() -> str:
    """Return local folder path of header files."""
    import os
    return os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/headers/'

def get_header_path ( ) -> str : import os return os . path . abspath ( os . path . dirname ( os . path . realpath ( __file__ ) ) ) + '/headers/'

Return local folder path of header files .


def _align_iteration_with_cl_boundary(self, iteration, subtract=True):
        """Align iteration with cacheline boundary."""
        # FIXME handle multiple datatypes
        element_size = self.kernel.datatypes_size[self.kernel.datatype]
        cacheline_size = self.machine['cacheline size']
        elements_per_cacheline = int(cacheline_size // element_size)

        # Gathering some loop information:
        inner_loop = list(self.kernel.get_loop_stack(subs_consts=True))[-1]
        inner_increment = inner_loop['increment']

        # do this by aligning either writes (preferred) or reads
        # Assumption: writes (and reads) increase linearly
        o = self.kernel.compile_global_offsets(iteration=iteration)[0]
        if len(o[1]):
            # we have a write to work with:
            first_offset = min(o[1])
        else:
            # we use reads
            first_offset = min(o[0])

        diff = first_offset - \
               (int(first_offset) >> self.csim.first_level.cl_bits << self.csim.first_level.cl_bits)
        if diff == 0:
            return iteration
        elif subtract:
            return iteration - (diff // element_size) // inner_increment
        else:
            return iteration + (elements_per_cacheline - diff // element_size) // inner_increment

def
_align_iteration_with_cl_boundary
(
self
,
iteration
,
subtract
=
True
)
:
element_size
=
self
.
kernel
.
datatypes_size
[
self
.
kernel
.
datatype
]
cacheline_size
=
self
.
machine
[
'cacheline size'
]
elements_per_cacheline
=
int
(
cacheline_size
//
element_size
)
inner_loop
=
list
(
self
.
kernel
.
get_loop_stack
(
subs_consts
=
True
)
)
[
1
]
inner_increment
=
inner_loop
[
'increment'
]
o
=
self
.
kernel
.
compile_global_offsets
(
iteration
=
iteration
)
[
0
]
if
len
(
o
[
1
]
)
:
first_offset
=
min
(
o
[
1
]
)
else
:
first_offset
=
min
(
o
[
0
]
)
diff
=
first_offset
(
int
(
first_offset
)
>>
self
.
csim
.
first_level
.
cl_bits
<<
self
.
csim
.
first_level
.
cl_bits
)
if
diff
==
0
:
return
iteration
elif
subtract
:
return
iteration
(
diff
//
element_size
)
//
inner_increment
else
:
return
iteration
+
(
elements_per_cacheline
diff
//
element_size
)
//
inner_increment

Align iteration with cacheline boundary .


def get_loads(self):
        """Return a list with number of loaded cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['LOAD_count'] / self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_loads
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'LOAD_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of loaded cache lines per memory hierarchy level .


def get_hits(self):
        """Return a list with number of hit cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['HIT_count']/self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_hits
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'HIT_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of hit cache lines per memory hierarchy level .


def get_misses(self):
        """Return a list with number of missed cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['MISS_count']/self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_misses
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'MISS_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of missed cache lines per memory hierarchy level .


def get_stores(self):
        """Return a list with number of stored cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['STORE_count']/self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_stores
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'STORE_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of stored cache lines per memory hierarchy level .


def get_evicts(self):
        """Return a list with number of evicted cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['EVICT_count']/self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_evicts
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'EVICT_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of evicted cache lines per memory hierarchy level .


def get_infos(self):
        """Return verbose information about the predictor."""
        first_dim_factor = self.first_dim_factor
        infos = {'memory hierarchy': [], 'cache stats': self.stats,
                 'cachelines in stats': first_dim_factor}
        for cache_level, cache_info in list(enumerate(self.machine['memory hierarchy'])):
            infos['memory hierarchy'].append({
                'index': len(infos['memory hierarchy']),
                'level': '{}'.format(cache_info['level']),
                'total loads': self.stats[cache_level]['LOAD_byte']/first_dim_factor,
                'total misses': self.stats[cache_level]['MISS_byte']/first_dim_factor,
                'total hits': self.stats[cache_level]['HIT_byte']/first_dim_factor,
                'total stores': self.stats[cache_level]['STORE_byte']/first_dim_factor,
                'total evicts': self.stats[cache_level]['EVICT_byte']/first_dim_factor,
                'total lines load': self.stats[cache_level]['LOAD_count']/first_dim_factor,
                'total lines misses': self.stats[cache_level]['MISS_count']/first_dim_factor,
                'total lines hits': self.stats[cache_level]['HIT_count']/first_dim_factor,
                'total lines stores': self.stats[cache_level]['STORE_count']/first_dim_factor,
                'total lines evicts': self.stats[cache_level]['EVICT_count']/first_dim_factor,
                'cycles': None})
        return infos

def
get_infos
(
self
)
:
first_dim_factor
=
self
.
first_dim_factor
infos
=
{
'memory hierarchy'
:
[
]
,
'cache stats'
:
self
.
stats
,
'cachelines in stats'
:
first_dim_factor
}
for
cache_level
,
cache_info
in
list
(
enumerate
(
self
.
machine
[
'memory hierarchy'
]
)
)
:
infos
[
'memory hierarchy'
]
.
append
(
{
'index'
:
len
(
infos
[
'memory hierarchy'
]
)
,
'level'
:
'{}'
.
format
(
cache_info
[
'level'
]
)
,
'total loads'
:
self
.
stats
[
cache_level
]
[
'LOAD_byte'
]
/
first_dim_factor
,
'total misses'
:
self
.
stats
[
cache_level
]
[
'MISS_byte'
]
/
first_dim_factor
,
'total hits'
:
self
.
stats
[
cache_level
]
[
'HIT_byte'
]
/
first_dim_factor
,
'total stores'
:
self
.
stats
[
cache_level
]
[
'STORE_byte'
]
/
first_dim_factor
,
'total evicts'
:
self
.
stats
[
cache_level
]
[
'EVICT_byte'
]
/
first_dim_factor
,
'total lines load'
:
self
.
stats
[
cache_level
]
[
'LOAD_count'
]
/
first_dim_factor
,
'total lines misses'
:
self
.
stats
[
cache_level
]
[
'MISS_count'
]
/
first_dim_factor
,
'total lines hits'
:
self
.
stats
[
cache_level
]
[
'HIT_count'
]
/
first_dim_factor
,
'total lines stores'
:
self
.
stats
[
cache_level
]
[
'STORE_count'
]
/
first_dim_factor
,
'total lines evicts'
:
self
.
stats
[
cache_level
]
[
'EVICT_count'
]
/
first_dim_factor
,
'cycles'
:
}
)
return
infos

Return verbose information about the predictor .