def _base_opration(self, method):
        """ Base method for storage operations.
        """
        uuids = self.uuids()

        while True:
            chunk = list(islice(uuids, 0, self.chunk_size))

            if not chunk:
                return

            rest_request(method, self.storage_url, chunk)

def _base_opration ( self , method ) : uuids = self . uuids ( ) while True : chunk = list ( islice ( uuids , 0 , self . chunk_size ) ) if not chunk : return rest_request ( method , self . storage_url , chunk )


def uuids(self):
        """ Extract uuid from each item of specified ``seq``.
        """
        for f in self._seq:
            if isinstance(f, File):
                yield f.uuid
            elif isinstance(f, six.string_types):
                yield f
            else:
                raise ValueError(
                    'Invalid type for sequence item: {0}'.format(type(f)))

def
uuids
(
self
)
:
for
f
in
self
.
_seq
:
if
isinstance
(
f
,
File
)
:
yield
f
.
uuid
elif
isinstance
(
f
,
six
.
string_types
)
:
yield
f
else
:
raise
ValueError
(
'Invalid type for sequence item: {0}'
.
format
(
type
(
f
)
)
)

Extract uuid from each item of specified seq .


def _list(api_list_class, arg_namespace, **extra):
    """ A common function for building methods of the "list showing".
    """
    if arg_namespace.starting_point:
        ordering_field = (arg_namespace.ordering or '').lstrip('-')
        if ordering_field in ('', 'datetime_uploaded', 'datetime_created'):
            arg_namespace.starting_point = parser.parse(
                arg_namespace.starting_point)

    items = api_list_class(
        starting_point=arg_namespace.starting_point,
        ordering=arg_namespace.ordering,
        limit=arg_namespace.limit,
        request_limit=arg_namespace.request_limit,
        **extra
    )
    items.constructor = lambda x: x

    try:
        pprint(list(items))
    except ValueError as e:
        print(e)

def _list ( api_list_class , arg_namespace , ** extra ) : if arg_namespace . starting_point : ordering_field = ( arg_namespace . ordering or '' ) . lstrip ( '-' ) if ordering_field in ( '' , 'datetime_uploaded' , 'datetime_created' ) : arg_namespace . starting_point = parser . parse ( arg_namespace . starting_point ) items = api_list_class ( starting_point = arg_namespace . starting_point , ordering = arg_namespace . ordering , limit = arg_namespace . limit , request_limit = arg_namespace . request_limit , ** extra ) items . constructor = lambda x : x try : pprint ( list ( items ) ) except ValueError as e : print ( e )

A common function for building methods of the list showing .


def bar(iter_content, parts, title=''):
    """ Iterates over the "iter_content" and draws a progress bar to stdout.
    """
    parts = max(float(parts), 1.0)
    cells = 10
    progress = 0
    step = cells / parts

    draw = lambda progress: sys.stdout.write(
        '\r[{0:10}] {1:.2f}% {2}'.format(
            '#'*int(progress), progress * cells, title))

    for chunk in iter_content:
        yield chunk

        progress += step
        draw(progress)
        sys.stdout.flush()

    draw(cells)
    print('')

def
bar
(
iter_content
,
parts
,
title
=
''
)
:
parts
=
max
(
float
(
parts
)
,
1.0
)
cells
=
10
progress
=
0
step
=
cells
/
parts
draw
=
lambda
progress
:
sys
.
stdout
.
write
(
'\r[{0:10}] {1:.2f}% {2}'
.
format
(
'#'
*
int
(
progress
)
,
progress
*
cells
,
title
)
)
for
chunk
in
iter_content
:
yield
chunk
progress
+=
step
draw
(
progress
)
sys
.
stdout
.
flush
(
)
draw
(
cells
)
print
(
''
)

Iterates over the iter_content and draws a progress bar to stdout .


def uploading_request(verb, path, data=None, files=None, timeout=conf.DEFAULT):
    """Makes Uploading API request and returns response as ``dict``.

    It takes settings from ``conf`` module.

    Make sure that given ``path`` does not contain leading slash.

    Usage example::

        >>> file_obj = open('photo.jpg', 'rb')
        >>> uploading_request('POST', 'base/', files={'file': file_obj})
        {
            'file': '9b9f4483-77b8-40ae-a198-272ba6280004'
        }
        >>> File('9b9f4483-77b8-40ae-a198-272ba6280004')

    """
    path = path.lstrip('/')
    url = urljoin(conf.upload_base, path)

    if data is None:
        data = {}
    data['pub_key'] = conf.pub_key
    data['UPLOADCARE_PUB_KEY'] = conf.pub_key

    headers = {
        'User-Agent': _build_user_agent(),
    }

    try:
        response = session.request(
            str(verb), url, allow_redirects=True,
            verify=conf.verify_upload_ssl, data=data, files=files,
            headers=headers, timeout=_get_timeout(timeout),
        )
    except requests.RequestException as exc:
        raise APIConnectionError(exc.args[0])

    # No content.
    if response.status_code == 204:
        return {}

    if 200 <= response.status_code < 300:
        if _content_type_from_response(response).endswith(('/json', '+json')):
            try:
                return response.json()
            except ValueError as exc:
                raise APIError(exc.args[0])

    if response.status_code in (400, 404):
        raise InvalidRequestError(response.content)

    # Not json or unknown status code.
    raise APIError(response.content)

def uploading_request ( verb , path , data = None , files = None , timeout = conf . DEFAULT ) : path = path . lstrip ( '/' ) url = urljoin ( conf . upload_base , path ) if data is None : data = { } data [ 'pub_key' ] = conf . pub_key data [ 'UPLOADCARE_PUB_KEY' ] = conf . pub_key headers = { 'User-Agent' : _build_user_agent ( ) , } try : response = session . request ( str ( verb ) , url , allow_redirects = True , verify = conf . verify_upload_ssl , data = data , files = files , headers = headers , timeout = _get_timeout ( timeout ) , ) except requests . RequestException as exc : raise APIConnectionError ( exc . args [ 0 ] ) if response . status_code == 204 : return { } if 200 <= response . status_code < 300 : if _content_type_from_response ( response ) . endswith ( ( '/json' , '+json' ) ) : try : return response . json ( ) except ValueError as exc : raise APIError ( exc . args [ 0 ] ) if response . status_code in ( 400 , 404 ) : raise InvalidRequestError ( response . content ) raise APIError ( response . content )

Makes Uploading API request and returns response as dict .


def home_mode_status(self, **kwargs):
        """Returns the status of Home Mode"""
        api = self._api_info['home_mode']
        payload = dict({
            'api': api['name'],
            'method': 'GetInfo',
            'version': api['version'],
            '_sid': self._sid
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        return response['data']['on']

def home_mode_status ( self , ** kwargs ) : api = self . _api_info [ 'home_mode' ] payload = dict ( { 'api' : api [ 'name' ] , 'method' : 'GetInfo' , 'version' : api [ 'version' ] , '_sid' : self . _sid } , ** kwargs ) response = self . _get_json_with_retry ( api [ 'url' ] , payload ) return response [ 'data' ] [ 'on' ]


def camera_list(self, **kwargs):
        """Return a list of cameras."""
        api = self._api_info['camera']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'List',
            'version': api['version'],
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        cameras = []

        for data in response['data']['cameras']:
            cameras.append(Camera(data, self._video_stream_url))

        return cameras

def camera_list ( self , ** kwargs ) : api = self . _api_info [ 'camera' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'List' , 'version' : api [ 'version' ] , } , ** kwargs ) response = self . _get_json_with_retry ( api [ 'url' ] , payload ) cameras = [ ] for data in response [ 'data' ] [ 'cameras' ] : cameras . append ( Camera ( data , self . _video_stream_url ) ) return cameras


def camera_info(self, camera_ids, **kwargs):
        """Return a list of cameras matching camera_ids."""
        api = self._api_info['camera']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'GetInfo',
            'version': api['version'],
            'cameraIds': ', '.join(str(id) for id in camera_ids),
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        cameras = []

        for data in response['data']['cameras']:
            cameras.append(Camera(data, self._video_stream_url))

        return cameras

def
camera_info
(
self
,
camera_ids
,
**
kwargs
)
:
api
=
self
.
_api_info
[
'camera'
]
payload
=
dict
(
{
'_sid'
:
self
.
_sid
,
'api'
:
api
[
'name'
]
,
'method'
:
'GetInfo'
,
'version'
:
api
[
'version'
]
,
'cameraIds'
:
', '
.
join
(
str
(
id
)
for
id
in
camera_ids
)
,
}
,
**
kwargs
)
response
=
self
.
_get_json_with_retry
(
api
[
'url'
]
,
payload
)
cameras
=
[
]
for
data
in
response
[
'data'
]
[
'cameras'
]
:
cameras
.
append
(
Camera
(
data
,
self
.
_video_stream_url
)
)
return
cameras

Return a list of cameras matching camera_ids .


def camera_snapshot(self, camera_id, **kwargs):
        """Return bytes of camera image."""
        api = self._api_info['camera']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'GetSnapshot',
            'version': api['version'],
            'cameraId': camera_id,
        }, **kwargs)
        response = self._get(api['url'], payload)

        return response.content

def camera_snapshot ( self , camera_id , ** kwargs ) : api = self . _api_info [ 'camera' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'GetSnapshot' , 'version' : api [ 'version' ] , 'cameraId' : camera_id , } , ** kwargs ) response = self . _get ( api [ 'url' ] , payload ) return response . content


def camera_disable(self, camera_id, **kwargs):
        """Disable camera."""
        api = self._api_info['camera']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'Disable',
            'version': 9,
            'idList': camera_id,
        }, **kwargs)
        print(api['url'])
        print(payload)
        response = self._get(api['url'], payload)

        return response['success']

def camera_disable ( self , camera_id , ** kwargs ) : api = self . _api_info [ 'camera' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'Disable' , 'version' : 9 , 'idList' : camera_id , } , ** kwargs ) print ( api [ 'url' ] ) print ( payload ) response = self . _get ( api [ 'url' ] , payload ) return response [ 'success' ]


def camera_event_motion_enum(self, camera_id, **kwargs):
        """Return motion settings matching camera_id."""
        api = self._api_info['camera_event']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'MotionEnum',
            'version': api['version'],
            'camId': camera_id,
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        return MotionSetting(camera_id, response['data']['MDParam'])

def camera_event_motion_enum ( self , camera_id , ** kwargs ) : api = self . _api_info [ 'camera_event' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'MotionEnum' , 'version' : api [ 'version' ] , 'camId' : camera_id , } , ** kwargs ) response = self . _get_json_with_retry ( api [ 'url' ] , payload ) return MotionSetting ( camera_id , response [ 'data' ] [ 'MDParam' ] )

Return motion settings matching camera_id .


def camera_event_md_param_save(self, camera_id, **kwargs):
        """Update motion settings matching camera_id with keyword args."""
        api = self._api_info['camera_event']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'MDParamSave',
            'version': api['version'],
            'camId': camera_id,
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        return response['data']['camId']

def camera_event_md_param_save ( self , camera_id , ** kwargs ) : api = self . _api_info [ 'camera_event' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'MDParamSave' , 'version' : api [ 'version' ] , 'camId' : camera_id , } , ** kwargs ) response = self . _get_json_with_retry ( api [ 'url' ] , payload ) return response [ 'data' ] [ 'camId' ]

Update motion settings matching camera_id with keyword args .


def update(self):
        """Update cameras and motion settings with latest from API."""
        cameras = self._api.camera_list()
        self._cameras_by_id = {v.camera_id: v for i, v in enumerate(cameras)}

        motion_settings = []
        for camera_id in self._cameras_by_id.keys():
            motion_setting = self._api.camera_event_motion_enum(camera_id)
            motion_settings.append(motion_setting)

        self._motion_settings_by_id = {
            v.camera_id: v for i, v in enumerate(motion_settings)}

def update ( self ) : cameras = self . _api . camera_list ( ) self . _cameras_by_id = { v . camera_id : v for i , v in enumerate ( cameras ) } motion_settings = [ ] for camera_id in self . _cameras_by_id . keys ( ) : motion_setting = self . _api . camera_event_motion_enum ( camera_id ) motion_settings . append ( motion_setting ) self . _motion_settings_by_id = { v . camera_id : v for i , v in enumerate ( motion_settings ) }

Update cameras and motion settings with latest from API .


def is_last_li(li, meta_data, current_numId):
    """
    Determine if ``li`` is the last list item for a given list
    """
    if not is_li(li, meta_data):
        return False
    w_namespace = get_namespace(li, 'w')
    next_el = li
    while True:
        # If we run out of element this must be the last list item
        if next_el is None:
            return True

        next_el = next_el.getnext()
        # Ignore elements that are not a list item
        if not is_li(next_el, meta_data):
            continue

        new_numId = get_numId(next_el, w_namespace)
        if current_numId != new_numId:
            return True
        # If we have gotten here then we have found another list item in the
        # current list, so ``li`` is not the last li in the list.
        return False

def is_last_li ( li , meta_data , current_numId ) : if not is_li ( li , meta_data ) : return False w_namespace = get_namespace ( li , 'w' ) next_el = li while True : if next_el is None : return True next_el = next_el . getnext ( ) if not is_li ( next_el , meta_data ) : continue new_numId = get_numId ( next_el , w_namespace ) if current_numId != new_numId : return True return False

Determine if li is the last list item for a given list


def get_single_list_nodes_data(li, meta_data):
    """
    Find consecutive li tags that have content that have the same list id.
    """
    yield li
    w_namespace = get_namespace(li, 'w')
    current_numId = get_numId(li, w_namespace)
    starting_ilvl = get_ilvl(li, w_namespace)
    el = li
    while True:
        el = el.getnext()
        if el is None:
            break
        # If the tag has no content ignore it.
        if not has_text(el):
            continue

        # Stop the lists if you come across a list item that should be a
        # heading.
        if _is_top_level_upper_roman(el, meta_data):
            break

        if (
                is_li(el, meta_data) and
                (starting_ilvl > get_ilvl(el, w_namespace))):
            break

        new_numId = get_numId(el, w_namespace)
        if new_numId is None or new_numId == -1:
            # Not a p tag or a list item
            yield el
            continue
        # If the list id of the next tag is different that the previous that
        # means a new list being made (not nested)
        if current_numId != new_numId:
            # Not a subsequent list.
            break
        if is_last_li(el, meta_data, current_numId):
            yield el
            break
        yield el

def get_single_list_nodes_data ( li , meta_data ) : yield li w_namespace = get_namespace ( li , 'w' ) current_numId = get_numId ( li , w_namespace ) starting_ilvl = get_ilvl ( li , w_namespace ) el = li while True : el = el . getnext ( ) if el is None : break if not has_text ( el ) : continue if _is_top_level_upper_roman ( el , meta_data ) : break if ( is_li ( el , meta_data ) and ( starting_ilvl > get_ilvl ( el , w_namespace ) ) ) : break new_numId = get_numId ( el , w_namespace ) if new_numId is None or new_numId == - 1 : yield el continue if current_numId != new_numId : break if is_last_li ( el , meta_data , current_numId ) : yield el break yield el

Find consecutive li tags that have content that have the same list id .


def get_ilvl(li, w_namespace):
    """
    The ilvl on an li tag tells the li tag at what level of indentation this
    tag is at. This is used to determine if the li tag needs to be nested or
    not.
    """
    ilvls = li.xpath('.//w:ilvl', namespaces=li.nsmap)
    if len(ilvls) == 0:
        return -1
    return int(ilvls[0].get('%sval' % w_namespace))

def get_ilvl ( li , w_namespace ) : ilvls = li . xpath ( './/w:ilvl' , namespaces = li . nsmap ) if len ( ilvls ) == 0 : return - 1 return int ( ilvls [ 0 ] . get ( '%sval' % w_namespace ) )

The ilvl on an li tag tells the li tag at what level of indentation this tag is at . This is used to determine if the li tag needs to be nested or not .


def get_v_merge(tc):
    """
    vMerge is what docx uses to denote that a table cell is part of a rowspan.
    The first cell to have a vMerge is the start of the rowspan, and the vMerge
    will be denoted with 'restart'. If it is anything other than restart then
    it is a continuation of another rowspan.
    """
    if tc is None:
        return None
    v_merges = tc.xpath('.//w:vMerge', namespaces=tc.nsmap)
    if len(v_merges) != 1:
        return None
    v_merge = v_merges[0]
    return v_merge

def get_v_merge ( tc ) : if tc is None : return None v_merges = tc . xpath ( './/w:vMerge' , namespaces = tc . nsmap ) if len ( v_merges ) != 1 : return None v_merge = v_merges [ 0 ] return v_merge

vMerge is what docx uses to denote that a table cell is part of a rowspan . The first cell to have a vMerge is the start of the rowspan and the vMerge will be denoted with restart . If it is anything other than restart then it is a continuation of another rowspan .


def get_grid_span(tc):
    """
    gridSpan is what docx uses to denote that a table cell has a colspan. This
    is much more simple than rowspans in that there is a one-to-one mapping
    from gridSpan to colspan.
    """
    w_namespace = get_namespace(tc, 'w')
    grid_spans = tc.xpath('.//w:gridSpan', namespaces=tc.nsmap)
    if len(grid_spans) != 1:
        return 1
    grid_span = grid_spans[0]
    return int(grid_span.get('%sval' % w_namespace))

def get_grid_span ( tc ) : w_namespace = get_namespace ( tc , 'w' ) grid_spans = tc . xpath ( './/w:gridSpan' , namespaces = tc . nsmap ) if len ( grid_spans ) != 1 : return 1 grid_span = grid_spans [ 0 ] return int ( grid_span . get ( '%sval' % w_namespace ) )

gridSpan is what docx uses to denote that a table cell has a colspan . This is much more simple than rowspans in that there is a one - to - one mapping from gridSpan to colspan .


def get_td_at_index(tr, index):
    """
    When calculating the rowspan for a given cell it is required to find all
    table cells 'below' the initial cell with a v_merge. This function will
    return the td element at the passed in index, taking into account colspans.
    """
    current = 0
    for td in tr.xpath('.//w:tc', namespaces=tr.nsmap):
        if index == current:
            return td
        current += get_grid_span(td)

def get_td_at_index ( tr , index ) : current = 0 for td in tr . xpath ( './/w:tc' , namespaces = tr . nsmap ) : if index == current : return td current += get_grid_span ( td )

When calculating the rowspan for a given cell it is required to find all table cells below the initial cell with a v_merge . This function will return the td element at the passed in index taking into account colspans .


def style_is_false(style):
    """
    For bold, italics and underline. Simply checking to see if the various tags
    are present will not suffice. If the tag is present and set to False then
    the style should not be present.
    """
    if style is None:
        return False
    w_namespace = get_namespace(style, 'w')
    return style.get('%sval' % w_namespace) != 'false'

def style_is_false ( style ) : if style is None : return False w_namespace = get_namespace ( style , 'w' ) return style . get ( '%sval' % w_namespace ) != 'false'

For bold italics and underline . Simply checking to see if the various tags are present will not suffice . If the tag is present and set to False then the style should not be present .


def is_bold(r):
    """
    The function will return True if the r tag passed in is considered bold.
    """
    w_namespace = get_namespace(r, 'w')
    rpr = r.find('%srPr' % w_namespace)
    if rpr is None:
        return False
    bold = rpr.find('%sb' % w_namespace)
    return style_is_false(bold)

def is_bold ( r ) : w_namespace = get_namespace ( r , 'w' ) rpr = r . find ( '%srPr' % w_namespace ) if rpr is None : return False bold = rpr . find ( '%sb' % w_namespace ) return style_is_false ( bold )

The function will return True if the r tag passed in is considered bold .


def is_italics(r):
    """
    The function will return True if the r tag passed in is considered
    italicized.
    """
    w_namespace = get_namespace(r, 'w')
    rpr = r.find('%srPr' % w_namespace)
    if rpr is None:
        return False
    italics = rpr.find('%si' % w_namespace)
    return style_is_false(italics)

def is_italics ( r ) : w_namespace = get_namespace ( r , 'w' ) rpr = r . find ( '%srPr' % w_namespace ) if rpr is None : return False italics = rpr . find ( '%si' % w_namespace ) return style_is_false ( italics )

The function will return True if the r tag passed in is considered italicized .


def is_underlined(r):
    """
    The function will return True if the r tag passed in is considered
    underlined.
    """
    w_namespace = get_namespace(r, 'w')
    rpr = r.find('%srPr' % w_namespace)
    if rpr is None:
        return False
    underline = rpr.find('%su' % w_namespace)
    return style_is_false(underline)

def is_underlined ( r ) : w_namespace = get_namespace ( r , 'w' ) rpr = r . find ( '%srPr' % w_namespace ) if rpr is None : return False underline = rpr . find ( '%su' % w_namespace ) return style_is_false ( underline )

The function will return True if the r tag passed in is considered underlined .


def is_title(p):
    """
    Certain p tags are denoted as ``Title`` tags. This function will return
    True if the passed in p tag is considered a title.
    """
    w_namespace = get_namespace(p, 'w')
    styles = p.xpath('.//w:pStyle', namespaces=p.nsmap)
    if len(styles) == 0:
        return False
    style = styles[0]
    return style.get('%sval' % w_namespace) == 'Title'

def is_title ( p ) : w_namespace = get_namespace ( p , 'w' ) styles = p . xpath ( './/w:pStyle' , namespaces = p . nsmap ) if len ( styles ) == 0 : return False style = styles [ 0 ] return style . get ( '%sval' % w_namespace ) == 'Title'

Certain p tags are denoted as Title tags . This function will return True if the passed in p tag is considered a title .


def get_text_run_content_data(r):
    """
    It turns out that r tags can contain both t tags and drawing tags. Since we
    need both, this function will return them in the order in which they are
    found.
    """
    w_namespace = get_namespace(r, 'w')
    valid_elements = (
        '%st' % w_namespace,
        '%sdrawing' % w_namespace,
        '%spict' % w_namespace,
        '%sbr' % w_namespace,
    )
    for el in r:
        if el.tag in valid_elements:
            yield el

def get_text_run_content_data ( r ) : w_namespace = get_namespace ( r , 'w' ) valid_elements = ( '%st' % w_namespace , '%sdrawing' % w_namespace , '%spict' % w_namespace , '%sbr' % w_namespace , ) for el in r : if el . tag in valid_elements : yield el

It turns out that r tags can contain both t tags and drawing tags . Since we need both this function will return them in the order in which they are found .


def get_relationship_info(tree, media, image_sizes):
    """
    There is a separate file holds the targets to links as well as the targets
    for images. Return a dictionary based on the relationship id and the
    target.
    """
    if tree is None:
        return {}
    result = {}
    # Loop through each relationship.
    for el in tree.iter():
        el_id = el.get('Id')
        if el_id is None:
            continue
        # Store the target in the result dict.
        target = el.get('Target')
        if any(
                target.lower().endswith(ext) for
                ext in IMAGE_EXTENSIONS_TO_SKIP):
            continue
        if target in media:
            image_size = image_sizes.get(el_id)
            target = convert_image(media[target], image_size)
        # cgi will replace things like & < > with & < >
        result[el_id] = cgi.escape(target)

    return result

def get_relationship_info ( tree , media , image_sizes ) : if tree is None : return { } result = { } for el in tree . iter ( ) : el_id = el . get ( 'Id' ) if el_id is None : continue target = el . get ( 'Target' ) if any ( target . lower ( ) . endswith ( ext ) for ext in IMAGE_EXTENSIONS_TO_SKIP ) : continue if target in media : image_size = image_sizes . get ( el_id ) target = convert_image ( media [ target ] , image_size ) result [ el_id ] = cgi . escape ( target ) return result

There is a separate file holds the targets to links as well as the targets for images . Return a dictionary based on the relationship id and the target .


def _get_document_data(f, image_handler=None):
    '''
    ``f`` is a ``ZipFile`` that is open
    Extract out the document data, numbering data and the relationship data.
    '''
    if image_handler is None:
        def image_handler(image_id, relationship_dict):
            return relationship_dict.get(image_id)

    document_xml = None
    numbering_xml = None
    relationship_xml = None
    styles_xml = None
    parser = etree.XMLParser(strip_cdata=False)
    path, _ = os.path.split(f.filename)
    media = {}
    image_sizes = {}
    # Loop through the files in the zip file.
    for item in f.infolist():
        # This file holds all the content of the document.
        if item.filename == 'word/document.xml':
            xml = f.read(item.filename)
            document_xml = etree.fromstring(xml, parser)
        # This file tells document.xml how lists should look.
        elif item.filename == 'word/numbering.xml':
            xml = f.read(item.filename)
            numbering_xml = etree.fromstring(xml, parser)
        elif item.filename == 'word/styles.xml':
            xml = f.read(item.filename)
            styles_xml = etree.fromstring(xml, parser)
        # This file holds the targets for hyperlinks and images.
        elif item.filename == 'word/_rels/document.xml.rels':
            xml = f.read(item.filename)
            try:
                relationship_xml = etree.fromstring(xml, parser)
            except XMLSyntaxError:
                relationship_xml = etree.fromstring('<xml></xml>', parser)
        if item.filename.startswith('word/media/'):
            # Strip off the leading word/
            media[item.filename[len('word/'):]] = f.extract(
                item.filename,
                path,
            )
    # Close the file pointer.
    f.close()

    # Get dictionaries for the numbering and the relationships.
    numbering_dict = get_numbering_info(numbering_xml)
    image_sizes = get_image_sizes(document_xml)
    relationship_dict = get_relationship_info(
        relationship_xml,
        media,
        image_sizes
    )
    styles_dict = get_style_dict(styles_xml)
    font_sizes_dict = defaultdict(int)
    if DETECT_FONT_SIZE:
        font_sizes_dict = get_font_sizes_dict(document_xml, styles_dict)
    meta_data = MetaData(
        numbering_dict=numbering_dict,
        relationship_dict=relationship_dict,
        styles_dict=styles_dict,
        font_sizes_dict=font_sizes_dict,
        image_handler=image_handler,
        image_sizes=image_sizes,
    )
    return document_xml, meta_data

def _get_document_data ( f , image_handler = None ) : if image_handler is None : def image_handler ( image_id , relationship_dict ) : return relationship_dict . get ( image_id ) document_xml = None numbering_xml = None relationship_xml = None styles_xml = None parser = etree . XMLParser ( strip_cdata = False ) path , _ = os . path . split ( f . filename ) media = { } image_sizes = { } for item in f . infolist ( ) : if item . filename == 'word/document.xml' : xml = f . read ( item . filename ) document_xml = etree . fromstring ( xml , parser ) elif item . filename == 'word/numbering.xml' : xml = f . read ( item . filename ) numbering_xml = etree . fromstring ( xml , parser ) elif item . filename == 'word/styles.xml' : xml = f . read ( item . filename ) styles_xml = etree . fromstring ( xml , parser ) elif item . filename == 'word/_rels/document.xml.rels' : xml = f . read ( item . filename ) try : relationship_xml = etree . fromstring ( xml , parser ) except XMLSyntaxError : relationship_xml = etree . fromstring ( '<xml></xml>' , parser ) if item . filename . startswith ( 'word/media/' ) : media [ item . filename [ len ( 'word/' ) : ] ] = f . extract ( item . filename , path , ) f . close ( ) numbering_dict = get_numbering_info ( numbering_xml ) image_sizes = get_image_sizes ( document_xml ) relationship_dict = get_relationship_info ( relationship_xml , media , image_sizes ) styles_dict = get_style_dict ( styles_xml ) font_sizes_dict = defaultdict ( int ) if DETECT_FONT_SIZE : font_sizes_dict = get_font_sizes_dict ( document_xml , styles_dict ) meta_data = MetaData ( numbering_dict = numbering_dict , relationship_dict = relationship_dict , styles_dict = styles_dict , font_sizes_dict = font_sizes_dict , image_handler = image_handler , image_sizes = image_sizes , ) return document_xml , meta_data

f is a ZipFile that is open Extract out the document data numbering data and the relationship data .


def get_ordered_list_type(meta_data, numId, ilvl):
    """
    Return the list type. If numId or ilvl not in the numbering dict then
    default to returning decimal.

    This function only cares about ordered lists, unordered lists get dealt
    with elsewhere.
    """

    # Early return if numId or ilvl are not valid
    numbering_dict = meta_data.numbering_dict
    if numId not in numbering_dict:
        return DEFAULT_LIST_NUMBERING_STYLE
    if ilvl not in numbering_dict[numId]:
        return DEFAULT_LIST_NUMBERING_STYLE
    return meta_data.numbering_dict[numId][ilvl]

def get_ordered_list_type ( meta_data , numId , ilvl ) : numbering_dict = meta_data . numbering_dict if numId not in numbering_dict : return DEFAULT_LIST_NUMBERING_STYLE if ilvl not in numbering_dict [ numId ] : return DEFAULT_LIST_NUMBERING_STYLE return meta_data . numbering_dict [ numId ] [ ilvl ]

Return the list type . If numId or ilvl not in the numbering dict then default to returning decimal .


def build_list(li_nodes, meta_data):
    """
    Build the list structure and return the root list
    """
    # Need to keep track of all incomplete nested lists.
    ol_dict = {}

    # Need to keep track of the current indentation level.
    current_ilvl = -1

    # Need to keep track of the current list id.
    current_numId = -1

    # Need to keep track of list that new li tags should be added too.
    current_ol = None

    # Store the first list created (the root list) for the return value.
    root_ol = None
    visited_nodes = []
    list_contents = []

    def _build_li(list_contents):
        data = '
'.join(t for t in list_contents if t is not None)
        return etree.XML('
%s
' % data)

    def _build_non_li_content(el, meta_data):
        w_namespace = get_namespace(el, 'w')
        if el.tag == '%stbl' % w_namespace:
            new_el, visited_nodes = build_table(el, meta_data)
            return etree.tostring(new_el), visited_nodes
        elif el.tag == '%sp' % w_namespace:
            return get_element_content(el, meta_data), [el]
        if has_text(el):
            raise UnintendedTag('Did not expect %s' % el.tag)

    def _merge_lists(ilvl, current_ilvl, ol_dict, current_ol):
        for i in reversed(range(ilvl, current_ilvl)):
            # Any list that is more indented that ilvl needs to
            # be merged to the list before it.
            if i not in ol_dict:
                continue
            if ol_dict[i] is not current_ol:
                if ol_dict[i] is current_ol:
                    continue
                ol_dict[i][-1].append(current_ol)
                current_ol = ol_dict[i]

        # Clean up finished nested lists.
        for key in list(ol_dict):
            if key > ilvl:
                del ol_dict[key]
        return current_ol

    for li_node in li_nodes:
        w_namespace = get_namespace(li_node, 'w')
        if not is_li(li_node, meta_data):
            # Get the content and visited nodes
            new_el, el_visited_nodes = _build_non_li_content(
                li_node,
                meta_data,
            )
            list_contents.append(new_el)
            visited_nodes.extend(el_visited_nodes)
            continue
        if list_contents:
            li_el = _build_li(list_contents)
            list_contents = []
            current_ol.append(li_el)
        # Get the data needed to build the current list item
        list_contents.append(get_element_content(
            li_node,
            meta_data,
        ))
        ilvl = get_ilvl(li_node, w_namespace)
        numId = get_numId(li_node, w_namespace)
        list_type = get_ordered_list_type(meta_data, numId, ilvl)

        # If the ilvl is greater than the current_ilvl or the list id is
        # changing then we have the first li tag in a nested list. We need to
        # create a new list object and update all of our variables for keeping
        # track.
        if (ilvl > current_ilvl) or (numId != current_numId):
            # Only create a new list
            ol_dict[ilvl] = create_list(list_type)
            current_ol = ol_dict[ilvl]
            current_ilvl = ilvl
            current_numId = numId
        # Both cases above are not True then we need to close all lists greater
        # than ilvl and then remove them from the ol_dict
        else:
            # Merge any nested lists that need to be merged.
            current_ol = _merge_lists(
                ilvl=ilvl,
                current_ilvl=current_ilvl,
                ol_dict=ol_dict,
                current_ol=current_ol,
            )

        # Set the root list after the first list is created.
        if root_ol is None:
            root_ol = current_ol

        # Set the current list.
        if ilvl in ol_dict:
            current_ol = ol_dict[ilvl]
        else:
            # In some instances the ilvl is not in the ol_dict, if that is the
            # case, create it here (not sure how this happens but it has
            # before.) Only do this if the current_ol is not the root_ol,
            # otherwise etree will crash.

            if current_ol is not root_ol:

                # Merge the current_ol into the root_ol. _merge_lists is not
                # equipped to handle this situation since the only way to get
                # into this block of code is to have mangled ilvls.
                root_ol[-1].append(current_ol)

                # Reset the current_ol
                current_ol = create_list(list_type)

        # Create the li element.
        visited_nodes.extend(list(li_node.iter()))

    # If a list item is the last thing in a document, then you will need to add
    # it here. Should probably figure out how to get the above logic to deal
    # with it.
    if list_contents:
        li_el = _build_li(list_contents)
        list_contents = []
        current_ol.append(li_el)

    # Merge up any nested lists that have not been merged.
    current_ol = _merge_lists(
        ilvl=0,
        current_ilvl=current_ilvl,
        ol_dict=ol_dict,
        current_ol=current_ol,
    )

    return root_ol, visited_nodes

def
build_list
(
li_nodes
,
meta_data
)
:
ol_dict
=
{
}
current_ilvl
=
1
current_numId
=
1
current_ol
=
root_ol
=
visited_nodes
=
[
]
list_contents
=
[
]
def
_build_li
(
list_contents
)
:
data
=
'
'
.
join
(
t
for
t
in
list_contents
if
t
is
not
)
return
etree
.
XML
(
'
%s
'
%
data
)
def
_build_non_li_content
(
el
,
meta_data
)
:
w_namespace
=
get_namespace
(
el
,
'w'
)
if
el
.
tag
==
'%stbl'
%
w_namespace
:
new_el
,
visited_nodes
=
build_table
(
el
,
meta_data
)
return
etree
.
tostring
(
new_el
)
,
visited_nodes
elif
el
.
tag
==
'%sp'
%
w_namespace
:
return
get_element_content
(
el
,
meta_data
)
,
[
el
]
if
has_text
(
el
)
:
raise
UnintendedTag
(
'Did not expect %s'
%
el
.
tag
)
def
_merge_lists
(
ilvl
,
current_ilvl
,
ol_dict
,
current_ol
)
:
for
i
in
reversed
(
range
(
ilvl
,
current_ilvl
)
)
:
if
i
not
in
ol_dict
:
continue
if
ol_dict
[
i
]
is
not
current_ol
:
if
ol_dict
[
i
]
is
current_ol
:
continue
ol_dict
[
i
]
[
1
]
.
append
(
current_ol
)
current_ol
=
ol_dict
[
i
]
for
key
in
list
(
ol_dict
)
:
if
key
>
ilvl
:
del
ol_dict
[
key
]
return
current_ol
for
li_node
in
li_nodes
:
w_namespace
=
get_namespace
(
li_node
,
'w'
)
if
not
is_li
(
li_node
,
meta_data
)
:
new_el
,
el_visited_nodes
=
_build_non_li_content
(
li_node
,
meta_data
,
)
list_contents
.
append
(
new_el
)
visited_nodes
.
extend
(
el_visited_nodes
)
continue
if
list_contents
:
li_el
=
_build_li
(
list_contents
)
list_contents
=
[
]
current_ol
.
append
(
li_el
)
list_contents
.
append
(
get_element_content
(
li_node
,
meta_data
,
)
)
ilvl
=
get_ilvl
(
li_node
,
w_namespace
)
numId
=
get_numId
(
li_node
,
w_namespace
)
list_type
=
get_ordered_list_type
(
meta_data
,
numId
,
ilvl
)
if
(
ilvl
>
current_ilvl
)
or
(
numId
!=
current_numId
)
:
ol_dict
[
ilvl
]
=
create_list
(
list_type
)
current_ol
=
ol_dict
[
ilvl
]
current_ilvl
=
ilvl
current_numId
=
numId
else
:
current_ol
=
_merge_lists
(
ilvl
=
ilvl
,
current_ilvl
=
current_ilvl
,
ol_dict
=
ol_dict
,
current_ol
=
current_ol
,
)
if
root_ol
is
:
root_ol
=
current_ol
if
ilvl
in
ol_dict
:
current_ol
=
ol_dict
[
ilvl
]
else
:
if
current_ol
is
not
root_ol
:
root_ol
[
1
]
.
append
(
current_ol
)
current_ol
=
create_list
(
list_type
)
visited_nodes
.
extend
(
list
(
li_node
.
iter
(
)
)
)
if
list_contents
:
li_el
=
_build_li
(
list_contents
)
list_contents
=
[
]
current_ol
.
append
(
li_el
)
current_ol
=
_merge_lists
(
ilvl
=
0
,
current_ilvl
=
current_ilvl
,
ol_dict
=
ol_dict
,
current_ol
=
current_ol
,
)
return
root_ol
,
visited_nodes


def build_tr(tr, meta_data, row_spans):
    """
    This will return a single tr element, with all tds already populated.
    """

    # Create a blank tr element.
    tr_el = etree.Element('tr')
    w_namespace = get_namespace(tr, 'w')
    visited_nodes = []
    for el in tr:
        if el in visited_nodes:
            continue
        visited_nodes.append(el)
        # Find the table cells.
        if el.tag == '%stc' % w_namespace:
            v_merge = get_v_merge(el)
            # If there is a v_merge and it is not restart then this cell can be
            # ignored.
            if (
                    v_merge is not None and
                    v_merge.get('%sval' % w_namespace) != 'restart'):
                continue

            # Loop through each and build a list of all the content.
            texts = []
            for td_content in el:
                # Since we are doing look-a-heads in this loop we need to check
                # again to see if we have already visited the node.
                if td_content in visited_nodes:
                    continue

                # Check to see if it is a list or a regular paragraph.
                if is_li(td_content, meta_data):
                    # If it is a list, create the list and update
                    # visited_nodes.
                    li_nodes = get_single_list_nodes_data(
                        td_content,
                        meta_data,
                    )
                    list_el, list_visited_nodes = build_list(
                        li_nodes,
                        meta_data,
                    )
                    visited_nodes.extend(list_visited_nodes)
                    texts.append(etree.tostring(list_el))
                elif td_content.tag == '%stbl' % w_namespace:
                    table_el, table_visited_nodes = build_table(
                        td_content,
                        meta_data,
                    )
                    visited_nodes.extend(table_visited_nodes)
                    texts.append(etree.tostring(table_el))
                elif td_content.tag == '%stcPr' % w_namespace:
                    # Do nothing
                    visited_nodes.append(td_content)
                    continue
                else:
                    text = get_element_content(
                        td_content,
                        meta_data,
                        is_td=True,
                    )
                    texts.append(text)

            data = '
'.join(t for t in texts if t is not None)
            td_el = etree.XML('%s	' % data)
            # if there is a colspan then set it here.
            colspan = get_grid_span(el)
            if colspan > 1:
                td_el.set('colspan', '%d' % colspan)
            v_merge = get_v_merge(el)

            # If this td has a v_merge and it is restart then set the rowspan
            # here.
            if (
                    v_merge is not None and
                    v_merge.get('%sval' % w_namespace) == 'restart'):
                rowspan = next(row_spans)
                td_el.set('rowspan', '%d' % rowspan)

            tr_el.append(td_el)
    return tr_el

def
build_tr
(
tr
,
meta_data
,
row_spans
)
:
tr_el
=
etree
.
Element
(
'tr'
)
w_namespace
=
get_namespace
(
tr
,
'w'
)
visited_nodes
=
[
]
for
el
in
tr
:
if
el
in
visited_nodes
:
continue
visited_nodes
.
append
(
el
)
if
el
.
tag
==
'%stc'
%
w_namespace
:
v_merge
=
get_v_merge
(
el
)
if
(
v_merge
is
not
and
v_merge
.
get
(
'%sval'
%
w_namespace
)
!=
'restart'
)
:
continue
texts
=
[
]
for
td_content
in
el
:
if
td_content
in
visited_nodes
:
continue
if
is_li
(
td_content
,
meta_data
)
:
li_nodes
=
get_single_list_nodes_data
(
td_content
,
meta_data
,
)
list_el
,
list_visited_nodes
=
build_list
(
li_nodes
,
meta_data
,
)
visited_nodes
.
extend
(
list_visited_nodes
)
texts
.
append
(
etree
.
tostring
(
list_el
)
)
elif
td_content
.
tag
==
'%stbl'
%
w_namespace
:
table_el
,
table_visited_nodes
=
build_table
(
td_content
,
meta_data
,
)
visited_nodes
.
extend
(
table_visited_nodes
)
texts
.
append
(
etree
.
tostring
(
table_el
)
)
elif
td_content
.
tag
==
'%stcPr'
%
w_namespace
:
visited_nodes
.
append
(
td_content
)
continue
else
:
text
=
get_element_content
(
td_content
,
meta_data
,
is_td
=
True
,
)
texts
.
append
(
text
)
data
=
'
'
.
join
(
t
for
t
in
texts
if
t
is
not
)
td_el
=
etree
.
XML
(
'%s	'
%
data
)
colspan
=
get_grid_span
(
el
)
if
colspan
>
1
:
td_el
.
set
(
'colspan'
,
'%d'
%
colspan
)
v_merge
=
get_v_merge
(
el
)
if
(
v_merge
is
not
and
v_merge
.
get
(
'%sval'
%
w_namespace
)
==
'restart'
)
:
rowspan
=
next
(
row_spans
)
td_el
.
set
(
'rowspan'
,
'%d'
%
rowspan
)
tr_el
.
append
(
td_el
)
return
tr_el

This will return a single tr element with all tds already populated .


def build_table(table, meta_data):
    """
    This returns a table object with all rows and cells correctly populated.
    """

    # Create a blank table element.
    table_el = etree.Element('table')
    w_namespace = get_namespace(table, 'w')

    # Get the rowspan values for cells that have a rowspan.
    row_spans = get_rowspan_data(table)
    for el in table:
        if el.tag == '%str' % w_namespace:
            # Create the tr element.
            tr_el = build_tr(
                el,
                meta_data,
                row_spans,
            )
            # And append it to the table.
            table_el.append(tr_el)

    visited_nodes = list(table.iter())
    return table_el, visited_nodes

def build_table ( table , meta_data ) : table_el = etree . Element ( 'table' ) w_namespace = get_namespace ( table , 'w' ) row_spans = get_rowspan_data ( table ) for el in table : if el . tag == '%str' % w_namespace : tr_el = build_tr ( el , meta_data , row_spans , ) table_el . append ( tr_el ) visited_nodes = list ( table . iter ( ) ) return table_el , visited_nodes

This returns a table object with all rows and cells correctly populated .


def get_t_tag_content(
        t, parent, remove_bold, remove_italics, meta_data):
    """
    Generate the string data that for this particular t tag.
    """
    if t is None or t.text is None:
        return ''

    # Need to escape the text so that we do not accidentally put in text
    # that is not valid XML.
    # cgi will replace things like & < > with & < >
    text = cgi.escape(t.text)

    # Wrap the text with any modifiers it might have (bold, italics or
    # underline)
    el_is_bold = not remove_bold and (
        is_bold(parent) or
        is_underlined(parent)
    )
    el_is_italics = not remove_italics and is_italics(parent)
    if el_is_bold:
        text = '%s' % text
    if el_is_italics:
        text = '%s' % text
    return text

def get_t_tag_content ( t , parent , remove_bold , remove_italics , meta_data ) : if t is None or t . text is None : return '' text = cgi . escape ( t . text ) el_is_bold = not remove_bold and ( is_bold ( parent ) or is_underlined ( parent ) ) el_is_italics = not remove_italics and is_italics ( parent ) if el_is_bold : text = '%s' % text if el_is_italics : text = '%s' % text return text

Generate the string data that for this particular t tag .


def _strip_tag(tree, tag):
    """
    Remove all tags that have the tag name ``tag``
    """
    for el in tree.iter():
        if el.tag == tag:
            el.getparent().remove(el)

def _strip_tag ( tree , tag ) : for el in tree . iter ( ) : if el . tag == tag : el . getparent ( ) . remove ( el )

Remove all tags that have the tag name tag


def find(dataset, url):
    '''Find the location of a dataset on disk, downloading if needed.'''
    fn = os.path.join(DATASETS, dataset)
    dn = os.path.dirname(fn)
    if not os.path.exists(dn):
        print('creating dataset directory: %s', dn)
        os.makedirs(dn)
    if not os.path.exists(fn):
        if sys.version_info < (3, ):
            urllib.urlretrieve(url, fn)
        else:
            urllib.request.urlretrieve(url, fn)
    return fn

def
find
(
dataset
,
url
)
:
fn
=
os
.
path
.
join
(
DATASETS
,
dataset
)
dn
=
os
.
path
.
dirname
(
fn
)
if
not
os
.
path
.
exists
(
dn
)
:
print
(
'creating dataset directory: %s'
,
dn
)
os
.
makedirs
(
dn
)
if
not
os
.
path
.
exists
(
fn
)
:
if
sys
.
version_info
<
(
3
,
)
:
urllib
.
urlretrieve
(
url
,
fn
)
else
:
urllib
.
request
.
urlretrieve
(
url
,
fn
)
return
fn

Find the location of a dataset on disk downloading if needed .


def load_mnist(flatten=True, labels=False):
    '''Load the MNIST digits dataset.'''
    fn = find('mnist.pkl.gz', 'http://deeplearning.net/data/mnist/mnist.pkl.gz')
    h = gzip.open(fn, 'rb')
    if sys.version_info < (3, ):
        (timg, tlab), (vimg, vlab), (simg, slab) = pickle.load(h)
    else:
        (timg, tlab), (vimg, vlab), (simg, slab) = pickle.load(h, encoding='bytes')
    h.close()
    if not flatten:
        timg = timg.reshape((-1, 28, 28, 1))
        vimg = vimg.reshape((-1, 28, 28, 1))
        simg = simg.reshape((-1, 28, 28, 1))
    if labels:
        return ((timg, tlab.astype('i')),
                (vimg, vlab.astype('i')),
                (simg, slab.astype('i')))
    return (timg, ), (vimg, ), (simg, )


def load_cifar(flatten=True, labels=False):
    '''Load the CIFAR10 image dataset.'''
    def extract(name):
        print('extracting data from {}'.format(name))
        h = tar.extractfile(name)
        if sys.version_info < (3, ):
            d = pickle.load(h)
        else:
            d = pickle.load(h, encoding='bytes')
            for k in list(d):
                d[k.decode('utf8')] = d[k]
        h.close()
        img = d['data'].reshape(
            (-1, 3, 32, 32)).transpose((0, 2, 3, 1)).astype('f') / 128 - 1
        if flatten:
            img = img.reshape((-1, 32 * 32 * 3))
        d['data'] = img
        return d

    fn = find('cifar10.tar.gz', 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz')
    tar = tarfile.open(fn)

    imgs = []
    labs = []
    for i in range(1, 6):
        d = extract('cifar-10-batches-py/data_batch_{}'.format(i))
        imgs.extend(d['data'])
        labs.extend(d['labels'])
    timg = np.asarray(imgs[:40000])
    tlab = np.asarray(labs[:40000], 'i')
    vimg = np.asarray(imgs[40000:])
    vlab = np.asarray(labs[40000:], 'i')

    d = extract('cifar-10-batches-py/test_batch')
    simg = d['data']
    slab = d['labels']

    tar.close()

    if labels:
        return (timg, tlab), (vimg, vlab), (simg, slab)
    return (timg, ), (vimg, ), (simg, )


def plot_images(imgs, loc, title=None, channels=1):
    '''Plot an array of images.

    We assume that we are given a matrix of data whose shape is (n*n, s*s*c) --
    that is, there are n^2 images along the first axis of the array, and each
    image is c squares measuring s pixels on a side. Each row of the input will
    be plotted as a sub-region within a single image array containing an n x n
    grid of images.
    '''
    n = int(np.sqrt(len(imgs)))
    assert n * n == len(imgs), 'images array must contain a square number of rows!'
    s = int(np.sqrt(len(imgs[0]) / channels))
    assert s * s == len(imgs[0]) / channels, 'images must be square!'

    img = np.zeros(((s+1) * n - 1, (s+1) * n - 1, channels), dtype=imgs[0].dtype)
    for i, pix in enumerate(imgs):
        r, c = divmod(i, n)
        img[r * (s+1):(r+1) * (s+1) - 1,
            c * (s+1):(c+1) * (s+1) - 1] = pix.reshape((s, s, channels))

    img -= img.min()
    img /= img.max()

    ax = plt.gcf().add_subplot(loc)
    ax.xaxis.set_visible(False)
    ax.yaxis.set_visible(False)
    ax.set_frame_on(False)
    ax.imshow(img.squeeze(), cmap=plt.cm.gray)
    if title:
        ax.set_title(title)


def plot_layers(weights, tied_weights=False, channels=1):
    '''Create a plot of weights, visualized as "bottom-level" pixel arrays.'''
    if hasattr(weights[0], 'get_value'):
        weights = [w.get_value() for w in weights]
    k = min(len(weights), 9)
    imgs = np.eye(weights[0].shape[0])
    for i, weight in enumerate(weights[:-1]):
        imgs = np.dot(weight.T, imgs)
        plot_images(imgs,
                    100 + 10 * k + i + 1,
                    channels=channels,
                    title='Layer {}'.format(i+1))
    weight = weights[-1]
    n = weight.shape[1] / channels
    if int(np.sqrt(n)) ** 2 != n:
        return
    if tied_weights:
        imgs = np.dot(weight.T, imgs)
        plot_images(imgs,
                    100 + 10 * k + k,
                    channels=channels,
                    title='Layer {}'.format(k))
    else:
        plot_images(weight,
                    100 + 10 * k + k,
                    channels=channels,
                    title='Decoding weights')

def
plot_layers
(
weights
,
tied_weights
=
False
,
channels
=
1
)
:
if
hasattr
(
weights
[
0
]
,
'get_value'
)
:
weights
=
[
w
.
get_value
(
)
for
w
in
weights
]
k
=
min
(
len
(
weights
)
,
9
)
imgs
=
np
.
eye
(
weights
[
0
]
.
shape
[
0
]
)
for
i
,
weight
in
enumerate
(
weights
[
:
1
]
)
:
imgs
=
np
.
dot
(
weight
.
T
,
imgs
)
plot_images
(
imgs
,
100
+
10
*
k
+
i
+
1
,
channels
=
channels
,
title
=
'Layer {}'
.
format
(
i
+
1
)
)
weight
=
weights
[
1
]
n
=
weight
.
shape
[
1
]
/
channels
if
int
(
np
.
sqrt
(
n
)
)
**
2
!=
n
:
return
if
tied_weights
:
imgs
=
np
.
dot
(
weight
.
T
,
imgs
)
plot_images
(
imgs
,
100
+
10
*
k
+
k
,
channels
=
channels
,
title
=
'Layer {}'
.
format
(
k
)
)
else
:
plot_images
(
weight
,
100
+
10
*
k
+
k
,
channels
=
channels
,
title
=
'Decoding weights'
)

Create a plot of weights visualized as bottom - level pixel arrays .


def plot_filters(filters):
    '''Create a plot of conv filters, visualized as pixel arrays.'''
    imgs = filters.get_value()

    N, channels, x, y = imgs.shape
    n = int(np.sqrt(N))
    assert n * n == N, 'filters must contain a square number of rows!'
    assert channels == 1 or channels == 3, 'can only plot grayscale or rgb filters!'

    img = np.zeros(((y+1) * n - 1, (x+1) * n - 1, channels), dtype=imgs[0].dtype)
    for i, pix in enumerate(imgs):
        r, c = divmod(i, n)
        img[r * (y+1):(r+1) * (y+1) - 1,
            c * (x+1):(c+1) * (x+1) - 1] = pix.transpose((1, 2, 0))

    img -= img.min()
    img /= img.max()

    ax = plt.gcf().add_subplot(111)
    ax.xaxis.set_visible(False)
    ax.yaxis.set_visible(False)
    ax.set_frame_on(False)
    ax.imshow(img.squeeze(), cmap=plt.cm.gray)

Create a plot of conv filters visualized as pixel arrays .


def batches(arrays, steps=100, batch_size=64, rng=None):
    '''Create a callable that generates samples from a dataset.

    Parameters
    ----------
    arrays : list of ndarray (time-steps, data-dimensions)
        Arrays of data. Rows in these arrays are assumed to correspond to time
        steps, and columns to variables. Multiple arrays can be given; in such
        a case, these arrays usually correspond to [input, output]---for
        example, for a recurrent regression problem---or [input, output,
        weights]---for a weighted regression or classification problem.
    steps : int, optional
        Generate samples of this many time steps. Defaults to 100.
    batch_size : int, optional
        Generate this many samples per call. Defaults to 64. This must match the
        batch_size parameter that was used when creating the recurrent network
        that will process the data.
    rng : :class:`numpy.random.RandomState` or int, optional
        A random number generator, or an integer seed for a random number
        generator. If not provided, the random number generator will be created
        with an automatically chosen seed.

    Returns
    -------
    callable :
        A callable that can be used inside a dataset for training a recurrent
        network.
    '''
    assert batch_size >= 2, 'batch_size must be at least 2!'
    assert isinstance(arrays, (tuple, list)), 'arrays must be a tuple or list!'

    if rng is None or isinstance(rng, int):
        rng = np.random.RandomState(rng)

    def sample():
        xs = [np.zeros((batch_size, steps, a.shape[1]), a.dtype) for a in arrays]
        for i in range(batch_size):
            j = rng.randint(len(arrays[0]) - steps)
            for x, a in zip(xs, arrays):
                x[i] = a[j:j+steps]
        return xs

    return sample

def
batches
(
arrays
,
steps
=
100
,
batch_size
=
64
,
rng
=
)
:
assert
batch_size
>=
2
,
'batch_size must be at least 2!'
assert
isinstance
(
arrays
,
(
tuple
,
list
)
)
,
'arrays must be a tuple or list!'
if
rng
is
or
isinstance
(
rng
,
int
)
:
rng
=
np
.
random
.
RandomState
(
rng
)
def
sample
(
)
:
xs
=
[
np
.
zeros
(
(
batch_size
,
steps
,
a
.
shape
[
1
]
)
,
a
.
dtype
)
for
a
in
arrays
]
for
i
in
range
(
batch_size
)
:
j
=
rng
.
randint
(
len
(
arrays
[
0
]
)
steps
)
for
x
,
a
in
zip
(
xs
,
arrays
)
:
x
[
i
]
=
a
[
j
:
j
+
steps
]
return
xs
return
sample

Create a callable that generates samples from a dataset .


def encode(self, txt):
        '''Encode a text string by replacing characters with alphabet index.

        Parameters
        ----------
        txt : str
            A string to encode.

        Returns
        -------
        classes : list of int
            A sequence of alphabet index values corresponding to the given text.
        '''
        return list(self._fwd_index.get(c, 0) for c in txt)

def encode ( self , txt ) : return list ( self . _fwd_index . get ( c , 0 ) for c in txt )

Encode a text string by replacing characters with alphabet index .


def classifier_batches(self, steps, batch_size, rng=None):
        '''Create a callable that returns a batch of training data.

        Parameters
        ----------
        steps : int
            Number of time steps in each batch.
        batch_size : int
            Number of training examples per batch.
        rng : :class:`numpy.random.RandomState` or int, optional
            A random number generator, or an integer seed for a random number
            generator. If not provided, the random number generator will be
            created with an automatically chosen seed.

        Returns
        -------
        batch : callable
            A callable that, when called, returns a batch of data that can be
            used to train a classifier model.
        '''
        assert batch_size >= 2, 'batch_size must be at least 2!'

        if rng is None or isinstance(rng, int):
            rng = np.random.RandomState(rng)

        T = np.arange(steps)

        def batch():
            inputs = np.zeros((batch_size, steps, 1 + len(self.alpha)), 'f')
            outputs = np.zeros((batch_size, steps), 'i')
            for b in range(batch_size):
                offset = rng.randint(len(self.text) - steps - 1)
                enc = self.encode(self.text[offset:offset + steps + 1])
                inputs[b, T, enc[:-1]] = 1
                outputs[b, T] = enc[1:]
            return [inputs, outputs]

        return batch

def
classifier_batches
(
self
,
steps
,
batch_size
,
rng
=
)
:
assert
batch_size
>=
2
,
'batch_size must be at least 2!'
if
rng
is
or
isinstance
(
rng
,
int
)
:
rng
=
np
.
random
.
RandomState
(
rng
)
T
=
np
.
arange
(
steps
)
def
batch
(
)
:
inputs
=
np
.
zeros
(
(
batch_size
,
steps
,
1
+
len
(
self
.
alpha
)
)
,
'f'
)
outputs
=
np
.
zeros
(
(
batch_size
,
steps
)
,
'i'
)
for
b
in
range
(
batch_size
)
:
offset
=
rng
.
randint
(
len
(
self
.
text
)
steps
1
)
enc
=
self
.
encode
(
self
.
text
[
offset
:
offset
+
steps
+
1
]
)
inputs
[
b
,
T
,
enc
[
:
1
]
]
=
1
outputs
[
b
,
T
]
=
enc
[
1
:
]
return
[
inputs
,
outputs
]
return
batch

Create a callable that returns a batch of training data .


def predict_sequence(self, labels, steps, streams=1, rng=None):
        '''Draw a sequential sample of class labels from this network.

        Parameters
        ----------
        labels : list of int
            A list of integer class labels to get the classifier started.
        steps : int
            The number of time steps to sample.
        streams : int, optional
            Number of parallel streams to sample from the model. Defaults to 1.
        rng : :class:`numpy.random.RandomState` or int, optional
            A random number generator, or an integer seed for a random number
            generator. If not provided, the random number generator will be
            created with an automatically chosen seed.

        Yields
        ------
        label(s) : int or list of int
            Yields at each time step an integer class label sampled sequentially
            from the model. If the number of requested streams is greater than
            1, this will be a list containing the corresponding number of class
            labels.
        '''
        if rng is None or isinstance(rng, int):
            rng = np.random.RandomState(rng)
        offset = len(labels)
        batch = max(2, streams)
        inputs = np.zeros((batch, offset + steps, self.layers[0].output_size), 'f')
        inputs[:, np.arange(offset), labels] = 1
        for i in range(offset, offset + steps):
            chars = []
            for pdf in self.predict_proba(inputs[:i])[:, -1]:
                try:
                    c = rng.multinomial(1, pdf).argmax(axis=-1)
                except ValueError:
                    # sometimes the pdf triggers a normalization error. just
                    # choose greedily in this case.
                    c = pdf.argmax(axis=-1)
                chars.append(int(c))
            inputs[np.arange(batch), i, chars] = 1
            yield chars[0] if streams == 1 else chars

def predict_sequence ( self , labels , steps , streams = 1 , rng = None ) : if rng is None or isinstance ( rng , int ) : rng = np . random . RandomState ( rng ) offset = len ( labels ) batch = max ( 2 , streams ) inputs = np . zeros ( ( batch , offset + steps , self . layers [ 0 ] . output_size ) , 'f' ) inputs [ : , np . arange ( offset ) , labels ] = 1 for i in range ( offset , offset + steps ) : chars = [ ] for pdf in self . predict_proba ( inputs [ : i ] ) [ : , - 1 ] : try : c = rng . multinomial ( 1 , pdf ) . argmax ( axis = - 1 ) except ValueError : c = pdf . argmax ( axis = - 1 ) chars . append ( int ( c ) ) inputs [ np . arange ( batch ) , i , chars ] = 1 yield chars [ 0 ] if streams == 1 else chars

Draw a sequential sample of class labels from this network .


def add_conv_weights(self, name, mean=0, std=None, sparsity=0):
        '''Add a convolutional weight array to this layer's parameters.

        Parameters
        ----------
        name : str
            Name of the parameter to add.
        mean : float, optional
            Mean value for randomly-initialized weights. Defaults to 0.
        std : float, optional
            Standard deviation of initial matrix values. Defaults to
            :math:`1 / sqrt(n_i + n_o)`.
        sparsity : float, optional
            Fraction of weights to set to zero. Defaults to 0.
        '''
        nin = self.input_size
        nout = self.output_size
        mean = self.kwargs.get(
            'mean_{}'.format(name),
            self.kwargs.get('mean', mean))
        std = self.kwargs.get(
            'std_{}'.format(name),
            self.kwargs.get('std', std or 1 / np.sqrt(nin + nout)))
        sparsity = self.kwargs.get(
            'sparsity_{}'.format(name),
            self.kwargs.get('sparsity', sparsity))
        arr = np.zeros((nout, nin) + self.filter_size, util.FLOAT)
        for r in range(self.filter_size[0]):
            for c in range(self.filter_size[1]):
                arr[:, :, r, c] = util.random_matrix(
                    nout, nin, mean, std, sparsity=sparsity, rng=self.rng)
        self._params.append(theano.shared(arr, name=self._fmt(name)))

def add_conv_weights ( self , name , mean = 0 , std = None , sparsity = 0 ) : nin = self . input_size nout = self . output_size mean = self . kwargs . get ( 'mean_{}' . format ( name ) , self . kwargs . get ( 'mean' , mean ) ) std = self . kwargs . get ( 'std_{}' . format ( name ) , self . kwargs . get ( 'std' , std or 1 / np . sqrt ( nin + nout ) ) ) sparsity = self . kwargs . get ( 'sparsity_{}' . format ( name ) , self . kwargs . get ( 'sparsity' , sparsity ) ) arr = np . zeros ( ( nout , nin ) + self . filter_size , util . FLOAT ) for r in range ( self . filter_size [ 0 ] ) : for c in range ( self . filter_size [ 1 ] ) : arr [ : , : , r , c ] = util . random_matrix ( nout , nin , mean , std , sparsity = sparsity , rng = self . rng ) self . _params . append ( theano . shared ( arr , name = self . _fmt ( name ) ) )

Add a convolutional weight array to this layer s parameters .


def encode(self, x, layer=None, sample=False, **kwargs):
        '''Encode a dataset using the hidden layer activations of our network.

        Parameters
        ----------
        x : ndarray
            A dataset to encode. Rows of this dataset capture individual data
            points, while columns represent the variables in each data point.

        layer : str, optional
            The name of the hidden layer output to use. By default, we use
            the "middle" hidden layer---for example, for a 4,2,4 or 4,3,2,3,4
            autoencoder, we use the layer with size 2.

        sample : bool, optional
            If True, then draw a sample using the hidden activations as
            independent Bernoulli probabilities for the encoded data. This
            assumes the hidden layer has a logistic sigmoid activation function.

        Returns
        -------
        ndarray :
            The given dataset, encoded by the appropriate hidden layer
            activation.
        '''
        enc = self.feed_forward(x, **kwargs)[self._find_output(layer)]
        if sample:
            return np.random.binomial(n=1, p=enc).astype(np.uint8)
        return enc

def encode ( self , x , layer = None , sample = False , ** kwargs ) : enc = self . feed_forward ( x , ** kwargs ) [ self . _find_output ( layer ) ] if sample : return np . random . binomial ( n = 1 , p = enc ) . astype ( np . uint8 ) return enc

Encode a dataset using the hidden layer activations of our network .


def decode(self, z, layer=None, **kwargs):
        '''Decode an encoded dataset by computing the output layer activation.

        Parameters
        ----------
        z : ndarray
            A matrix containing encoded data from this autoencoder.
        layer : int or str or :class:`Layer <layers.Layer>`, optional
            The index or name of the hidden layer that was used to encode `z`.

        Returns
        -------
        decoded : ndarray
            The decoded dataset.
        '''
        key = self._find_output(layer)
        if key not in self._functions:
            regs = regularizers.from_kwargs(self, **kwargs)
            outputs, updates = self.build_graph(regs)
            self._functions[key] = theano.function(
                [outputs[key]],
                [outputs[self.layers[-1].output_name]],
                updates=updates)
        return self._functions[key](z)[0]

def decode ( self , z , layer = None , ** kwargs ) : key = self . _find_output ( layer ) if key not in self . _functions : regs = regularizers . from_kwargs ( self , ** kwargs ) outputs , updates = self . build_graph ( regs ) self . _functions [ key ] = theano . function ( [ outputs [ key ] ] , [ outputs [ self . layers [ - 1 ] . output_name ] ] , updates = updates ) return self . _functions [ key ] ( z ) [ 0 ]

Decode an encoded dataset by computing the output layer activation .


def _find_output(self, layer):
        '''Find a layer output name for the given layer specifier.

        Parameters
        ----------
        layer : None, int, str, or :class:`theanets.layers.Layer`
            A layer specification. If this is None, the "middle" layer in the
            network will be used (i.e., the layer at the middle index in the
            list of network layers). If this is an integer, the corresponding
            layer in the network's layer list will be used. If this is a string,
            the layer with the corresponding name will be returned.

        Returns
        -------
        name : str
            The fully-scoped output name for the desired layer.
        '''
        if layer is None:
            layer = len(self.layers) // 2
        if isinstance(layer, int):
            layer = self.layers[layer]
        if isinstance(layer, util.basestring):
            try:
                layer = [l for l in self.layers if l.name == layer][0]
            except IndexError:
                pass
        if isinstance(layer, layers.Layer):
            layer = layer.output_name
        return layer

def _find_output ( self , layer ) : if layer is None : layer = len ( self . layers ) // 2 if isinstance ( layer , int ) : layer = self . layers [ layer ] if isinstance ( layer , util . basestring ) : try : layer = [ l for l in self . layers if l . name == layer ] [ 0 ] except IndexError : pass if isinstance ( layer , layers . Layer ) : layer = layer . output_name return layer

Find a layer output name for the given layer specifier .


def score(self, x, w=None, **kwargs):
        '''Compute R^2 coefficient of determination for a given input.

        Parameters
        ----------
        x : ndarray (num-examples, num-inputs)
            An array containing data to be fed into the network. Multiple
            examples are arranged as rows in this array, with columns containing
            the variables for each example.

        Returns
        -------
        r2 : float
            The R^2 correlation between the prediction of this netork and its
            input. This can serve as one measure of the information loss of the
            autoencoder.
        '''
        return super(Autoencoder, self).score(x, x, w=w, **kwargs)

def score ( self , x , w = None , ** kwargs ) : return super ( Autoencoder , self ) . score ( x , x , w = w , ** kwargs )

Compute R^2 coefficient of determination for a given input .


def predict(self, x, **kwargs):
        '''Compute a greedy classification for the given set of data.

        Parameters
        ----------
        x : ndarray (num-examples, num-variables)
            An array containing examples to classify. Examples are given as the
            rows in this array.

        Returns
        -------
        k : ndarray (num-examples, )
            A vector of class index values, one per row of input data.
        '''
        outputs = self.feed_forward(x, **kwargs)
        return outputs[self.layers[-1].output_name].argmax(axis=-1)

def predict ( self , x , ** kwargs ) : outputs = self . feed_forward ( x , ** kwargs ) return outputs [ self . layers [ - 1 ] . output_name ] . argmax ( axis = - 1 )

Compute a greedy classification for the given set of data .


def predict_proba(self, x, **kwargs):
        '''Compute class posterior probabilities for the given set of data.

        Parameters
        ----------
        x : ndarray (num-examples, num-variables)
            An array containing examples to predict. Examples are given as the
            rows in this array.

        Returns
        -------
        p : ndarray (num-examples, num-classes)
            An array of class posterior probability values, one per row of input
            data.
        '''
        return self.feed_forward(x, **kwargs)[self.layers[-1].output_name]

def predict_proba ( self , x , ** kwargs ) : return self . feed_forward ( x , ** kwargs ) [ self . layers [ - 1 ] . output_name ]

Compute class posterior probabilities for the given set of data .


def predict_logit(self, x, **kwargs):
        '''Compute the logit values that underlie the softmax output.

        Parameters
        ----------
        x : ndarray (num-examples, num-variables)
            An array containing examples to classify. Examples are given as the
            rows in this array.

        Returns
        -------
        l : ndarray (num-examples, num-classes)
            An array of posterior class logit values, one row of logit values
            per row of input data.
        '''
        return self.feed_forward(x, **kwargs)[self.layers[-1].full_name('pre')]

def predict_logit ( self , x , ** kwargs ) : return self . feed_forward ( x , ** kwargs ) [ self . layers [ - 1 ] . full_name ( 'pre' ) ]

Compute the logit values that underlie the softmax output .


def score(self, x, y, w=None, **kwargs):
        '''Compute the mean accuracy on a set of labeled data.

        Parameters
        ----------
        x : ndarray (num-examples, num-variables)
            An array containing examples to classify. Examples are given as the
            rows in this array.
        y : ndarray (num-examples, )
            A vector of integer class labels, one for each row of input data.
        w : ndarray (num-examples, )
            A vector of weights, one for each row of input data.

        Returns
        -------
        score : float
            The (possibly weighted) mean accuracy of the model on the data.
        '''
        eq = y == self.predict(x, **kwargs)
        if w is not None:
            return (w * eq).sum() / w.sum()
        return eq.mean()

def score ( self , x , y , w = None , ** kwargs ) : eq = y == self . predict ( x , ** kwargs ) if w is not None : return ( w * eq ) . sum ( ) / w . sum ( ) return eq . mean ( )

Compute the mean accuracy on a set of labeled data .


def batch_at(features, labels, seq_begins, seq_lengths):
    '''Extract a single batch of data to pass to the model being trained.

    Parameters
    ----------
    features, labels : ndarray
        Arrays of the input features and target labels.
    seq_begins : ndarray
        Array of the start offsets of the speech segments to include.
    seq_lengths : ndarray
        Array of the lengths of the speech segments to include in the batch.

    Returns
    -------
    features, labels, mask : ndarrays
        A triple of arrays for training a network. The first element contains
        input features, the second contains target labels, and the third
        contains a "mask" consisting of ones where there is valid data and zeros
        everywhere else.
    '''
    length = seq_lengths.max()
    feat = np.zeros((BATCH_SIZE, length, features.shape[-1]), 'f')
    labl = np.zeros((BATCH_SIZE, length), 'i')
    mask = np.zeros((BATCH_SIZE, length), 'f')
    for b, (begin, length) in enumerate(zip(seq_begins, seq_lengths)):
        feat[b, :length] = features[begin:begin+length]
        labl[b, :length] = labels[begin:begin+length]
        mask[b, :length] = 1
    return [feat, labl, mask]

def batch_at ( features , labels , seq_begins , seq_lengths ) : length = seq_lengths . max ( ) feat = np . zeros ( ( BATCH_SIZE , length , features . shape [ - 1 ] ) , 'f' ) labl = np . zeros ( ( BATCH_SIZE , length ) , 'i' ) mask = np . zeros ( ( BATCH_SIZE , length ) , 'f' ) for b , ( begin , length ) in enumerate ( zip ( seq_begins , seq_lengths ) ) : feat [ b , : length ] = features [ begin : begin + length ] labl [ b , : length ] = labels [ begin : begin + length ] mask [ b , : length ] = 1 return [ feat , labl , mask ]

Extract a single batch of data to pass to the model being trained .


def batches(dataset):
    '''Returns a callable that chooses sequences from netcdf data.'''
    seq_lengths = dataset.variables['seqLengths'].data
    seq_begins = np.concatenate(([0], np.cumsum(seq_lengths)[:-1]))

    def sample():
        chosen = np.random.choice(
            list(range(len(seq_lengths))), BATCH_SIZE, replace=False)
        return batch_at(dataset.variables['inputs'].data,
                        dataset.variables['targetClasses'].data,
                        seq_begins[chosen],
                        seq_lengths[chosen])

    return sample

def batches ( dataset ) : seq_lengths = dataset . variables [ 'seqLengths' ] . data seq_begins = np . concatenate ( ( [ 0 ] , np . cumsum ( seq_lengths ) [ : - 1 ] ) ) def sample ( ) : chosen = np . random . choice ( list ( range ( len ( seq_lengths ) ) ) , BATCH_SIZE , replace = False ) return batch_at ( dataset . variables [ 'inputs' ] . data , dataset . variables [ 'targetClasses' ] . data , seq_begins [ chosen ] , seq_lengths [ chosen ] ) return sample

Returns a callable that chooses sequences from netcdf data .


def load(self, path):
        '''Load a saved network from a pickle file on disk.

        This method sets the ``network`` attribute of the experiment to the
        loaded network model.

        Parameters
        ----------
        filename : str
            Load the keyword arguments and parameters of a network from a pickle
            file at the named path. If this name ends in ".gz" then the input
            will automatically be gunzipped; otherwise the input will be treated
            as a "raw" pickle.

        Returns
        -------
        network : :class:`Network <graph.Network>`
            A newly-constructed network, with topology and parameters loaded
            from the given pickle file.
        '''
        self.network = graph.Network.load(path)
        return self.network

def load ( self , path ) : self . network = graph . Network . load ( path ) return self . network

Load a saved network from a pickle file on disk .


def random_matrix(rows, cols, mean=0, std=1, sparsity=0, radius=0, diagonal=0, rng=None):
    '''Create a matrix of randomly-initialized weights.

    Parameters
    ----------
    rows : int
        Number of rows of the weight matrix -- equivalently, the number of
        "input" units that the weight matrix connects.
    cols : int
        Number of columns of the weight matrix -- equivalently, the number
        of "output" units that the weight matrix connects.
    mean : float, optional
        Draw initial weight values from a normal with this mean. Defaults to 0.
    std : float, optional
        Draw initial weight values from a normal with this standard deviation.
        Defaults to 1.
    sparsity : float in (0, 1), optional
        If given, ensure that the given fraction of the weight matrix is
        set to zero. Defaults to 0, meaning all weights are nonzero.
    radius : float, optional
        If given, rescale the initial weights to have this spectral radius.
        No scaling is performed by default.
    diagonal : float, optional
        If nonzero, create a matrix containing all zeros except for this value
        along the diagonal. If nonzero, other arguments (except for rows and
        cols) will be ignored.
    rng : :class:`numpy.random.RandomState` or int, optional
        A random number generator, or an integer seed for a random number
        generator. If not provided, the random number generator will be created
        with an automatically chosen seed.

    Returns
    -------
    matrix : numpy array
        An array containing random values. These often represent the weights
        connecting each "input" unit to each "output" unit in a layer.
    '''
    if rng is None or isinstance(rng, int):
        rng = np.random.RandomState(rng)
    arr = mean + std * rng.randn(rows, cols)
    if 1 > sparsity > 0:
        k = min(rows, cols)
        mask = rng.binomial(n=1, p=1 - sparsity, size=(rows, cols)).astype(bool)
        mask[:k, :k] |= np.eye(k).astype(bool)
        arr *= mask
    if radius > 0:
        # rescale weights to have the appropriate spectral radius.
        u, s, vT = np.linalg.svd(arr, full_matrices=False)
        arr = np.dot(np.dot(u, np.diag(radius * s / abs(s[0]))), vT)
    if diagonal != 0:
        # generate a diagonal weight matrix. ignore other options.
        arr = diagonal * np.eye(max(rows, cols))[:rows, :cols]
    return arr.astype(FLOAT)

def random_matrix ( rows , cols , mean = 0 , std = 1 , sparsity = 0 , radius = 0 , diagonal = 0 , rng = None ) : if rng is None or isinstance ( rng , int ) : rng = np . random . RandomState ( rng ) arr = mean + std * rng . randn ( rows , cols ) if 1 > sparsity > 0 : k = min ( rows , cols ) mask = rng . binomial ( n = 1 , p = 1 - sparsity , size = ( rows , cols ) ) . astype ( bool ) mask [ : k , : k ] |= np . eye ( k ) . astype ( bool ) arr *= mask if radius > 0 : u , s , vT = np . linalg . svd ( arr , full_matrices = False ) arr = np . dot ( np . dot ( u , np . diag ( radius * s / abs ( s [ 0 ] ) ) ) , vT ) if diagonal != 0 : arr = diagonal * np . eye ( max ( rows , cols ) ) [ : rows , : cols ] return arr . astype ( FLOAT )

Create a matrix of randomly - initialized weights .


def random_vector(size, mean=0, std=1, rng=None):
    '''Create a vector of randomly-initialized values.

    Parameters
    ----------
    size : int
        Length of vecctor to create.
    mean : float, optional
        Mean value for initial vector values. Defaults to 0.
    std : float, optional
        Standard deviation for initial vector values. Defaults to 1.
    rng : :class:`numpy.random.RandomState` or int, optional
        A random number generator, or an integer seed for a random number
        generator. If not provided, the random number generator will be created
        with an automatically chosen seed.

    Returns
    -------
    vector : numpy array
        An array containing random values. This often represents the bias for a
        layer of computation units.
    '''
    if rng is None or isinstance(rng, int):
        rng = np.random.RandomState(rng)
    return (mean + std * rng.randn(size)).astype(FLOAT)

def random_vector ( size , mean = 0 , std = 1 , rng = None ) : if rng is None or isinstance ( rng , int ) : rng = np . random . RandomState ( rng ) return ( mean + std * rng . randn ( size ) ) . astype ( FLOAT )

Create a vector of randomly - initialized values .


def outputs_matching(outputs, patterns):
    '''Get the outputs from a network that match a pattern.

    Parameters
    ----------
    outputs : dict or sequence of (str, theano expression)
        Output expressions to filter for matches. If this is a dictionary, its
        ``items()`` will be processed for matches.
    patterns : sequence of str
        A sequence of glob-style patterns to match against. Any parameter
        matching any pattern in this sequence will be included in the match.

    Yields
    ------
    matches : pair of str, theano expression
        Generates a sequence of (name, expression) pairs. The name is the name
        of the output that matched, and the expression is the symbolic output in
        the network graph.
    '''
    if isinstance(patterns, basestring):
        patterns = (patterns, )
    if isinstance(outputs, dict):
        outputs = outputs.items()
    for name, expr in outputs:
        for pattern in patterns:
            if fnmatch.fnmatch(name, pattern):
                yield name, expr
                break

def outputs_matching ( outputs , patterns ) : if isinstance ( patterns , basestring ) : patterns = ( patterns , ) if isinstance ( outputs , dict ) : outputs = outputs . items ( ) for name , expr in outputs : for pattern in patterns : if fnmatch . fnmatch ( name , pattern ) : yield name , expr break

Get the outputs from a network that match a pattern .


def params_matching(layers, patterns):
    '''Get the parameters from a network that match a pattern.

    Parameters
    ----------
    layers : list of :class:`theanets.layers.Layer`
        A list of network layers to retrieve parameters from.
    patterns : sequence of str
        A sequence of glob-style patterns to match against. Any parameter
        matching any pattern in this sequence will be included in the match.

    Yields
    ------
    matches : pair of str, theano expression
        Generates a sequence of (name, expression) pairs. The name is the name
        of the parameter that matched, and the expression represents the
        parameter symbolically.
    '''
    if isinstance(patterns, basestring):
        patterns = (patterns, )
    for layer in layers:
        for param in layer.params:
            name = param.name
            for pattern in patterns:
                if fnmatch.fnmatch(name, pattern):
                    yield name, param
                    break

def params_matching ( layers , patterns ) : if isinstance ( patterns , basestring ) : patterns = ( patterns , ) for layer in layers : for param in layer . params : name = param . name for pattern in patterns : if fnmatch . fnmatch ( name , pattern ) : yield name , param break

Get the parameters from a network that match a pattern .


def from_kwargs(graph, **kwargs):
    '''Construct common regularizers from a set of keyword arguments.

    Keyword arguments not listed below will be passed to
    :func:`Regularizer.build` if they specify the name of a registered
    :class:`Regularizer`.

    Parameters
    ----------
    graph : :class:`theanets.graph.Network`
        A network graph to regularize.

    regularizers : dict or tuple/list of :class:`Regularizer`, optional
        If this is a list or a tuple, the contents of the list will be returned
        as the regularizers. This is to permit custom lists of regularizers to
        be passed easily.

        If this is a dict, its contents will be added to the other keyword
        arguments passed in.

    rng : int or theano RandomStreams, optional
        If an integer is provided, it will be used to seed the random number
        generators for the dropout or noise regularizers. If a theano
        RandomStreams object is provided, it will be used directly. Defaults to
        13.

    input_dropout : float, optional
        Apply dropout to input layers in the network graph, with this dropout
        rate. Defaults to 0 (no dropout).

    hidden_dropout : float, optional
        Apply dropout to hidden layers in the network graph, with this dropout
        rate. Defaults to 0 (no dropout).

    output_dropout : float, optional
        Apply dropout to the output layer in the network graph, with this
        dropout rate. Defaults to 0 (no dropout).

    input_noise : float, optional
        Apply noise to input layers in the network graph, with this standard
        deviation. Defaults to 0 (no noise).

    hidden_noise : float, optional
        Apply noise to hidden layers in the network graph, with this standard
        deviation. Defaults to 0 (no noise).

    output_noise : float, optional
        Apply noise to the output layer in the network graph, with this
        standard deviation. Defaults to 0 (no noise).

    Returns
    -------
    regs : list of :class:`Regularizer`
        A list of regularizers to apply to the given network graph.
    '''
    if 'regularizers' in kwargs:
        regs = kwargs['regularizers']
        if isinstance(regs, (tuple, list)):
            return regs
        if isinstance(regs, dict):
            kwargs.update(regs)

    regs = []

    rng = kwargs.get('rng', 13)

    def pattern(ls):
        return tuple(l.output_name for l in ls)

    inputs = pattern([l for l in graph.layers if isinstance(l, layers.Input)])
    hiddens = pattern(graph.layers[1:-1])
    outputs = pattern([graph.layers[-1]])

    # create regularizers for different types of canned dropout.
    spec = {inputs: kwargs.get('input_dropout', 0),
            hiddens: kwargs.get('hidden_dropout', 0),
            outputs: kwargs.get('output_dropout', 0)}
    spec.update(kwargs.get('dropout', {}))
    for pattern, w in spec.items():
        if w:
            regs.append(BernoulliDropout(pattern=pattern, weight=w, rng=rng))

    # create regularizers for different types of canned noise.
    spec = {inputs: kwargs.get('input_noise', 0),
            hiddens: kwargs.get('hidden_noise', 0),
            outputs: kwargs.get('output_noise', 0)}
    spec.update(kwargs.get('noise', {}))
    for pattern, w in spec.items():
        if w:
            regs.append(GaussianNoise(pattern=pattern, weight=w, rng=rng))

    # create regularizers based on other keyword arguments.
    for key, value in kwargs.items():
        if Regularizer.is_registered(key):
            if not isinstance(value, dict):
                value = dict(weight=value)
            regs.append(Regularizer.build(key, **value))

    return regs

def from_kwargs ( graph , ** kwargs ) : if 'regularizers' in kwargs : regs = kwargs [ 'regularizers' ] if isinstance ( regs , ( tuple , list ) ) : return regs if isinstance ( regs , dict ) : kwargs . update ( regs ) regs = [ ] rng = kwargs . get ( 'rng' , 13 ) def pattern ( ls ) : return tuple ( l . output_name for l in ls ) inputs = pattern ( [ l for l in graph . layers if isinstance ( l , layers . Input ) ] ) hiddens = pattern ( graph . layers [ 1 : - 1 ] ) outputs = pattern ( [ graph . layers [ - 1 ] ] ) spec = { inputs : kwargs . get ( 'input_dropout' , 0 ) , hiddens : kwargs . get ( 'hidden_dropout' , 0 ) , outputs : kwargs . get ( 'output_dropout' , 0 ) } spec . update ( kwargs . get ( 'dropout' , { } ) ) for pattern , w in spec . items ( ) : if w : regs . append ( BernoulliDropout ( pattern = pattern , weight = w , rng = rng ) ) spec = { inputs : kwargs . get ( 'input_noise' , 0 ) , hiddens : kwargs . get ( 'hidden_noise' , 0 ) , outputs : kwargs . get ( 'output_noise' , 0 ) } spec . update ( kwargs . get ( 'noise' , { } ) ) for pattern , w in spec . items ( ) : if w : regs . append ( GaussianNoise ( pattern = pattern , weight = w , rng = rng ) ) for key , value in kwargs . items ( ) : if Regularizer . is_registered ( key ) : if not isinstance ( value , dict ) : value = dict ( weight = value ) regs . append ( Regularizer . build ( key , ** value ) ) return regs

Construct common regularizers from a set of keyword arguments .


def variables(self):
        '''A list of Theano variables used in this loss.'''
        result = [self._target]
        if self._weights is not None:
            result.append(self._weights)
        return result

def variables ( self ) : result = [ self . _target ] if self . _weights is not None : result . append ( self . _weights ) return result

A list of Theano variables used in this loss .


def accuracy(self, outputs):
        '''Build a Theano expression for computing the accuracy of graph output.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        acc : Theano expression
            A Theano expression representing the accuracy of the output compared
            to the target data.
        '''
        output = outputs[self.output_name]
        predict = TT.argmax(output, axis=-1)
        correct = TT.eq(predict, self._target)
        acc = correct.mean()
        if self._weights is not None:
            acc = (self._weights * correct).sum() / self._weights.sum()
        return acc

def accuracy ( self , outputs ) : output = outputs [ self . output_name ] predict = TT . argmax ( output , axis = - 1 ) correct = TT . eq ( predict , self . _target ) acc = correct . mean ( ) if self . _weights is not None : acc = ( self . _weights * correct ) . sum ( ) / self . _weights . sum ( ) return acc

Build a Theano expression for computing the accuracy of graph output .


def _scan(self, inputs, outputs, name='scan', step=None, constants=None):
        '''Helper method for defining a basic loop in theano.

        Parameters
        ----------
        inputs : sequence of theano expressions
            Inputs to the scan operation.
        outputs : sequence of output specifiers
            Specifiers for the outputs of the scan operation. This should be a
            sequence containing:
            - None for values that are output by the scan but not tapped as
              inputs,
            - an integer or theano scalar (``ndim == 0``) indicating the batch
              size for initial zero state,
            - a theano tensor variable (``ndim > 0``) containing initial state
              data, or
            - a dictionary containing a full output specifier. See
              ``outputs_info`` in the Theano documentation for ``scan``.
        name : str, optional
            Name of the scan variable to create. Defaults to ``'scan'``.
        step : callable, optional
            The callable to apply in the loop. Defaults to :func:`self._step`.
        constants : sequence of tensor, optional
            A sequence of parameters, if any, needed by the step function.

        Returns
        -------
        output(s) : theano expression(s)
            Theano expression(s) representing output(s) from the scan.
        updates : sequence of update tuples
            A sequence of updates to apply inside a theano function.
        '''
        init = []
        for i, x in enumerate(outputs):
            ndim = getattr(x, 'ndim', -1)
            if x is None or isinstance(x, dict) or ndim > 0:
                init.append(x)
                continue
            if isinstance(x, int) or ndim == 0:
                init.append(TT.repeat(theano.shared(
                    np.zeros((1, self.output_size), util.FLOAT),
                    name=self._fmt('init{}'.format(i))), x, axis=0))
                continue
            raise ValueError('cannot handle input {} for scan!'.format(x))
        return theano.scan(
            step or self._step,
            name=self._fmt(name),
            sequences=inputs,
            outputs_info=init,
            non_sequences=constants,
            go_backwards='back' in self.kwargs.get('direction', '').lower(),
            truncate_gradient=self.kwargs.get('bptt_limit', -1),
        )

def
_scan
(
self
,
inputs
,
outputs
,
name
=
'scan'
,
step
=
,
constants
=
)
:
init
=
[
]
for
i
,
x
in
enumerate
(
outputs
)
:
ndim
=
getattr
(
x
,
'ndim'
,
1
)
if
x
is
or
isinstance
(
x
,
dict
)
or
ndim
>
0
:
init
.
append
(
x
)
continue
if
isinstance
(
x
,
int
)
or
ndim
==
0
:
init
.
append
(
TT
.
repeat
(
theano
.
shared
(
np
.
zeros
(
(
1
,
self
.
output_size
)
,
util
.
FLOAT
)
,
name
=
self
.
_fmt
(
'init{}'
.
format
(
i
)
)
)
,
x
,
axis
=
0
)
)
continue
raise
ValueError
(
'cannot handle input {} for scan!'
.
format
(
x
)
)
return
theano
.
scan
(
step
or
self
.
_step
,
name
=
self
.
_fmt
(
name
)
,
sequences
=
inputs
,
outputs_info
=
init
,
non_sequences
=
constants
,
go_backwards
=
'back'
in
self
.
kwargs
.
get
(
'direction'
,
''
)
.
lower
(
)
,
truncate_gradient
=
self
.
kwargs
.
get
(
'bptt_limit'
,
1
)
,
)

Helper method for defining a basic loop in theano .


def build(name, layer, **kwargs):
    '''Construct an activation function by name.

    Parameters
    ----------
    name : str or :class:`Activation`
        The name of the type of activation function to build, or an
        already-created instance of an activation function.
    layer : :class:`theanets.layers.Layer`
        The layer to which this activation will be applied.
    kwargs : dict
        Additional named arguments to pass to the activation constructor.

    Returns
    -------
    activation : :class:`Activation`
        A neural network activation function instance.
    '''
    if isinstance(name, Activation):
        return name

    if '+' in name:
        return functools.reduce(
            Compose, (build(n, layer, **kwargs) for n in name.split('+')))

    act = COMMON.get(name)
    if act is not None:
        act.name = name
        act.params = []
        return act

    if name.lower().startswith('maxout') and ':' in name:
        name, pieces = name.split(':', 1)
        kwargs['pieces'] = int(pieces)
    kwargs['name'] = name
    kwargs['layer'] = layer
    return Activation.build(name, **kwargs)

def build ( name , layer , ** kwargs ) : if isinstance ( name , Activation ) : return name if '+' in name : return functools . reduce ( Compose , ( build ( n , layer , ** kwargs ) for n in name . split ( '+' ) ) ) act = COMMON . get ( name ) if act is not None : act . name = name act . params = [ ] return act if name . lower ( ) . startswith ( 'maxout' ) and ':' in name : name , pieces = name . split ( ':' , 1 ) kwargs [ 'pieces' ] = int ( pieces ) kwargs [ 'name' ] = name kwargs [ 'layer' ] = layer return Activation . build ( name , ** kwargs )

Construct an activation function by name .


def reservoir(xs, n, rng):
        '''Select a random sample of n items from xs.'''
        pool = []
        for i, x in enumerate(xs):
            if len(pool) < n:
                pool.append(x / np.linalg.norm(x))
                continue
            j = rng.randint(i + 1)
            if j < n:
                pool[j] = x / np.linalg.norm(x)
        # if the pool still has fewer than n items, pad with distorted random
        # duplicates from the source data.
        L = len(pool)
        S = np.std(pool, axis=0)
        while len(pool) < n:
            x = pool[rng.randint(L)]
            pool.append(x + S * rng.randn(*x.shape))
        return np.array(pool, dtype=pool[0].dtype)

def reservoir ( xs , n , rng ) : pool = [ ] for i , x in enumerate ( xs ) : if len ( pool ) < n : pool . append ( x / np . linalg . norm ( x ) ) continue j = rng . randint ( i + 1 ) if j < n : pool [ j ] = x / np . linalg . norm ( x ) L = len ( pool ) S = np . std ( pool , axis = 0 ) while len ( pool ) < n : x = pool [ rng . randint ( L ) ] pool . append ( x + S * rng . randn ( * x . shape ) ) return np . array ( pool , dtype = pool [ 0 ] . dtype )

Select a random sample of n items from xs .


def set_loss(self, *args, **kwargs):
        '''Clear the current loss functions from the network and add a new one.

        All parameters and keyword arguments are passed to :func:`add_loss`
        after clearing the current losses.
        '''
        self.losses = []
        self.add_loss(*args, **kwargs)

def set_loss ( self , * args , ** kwargs ) : self . losses = [ ] self . add_loss ( * args , ** kwargs )

Clear the current loss functions from the network and add a new one .