def source_token_lines(source):
    """Generate a series of lines, one for each line in `source`.

    Each line is a list of pairs, each pair is a token::

        [('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ]

    Each pair has a token class, and the token text.

    If you concatenate all the token texts, and then join them with newlines,
    you should have your original `source` back, with two differences:
    trailing whitespace is not preserved, and a final line with no newline
    is indistinguishable from a final line with a newline.

    """
    ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])
    line = []
    col = 0
    source = source.expandtabs(8).replace('\r\n', '\n')
    tokgen = generate_tokens(source)
    for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):
        mark_start = True
        for part in re.split('(\n)', ttext):
            if part == '\n':
                yield line
                line = []
                col = 0
                mark_end = False
            elif part == '':
                mark_end = False
            elif ttype in ws_tokens:
                mark_end = False
            else:
                if mark_start and scol > col:
                    line.append(("ws", " " * (scol - col)))
                    mark_start = False
                tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3]
                if ttype == token.NAME and keyword.iskeyword(ttext):
                    tok_class = "key"
                line.append((tok_class, part))
                mark_end = True
            scol = 0
        if mark_end:
            col = ecol

    if line:
        yield line

def
source_token_lines
(
source
)
:
ws_tokens
=
set
(
[
token
.
INDENT
,
token
.
DEDENT
,
token
.
NEWLINE
,
tokenize
.
NL
]
)
line
=
[
]
col
=
0
source
=
source
.
expandtabs
(
8
)
.
replace
(
'\r\n'
,
'\n'
)
tokgen
=
generate_tokens
(
source
)
for
ttype
,
ttext
,
(
_
,
scol
)
,
(
_
,
ecol
)
,
_
in
phys_tokens
(
tokgen
)
:
mark_start
=
True
for
part
in
re
.
split
(
'(\n)'
,
ttext
)
:
if
part
==
'\n'
:
yield
line
line
=
[
]
col
=
0
mark_end
=
False
elif
part
==
''
:
mark_end
=
False
elif
ttype
in
ws_tokens
:
mark_end
=
False
else
:
if
mark_start
and
scol
>
col
:
line
.
append
(
(
"ws"
,
" "
*
(
scol
col
)
)
)
mark_start
=
False
tok_class
=
tokenize
.
tok_name
.
get
(
ttype
,
'xx'
)
.
lower
(
)
[
:
3
]
if
ttype
==
token
.
NAME
and
keyword
.
iskeyword
(
ttext
)
:
tok_class
=
"key"
line
.
append
(
(
tok_class
,
part
)
)
mark_end
=
True
scol
=
0
if
mark_end
:
col
=
ecol
if
line
:
yield
line

Generate a series of lines one for each line in source .


def load_default_config(ipython_dir=None):
    """Load the default config file from the default ipython_dir.

    This is useful for embedded shells.
    """
    if ipython_dir is None:
        ipython_dir = get_ipython_dir()
    profile_dir = os.path.join(ipython_dir, 'profile_default')
    cl = PyFileConfigLoader(default_config_file_name, profile_dir)
    try:
        config = cl.load_config()
    except ConfigFileNotFound:
        # no config found
        config = Config()
    return config

def load_default_config ( ipython_dir = None ) : if ipython_dir is None : ipython_dir = get_ipython_dir ( ) profile_dir = os . path . join ( ipython_dir , 'profile_default' ) cl = PyFileConfigLoader ( default_config_file_name , profile_dir ) try : config = cl . load_config ( ) except ConfigFileNotFound : config = Config ( ) return config

Load the default config file from the default ipython_dir .


def _classes_default(self):
        """This has to be in a method, for TerminalIPythonApp to be available."""
        return [
            InteractiveShellApp, # ShellApp comes before TerminalApp, because
            self.__class__,      # it will also affect subclasses (e.g. QtConsole)
            TerminalInteractiveShell,
            PromptManager,
            HistoryManager,
            ProfileDir,
            PlainTextFormatter,
            IPCompleter,
            ScriptMagics,
        ]

def _classes_default ( self ) : return [ InteractiveShellApp , self . __class__ , TerminalInteractiveShell , PromptManager , HistoryManager , ProfileDir , PlainTextFormatter , IPCompleter , ScriptMagics , ]

This has to be in a method for TerminalIPythonApp to be available .


def parse_command_line(self, argv=None):
        """override to allow old '-pylab' flag with deprecation warning"""

        argv = sys.argv[1:] if argv is None else argv

        if '-pylab' in argv:
            # deprecated `-pylab` given,
            # warn and transform into current syntax
            argv = argv[:] # copy, don't clobber
            idx = argv.index('-pylab')
            warn.warn("`-pylab` flag has been deprecated.\n"
            "    Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
            sub = '--pylab'
            if len(argv) > idx+1:
                # check for gui arg, as in '-pylab qt'
                gui = argv[idx+1]
                if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
                    sub = '--pylab='+gui
                    argv.pop(idx+1)
            argv[idx] = sub

        return super(TerminalIPythonApp, self).parse_command_line(argv)

override to allow old - pylab flag with deprecation warning


def initialize(self, argv=None):
        """Do actions after construct, but before starting the app."""
        super(TerminalIPythonApp, self).initialize(argv)
        if self.subapp is not None:
            # don't bother initializing further, starting subapp
            return
        if not self.ignore_old_config:
            check_for_old_config(self.ipython_dir)
        # print self.extra_args
        if self.extra_args and not self.something_to_run:
            self.file_to_run = self.extra_args[0]
        self.init_path()
        # create the shell
        self.init_shell()
        # and draw the banner
        self.init_banner()
        # Now a variety of things that happen after the banner is printed.
        self.init_gui_pylab()
        self.init_extensions()
        self.init_code()

def initialize ( self , argv = None ) : super ( TerminalIPythonApp , self ) . initialize ( argv ) if self . subapp is not None : return if not self . ignore_old_config : check_for_old_config ( self . ipython_dir ) if self . extra_args and not self . something_to_run : self . file_to_run = self . extra_args [ 0 ] self . init_path ( ) self . init_shell ( ) self . init_banner ( ) self . init_gui_pylab ( ) self . init_extensions ( ) self . init_code ( )

Do actions after construct but before starting the app .


def init_shell(self):
        """initialize the InteractiveShell instance"""
        # Create an InteractiveShell instance.
        # shell.display_banner should always be False for the terminal
        # based app, because we call shell.show_banner() by hand below
        # so the banner shows *before* all extension loading stuff.
        self.shell = TerminalInteractiveShell.instance(config=self.config,
                        display_banner=False, profile_dir=self.profile_dir,
                        ipython_dir=self.ipython_dir)
        self.shell.configurables.append(self)

def init_shell ( self ) : self . shell = TerminalInteractiveShell . instance ( config = self . config , display_banner = False , profile_dir = self . profile_dir , ipython_dir = self . ipython_dir ) self . shell . configurables . append ( self )


def init_banner(self):
        """optionally display the banner"""
        if self.display_banner and self.interact:
            self.shell.show_banner()
        # Make sure there is a space below the banner.
        if self.log_level <= logging.INFO: print

def init_banner ( self ) : if self . display_banner and self . interact : self . shell . show_banner ( ) if self . log_level <= logging . INFO : print


def repr_type(obj):
    """ Return a string representation of a value and its type for readable
    error messages.
    """
    the_type = type(obj)
    if (not py3compat.PY3) and the_type is InstanceType:
        # Old-style class.
        the_type = obj.__class__
    msg = '%r %r' % (obj, the_type)
    return msg

def
repr_type
(
obj
)
:
the_type
=
type
(
obj
)
if
(
not
py3compat
.
PY3
)
and
the_type
is
InstanceType
:
the_type
=
obj
.
__class__
msg
=
'%r %r'
%
(
obj
,
the_type
)
return
msg

Return a string representation of a value and its type for readable error messages .


def parse_notifier_name(name):
    """Convert the name argument to a list of names.

    Examples
    --------

    >>> parse_notifier_name('a')
    ['a']
    >>> parse_notifier_name(['a','b'])
    ['a', 'b']
    >>> parse_notifier_name(None)
    ['anytrait']
    """
    if isinstance(name, str):
        return [name]
    elif name is None:
        return ['anytrait']
    elif isinstance(name, (list, tuple)):
        for n in name:
            assert isinstance(n, str), "names must be strings"
        return name

def
parse_notifier_name
(
name
)
:
if
isinstance
(
name
,
str
)
:
return
[
name
]
elif
name
is
:
return
[
'anytrait'
]
elif
isinstance
(
name
,
(
list
,
tuple
)
)
:
for
n
in
name
:
assert
isinstance
(
n
,
str
)
,
"names must be strings"
return
name

Convert the name argument to a list of names .


def set_default_value(self, obj):
        """Set the default value on a per instance basis.

        This method is called by :meth:`instance_init` to create and
        validate the default value.  The creation and validation of
        default values must be delayed until the parent :class:`HasTraits`
        class has been instantiated.
        """
        # Check for a deferred initializer defined in the same class as the
        # trait declaration or above.
        mro = type(obj).mro()
        meth_name = '_%s_default' % self.name
        for cls in mro[:mro.index(self.this_class)+1]:
            if meth_name in cls.__dict__:
                break
        else:
            # We didn't find one. Do static initialization.
            dv = self.get_default_value()
            newdv = self._validate(obj, dv)
            obj._trait_values[self.name] = newdv
            return
        # Complete the dynamic initialization.
        obj._trait_dyn_inits[self.name] = cls.__dict__[meth_name]

def set_default_value ( self , obj ) : mro = type ( obj ) . mro ( ) meth_name = '_%s_default' % self . name for cls in mro [ : mro . index ( self . this_class ) + 1 ] : if meth_name in cls . __dict__ : break else : dv = self . get_default_value ( ) newdv = self . _validate ( obj , dv ) obj . _trait_values [ self . name ] = newdv return obj . _trait_dyn_inits [ self . name ] = cls . __dict__ [ meth_name ]

Set the default value on a per instance basis .


def on_trait_change(self, handler, name=None, remove=False):
        """Setup a handler to be called when a trait changes.

        This is used to setup dynamic notifications of trait changes.

        Static handlers can be created by creating methods on a HasTraits
        subclass with the naming convention '_[traitname]_changed'.  Thus,
        to create static handler for the trait 'a', create the method
        _a_changed(self, name, old, new) (fewer arguments can be used, see
        below).

        Parameters
        ----------
        handler : callable
            A callable that is called when a trait changes.  Its
            signature can be handler(), handler(name), handler(name, new)
            or handler(name, old, new).
        name : list, str, None
            If None, the handler will apply to all traits.  If a list
            of str, handler will apply to all names in the list.  If a
            str, the handler will apply just to that name.
        remove : bool
            If False (the default), then install the handler.  If True
            then unintall it.
        """
        if remove:
            names = parse_notifier_name(name)
            for n in names:
                self._remove_notifiers(handler, n)
        else:
            names = parse_notifier_name(name)
            for n in names:
                self._add_notifiers(handler, n)

def on_trait_change ( self , handler , name = None , remove = False ) : if remove : names = parse_notifier_name ( name ) for n in names : self . _remove_notifiers ( handler , n ) else : names = parse_notifier_name ( name ) for n in names : self . _add_notifiers ( handler , n )

Setup a handler to be called when a trait changes .


def class_traits(cls, **metadata):
        """Get a list of all the traits of this class.

        This method is just like the :meth:`traits` method, but is unbound.

        The TraitTypes returned don't know anything about the values
        that the various HasTrait's instances are holding.

        This follows the same algorithm as traits does and does not allow
        for any simple way of specifying merely that a metadata name
        exists, but has any value.  This is because get_metadata returns
        None if a metadata key doesn't exist.
        """
        traits = dict([memb for memb in getmembers(cls) if \
                     isinstance(memb[1], TraitType)])

        if len(metadata) == 0:
            return traits

        for meta_name, meta_eval in metadata.items():
            if type(meta_eval) is not FunctionType:
                metadata[meta_name] = _SimpleTest(meta_eval)

        result = {}
        for name, trait in traits.items():
            for meta_name, meta_eval in metadata.items():
                if not meta_eval(trait.get_metadata(meta_name)):
                    break
            else:
                result[name] = trait

        return result

def class_traits ( cls , ** metadata ) : traits = dict ( [ memb for memb in getmembers ( cls ) if isinstance ( memb [ 1 ] , TraitType ) ] ) if len ( metadata ) == 0 : return traits for meta_name , meta_eval in metadata . items ( ) : if type ( meta_eval ) is not FunctionType : metadata [ meta_name ] = _SimpleTest ( meta_eval ) result = { } for name , trait in traits . items ( ) : for meta_name , meta_eval in metadata . items ( ) : if not meta_eval ( trait . get_metadata ( meta_name ) ) : break else : result [ name ] = trait return result

Get a list of all the traits of this class .


def trait_metadata(self, traitname, key):
        """Get metadata values for trait by key."""
        try:
            trait = getattr(self.__class__, traitname)
        except AttributeError:
            raise TraitError("Class %s does not have a trait named %s" %
                                (self.__class__.__name__, traitname))
        else:
            return trait.get_metadata(key)


def validate(self, obj, value):
        """Validates that the value is a valid object instance."""
        try:
            if issubclass(value, self.klass):
                return value
        except:
            if (value is None) and (self._allow_none):
                return value

        self.error(obj, value)

def validate ( self , obj , value ) : try : if issubclass ( value , self . klass ) : return value except : if ( value is None ) and ( self . _allow_none ) : return value self . error ( obj , value )

Validates that the value is a valid object instance .


def get_default_value(self):
        """Instantiate a default value instance.

        This is called when the containing HasTraits classes'
        :meth:`__new__` method is called to ensure that a unique instance
        is created for each HasTraits instance.
        """
        dv  = self.default_value
        if isinstance(dv, DefaultValueGenerator):
            return dv.generate(self.klass)
        else:
            return dv

def get_default_value ( self ) : dv = self . default_value if isinstance ( dv , DefaultValueGenerator ) : return dv . generate ( self . klass ) else : return dv


def check(self, completed, failed=None):
        """check whether our dependencies have been met."""
        if len(self) == 0:
            return True
        against = set()
        if self.success:
            against = completed
        if failed is not None and self.failure:
            against = against.union(failed)
        if self.all:
            return self.issubset(against)
        else:
            return not self.isdisjoint(against)

def check ( self , completed , failed = None ) : if len ( self ) == 0 : return True against = set ( ) if self . success : against = completed if failed is not None and self . failure : against = against . union ( failed ) if self . all : return self . issubset ( against ) else : return not self . isdisjoint ( against )

check whether our dependencies have been met .


def unreachable(self, completed, failed=None):
        """return whether this dependency has become impossible."""
        if len(self) == 0:
            return False
        against = set()
        if not self.success:
            against = completed
        if failed is not None and not self.failure:
            against = against.union(failed)
        if self.all:
            return not self.isdisjoint(against)
        else:
            return self.issubset(against)

def unreachable ( self , completed , failed = None ) : if len ( self ) == 0 : return False against = set ( ) if not self . success : against = completed if failed is not None and not self . failure : against = against . union ( failed ) if self . all : return not self . isdisjoint ( against ) else : return self . issubset ( against )

return whether this dependency has become impossible .


def as_dict(self):
        """Represent this dependency as a dict. For json compatibility."""
        return dict(
            dependencies=list(self),
            all=self.all,
            success=self.success,
            failure=self.failure
        )

def as_dict ( self ) : return dict ( dependencies = list ( self ) , all = self . all , success = self . success , failure = self . failure )

Represent this dependency as a dict . For json compatibility .


def depth(n, tree):
    """get depth of an element in the tree"""
    d = 0
    parent = tree[n]
    while parent is not None:
        d += 1
        parent = tree[parent]
    return d

def depth ( n , tree ) : d = 0 parent = tree [ n ] while parent is not None : d += 1 parent = tree [ parent ] return d


def print_bintree(tree, indent='  '):
    """print a binary tree"""
    for n in sorted(tree.keys()):
        print "%s%s" % (indent * depth(n,tree), n)

def
print_bintree
(
tree
,
indent
=
'  '
)
:
for
n
in
sorted
(
tree
.
keys
(
)
)
:
print
"%s%s"
%
(
indent
*
depth
(
n
,
tree
)
,
n
)


def disambiguate_dns_url(url, location):
    """accept either IP address or dns name, and return IP"""
    if not ip_pat.match(location):
        location = socket.gethostbyname(location)
    return disambiguate_url(url, location)

def disambiguate_dns_url ( url , location ) : if not ip_pat . match ( location ) : location = socket . gethostbyname ( location ) return disambiguate_url ( url , location )

accept either IP address or dns name and return IP


def allreduce(self, f, value, flat=True):
        """parallel reduce followed by broadcast of the result"""
        return self.reduce(f, value, flat=flat, all=True)

def allreduce ( self , f , value , flat = True ) : return self . reduce ( f , value , flat = flat , all = True )


def _validate_targets(self, targets):
        """turn any valid targets argument into a list of integer ids"""
        if targets is None:
            # default to all
            return self.ids

        if isinstance(targets, (int,str,unicode)):
            # only one target specified
            targets = [targets]
        _targets = []
        for t in targets:
            # map raw identities to ids
            if isinstance(t, (str,unicode)):
                t = self.by_ident.get(cast_bytes(t), t)
            _targets.append(t)
        targets = _targets
        bad_targets = [ t for t in targets if t not in self.ids ]
        if bad_targets:
            raise IndexError("No Such Engine: %r" % bad_targets)
        if not targets:
            raise IndexError("No Engines Registered")
        return targets

def
_validate_targets
(
self
,
targets
)
:
if
targets
is
:
return
self
.
ids
if
isinstance
(
targets
,
(
int
,
str
,
unicode
)
)
:
targets
=
[
targets
]
_targets
=
[
]
for
t
in
targets
:
if
isinstance
(
t
,
(
str
,
unicode
)
)
:
t
=
self
.
by_ident
.
get
(
cast_bytes
(
t
)
,
t
)
_targets
.
append
(
t
)
targets
=
_targets
bad_targets
=
[
t
for
t
in
targets
if
t
not
in
self
.
ids
]
if
bad_targets
:
raise
IndexError
(
"No Such Engine: %r"
%
bad_targets
)
if
not
targets
:
raise
IndexError
(
"No Engines Registered"
)
return
targets


def dispatch_monitor_traffic(self, msg):
        """all ME and Task queue messages come through here, as well as
        IOPub traffic."""
        self.log.debug("monitor traffic: %r", msg[0])
        switch = msg[0]
        try:
            idents, msg = self.session.feed_identities(msg[1:])
        except ValueError:
            idents=[]
        if not idents:
            self.log.error("Monitor message without topic: %r", msg)
            return
        handler = self.monitor_handlers.get(switch, None)
        if handler is not None:
            handler(idents, msg)
        else:
            self.log.error("Unrecognized monitor topic: %r", switch)

def
dispatch_monitor_traffic
(
self
,
msg
)
:
self
.
log
.
debug
(
"monitor traffic: %r"
,
msg
[
0
]
)
switch
=
msg
[
0
]
try
:
idents
,
msg
=
self
.
session
.
feed_identities
(
msg
[
1
:
]
)
except
ValueError
:
idents
=
[
]
if
not
idents
:
self
.
log
.
error
(
"Monitor message without topic: %r"
,
msg
)
return
handler
=
self
.
monitor_handlers
.
get
(
switch
,
)
if
handler
is
not
:
handler
(
idents
,
msg
)
else
:
self
.
log
.
error
(
"Unrecognized monitor topic: %r"
,
switch
)

all ME and Task queue messages come through here as well as IOPub traffic .


def dispatch_query(self, msg):
        """Route registration requests and queries from clients."""
        try:
            idents, msg = self.session.feed_identities(msg)
        except ValueError:
            idents = []
        if not idents:
            self.log.error("Bad Query Message: %r", msg)
            return
        client_id = idents[0]
        try:
            msg = self.session.unserialize(msg, content=True)
        except Exception:
            content = error.wrap_exception()
            self.log.error("Bad Query Message: %r", msg, exc_info=True)
            self.session.send(self.query, "hub_error", ident=client_id,
                    content=content)
            return
        # print client_id, header, parent, content
        #switch on message type:
        msg_type = msg['header']['msg_type']
        self.log.info("client::client %r requested %r", client_id, msg_type)
        handler = self.query_handlers.get(msg_type, None)
        try:
            assert handler is not None, "Bad Message Type: %r" % msg_type
        except:
            content = error.wrap_exception()
            self.log.error("Bad Message Type: %r", msg_type, exc_info=True)
            self.session.send(self.query, "hub_error", ident=client_id,
                    content=content)
            return

        else:
            handler(idents, msg)

def
dispatch_query
(
self
,
msg
)
:
try
:
idents
,
msg
=
self
.
session
.
feed_identities
(
msg
)
except
ValueError
:
idents
=
[
]
if
not
idents
:
self
.
log
.
error
(
"Bad Query Message: %r"
,
msg
)
return
client_id
=
idents
[
0
]
try
:
msg
=
self
.
session
.
unserialize
(
msg
,
content
=
True
)
except
Exception
:
content
=
error
.
wrap_exception
(
)
self
.
log
.
error
(
"Bad Query Message: %r"
,
msg
,
exc_info
=
True
)
self
.
session
.
send
(
self
.
query
,
"hub_error"
,
ident
=
client_id
,
content
=
content
)
return
msg_type
=
msg
[
'header'
]
[
'msg_type'
]
self
.
log
.
info
(
"client::client %r requested %r"
,
client_id
,
msg_type
)
handler
=
self
.
query_handlers
.
get
(
msg_type
,
)
try
:
assert
handler
is
not
,
"Bad Message Type: %r"
%
msg_type
except
:
content
=
error
.
wrap_exception
(
)
self
.
log
.
error
(
"Bad Message Type: %r"
,
msg_type
,
exc_info
=
True
)
self
.
session
.
send
(
self
.
query
,
"hub_error"
,
ident
=
client_id
,
content
=
content
)
return
else
:
handler
(
idents
,
msg
)

Route registration requests and queries from clients .


def handle_new_heart(self, heart):
        """handler to attach to heartbeater.
        Called when a new heart starts to beat.
        Triggers completion of registration."""
        self.log.debug("heartbeat::handle_new_heart(%r)", heart)
        if heart not in self.incoming_registrations:
            self.log.info("heartbeat::ignoring new heart: %r", heart)
        else:
            self.finish_registration(heart)

def
handle_new_heart
(
self
,
heart
)
:
self
.
log
.
debug
(
"heartbeat::handle_new_heart(%r)"
,
heart
)
if
heart
not
in
self
.
incoming_registrations
:
self
.
log
.
info
(
"heartbeat::ignoring new heart: %r"
,
heart
)
else
:
self
.
finish_registration
(
heart
)

handler to attach to heartbeater . Called when a new heart starts to beat . Triggers completion of registration .


def handle_heart_failure(self, heart):
        """handler to attach to heartbeater.
        called when a previously registered heart fails to respond to beat request.
        triggers unregistration"""
        self.log.debug("heartbeat::handle_heart_failure(%r)", heart)
        eid = self.hearts.get(heart, None)
        queue = self.engines[eid].queue
        if eid is None or self.keytable[eid] in self.dead_engines:
            self.log.info("heartbeat::ignoring heart failure %r (not an engine or already dead)", heart)
        else:
            self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue)))

handler to attach to heartbeater . called when a previously registered heart fails to respond to beat request . triggers unregistration


def save_task_request(self, idents, msg):
        """Save the submission of a task."""
        client_id = idents[0]

        try:
            msg = self.session.unserialize(msg)
        except Exception:
            self.log.error("task::client %r sent invalid task message: %r",
                    client_id, msg, exc_info=True)
            return
        record = init_record(msg)

        record['client_uuid'] = client_id.decode('ascii')
        record['queue'] = 'task'
        header = msg['header']
        msg_id = header['msg_id']
        self.pending.add(msg_id)
        self.unassigned.add(msg_id)
        try:
            # it's posible iopub arrived first:
            existing = self.db.get_record(msg_id)
            if existing['resubmitted']:
                for key in ('submitted', 'client_uuid', 'buffers'):
                    # don't clobber these keys on resubmit
                    # submitted and client_uuid should be different
                    # and buffers might be big, and shouldn't have changed
                    record.pop(key)
                    # still check content,header which should not change
                    # but are not expensive to compare as buffers

            for key,evalue in existing.iteritems():
                if key.endswith('buffers'):
                    # don't compare buffers
                    continue
                rvalue = record.get(key, None)
                if evalue and rvalue and evalue != rvalue:
                    self.log.warn("conflicting initial state for record: %r:%r <%r> %r", msg_id, rvalue, key, evalue)
                elif evalue and not rvalue:
                    record[key] = evalue
            try:
                self.db.update_record(msg_id, record)
            except Exception:
                self.log.error("DB Error updating record %r", msg_id, exc_info=True)
        except KeyError:
            try:
                self.db.add_record(msg_id, record)
            except Exception:
                self.log.error("DB Error adding record %r", msg_id, exc_info=True)
        except Exception:
            self.log.error("DB Error saving task request %r", msg_id, exc_info=True)


def save_task_result(self, idents, msg):
        """save the result of a completed task."""
        client_id = idents[0]
        try:
            msg = self.session.unserialize(msg)
        except Exception:
            self.log.error("task::invalid task result message send to %r: %r",
                    client_id, msg, exc_info=True)
            return

        parent = msg['parent_header']
        if not parent:
            # print msg
            self.log.warn("Task %r had no parent!", msg)
            return
        msg_id = parent['msg_id']
        if msg_id in self.unassigned:
            self.unassigned.remove(msg_id)

        header = msg['header']
        engine_uuid = header.get('engine', u'')
        eid = self.by_ident.get(cast_bytes(engine_uuid), None)

        status = header.get('status', None)

        if msg_id in self.pending:
            self.log.info("task::task %r finished on %s", msg_id, eid)
            self.pending.remove(msg_id)
            self.all_completed.add(msg_id)
            if eid is not None:
                if status != 'aborted':
                    self.completed[eid].append(msg_id)
                if msg_id in self.tasks[eid]:
                    self.tasks[eid].remove(msg_id)
            completed = header['date']
            started = header.get('started', None)
            result = {
                'result_header' : header,
                'result_content': msg['content'],
                'started' : started,
                'completed' : completed,
                'received' : datetime.now(),
                'engine_uuid': engine_uuid,
            }

            result['result_buffers'] = msg['buffers']
            try:
                self.db.update_record(msg_id, result)
            except Exception:
                self.log.error("DB Error saving task request %r", msg_id, exc_info=True)

        else:
            self.log.debug("task::unknown task %r finished", msg_id)


def save_iopub_message(self, topics, msg):
        """save an iopub message into the db"""
        # print (topics)
        try:
            msg = self.session.unserialize(msg, content=True)
        except Exception:
            self.log.error("iopub::invalid IOPub message", exc_info=True)
            return

        parent = msg['parent_header']
        if not parent:
            self.log.warn("iopub::IOPub message lacks parent: %r", msg)
            return
        msg_id = parent['msg_id']
        msg_type = msg['header']['msg_type']
        content = msg['content']

        # ensure msg_id is in db
        try:
            rec = self.db.get_record(msg_id)
        except KeyError:
            rec = empty_record()
            rec['msg_id'] = msg_id
            self.db.add_record(msg_id, rec)
        # stream
        d = {}
        if msg_type == 'stream':
            name = content['name']
            s = rec[name] or ''
            d[name] = s + content['data']

        elif msg_type == 'pyerr':
            d['pyerr'] = content
        elif msg_type == 'pyin':
            d['pyin'] = content['code']
        elif msg_type in ('display_data', 'pyout'):
            d[msg_type] = content
        elif msg_type == 'status':
            pass
        else:
            self.log.warn("unhandled iopub msg_type: %r", msg_type)

        if not d:
            return

        try:
            self.db.update_record(msg_id, d)
        except Exception:
            self.log.error("DB Error saving iopub message %r", msg_id, exc_info=True)

def
save_iopub_message
(
self
,
topics
,
msg
)
:
try
:
msg
=
self
.
session
.
unserialize
(
msg
,
content
=
True
)
except
Exception
:
self
.
log
.
error
(
"iopub::invalid IOPub message"
,
exc_info
=
True
)
return
parent
=
msg
[
'parent_header'
]
if
not
parent
:
self
.
log
.
warn
(
"iopub::IOPub message lacks parent: %r"
,
msg
)
return
msg_id
=
parent
[
'msg_id'
]
msg_type
=
msg
[
'header'
]
[
'msg_type'
]
content
=
msg
[
'content'
]
try
:
rec
=
self
.
db
.
get_record
(
msg_id
)
except
KeyError
:
rec
=
empty_record
(
)
rec
[
'msg_id'
]
=
msg_id
self
.
db
.
add_record
(
msg_id
,
rec
)
d
=
{
}
if
msg_type
==
'stream'
:
name
=
content
[
'name'
]
s
=
rec
[
name
]
or
''
d
[
name
]
=
s
+
content
[
'data'
]
elif
msg_type
==
'pyerr'
:
d
[
'pyerr'
]
=
content
elif
msg_type
==
'pyin'
:
d
[
'pyin'
]
=
content
[
'code'
]
elif
msg_type
in
(
'display_data'
,
'pyout'
)
:
d
[
msg_type
]
=
content
elif
msg_type
==
'status'
:
pass
else
:
self
.
log
.
warn
(
"unhandled iopub msg_type: %r"
,
msg_type
)
if
not
d
:
return
try
:
self
.
db
.
update_record
(
msg_id
,
d
)
except
Exception
:
self
.
log
.
error
(
"DB Error saving iopub message %r"
,
msg_id
,
exc_info
=
True
)


def connection_request(self, client_id, msg):
        """Reply with connection addresses for clients."""
        self.log.info("client::client %r connected", client_id)
        content = dict(status='ok')
        content.update(self.client_info)
        jsonable = {}
        for k,v in self.keytable.iteritems():
            if v not in self.dead_engines:
                jsonable[str(k)] = v.decode('ascii')
        content['engines'] = jsonable
        self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id)

def
connection_request
(
self
,
client_id
,
msg
)
:
self
.
log
.
info
(
"client::client %r connected"
,
client_id
)
content
=
dict
(
status
=
'ok'
)
content
.
update
(
self
.
client_info
)
jsonable
=
{
}
for
k
,
v
in
self
.
keytable
.
iteritems
(
)
:
if
v
not
in
self
.
dead_engines
:
jsonable
[
str
(
k
)
]
=
v
.
decode
(
'ascii'
)
content
[
'engines'
]
=
jsonable
self
.
session
.
send
(
self
.
query
,
'connection_reply'
,
content
,
parent
=
msg
,
ident
=
client_id
)

Reply with connection addresses for clients .


def register_engine(self, reg, msg):
        """Register a new engine."""
        content = msg['content']
        try:
            queue = cast_bytes(content['queue'])
        except KeyError:
            self.log.error("registration::queue not specified", exc_info=True)
            return
        heart = content.get('heartbeat', None)
        if heart:
            heart = cast_bytes(heart)
        """register a new engine, and create the socket(s) necessary"""
        eid = self._next_id
        # print (eid, queue, reg, heart)

        self.log.debug("registration::register_engine(%i, %r, %r, %r)", eid, queue, reg, heart)

        content = dict(id=eid,status='ok')
        content.update(self.engine_info)
        # check if requesting available IDs:
        if queue in self.by_ident:
            try:
                raise KeyError("queue_id %r in use" % queue)
            except:
                content = error.wrap_exception()
                self.log.error("queue_id %r in use", queue, exc_info=True)
        elif heart in self.hearts: # need to check unique hearts?
            try:
                raise KeyError("heart_id %r in use" % heart)
            except:
                self.log.error("heart_id %r in use", heart, exc_info=True)
                content = error.wrap_exception()
        else:
            for h, pack in self.incoming_registrations.iteritems():
                if heart == h:
                    try:
                        raise KeyError("heart_id %r in use" % heart)
                    except:
                        self.log.error("heart_id %r in use", heart, exc_info=True)
                        content = error.wrap_exception()
                    break
                elif queue == pack[1]:
                    try:
                        raise KeyError("queue_id %r in use" % queue)
                    except:
                        self.log.error("queue_id %r in use", queue, exc_info=True)
                        content = error.wrap_exception()
                    break

        msg = self.session.send(self.query, "registration_reply",
                content=content,
                ident=reg)

        if content['status'] == 'ok':
            if heart in self.heartmonitor.hearts:
                # already beating
                self.incoming_registrations[heart] = (eid,queue,reg[0],None)
                self.finish_registration(heart)
            else:
                purge = lambda : self._purge_stalled_registration(heart)
                dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop)
                dc.start()
                self.incoming_registrations[heart] = (eid,queue,reg[0],dc)
        else:
            self.log.error("registration::registration %i failed: %r", eid, content['evalue'])
        return eid


def unregister_engine(self, ident, msg):
        """Unregister an engine that explicitly requested to leave."""
        try:
            eid = msg['content']['id']
        except:
            self.log.error("registration::bad engine id for unregistration: %r", ident, exc_info=True)
            return
        self.log.info("registration::unregister_engine(%r)", eid)
        # print (eid)
        uuid = self.keytable[eid]
        content=dict(id=eid, queue=uuid.decode('ascii'))
        self.dead_engines.add(uuid)
        # self.ids.remove(eid)
        # uuid = self.keytable.pop(eid)
        #
        # ec = self.engines.pop(eid)
        # self.hearts.pop(ec.heartbeat)
        # self.by_ident.pop(ec.queue)
        # self.completed.pop(eid)
        handleit = lambda : self._handle_stranded_msgs(eid, uuid)
        dc = ioloop.DelayedCallback(handleit, self.registration_timeout, self.loop)
        dc.start()
        ############## TODO: HANDLE IT ################

        if self.notifier:
            self.session.send(self.notifier, "unregistration_notification", content=content)

Unregister an engine that explicitly requested to leave .


def finish_registration(self, heart):
        """Second half of engine registration, called after our HeartMonitor
        has received a beat from the Engine's Heart."""
        try:
            (eid,queue,reg,purge) = self.incoming_registrations.pop(heart)
        except KeyError:
            self.log.error("registration::tried to finish nonexistant registration", exc_info=True)
            return
        self.log.info("registration::finished registering engine %i:%r", eid, queue)
        if purge is not None:
            purge.stop()
        control = queue
        self.ids.add(eid)
        self.keytable[eid] = queue
        self.engines[eid] = EngineConnector(id=eid, queue=queue, registration=reg,
                                    control=control, heartbeat=heart)
        self.by_ident[queue] = eid
        self.queues[eid] = list()
        self.tasks[eid] = list()
        self.completed[eid] = list()
        self.hearts[heart] = eid
        content = dict(id=eid, queue=self.engines[eid].queue.decode('ascii'))
        if self.notifier:
            self.session.send(self.notifier, "registration_notification", content=content)
        self.log.info("engine::Engine Connected: %i", eid)

Second half of engine registration called after our HeartMonitor has received a beat from the Engine s Heart .


def shutdown_request(self, client_id, msg):
        """handle shutdown request."""
        self.session.send(self.query, 'shutdown_reply', content={'status': 'ok'}, ident=client_id)
        # also notify other clients of shutdown
        self.session.send(self.notifier, 'shutdown_notice', content={'status': 'ok'})
        dc = ioloop.DelayedCallback(lambda : self._shutdown(), 1000, self.loop)
        dc.start()

def shutdown_request ( self , client_id , msg ) : self . session . send ( self . query , 'shutdown_reply' , content = { 'status' : 'ok' } , ident = client_id ) self . session . send ( self . notifier , 'shutdown_notice' , content = { 'status' : 'ok' } ) dc = ioloop . DelayedCallback ( lambda : self . _shutdown ( ) , 1000 , self . loop ) dc . start ( )


def purge_results(self, client_id, msg):
        """Purge results from memory. This method is more valuable before we move
        to a DB based message storage mechanism."""
        content = msg['content']
        self.log.info("Dropping records with %s", content)
        msg_ids = content.get('msg_ids', [])
        reply = dict(status='ok')
        if msg_ids == 'all':
            try:
                self.db.drop_matching_records(dict(completed={'$ne':None}))
            except Exception:
                reply = error.wrap_exception()
        else:
            pending = filter(lambda m: m in self.pending, msg_ids)
            if pending:
                try:
                    raise IndexError("msg pending: %r" % pending[0])
                except:
                    reply = error.wrap_exception()
            else:
                try:
                    self.db.drop_matching_records(dict(msg_id={'$in':msg_ids}))
                except Exception:
                    reply = error.wrap_exception()

            if reply['status'] == 'ok':
                eids = content.get('engine_ids', [])
                for eid in eids:
                    if eid not in self.engines:
                        try:
                            raise IndexError("No such engine: %i" % eid)
                        except:
                            reply = error.wrap_exception()
                        break
                    uid = self.engines[eid].queue
                    try:
                        self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None}))
                    except Exception:
                        reply = error.wrap_exception()
                        break

        self.session.send(self.query, 'purge_reply', content=reply, ident=client_id)

def
purge_results
(
self
,
client_id
,
msg
)
:
content
=
msg
[
'content'
]
self
.
log
.
info
(
"Dropping records with %s"
,
content
)
msg_ids
=
content
.
get
(
'msg_ids'
,
[
]
)
reply
=
dict
(
status
=
'ok'
)
if
msg_ids
==
'all'
:
try
:
self
.
db
.
drop_matching_records
(
dict
(
completed
=
{
'$ne'
:
}
)
)
except
Exception
:
reply
=
error
.
wrap_exception
(
)
else
:
pending
=
filter
(
lambda
m
:
m
in
self
.
pending
,
msg_ids
)
if
pending
:
try
:
raise
IndexError
(
"msg pending: %r"
%
pending
[
0
]
)
except
:
reply
=
error
.
wrap_exception
(
)
else
:
try
:
self
.
db
.
drop_matching_records
(
dict
(
msg_id
=
{
'$in'
:
msg_ids
}
)
)
except
Exception
:
reply
=
error
.
wrap_exception
(
)
if
reply
[
'status'
]
==
'ok'
:
eids
=
content
.
get
(
'engine_ids'
,
[
]
)
for
eid
in
eids
:
if
eid
not
in
self
.
engines
:
try
:
raise
IndexError
(
"No such engine: %i"
%
eid
)
except
:
reply
=
error
.
wrap_exception
(
)
break
uid
=
self
.
engines
[
eid
]
.
queue
try
:
self
.
db
.
drop_matching_records
(
dict
(
engine_uuid
=
uid
,
completed
=
{
'$ne'
:
}
)
)
except
Exception
:
reply
=
error
.
wrap_exception
(
)
break
self
.
session
.
send
(
self
.
query
,
'purge_reply'
,
content
=
reply
,
ident
=
client_id
)

Purge results from memory . This method is more valuable before we move to a DB based message storage mechanism .


def _extract_record(self, rec):
        """decompose a TaskRecord dict into subsection of reply for get_result"""
        io_dict = {}
        for key in ('pyin', 'pyout', 'pyerr', 'stdout', 'stderr'):
                io_dict[key] = rec[key]
        content = { 'result_content': rec['result_content'],
                            'header': rec['header'],
                            'result_header' : rec['result_header'],
                            'received' : rec['received'],
                            'io' : io_dict,
                          }
        if rec['result_buffers']:
            buffers = map(bytes, rec['result_buffers'])
        else:
            buffers = []

        return content, buffers

def _extract_record ( self , rec ) : io_dict = { } for key in ( 'pyin' , 'pyout' , 'pyerr' , 'stdout' , 'stderr' ) : io_dict [ key ] = rec [ key ] content = { 'result_content' : rec [ 'result_content' ] , 'header' : rec [ 'header' ] , 'result_header' : rec [ 'result_header' ] , 'received' : rec [ 'received' ] , 'io' : io_dict , } if rec [ 'result_buffers' ] : buffers = map ( bytes , rec [ 'result_buffers' ] ) else : buffers = [ ] return content , buffers


def get_results(self, client_id, msg):
        """Get the result of 1 or more messages."""
        content = msg['content']
        msg_ids = sorted(set(content['msg_ids']))
        statusonly = content.get('status_only', False)
        pending = []
        completed = []
        content = dict(status='ok')
        content['pending'] = pending
        content['completed'] = completed
        buffers = []
        if not statusonly:
            try:
                matches = self.db.find_records(dict(msg_id={'$in':msg_ids}))
                # turn match list into dict, for faster lookup
                records = {}
                for rec in matches:
                    records[rec['msg_id']] = rec
            except Exception:
                content = error.wrap_exception()
                self.session.send(self.query, "result_reply", content=content,
                                                    parent=msg, ident=client_id)
                return
        else:
            records = {}
        for msg_id in msg_ids:
            if msg_id in self.pending:
                pending.append(msg_id)
            elif msg_id in self.all_completed:
                completed.append(msg_id)
                if not statusonly:
                    c,bufs = self._extract_record(records[msg_id])
                    content[msg_id] = c
                    buffers.extend(bufs)
            elif msg_id in records:
                if rec['completed']:
                    completed.append(msg_id)
                    c,bufs = self._extract_record(records[msg_id])
                    content[msg_id] = c
                    buffers.extend(bufs)
                else:
                    pending.append(msg_id)
            else:
                try:
                    raise KeyError('No such message: '+msg_id)
                except:
                    content = error.wrap_exception()
                break
        self.session.send(self.query, "result_reply", content=content,
                                            parent=msg, ident=client_id,
                                            buffers=buffers)

def
get_results
(
self
,
client_id
,
msg
)
:
content
=
msg
[
'content'
]
msg_ids
=
sorted
(
set
(
content
[
'msg_ids'
]
)
)
statusonly
=
content
.
get
(
'status_only'
,
False
)
pending
=
[
]
completed
=
[
]
content
=
dict
(
status
=
'ok'
)
content
[
'pending'
]
=
pending
content
[
'completed'
]
=
completed
buffers
=
[
]
if
not
statusonly
:
try
:
matches
=
self
.
db
.
find_records
(
dict
(
msg_id
=
{
'$in'
:
msg_ids
}
)
)
records
=
{
}
for
rec
in
matches
:
records
[
rec
[
'msg_id'
]
]
=
rec
except
Exception
:
content
=
error
.
wrap_exception
(
)
self
.
session
.
send
(
self
.
query
,
"result_reply"
,
content
=
content
,
parent
=
msg
,
ident
=
client_id
)
return
else
:
records
=
{
}
for
msg_id
in
msg_ids
:
if
msg_id
in
self
.
pending
:
pending
.
append
(
msg_id
)
elif
msg_id
in
self
.
all_completed
:
completed
.
append
(
msg_id
)
if
not
statusonly
:
c
,
bufs
=
self
.
_extract_record
(
records
[
msg_id
]
)
content
[
msg_id
]
=
c
buffers
.
extend
(
bufs
)
elif
msg_id
in
records
:
if
rec
[
'completed'
]
:
completed
.
append
(
msg_id
)
c
,
bufs
=
self
.
_extract_record
(
records
[
msg_id
]
)
content
[
msg_id
]
=
c
buffers
.
extend
(
bufs
)
else
:
pending
.
append
(
msg_id
)
else
:
try
:
raise
KeyError
(
'No such message: '
+
msg_id
)
except
:
content
=
error
.
wrap_exception
(
)
break
self
.
session
.
send
(
self
.
query
,
"result_reply"
,
content
=
content
,
parent
=
msg
,
ident
=
client_id
,
buffers
=
buffers
)


def get_history(self, client_id, msg):
        """Get a list of all msg_ids in our DB records"""
        try:
            msg_ids = self.db.get_history()
        except Exception as e:
            content = error.wrap_exception()
        else:
            content = dict(status='ok', history=msg_ids)

        self.session.send(self.query, "history_reply", content=content,
                                            parent=msg, ident=client_id)

def get_history ( self , client_id , msg ) : try : msg_ids = self . db . get_history ( ) except Exception as e : content = error . wrap_exception ( ) else : content = dict ( status = 'ok' , history = msg_ids ) self . session . send ( self . query , "history_reply" , content = content , parent = msg , ident = client_id )


def db_query(self, client_id, msg):
        """Perform a raw query on the task record database."""
        content = msg['content']
        query = content.get('query', {})
        keys = content.get('keys', None)
        buffers = []
        empty = list()
        try:
            records = self.db.find_records(query, keys)
        except Exception as e:
            content = error.wrap_exception()
        else:
            # extract buffers from reply content:
            if keys is not None:
                buffer_lens = [] if 'buffers' in keys else None
                result_buffer_lens = [] if 'result_buffers' in keys else None
            else:
                buffer_lens = None
                result_buffer_lens = None

            for rec in records:
                # buffers may be None, so double check
                b = rec.pop('buffers', empty) or empty
                if buffer_lens is not None:
                    buffer_lens.append(len(b))
                    buffers.extend(b)
                rb = rec.pop('result_buffers', empty) or empty
                if result_buffer_lens is not None:
                    result_buffer_lens.append(len(rb))
                    buffers.extend(rb)
            content = dict(status='ok', records=records, buffer_lens=buffer_lens,
                                    result_buffer_lens=result_buffer_lens)
        # self.log.debug (content)
        self.session.send(self.query, "db_reply", content=content,
                                            parent=msg, ident=client_id,
                                            buffers=buffers)

def db_query ( self , client_id , msg ) : content = msg [ 'content' ] query = content . get ( 'query' , { } ) keys = content . get ( 'keys' , None ) buffers = [ ] empty = list ( ) try : records = self . db . find_records ( query , keys ) except Exception as e : content = error . wrap_exception ( ) else : if keys is not None : buffer_lens = [ ] if 'buffers' in keys else None result_buffer_lens = [ ] if 'result_buffers' in keys else None else : buffer_lens = None result_buffer_lens = None for rec in records : b = rec . pop ( 'buffers' , empty ) or empty if buffer_lens is not None : buffer_lens . append ( len ( b ) ) buffers . extend ( b ) rb = rec . pop ( 'result_buffers' , empty ) or empty if result_buffer_lens is not None : result_buffer_lens . append ( len ( rb ) ) buffers . extend ( rb ) content = dict ( status = 'ok' , records = records , buffer_lens = buffer_lens , result_buffer_lens = result_buffer_lens ) self . session . send ( self . query , "db_reply" , content = content , parent = msg , ident = client_id , buffers = buffers )

Perform a raw query on the task record database .


def cd(self, newdir):
        """
        go to the path
        """
        prevdir = os.getcwd()
        os.chdir(newdir)
        try:
            yield
        finally:
            os.chdir(prevdir)

def cd ( self , newdir ) : prevdir = os . getcwd ( ) os . chdir ( newdir ) try : yield finally : os . chdir ( prevdir )


def decode_cmd_out(self, completed_cmd):
        """
        return a standard message
        """
        try:
            stdout = completed_cmd.stdout.encode('utf-8').decode()
        except AttributeError:
            try:
                stdout = str(bytes(completed_cmd.stdout), 'big5').strip()
            except AttributeError:
                stdout = str(bytes(completed_cmd.stdout).decode('utf-8')).strip()
        try:
            stderr = completed_cmd.stderr.encode('utf-8').decode()
        except AttributeError:
            try:
                stderr = str(bytes(completed_cmd.stderr), 'big5').strip()
            except AttributeError:
                stderr = str(bytes(completed_cmd.stderr).decode('utf-8')).strip()
        return ParsedCompletedCommand(
            completed_cmd.returncode,
            completed_cmd.args,
            stdout,
            stderr
        )

def decode_cmd_out ( self , completed_cmd ) : try : stdout = completed_cmd . stdout . encode ( 'utf-8' ) . decode ( ) except AttributeError : try : stdout = str ( bytes ( completed_cmd . stdout ) , 'big5' ) . strip ( ) except AttributeError : stdout = str ( bytes ( completed_cmd . stdout ) . decode ( 'utf-8' ) ) . strip ( ) try : stderr = completed_cmd . stderr . encode ( 'utf-8' ) . decode ( ) except AttributeError : try : stderr = str ( bytes ( completed_cmd . stderr ) , 'big5' ) . strip ( ) except AttributeError : stderr = str ( bytes ( completed_cmd . stderr ) . decode ( 'utf-8' ) ) . strip ( ) return ParsedCompletedCommand ( completed_cmd . returncode , completed_cmd . args , stdout , stderr )


def run_command_under_r_root(self, cmd, catched=True):
        """
        subprocess run on here
        """
        RPATH = self.path
        with self.cd(newdir=RPATH):
            if catched:
                process = sp.run(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
            else:
                process = sp.run(cmd)
            return process

def run_command_under_r_root ( self , cmd , catched = True ) : RPATH = self . path with self . cd ( newdir = RPATH ) : if catched : process = sp . run ( cmd , stdout = sp . PIPE , stderr = sp . PIPE ) else : process = sp . run ( cmd ) return process


def execute(self):
        """
        Execute R script
        """
        rprocess = OrderedDict()
        commands = OrderedDict([
            (self.file, ['Rscript', self.file] + self.cmd),
        ])
        for cmd_name, cmd in commands.items():
            rprocess[cmd_name] = self.run_command_under_r_root(cmd)

        return self.decode_cmd_out(completed_cmd=rprocess[self.file])

def execute ( self ) : rprocess = OrderedDict ( ) commands = OrderedDict ( [ ( self . file , [ 'Rscript' , self . file ] + self . cmd ) , ] ) for cmd_name , cmd in commands . items ( ) : rprocess [ cmd_name ] = self . run_command_under_r_root ( cmd ) return self . decode_cmd_out ( completed_cmd = rprocess [ self . file ] )


def _dispatch(self, msg):
        """ Calls the frontend handler associated with the message type of the
            given message.
        """
        msg_type = msg['header']['msg_type']
        handler = getattr(self, '_handle_' + msg_type, None)
        if handler:
            handler(msg)

def _dispatch ( self , msg ) : msg_type = msg [ 'header' ] [ 'msg_type' ] handler = getattr ( self , '_handle_' + msg_type , None ) if handler : handler ( msg )

Calls the frontend handler associated with the message type of the given message .


def _is_from_this_session(self, msg):
        """ Returns whether a reply from the kernel originated from a request
            from this frontend.
        """
        session = self._kernel_manager.session.session
        parent = msg['parent_header']
        if not parent:
            # if the message has no parent, assume it is meant for all frontends
            return True
        else:
            return parent.get('session') == session

def _is_from_this_session ( self , msg ) : session = self . _kernel_manager . session . session parent = msg [ 'parent_header' ] if not parent : return True else : return parent . get ( 'session' ) == session

Returns whether a reply from the kernel originated from a request from this frontend .


def report(self, morfs, directory=None):
        """Run the report.

        See `coverage.report()` for arguments.

        """
        self.report_files(self.annotate_file, morfs, directory)

def report ( self , morfs , directory = None ) : self . report_files ( self . annotate_file , morfs , directory )


def annotate_file(self, cu, analysis):
        """Annotate a single file.

        `cu` is the CodeUnit for the file to annotate.

        """
        if not cu.relative:
            return

        filename = cu.filename
        source = cu.source_file()
        if self.directory:
            dest_file = os.path.join(self.directory, cu.flat_rootname())
            dest_file += ".py,cover"
        else:
            dest_file = filename + ",cover"
        dest = open(dest_file, 'w')

        statements = sorted(analysis.statements)
        missing = sorted(analysis.missing)
        excluded = sorted(analysis.excluded)

        lineno = 0
        i = 0
        j = 0
        covered = True
        while True:
            line = source.readline()
            if line == '':
                break
            lineno += 1
            while i < len(statements) and statements[i] < lineno:
                i += 1
            while j < len(missing) and missing[j] < lineno:
                j += 1
            if i < len(statements) and statements[i] == lineno:
                covered = j >= len(missing) or missing[j] > lineno
            if self.blank_re.match(line):
                dest.write('  ')
            elif self.else_re.match(line):
                # Special logic for lines containing only 'else:'.
                if i >= len(statements) and j >= len(missing):
                    dest.write('! ')
                elif i >= len(statements) or j >= len(missing):
                    dest.write('> ')
                elif statements[i] == missing[j]:
                    dest.write('! ')
                else:
                    dest.write('> ')
            elif lineno in excluded:
                dest.write('- ')
            elif covered:
                dest.write('> ')
            else:
                dest.write('! ')
            dest.write(line)
        source.close()
        dest.close()

def
annotate_file
(
self
,
cu
,
analysis
)
:
if
not
cu
.
relative
:
return
filename
=
cu
.
filename
source
=
cu
.
source_file
(
)
if
self
.
directory
:
dest_file
=
os
.
path
.
join
(
self
.
directory
,
cu
.
flat_rootname
(
)
)
dest_file
+=
".py,cover"
else
:
dest_file
=
filename
+
",cover"
dest
=
open
(
dest_file
,
'w'
)
statements
=
sorted
(
analysis
.
statements
)
missing
=
sorted
(
analysis
.
missing
)
excluded
=
sorted
(
analysis
.
excluded
)
lineno
=
0
i
=
0
j
=
0
covered
=
True
while
True
:
line
=
source
.
readline
(
)
if
line
==
''
:
break
lineno
+=
1
while
i
<
len
(
statements
)
and
statements
[
i
]
<
lineno
:
i
+=
1
while
j
<
len
(
missing
)
and
missing
[
j
]
<
lineno
:
j
+=
1
if
i
<
len
(
statements
)
and
statements
[
i
]
==
lineno
:
covered
=
j
>=
len
(
missing
)
or
missing
[
j
]
>
lineno
if
self
.
blank_re
.
match
(
line
)
:
dest
.
write
(
'  '
)
elif
self
.
else_re
.
match
(
line
)
:
if
i
>=
len
(
statements
)
and
j
>=
len
(
missing
)
:
dest
.
write
(
'! '
)
elif
i
>=
len
(
statements
)
or
j
>=
len
(
missing
)
:
dest
.
write
(
'> '
)
elif
statements
[
i
]
==
missing
[
j
]
:
dest
.
write
(
'! '
)
else
:
dest
.
write
(
'> '
)
elif
lineno
in
excluded
:
dest
.
write
(
'- '
)
elif
covered
:
dest
.
write
(
'> '
)
else
:
dest
.
write
(
'! '
)
dest
.
write
(
line
)
source
.
close
(
)
dest
.
close
(
)


def get_installed_version(name):
    '''
        returns installed package version and None if package is not installed
    '''
    pattern = re.compile(r'''Installed:\s+(?P<version>.*)''')
    cmd = 'apt-cache policy %s' % name
    args = shlex.split(cmd)
    try:
        output = subprocess.check_output(args)
        if not output:
            return None
    except CalledProcessError:
        return None
    # check output
    match = pattern.search(output)
    if match:
        version = match.groupdict()['version']
        if version == '(none)':
            return None
        else:
            return version

def
get_installed_version
(
name
)
:
pattern
=
re
.
compile
(
r
)
cmd
=
'apt-cache policy %s'
%
name
args
=
shlex
.
split
(
cmd
)
try
:
output
=
subprocess
.
check_output
(
args
)
if
not
output
:
return
except
CalledProcessError
:
return
match
=
pattern
.
search
(
output
)
if
match
:
version
=
match
.
groupdict
(
)
[
'version'
]
if
version
==
'(none)'
:
return
else
:
return
version


def squash_unicode(obj):
    """coerce unicode back to bytestrings."""
    if isinstance(obj,dict):
        for key in obj.keys():
            obj[key] = squash_unicode(obj[key])
            if isinstance(key, unicode):
                obj[squash_unicode(key)] = obj.pop(key)
    elif isinstance(obj, list):
        for i,v in enumerate(obj):
            obj[i] = squash_unicode(v)
    elif isinstance(obj, unicode):
        obj = obj.encode('utf8')
    return obj

def squash_unicode ( obj ) : if isinstance ( obj , dict ) : for key in obj . keys ( ) : obj [ key ] = squash_unicode ( obj [ key ] ) if isinstance ( key , unicode ) : obj [ squash_unicode ( key ) ] = obj . pop ( key ) elif isinstance ( obj , list ) : for i , v in enumerate ( obj ) : obj [ i ] = squash_unicode ( v ) elif isinstance ( obj , unicode ) : obj = obj . encode ( 'utf8' ) return obj


def extract_header(msg_or_header):
    """Given a message or header, return the header."""
    if not msg_or_header:
        return {}
    try:
        # See if msg_or_header is the entire message.
        h = msg_or_header['header']
    except KeyError:
        try:
            # See if msg_or_header is just the header
            h = msg_or_header['msg_id']
        except KeyError:
            raise
        else:
            h = msg_or_header
    if not isinstance(h, dict):
        h = dict(h)
    return h

def extract_header ( msg_or_header ) : if not msg_or_header : return { } try : h = msg_or_header [ 'header' ] except KeyError : try : h = msg_or_header [ 'msg_id' ] except KeyError : raise else : h = msg_or_header if not isinstance ( h , dict ) : h = dict ( h ) return h

Given a message or header return the header .


def _check_packers(self):
        """check packers for binary data and datetime support."""
        pack = self.pack
        unpack = self.unpack

        # check simple serialization
        msg = dict(a=[1,'hi'])
        try:
            packed = pack(msg)
        except Exception:
            raise ValueError("packer could not serialize a simple message")

        # ensure packed message is bytes
        if not isinstance(packed, bytes):
            raise ValueError("message packed to %r, but bytes are required"%type(packed))

        # check that unpack is pack's inverse
        try:
            unpacked = unpack(packed)
        except Exception:
            raise ValueError("unpacker could not handle the packer's output")

        # check datetime support
        msg = dict(t=datetime.now())
        try:
            unpacked = unpack(pack(msg))
        except Exception:
            self.pack = lambda o: pack(squash_dates(o))
            self.unpack = lambda s: extract_dates(unpack(s))

check packers for binary data and datetime support .


def msg(self, msg_type, content=None, parent=None, subheader=None, header=None):
        """Return the nested message dict.

        This format is different from what is sent over the wire. The
        serialize/unserialize methods converts this nested message dict to the wire
        format, which is a list of message parts.
        """
        msg = {}
        header = self.msg_header(msg_type) if header is None else header
        msg['header'] = header
        msg['msg_id'] = header['msg_id']
        msg['msg_type'] = header['msg_type']
        msg['parent_header'] = {} if parent is None else extract_header(parent)
        msg['content'] = {} if content is None else content
        sub = {} if subheader is None else subheader
        msg['header'].update(sub)
        return msg

def msg ( self , msg_type , content = None , parent = None , subheader = None , header = None ) : msg = { } header = self . msg_header ( msg_type ) if header is None else header msg [ 'header' ] = header msg [ 'msg_id' ] = header [ 'msg_id' ] msg [ 'msg_type' ] = header [ 'msg_type' ] msg [ 'parent_header' ] = { } if parent is None else extract_header ( parent ) msg [ 'content' ] = { } if content is None else content sub = { } if subheader is None else subheader msg [ 'header' ] . update ( sub ) return msg


def sign(self, msg_list):
        """Sign a message with HMAC digest. If no auth, return b''.

        Parameters
        ----------
        msg_list : list
            The [p_header,p_parent,p_content] part of the message list.
        """
        if self.auth is None:
            return b''
        h = self.auth.copy()
        for m in msg_list:
            h.update(m)
        return str_to_bytes(h.hexdigest())

def sign ( self , msg_list ) : if self . auth is None : return b'' h = self . auth . copy ( ) for m in msg_list : h . update ( m ) return str_to_bytes ( h . hexdigest ( ) )

Sign a message with HMAC digest . If no auth return b .


def serialize(self, msg, ident=None):
        """Serialize the message components to bytes.

        This is roughly the inverse of unserialize. The serialize/unserialize
        methods work with full message lists, whereas pack/unpack work with
        the individual message parts in the message list.

        Parameters
        ----------
        msg : dict or Message
            The nexted message dict as returned by the self.msg method.

        Returns
        -------
        msg_list : list
            The list of bytes objects to be sent with the format:
            [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,
             buffer1,buffer2,...]. In this list, the p_* entities are
            the packed or serialized versions, so if JSON is used, these
            are utf8 encoded JSON strings.
        """
        content = msg.get('content', {})
        if content is None:
            content = self.none
        elif isinstance(content, dict):
            content = self.pack(content)
        elif isinstance(content, bytes):
            # content is already packed, as in a relayed message
            pass
        elif isinstance(content, unicode):
            # should be bytes, but JSON often spits out unicode
            content = content.encode('utf8')
        else:
            raise TypeError("Content incorrect type: %s"%type(content))

        real_message = [self.pack(msg['header']),
                        self.pack(msg['parent_header']),
                        content
        ]

        to_send = []

        if isinstance(ident, list):
            # accept list of idents
            to_send.extend(ident)
        elif ident is not None:
            to_send.append(ident)
        to_send.append(DELIM)

        signature = self.sign(real_message)
        to_send.append(signature)

        to_send.extend(real_message)

        return to_send

def
serialize
(
self
,
msg
,
ident
=
)
:
content
=
msg
.
get
(
'content'
,
{
}
)
if
content
is
:
content
=
self
.
elif
isinstance
(
content
,
dict
)
:
content
=
self
.
pack
(
content
)
elif
isinstance
(
content
,
bytes
)
:
pass
elif
isinstance
(
content
,
unicode
)
:
content
=
content
.
encode
(
'utf8'
)
else
:
raise
TypeError
(
"Content incorrect type: %s"
%
type
(
content
)
)
real_message
=
[
self
.
pack
(
msg
[
'header'
]
)
,
self
.
pack
(
msg
[
'parent_header'
]
)
,
content
]
to_send
=
[
]
if
isinstance
(
ident
,
list
)
:
to_send
.
extend
(
ident
)
elif
ident
is
not
:
to_send
.
append
(
ident
)
to_send
.
append
(
DELIM
)
signature
=
self
.
sign
(
real_message
)
to_send
.
append
(
signature
)
to_send
.
extend
(
real_message
)
return
to_send

Serialize the message components to bytes .


def send(self, stream, msg_or_type, content=None, parent=None, ident=None,
             buffers=None, subheader=None, track=False, header=None):
        """Build and send a message via stream or socket.

        The message format used by this function internally is as follows:

        [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,
         buffer1,buffer2,...]

        The serialize/unserialize methods convert the nested message dict into this
        format.

        Parameters
        ----------

        stream : zmq.Socket or ZMQStream
            The socket-like object used to send the data.
        msg_or_type : str or Message/dict
            Normally, msg_or_type will be a msg_type unless a message is being
            sent more than once. If a header is supplied, this can be set to
            None and the msg_type will be pulled from the header.

        content : dict or None
            The content of the message (ignored if msg_or_type is a message).
        header : dict or None
            The header dict for the message (ignores if msg_to_type is a message).
        parent : Message or dict or None
            The parent or parent header describing the parent of this message
            (ignored if msg_or_type is a message).
        ident : bytes or list of bytes
            The zmq.IDENTITY routing path.
        subheader : dict or None
            Extra header keys for this message's header (ignored if msg_or_type
            is a message).
        buffers : list or None
            The already-serialized buffers to be appended to the message.
        track : bool
            Whether to track.  Only for use with Sockets, because ZMQStream
            objects cannot track messages.

        Returns
        -------
        msg : dict
            The constructed message.
        (msg,tracker) : (dict, MessageTracker)
            if track=True, then a 2-tuple will be returned,
            the first element being the constructed
            message, and the second being the MessageTracker

        """

        if not isinstance(stream, (zmq.Socket, ZMQStream)):
            raise TypeError("stream must be Socket or ZMQStream, not %r"%type(stream))
        elif track and isinstance(stream, ZMQStream):
            raise TypeError("ZMQStream cannot track messages")

        if isinstance(msg_or_type, (Message, dict)):
            # We got a Message or message dict, not a msg_type so don't
            # build a new Message.
            msg = msg_or_type
        else:
            msg = self.msg(msg_or_type, content=content, parent=parent,
                           subheader=subheader, header=header)

        buffers = [] if buffers is None else buffers
        to_send = self.serialize(msg, ident)
        flag = 0
        if buffers:
            flag = zmq.SNDMORE
            _track = False
        else:
            _track=track
        if track:
            tracker = stream.send_multipart(to_send, flag, copy=False, track=_track)
        else:
            tracker = stream.send_multipart(to_send, flag, copy=False)
        for b in buffers[:-1]:
            stream.send(b, flag, copy=False)
        if buffers:
            if track:
                tracker = stream.send(buffers[-1], copy=False, track=track)
            else:
                tracker = stream.send(buffers[-1], copy=False)

        # omsg = Message(msg)
        if self.debug:
            pprint.pprint(msg)
            pprint.pprint(to_send)
            pprint.pprint(buffers)

        msg['tracker'] = tracker

        return msg

Build and send a message via stream or socket .


def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None):
        """Send a raw message via ident path.

        This method is used to send a already serialized message.

        Parameters
        ----------
        stream : ZMQStream or Socket
            The ZMQ stream or socket to use for sending the message.
        msg_list : list
            The serialized list of messages to send. This only includes the
            [p_header,p_parent,p_content,buffer1,buffer2,...] portion of
            the message.
        ident : ident or list
            A single ident or a list of idents to use in sending.
        """
        to_send = []
        if isinstance(ident, bytes):
            ident = [ident]
        if ident is not None:
            to_send.extend(ident)

        to_send.append(DELIM)
        to_send.append(self.sign(msg_list))
        to_send.extend(msg_list)
        stream.send_multipart(msg_list, flags, copy=copy)

def send_raw ( self , stream , msg_list , flags = 0 , copy = True , ident = None ) : to_send = [ ] if isinstance ( ident , bytes ) : ident = [ ident ] if ident is not None : to_send . extend ( ident ) to_send . append ( DELIM ) to_send . append ( self . sign ( msg_list ) ) to_send . extend ( msg_list ) stream . send_multipart ( msg_list , flags , copy = copy )


def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
        """Receive and unpack a message.

        Parameters
        ----------
        socket : ZMQStream or Socket
            The socket or stream to use in receiving.

        Returns
        -------
        [idents], msg
            [idents] is a list of idents and msg is a nested message dict of
            same format as self.msg returns.
        """
        if isinstance(socket, ZMQStream):
            socket = socket.socket
        try:
            msg_list = socket.recv_multipart(mode, copy=copy)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                # We can convert EAGAIN to None as we know in this case
                # recv_multipart won't return None.
                return None,None
            else:
                raise
        # split multipart message into identity list and message dict
        # invalid large messages can cause very expensive string comparisons
        idents, msg_list = self.feed_identities(msg_list, copy)
        try:
            return idents, self.unserialize(msg_list, content=content, copy=copy)
        except Exception as e:
            # TODO: handle it
            raise e

def recv ( self , socket , mode = zmq . NOBLOCK , content = True , copy = True ) : if isinstance ( socket , ZMQStream ) : socket = socket . socket try : msg_list = socket . recv_multipart ( mode , copy = copy ) except zmq . ZMQError as e : if e . errno == zmq . EAGAIN : return None , None else : raise idents , msg_list = self . feed_identities ( msg_list , copy ) try : return idents , self . unserialize ( msg_list , content = content , copy = copy ) except Exception as e : raise e


def feed_identities(self, msg_list, copy=True):
        """Split the identities from the rest of the message.

        Feed until DELIM is reached, then return the prefix as idents and
        remainder as msg_list. This is easily broken by setting an IDENT to DELIM,
        but that would be silly.

        Parameters
        ----------
        msg_list : a list of Message or bytes objects
            The message to be split.
        copy : bool
            flag determining whether the arguments are bytes or Messages

        Returns
        -------
        (idents, msg_list) : two lists
            idents will always be a list of bytes, each of which is a ZMQ
            identity. msg_list will be a list of bytes or zmq.Messages of the
            form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and
            should be unpackable/unserializable via self.unserialize at this
            point.
        """
        if copy:
            idx = msg_list.index(DELIM)
            return msg_list[:idx], msg_list[idx+1:]
        else:
            failed = True
            for idx,m in enumerate(msg_list):
                if m.bytes == DELIM:
                    failed = False
                    break
            if failed:
                raise ValueError("DELIM not in msg_list")
            idents, msg_list = msg_list[:idx], msg_list[idx+1:]
            return [m.bytes for m in idents], msg_list

def
feed_identities
(
self
,
msg_list
,
copy
=
True
)
:
if
copy
:
idx
=
msg_list
.
index
(
DELIM
)
return
msg_list
[
:
idx
]
,
msg_list
[
idx
+
1
:
]
else
:
failed
=
True
for
idx
,
m
in
enumerate
(
msg_list
)
:
if
m
.
bytes
==
DELIM
:
failed
=
False
break
if
failed
:
raise
ValueError
(
"DELIM not in msg_list"
)
idents
,
msg_list
=
msg_list
[
:
idx
]
,
msg_list
[
idx
+
1
:
]
return
[
m
.
bytes
for
m
in
idents
]
,
msg_list

Split the identities from the rest of the message .


def unserialize(self, msg_list, content=True, copy=True):
        """Unserialize a msg_list to a nested message dict.

        This is roughly the inverse of serialize. The serialize/unserialize
        methods work with full message lists, whereas pack/unpack work with
        the individual message parts in the message list.

        Parameters:
        -----------
        msg_list : list of bytes or Message objects
            The list of message parts of the form [HMAC,p_header,p_parent,
            p_content,buffer1,buffer2,...].
        content : bool (True)
            Whether to unpack the content dict (True), or leave it packed
            (False).
        copy : bool (True)
            Whether to return the bytes (True), or the non-copying Message
            object in each place (False).

        Returns
        -------
        msg : dict
            The nested message dict with top-level keys [header, parent_header,
            content, buffers].
        """
        minlen = 4
        message = {}
        if not copy:
            for i in range(minlen):
                msg_list[i] = msg_list[i].bytes
        if self.auth is not None:
            signature = msg_list[0]
            if not signature:
                raise ValueError("Unsigned Message")
            if signature in self.digest_history:
                raise ValueError("Duplicate Signature: %r"%signature)
            self.digest_history.add(signature)
            check = self.sign(msg_list[1:4])
            if not signature == check:
                raise ValueError("Invalid Signature: %r"%signature)
        if not len(msg_list) >= minlen:
            raise TypeError("malformed message, must have at least %i elements"%minlen)
        header = self.unpack(msg_list[1])
        message['header'] = header
        message['msg_id'] = header['msg_id']
        message['msg_type'] = header['msg_type']
        message['parent_header'] = self.unpack(msg_list[2])
        if content:
            message['content'] = self.unpack(msg_list[3])
        else:
            message['content'] = msg_list[3]

        message['buffers'] = msg_list[4:]
        return message

Unserialize a msg_list to a nested message dict .


def save_svg(string, parent=None):
    """ Prompts the user to save an SVG document to disk.

    Parameters:
    -----------
    string : basestring
        A Python string containing a SVG document.

    parent : QWidget, optional
        The parent to use for the file dialog.

    Returns:
    --------
    The name of the file to which the document was saved, or None if the save
    was cancelled.
    """
    if isinstance(string, unicode):
        string = string.encode('utf-8')

    dialog = QtGui.QFileDialog(parent, 'Save SVG Document')
    dialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
    dialog.setDefaultSuffix('svg')
    dialog.setNameFilter('SVG document (*.svg)')
    if dialog.exec_():
        filename = dialog.selectedFiles()[0]
        f = open(filename, 'w')
        try:
            f.write(string)
        finally:
            f.close()
        return filename
    return None

def
save_svg
(
string
,
parent
=
)
:
if
isinstance
(
string
,
unicode
)
:
string
=
string
.
encode
(
'utf-8'
)
dialog
=
QtGui
.
QFileDialog
(
parent
,
'Save SVG Document'
)
dialog
.
setAcceptMode
(
QtGui
.
QFileDialog
.
AcceptSave
)
dialog
.
setDefaultSuffix
(
'svg'
)
dialog
.
setNameFilter
(
'SVG document (*.svg)'
)
if
dialog
.
exec_
(
)
:
filename
=
dialog
.
selectedFiles
(
)
[
0
]
f
=
open
(
filename
,
'w'
)
try
:
f
.
write
(
string
)
finally
:
f
.
close
(
)
return
filename
return

Prompts the user to save an SVG document to disk .


def svg_to_clipboard(string):
    """ Copy a SVG document to the clipboard.

    Parameters:
    -----------
    string : basestring
        A Python string containing a SVG document.
    """
    if isinstance(string, unicode):
        string = string.encode('utf-8')

    mime_data = QtCore.QMimeData()
    mime_data.setData('image/svg+xml', string)
    QtGui.QApplication.clipboard().setMimeData(mime_data)

def svg_to_clipboard ( string ) : if isinstance ( string , unicode ) : string = string . encode ( 'utf-8' ) mime_data = QtCore . QMimeData ( ) mime_data . setData ( 'image/svg+xml' , string ) QtGui . QApplication . clipboard ( ) . setMimeData ( mime_data )


def svg_to_image(string, size=None):
    """ Convert a SVG document to a QImage.

    Parameters:
    -----------
    string : basestring
        A Python string containing a SVG document.

    size : QSize, optional
        The size of the image that is produced. If not specified, the SVG
        document's default size is used.

    Raises:
    -------
    ValueError
        If an invalid SVG string is provided.

    Returns:
    --------
    A QImage of format QImage.Format_ARGB32.
    """
    if isinstance(string, unicode):
        string = string.encode('utf-8')

    renderer = QtSvg.QSvgRenderer(QtCore.QByteArray(string))
    if not renderer.isValid():
        raise ValueError('Invalid SVG data.')

    if size is None:
        size = renderer.defaultSize()
    image = QtGui.QImage(size, QtGui.QImage.Format_ARGB32)
    painter = QtGui.QPainter(image)
    renderer.render(painter)
    return image

def
svg_to_image
(
string
,
size
=
)
:
if
isinstance
(
string
,
unicode
)
:
string
=
string
.
encode
(
'utf-8'
)
renderer
=
QtSvg
.
QSvgRenderer
(
QtCore
.
QByteArray
(
string
)
)
if
not
renderer
.
isValid
(
)
:
raise
ValueError
(
'Invalid SVG data.'
)
if
size
is
:
size
=
renderer
.
defaultSize
(
)
image
=
QtGui
.
QImage
(
size
,
QtGui
.
QImage
.
Format_ARGB32
)
painter
=
QtGui
.
QPainter
(
image
)
renderer
.
render
(
painter
)
return
image


def object_info(**kw):
    """Make an object info dict with all fields present."""
    infodict = dict(izip_longest(info_fields, [None]))
    infodict.update(kw)
    return infodict

def object_info ( ** kw ) : infodict = dict ( izip_longest ( info_fields , [ None ] ) ) infodict . update ( kw ) return infodict

Make an object info dict with all fields present .


def getdoc(obj):
    """Stable wrapper around inspect.getdoc.

    This can't crash because of attribute problems.

    It also attempts to call a getdoc() method on the given object.  This
    allows objects which provide their docstrings via non-standard mechanisms
    (like Pyro proxies) to still be inspected by ipython's ? system."""
    # Allow objects to offer customized documentation via a getdoc method:
    try:
        ds = obj.getdoc()
    except Exception:
        pass
    else:
        # if we get extra info, we add it to the normal docstring.
        if isinstance(ds, basestring):
            return inspect.cleandoc(ds)

    try:
        return inspect.getdoc(obj)
    except Exception:
        # Harden against an inspect failure, which can occur with
        # SWIG-wrapped extensions.
        return None

def getdoc ( obj ) : try : ds = obj . getdoc ( ) except Exception : pass else : if isinstance ( ds , basestring ) : return inspect . cleandoc ( ds ) try : return inspect . getdoc ( obj ) except Exception : return None

Stable wrapper around inspect . getdoc .


def getsource(obj,is_binary=False):
    """Wrapper around inspect.getsource.

    This can be modified by other projects to provide customized source
    extraction.

    Inputs:

    - obj: an object whose source code we will attempt to extract.

    Optional inputs:

    - is_binary: whether the object is known to come from a binary source.
    This implementation will skip returning any output for binary objects, but
    custom extractors may know how to meaningfully process them."""

    if is_binary:
        return None
    else:
        # get source if obj was decorated with @decorator
        if hasattr(obj,"__wrapped__"):
            obj = obj.__wrapped__
        try:
            src = inspect.getsource(obj)
        except TypeError:
            if hasattr(obj,'__class__'):
                src = inspect.getsource(obj.__class__)
        return src

def getsource ( obj , is_binary = False ) : if is_binary : return None else : if hasattr ( obj , "__wrapped__" ) : obj = obj . __wrapped__ try : src = inspect . getsource ( obj ) except TypeError : if hasattr ( obj , '__class__' ) : src = inspect . getsource ( obj . __class__ ) return src


def getargspec(obj):
    """Get the names and default values of a function's arguments.

    A tuple of four things is returned: (args, varargs, varkw, defaults).
    'args' is a list of the argument names (it may contain nested lists).
    'varargs' and 'varkw' are the names of the * and ** arguments or None.
    'defaults' is an n-tuple of the default values of the last n arguments.

    Modified version of inspect.getargspec from the Python Standard
    Library."""

    if inspect.isfunction(obj):
        func_obj = obj
    elif inspect.ismethod(obj):
        func_obj = obj.im_func
    elif hasattr(obj, '__call__'):
        func_obj = obj.__call__
    else:
        raise TypeError('arg is not a Python function')
    args, varargs, varkw = inspect.getargs(func_obj.func_code)
    return args, varargs, varkw, func_obj.func_defaults

def
getargspec
(
obj
)
:
if
inspect
.
isfunction
(
obj
)
:
func_obj
=
obj
elif
inspect
.
ismethod
(
obj
)
:
func_obj
=
obj
.
im_func
elif
hasattr
(
obj
,
'__call__'
)
:
func_obj
=
obj
.
__call__
else
:
raise
TypeError
(
'arg is not a Python function'
)
args
,
varargs
,
varkw
=
inspect
.
getargs
(
func_obj
.
func_code
)
return
args
,
varargs
,
varkw
,
func_obj
.
func_defaults

Get the names and default values of a function s arguments .


def call_tip(oinfo, format_call=True):
    """Extract call tip data from an oinfo dict.

    Parameters
    ----------
    oinfo : dict

    format_call : bool, optional
      If True, the call line is formatted and returned as a string.  If not, a
      tuple of (name, argspec) is returned.

    Returns
    -------
    call_info : None, str or (str, dict) tuple.
      When format_call is True, the whole call information is formattted as a
      single string.  Otherwise, the object's name and its argspec dict are
      returned.  If no call information is available, None is returned.

    docstring : str or None
      The most relevant docstring for calling purposes is returned, if
      available.  The priority is: call docstring for callable instances, then
      constructor docstring for classes, then main object's docstring otherwise
      (regular functions).
    """
    # Get call definition
    argspec = oinfo.get('argspec')
    if argspec is None:
        call_line = None
    else:
        # Callable objects will have 'self' as their first argument, prune
        # it out if it's there for clarity (since users do *not* pass an
        # extra first argument explicitly).
        try:
            has_self = argspec['args'][0] == 'self'
        except (KeyError, IndexError):
            pass
        else:
            if has_self:
                argspec['args'] = argspec['args'][1:]

        call_line = oinfo['name']+format_argspec(argspec)

    # Now get docstring.
    # The priority is: call docstring, constructor docstring, main one.
    doc = oinfo.get('call_docstring')
    if doc is None:
        doc = oinfo.get('init_docstring')
    if doc is None:
        doc = oinfo.get('docstring','')

    return call_line, doc

def call_tip ( oinfo , format_call = True ) : argspec = oinfo . get ( 'argspec' ) if argspec is None : call_line = None else : try : has_self = argspec [ 'args' ] [ 0 ] == 'self' except ( KeyError , IndexError ) : pass else : if has_self : argspec [ 'args' ] = argspec [ 'args' ] [ 1 : ] call_line = oinfo [ 'name' ] + format_argspec ( argspec ) doc = oinfo . get ( 'call_docstring' ) if doc is None : doc = oinfo . get ( 'init_docstring' ) if doc is None : doc = oinfo . get ( 'docstring' , '' ) return call_line , doc

Extract call tip data from an oinfo dict .


def find_file(obj):
    """Find the absolute path to the file where an object was defined.

    This is essentially a robust wrapper around `inspect.getabsfile`.

    Returns None if no file can be found.

    Parameters
    ----------
    obj : any Python object

    Returns
    -------
    fname : str
      The absolute path to the file where the object was defined.
    """
    # get source if obj was decorated with @decorator
    if hasattr(obj, '__wrapped__'):
        obj = obj.__wrapped__

    fname = None
    try:
        fname = inspect.getabsfile(obj)
    except TypeError:
        # For an instance, the file that matters is where its class was
        # declared.
        if hasattr(obj, '__class__'):
            try:
                fname = inspect.getabsfile(obj.__class__)
            except TypeError:
                # Can happen for builtins
                pass
    except:
        pass
    return fname

def find_file ( obj ) : if hasattr ( obj , '__wrapped__' ) : obj = obj . __wrapped__ fname = None try : fname = inspect . getabsfile ( obj ) except TypeError : if hasattr ( obj , '__class__' ) : try : fname = inspect . getabsfile ( obj . __class__ ) except TypeError : pass except : pass return fname

Find the absolute path to the file where an object was defined .


def find_source_lines(obj):
    """Find the line number in a file where an object was defined.

    This is essentially a robust wrapper around `inspect.getsourcelines`.

    Returns None if no file can be found.

    Parameters
    ----------
    obj : any Python object

    Returns
    -------
    lineno : int
      The line number where the object definition starts.
    """
    # get source if obj was decorated with @decorator
    if hasattr(obj, '__wrapped__'):
        obj = obj.__wrapped__

    try:
        try:
            lineno = inspect.getsourcelines(obj)[1]
        except TypeError:
            # For instances, try the class object like getsource() does
            if hasattr(obj, '__class__'):
                lineno = inspect.getsourcelines(obj.__class__)[1]
    except:
        return None

    return lineno

def find_source_lines ( obj ) : if hasattr ( obj , '__wrapped__' ) : obj = obj . __wrapped__ try : try : lineno = inspect . getsourcelines ( obj ) [ 1 ] except TypeError : if hasattr ( obj , '__class__' ) : lineno = inspect . getsourcelines ( obj . __class__ ) [ 1 ] except : return None return lineno

Find the line number in a file where an object was defined .


def _getdef(self,obj,oname=''):
        """Return the definition header for any callable object.

        If any exception is generated, None is returned instead and the
        exception is suppressed."""

        try:
            # We need a plain string here, NOT unicode!
            hdef = oname + inspect.formatargspec(*getargspec(obj))
            return py3compat.unicode_to_str(hdef, 'ascii')
        except:
            return None

def _getdef ( self , obj , oname = '' ) : try : hdef = oname + inspect . formatargspec ( * getargspec ( obj ) ) return py3compat . unicode_to_str ( hdef , 'ascii' ) except : return None

Return the definition header for any callable object .


def __head(self,h):
        """Return a header string with proper colors."""
        return '%s%s%s' % (self.color_table.active_colors.header,h,
                           self.color_table.active_colors.normal)

def __head ( self , h ) : return '%s%s%s' % ( self . color_table . active_colors . header , h , self . color_table . active_colors . normal )

Return a header string with proper colors .


def noinfo(self, msg, oname):
        """Generic message when no information is found."""
        print 'No %s found' % msg,
        if oname:
            print 'for %s' % oname
        else:
            print

def
noinfo
(
self
,
msg
,
oname
)
:
print
'No %s found'
%
msg
,
if
oname
:
print
'for %s'
%
oname
else
:
print

Generic message when no information is found .


def pdef(self, obj, oname=''):
        """Print the definition header for any callable object.

        If the object is a class, print the constructor information."""

        if not callable(obj):
            print 'Object is not callable.'
            return

        header = ''

        if inspect.isclass(obj):
            header = self.__head('Class constructor information:\n')
            obj = obj.__init__
        elif (not py3compat.PY3) and type(obj) is types.InstanceType:
            obj = obj.__call__

        output = self._getdef(obj,oname)
        if output is None:
            self.noinfo('definition header',oname)
        else:
            print >>io.stdout, header,self.format(output),

def
pdef
(
self
,
obj
,
oname
=
''
)
:
if
not
callable
(
obj
)
:
print
'Object is not callable.'
return
header
=
''
if
inspect
.
isclass
(
obj
)
:
header
=
self
.
__head
(
'Class constructor information:\n'
)
obj
=
obj
.
__init__
elif
(
not
py3compat
.
PY3
)
and
type
(
obj
)
is
types
.
InstanceType
:
obj
=
obj
.
__call__
output
=
self
.
_getdef
(
obj
,
oname
)
if
output
is
:
self
.
noinfo
(
'definition header'
,
oname
)
else
:
print
>>
io
.
stdout
,
header
,
self
.
format
(
output
)
,

Print the definition header for any callable object .


def pdoc(self,obj,oname='',formatter = None):
        """Print the docstring for any object.

        Optional:
        -formatter: a function to run the docstring through for specially
        formatted docstrings.

        Examples
        --------

        In [1]: class NoInit:
           ...:     pass

        In [2]: class NoDoc:
           ...:     def __init__(self):
           ...:         pass

        In [3]: %pdoc NoDoc
        No documentation found for NoDoc

        In [4]: %pdoc NoInit
        No documentation found for NoInit

        In [5]: obj = NoInit()

        In [6]: %pdoc obj
        No documentation found for obj

        In [5]: obj2 = NoDoc()

        In [6]: %pdoc obj2
        No documentation found for obj2
        """

        head = self.__head  # For convenience
        lines = []
        ds = getdoc(obj)
        if formatter:
            ds = formatter(ds)
        if ds:
            lines.append(head("Class Docstring:"))
            lines.append(indent(ds))
        if inspect.isclass(obj) and hasattr(obj, '__init__'):
            init_ds = getdoc(obj.__init__)
            if init_ds is not None:
                lines.append(head("Constructor Docstring:"))
                lines.append(indent(init_ds))
        elif hasattr(obj,'__call__'):
            call_ds = getdoc(obj.__call__)
            if call_ds:
                lines.append(head("Calling Docstring:"))
                lines.append(indent(call_ds))

        if not lines:
            self.noinfo('documentation',oname)
        else:
            page.page('\n'.join(lines))

def
pdoc
(
self
,
obj
,
oname
=
''
,
formatter
=
)
:
head
=
self
.
__head
lines
=
[
]
ds
=
getdoc
(
obj
)
if
formatter
:
ds
=
formatter
(
ds
)
if
ds
:
lines
.
append
(
head
(
"Class Docstring:"
)
)
lines
.
append
(
indent
(
ds
)
)
if
inspect
.
isclass
(
obj
)
and
hasattr
(
obj
,
'__init__'
)
:
init_ds
=
getdoc
(
obj
.
__init__
)
if
init_ds
is
not
:
lines
.
append
(
head
(
"Constructor Docstring:"
)
)
lines
.
append
(
indent
(
init_ds
)
)
elif
hasattr
(
obj
,
'__call__'
)
:
call_ds
=
getdoc
(
obj
.
__call__
)
if
call_ds
:
lines
.
append
(
head
(
"Calling Docstring:"
)
)
lines
.
append
(
indent
(
call_ds
)
)
if
not
lines
:
self
.
noinfo
(
'documentation'
,
oname
)
else
:
page
.
page
(
'\n'
.
join
(
lines
)
)


def psource(self,obj,oname=''):
        """Print the source code for an object."""

        # Flush the source cache because inspect can return out-of-date source
        linecache.checkcache()
        try:
            src = getsource(obj)
        except:
            self.noinfo('source',oname)
        else:
            page.page(self.format(py3compat.unicode_to_str(src)))

def psource ( self , obj , oname = '' ) : linecache . checkcache ( ) try : src = getsource ( obj ) except : self . noinfo ( 'source' , oname ) else : page . page ( self . format ( py3compat . unicode_to_str ( src ) ) )


def pfile(self, obj, oname=''):
        """Show the whole file where an object was defined."""

        lineno = find_source_lines(obj)
        if lineno is None:
            self.noinfo('file', oname)
            return

        ofile = find_file(obj)
        # run contents of file through pager starting at line where the object
        # is defined, as long as the file isn't binary and is actually on the
        # filesystem.
        if ofile.endswith(('.so', '.dll', '.pyd')):
            print 'File %r is binary, not printing.' % ofile
        elif not os.path.isfile(ofile):
            print 'File %r does not exist, not printing.' % ofile
        else:
            # Print only text files, not extension binaries.  Note that
            # getsourcelines returns lineno with 1-offset and page() uses
            # 0-offset, so we must adjust.
            page.page(self.format(open(ofile).read()), lineno-1)

def
pfile
(
self
,
obj
,
oname
=
''
)
:
lineno
=
find_source_lines
(
obj
)
if
lineno
is
:
self
.
noinfo
(
'file'
,
oname
)
return
ofile
=
find_file
(
obj
)
if
ofile
.
endswith
(
(
'.so'
,
'.dll'
,
'.pyd'
)
)
:
print
'File %r is binary, not printing.'
%
ofile
elif
not
os
.
path
.
isfile
(
ofile
)
:
print
'File %r does not exist, not printing.'
%
ofile
else
:
page
.
page
(
self
.
format
(
open
(
ofile
)
.
read
(
)
)
,
lineno
1
)

Show the whole file where an object was defined .


def _format_fields(self, fields, title_width=12):
        """Formats a list of fields for display.

        Parameters
        ----------
        fields : list
          A list of 2-tuples: (field_title, field_content)
        title_width : int
          How many characters to pad titles to. Default 12.
        """
        out = []
        header = self.__head
        for title, content in fields:
            if len(content.splitlines()) > 1:
                title = header(title + ":") + "\n"
            else:
                title = header((title+":").ljust(title_width))
            out.append(title + content)
        return "\n".join(out)

def _format_fields ( self , fields , title_width = 12 ) : out = [ ] header = self . __head for title , content in fields : if len ( content . splitlines ( ) ) > 1 : title = header ( title + ":" ) + "\n" else : title = header ( ( title + ":" ) . ljust ( title_width ) ) out . append ( title + content ) return "\n" . join ( out )