def get_ip_address_country(ip_address, parallel=False):
    """
    Uses the MaxMind Geolite2 Country database to return the ISO code for the
    country associated with the given IPv4 or IPv6 address

    Args:
        ip_address (str): The IP address to query for
        parallel (bool): Parallel processing

    Returns:
        str: And ISO country code associated with the given IP address
    """
    def download_country_database(location="GeoLite2-Country.mmdb"):
        """Downloads the MaxMind Geolite2 Country database

        Args:
            location (str): Local location for the database file
        """
        if parallel:
            logging.warning("Cannot download GeoIP database in parallel mode")
            return
        url = "https://geolite.maxmind.com/download/geoip/database/" \
              "GeoLite2-Country.tar.gz"
        # Use a browser-like user agent string to bypass some proxy blocks
        headers = {"User-Agent": USER_AGENT}
        original_filename = "GeoLite2-Country.mmdb"
        try:
            response = requests.get(url, headers=headers)
            response.raise_for_status()
            tar_bytes = response.content
            tar_file = tarfile.open(fileobj=BytesIO(tar_bytes), mode="r:gz")
            tar_dir = tar_file.getnames()[0]
            tar_path = "{0}/{1}".format(tar_dir, original_filename)
            tar_file.extract(tar_path)
            shutil.move(tar_path, location)
            shutil.rmtree(tar_dir)
        except Exception as e:
            logger.warning("Error downloading {0}: {1}".format(url,
                                                               e.__str__()))

    system_paths = [
        "GeoLite2-Country.mmdb",
        "/usr/local/share/GeoIP/GeoLite2-Country.mmdb",
        "/usr/share/GeoIP/GeoLite2-Country.mmdb",
        "/var/lib/GeoIP/GeoLite2-Country.mmdb",
        "/var/local/lib/GeoIP/GeoLite2-Country.mmdb",
        "C:\\GeoIP\\GeoLite2-Country.mmdb"
    ]

    db_path = None

    for system_path in system_paths:
        if os.path.exists(system_path):
            db_path = system_path
            break

    if db_path is None:
        db_path = os.path.join(tempdir, "GeoLite2-Country.mmdb")
        if not os.path.exists(db_path):
            download_country_database(db_path)
            if not os.path.exists(db_path):
                return None
        else:
            db_age = datetime.now() - datetime.fromtimestamp(
                os.stat(db_path).st_mtime)
            if db_age > timedelta(days=7):
                download_country_database()
        db_path = db_path

    db_reader = geoip2.database.Reader(db_path)

    country = None

    try:
        country = db_reader.country(ip_address).country.iso_code
    except geoip2.errors.AddressNotFoundError:
        pass

    return country


def get_ip_address_info(ip_address, cache=None, nameservers=None,
                        timeout=2.0, parallel=False):
    """
    Returns reverse DNS and country information for the given IP address

    Args:
        ip_address (str): The IP address to check
        cache (ExpiringDict): Cache storage
        nameservers (list): A list of one or more nameservers to use
        (Cloudflare's public DNS resolvers by default)
        timeout (float): Sets the DNS timeout in seconds
        parallel (bool): parallel processing

    Returns:
        OrderedDict: ``ip_address``, ``reverse_dns``

    """
    ip_address = ip_address.lower()
    if cache:
        info = cache.get(ip_address, None)
        if info:
            return info
    info = OrderedDict()
    info["ip_address"] = ip_address
    reverse_dns = get_reverse_dns(ip_address,
                                  nameservers=nameservers,
                                  timeout=timeout)
    country = get_ip_address_country(ip_address, parallel=parallel)
    info["country"] = country
    info["reverse_dns"] = reverse_dns
    info["base_domain"] = None
    if reverse_dns is not None:
        base_domain = get_base_domain(reverse_dns)
        info["base_domain"] = base_domain

    return info

def get_ip_address_info ( ip_address , cache = None , nameservers = None , timeout = 2.0 , parallel = False ) : ip_address = ip_address . lower ( ) if cache : info = cache . get ( ip_address , None ) if info : return info info = OrderedDict ( ) info [ "ip_address" ] = ip_address reverse_dns = get_reverse_dns ( ip_address , nameservers = nameservers , timeout = timeout ) country = get_ip_address_country ( ip_address , parallel = parallel ) info [ "country" ] = country info [ "reverse_dns" ] = reverse_dns info [ "base_domain" ] = None if reverse_dns is not None : base_domain = get_base_domain ( reverse_dns ) info [ "base_domain" ] = base_domain return info


def convert_outlook_msg(msg_bytes):
    """
    Uses the ``msgconvert`` Perl utility to convert an Outlook MS file to
    standard RFC 822 format

    Args:
        msg_bytes (bytes): the content of the .msg file

    Returns:
        A RFC 822 string
    """
    if not is_outlook_msg(msg_bytes):
        raise ValueError("The supplied bytes are not an Outlook MSG file")
    orig_dir = os.getcwd()
    tmp_dir = tempfile.mkdtemp()
    os.chdir(tmp_dir)
    with open("sample.msg", "wb") as msg_file:
        msg_file.write(msg_bytes)
    try:
        subprocess.check_call(["msgconvert", "sample.msg"],
                              stdout=null_file, stderr=null_file)
        eml_path = "sample.eml"
        with open(eml_path, "rb") as eml_file:
            rfc822 = eml_file.read()
    except FileNotFoundError:
        raise EmailParserError(
            "Failed to convert Outlook MSG: msgconvert utility not found")
    finally:
        os.chdir(orig_dir)
        shutil.rmtree(tmp_dir)

    return rfc822

Uses the msgconvert Perl utility to convert an Outlook MS file to standard RFC 822 format


def _str_to_list(s):
    """Converts a comma separated string to a list"""
    _list = s.split(",")
    return list(map(lambda i: i.lstrip(), _list))

def _str_to_list ( s ) : _list = s . split ( "," ) return list ( map ( lambda i : i . lstrip ( ) , _list ) )


def cli_parse(file_path, sa, nameservers, dns_timeout, parallel=False):
    """Separated this function for multiprocessing"""
    try:
        file_results = parse_report_file(file_path,
                                         nameservers=nameservers,
                                         dns_timeout=dns_timeout,
                                         strip_attachment_payloads=sa,
                                         parallel=parallel)
    except ParserError as error:
        return error, file_path
    finally:
        global counter
        with counter.get_lock():
            counter.value += 1
    return file_results, file_path

def cli_parse ( file_path , sa , nameservers , dns_timeout , parallel = False ) : try : file_results = parse_report_file ( file_path , nameservers = nameservers , dns_timeout = dns_timeout , strip_attachment_payloads = sa , parallel = parallel ) except ParserError as error : return error , file_path finally : global counter with counter . get_lock ( ) : counter . value += 1 return file_results , file_path


def drain(self, sid=None):
        """
        Drain will put a connection into a drain state. All subscriptions will
        immediately be put into a drain state. Upon completion, the publishers
        will be drained and can not publish any additional messages. Upon draining
        of the publishers, the connection will be closed. Use the `closed_cb'
        option to know when the connection has moved from draining to closed.

        If a sid is passed, just the subscription with that sid will be drained
        without closing the connection.
        """
        if self.is_draining:
            return
        if self.is_closed:
            raise ErrConnectionClosed
        if self.is_connecting or self.is_reconnecting:
            raise ErrConnectionReconnecting

        if sid is not None:
            return self._drain_sub(sid)

        # Start draining the subscriptions
        self._status = Client.DRAINING_SUBS

        drain_tasks = []
        for ssid, sub in self._subs.items():
            task = self._drain_sub(ssid)
            drain_tasks.append(task)

        drain_is_done = asyncio.gather(*drain_tasks)
        try:
            yield from asyncio.wait_for(drain_is_done, self.options["drain_timeout"])
        except asyncio.TimeoutError:
            drain_is_done.exception()
            drain_is_done.cancel()
            if self._error_cb is not None:
                yield from self._error_cb(ErrDrainTimeout)
        except asyncio.CancelledError:
            pass
        finally:
            self._status = Client.DRAINING_PUBS
            yield from self.flush()
            yield from self._close(Client.CLOSED)

def drain ( self , sid = None ) : if self . is_draining : return if self . is_closed : raise ErrConnectionClosed if self . is_connecting or self . is_reconnecting : raise ErrConnectionReconnecting if sid is not None : return self . _drain_sub ( sid ) self . _status = Client . DRAINING_SUBS drain_tasks = [ ] for ssid , sub in self . _subs . items ( ) : task = self . _drain_sub ( ssid ) drain_tasks . append ( task ) drain_is_done = asyncio . gather ( * drain_tasks ) try : yield from asyncio . wait_for ( drain_is_done , self . options [ "drain_timeout" ] ) except asyncio . TimeoutError : drain_is_done . exception ( ) drain_is_done . cancel ( ) if self . _error_cb is not None : yield from self . _error_cb ( ErrDrainTimeout ) except asyncio . CancelledError : pass finally : self . _status = Client . DRAINING_PUBS yield from self . flush ( ) yield from self . _close ( Client . CLOSED )

Drain will put a connection into a drain state . All subscriptions will immediately be put into a drain state . Upon completion the publishers will be drained and can not publish any additional messages . Upon draining of the publishers the connection will be closed . Use the closed_cb option to know when the connection has moved from draining to closed .


def publish(self, subject, payload):
        """
        Sends a PUB command to the server on the specified subject.

          ->> PUB hello 5
          ->> MSG_PAYLOAD: world
          <<- MSG hello 2 5

        """
        if self.is_closed:
            raise ErrConnectionClosed
        if self.is_draining_pubs:
            raise ErrConnectionDraining

        payload_size = len(payload)
        if payload_size > self._max_payload:
            raise ErrMaxPayload
        yield from self._publish(subject, _EMPTY_, payload, payload_size)

def publish ( self , subject , payload ) : if self . is_closed : raise ErrConnectionClosed if self . is_draining_pubs : raise ErrConnectionDraining payload_size = len ( payload ) if payload_size > self . _max_payload : raise ErrMaxPayload yield from self . _publish ( subject , _EMPTY_ , payload , payload_size )

Sends a PUB command to the server on the specified subject .


def publish_request(self, subject, reply, payload):
        """
        Publishes a message tagging it with a reply subscription
        which can be used by those receiving the message to respond.

           ->> PUB hello   _INBOX.2007314fe0fcb2cdc2a2914c1 5
           ->> MSG_PAYLOAD: world
           <<- MSG hello 2 _INBOX.2007314fe0fcb2cdc2a2914c1 5

        """
        if self.is_closed:
            raise ErrConnectionClosed
        if self.is_draining_pubs:
            raise ErrConnectionDraining

        payload_size = len(payload)
        if payload_size > self._max_payload:
            raise ErrMaxPayload
        yield from self._publish(subject, reply.encode(), payload, payload_size)

def publish_request ( self , subject , reply , payload ) : if self . is_closed : raise ErrConnectionClosed if self . is_draining_pubs : raise ErrConnectionDraining payload_size = len ( payload ) if payload_size > self . _max_payload : raise ErrMaxPayload yield from self . _publish ( subject , reply . encode ( ) , payload , payload_size )

Publishes a message tagging it with a reply subscription which can be used by those receiving the message to respond .


def _publish(self, subject, reply, payload, payload_size):
        """
        Sends PUB command to the NATS server.
        """
        if subject == "":
            # Avoid sending messages with empty replies.
            raise ErrBadSubject

        payload_size_bytes = ("%d" % payload_size).encode()
        pub_cmd = b''.join([PUB_OP, _SPC_, subject.encode(
        ), _SPC_, reply, _SPC_, payload_size_bytes, _CRLF_, payload, _CRLF_])
        self.stats['out_msgs'] += 1
        self.stats['out_bytes'] += payload_size
        yield from self._send_command(pub_cmd)
        if self._flush_queue.empty():
            yield from self._flush_pending()

def _publish ( self , subject , reply , payload , payload_size ) : if subject == "" : raise ErrBadSubject payload_size_bytes = ( "%d" % payload_size ) . encode ( ) pub_cmd = b'' . join ( [ PUB_OP , _SPC_ , subject . encode ( ) , _SPC_ , reply , _SPC_ , payload_size_bytes , _CRLF_ , payload , _CRLF_ ] ) self . stats [ 'out_msgs' ] += 1 self . stats [ 'out_bytes' ] += payload_size yield from self . _send_command ( pub_cmd ) if self . _flush_queue . empty ( ) : yield from self . _flush_pending ( )


def subscribe_async(self, subject, **kwargs):
        """
        Sets the subcription to use a task per message to be processed.

        ..deprecated:: 7.0
          Will be removed 9.0.
        """
        kwargs["is_async"] = True
        sid = yield from self.subscribe(subject, **kwargs)
        return sid

def subscribe_async ( self , subject , ** kwargs ) : kwargs [ "is_async" ] = True sid = yield from self . subscribe ( subject , ** kwargs ) return sid

Sets the subcription to use a task per message to be processed .


def unsubscribe(self, ssid, max_msgs=0):
        """
        Takes a subscription sequence id and removes the subscription
        from the client, optionally after receiving more than max_msgs.
        """
        if self.is_closed:
            raise ErrConnectionClosed
        if self.is_draining:
            raise ErrConnectionDraining

        self._remove_sub(ssid, max_msgs)

        # We will send these for all subs when we reconnect anyway,
        # so that we can suppress here.
        if not self.is_reconnecting:
            yield from self.auto_unsubscribe(ssid, max_msgs)

def unsubscribe ( self , ssid , max_msgs = 0 ) : if self . is_closed : raise ErrConnectionClosed if self . is_draining : raise ErrConnectionDraining self . _remove_sub ( ssid , max_msgs ) if not self . is_reconnecting : yield from self . auto_unsubscribe ( ssid , max_msgs )

Takes a subscription sequence id and removes the subscription from the client optionally after receiving more than max_msgs .


def flush(self, timeout=60):
        """
        Sends a ping to the server expecting a pong back ensuring
        what we have written so far has made it to the server and
        also enabling measuring of roundtrip time.
        In case a pong is not returned within the allowed timeout,
        then it will raise ErrTimeout.
        """
        if timeout <= 0:
            raise ErrBadTimeout

        if self.is_closed:
            raise ErrConnectionClosed

        future = asyncio.Future(loop=self._loop)
        try:
            yield from self._send_ping(future)
            yield from asyncio.wait_for(future, timeout, loop=self._loop)
        except asyncio.TimeoutError:
            future.cancel()
            raise ErrTimeout

def flush ( self , timeout = 60 ) : if timeout <= 0 : raise ErrBadTimeout if self . is_closed : raise ErrConnectionClosed future = asyncio . Future ( loop = self . _loop ) try : yield from self . _send_ping ( future ) yield from asyncio . wait_for ( future , timeout , loop = self . _loop ) except asyncio . TimeoutError : future . cancel ( ) raise ErrTimeout

Sends a ping to the server expecting a pong back ensuring what we have written so far has made it to the server and also enabling measuring of roundtrip time . In case a pong is not returned within the allowed timeout then it will raise ErrTimeout .


def _select_next_server(self):
        """
        Looks up in the server pool for an available server
        and attempts to connect.
        """

        while True:
            if len(self._server_pool) == 0:
                self._current_server = None
                raise ErrNoServers

            now = time.monotonic()
            s = self._server_pool.pop(0)
            if self.options["max_reconnect_attempts"] > 0:
                if s.reconnects > self.options["max_reconnect_attempts"]:
                    # Discard server since already tried to reconnect too many times
                    continue

            # Not yet exceeded max_reconnect_attempts so can still use
            # this server in the future.
            self._server_pool.append(s)
            if s.last_attempt is not None and now < s.last_attempt + self.options["reconnect_time_wait"]:
                # Backoff connecting to server if we attempted recently.
                yield from asyncio.sleep(self.options["reconnect_time_wait"], loop=self._loop)
            try:
                s.last_attempt = time.monotonic()
                r, w = yield from asyncio.open_connection(
                    s.uri.hostname,
                    s.uri.port,
                    loop=self._loop,
                    limit=DEFAULT_BUFFER_SIZE)
                self._current_server = s

                # We keep a reference to the initial transport we used when
                # establishing the connection in case we later upgrade to TLS
                # after getting the first INFO message. This is in order to
                # prevent the GC closing the socket after we send CONNECT
                # and replace the transport.
                #
                # See https://github.com/nats-io/asyncio-nats/issues/43
                self._bare_io_reader = self._io_reader = r
                self._bare_io_writer = self._io_writer = w
                break
            except Exception as e:
                s.last_attempt = time.monotonic()
                s.reconnects += 1

                self._err = e
                if self._error_cb is not None:
                    yield from self._error_cb(e)
                continue

def _select_next_server ( self ) : while True : if len ( self . _server_pool ) == 0 : self . _current_server = None raise ErrNoServers now = time . monotonic ( ) s = self . _server_pool . pop ( 0 ) if self . options [ "max_reconnect_attempts" ] > 0 : if s . reconnects > self . options [ "max_reconnect_attempts" ] : continue self . _server_pool . append ( s ) if s . last_attempt is not None and now < s . last_attempt + self . options [ "reconnect_time_wait" ] : yield from asyncio . sleep ( self . options [ "reconnect_time_wait" ] , loop = self . _loop ) try : s . last_attempt = time . monotonic ( ) r , w = yield from asyncio . open_connection ( s . uri . hostname , s . uri . port , loop = self . _loop , limit = DEFAULT_BUFFER_SIZE ) self . _current_server = s self . _bare_io_reader = self . _io_reader = r self . _bare_io_writer = self . _io_writer = w break except Exception as e : s . last_attempt = time . monotonic ( ) s . reconnects += 1 self . _err = e if self . _error_cb is not None : yield from self . _error_cb ( e ) continue

Looks up in the server pool for an available server and attempts to connect .


def _process_err(self, err_msg):
        """
        Processes the raw error message sent by the server
        and close connection with current server.
        """
        if STALE_CONNECTION in err_msg:
            yield from self._process_op_err(ErrStaleConnection)
            return

        if AUTHORIZATION_VIOLATION in err_msg:
            self._err = ErrAuthorization
        else:
            m = b'nats: ' + err_msg[0]
            self._err = NatsError(m.decode())

        do_cbs = False
        if not self.is_connecting:
            do_cbs = True

        # FIXME: Some errors such as 'Invalid Subscription'
        # do not cause the server to close the connection.
        # For now we handle similar as other clients and close.
        self._loop.create_task(self._close(Client.CLOSED, do_cbs))

def
_process_err
(
self
,
err_msg
)
:
if
STALE_CONNECTION
in
err_msg
:
yield
from
self
.
_process_op_err
(
ErrStaleConnection
)
return
if
AUTHORIZATION_VIOLATION
in
err_msg
:
self
.
_err
=
ErrAuthorization
else
:
m
=
b'nats: '
+
err_msg
[
0
]
self
.
_err
=
NatsError
(
m
.
decode
(
)
)
do_cbs
=
False
if
not
self
.
is_connecting
:
do_cbs
=
True
self
.
_loop
.
create_task
(
self
.
_close
(
Client
.
CLOSED
,
do_cbs
)
)

Processes the raw error message sent by the server and close connection with current server .


def _process_op_err(self, e):
        """
        Process errors which occured while reading or parsing
        the protocol. If allow_reconnect is enabled it will
        try to switch the server to which it is currently connected
        otherwise it will disconnect.
        """
        if self.is_connecting or self.is_closed or self.is_reconnecting:
            return

        if self.options["allow_reconnect"] and self.is_connected:
            self._status = Client.RECONNECTING
            self._ps.reset()

            if self._reconnection_task is not None and not self._reconnection_task.cancelled():
                # Cancel the previous task in case it may still be running.
                self._reconnection_task.cancel()

            self._reconnection_task = self._loop.create_task(self._attempt_reconnect())
        else:
            self._process_disconnect()
            self._err = e
            yield from self._close(Client.CLOSED, True)

def _process_op_err ( self , e ) : if self . is_connecting or self . is_closed or self . is_reconnecting : return if self . options [ "allow_reconnect" ] and self . is_connected : self . _status = Client . RECONNECTING self . _ps . reset ( ) if self . _reconnection_task is not None and not self . _reconnection_task . cancelled ( ) : self . _reconnection_task . cancel ( ) self . _reconnection_task = self . _loop . create_task ( self . _attempt_reconnect ( ) ) else : self . _process_disconnect ( ) self . _err = e yield from self . _close ( Client . CLOSED , True )

Process errors which occured while reading or parsing the protocol . If allow_reconnect is enabled it will try to switch the server to which it is currently connected otherwise it will disconnect .


def _connect_command(self):
        '''
        Generates a JSON string with the params to be used
        when sending CONNECT to the server.

          ->> CONNECT {"lang": "python3"}

        '''
        options = {
            "verbose": self.options["verbose"],
            "pedantic": self.options["pedantic"],
            "lang": __lang__,
            "version": __version__,
            "protocol": PROTOCOL
        }
        if "auth_required" in self._server_info:
            if self._server_info["auth_required"]:
                # In case there is no password, then consider handle
                # sending a token instead.
                if self.options["user"] is not None and self.options["password"] is not None:
                    options["user"] = self.options["user"]
                    options["pass"] = self.options["password"]
                elif self.options["token"] is not None:
                    options["auth_token"] = self.options["token"]
                elif self._current_server.uri.password is None:
                    options["auth_token"] = self._current_server.uri.username
                else:
                    options["user"] = self._current_server.uri.username
                    options["pass"] = self._current_server.uri.password
        if self.options["name"] is not None:
            options["name"] = self.options["name"]
        if self.options["no_echo"] is not None:
            options["echo"] = not self.options["no_echo"]

        connect_opts = json.dumps(options, sort_keys=True)
        return b''.join([CONNECT_OP + _SPC_ + connect_opts.encode() + _CRLF_])

def _connect_command ( self ) : options = { "verbose" : self . options [ "verbose" ] , "pedantic" : self . options [ "pedantic" ] , "lang" : __lang__ , "version" : __version__ , "protocol" : PROTOCOL } if "auth_required" in self . _server_info : if self . _server_info [ "auth_required" ] : if self . options [ "user" ] is not None and self . options [ "password" ] is not None : options [ "user" ] = self . options [ "user" ] options [ "pass" ] = self . options [ "password" ] elif self . options [ "token" ] is not None : options [ "auth_token" ] = self . options [ "token" ] elif self . _current_server . uri . password is None : options [ "auth_token" ] = self . _current_server . uri . username else : options [ "user" ] = self . _current_server . uri . username options [ "pass" ] = self . _current_server . uri . password if self . options [ "name" ] is not None : options [ "name" ] = self . options [ "name" ] if self . options [ "no_echo" ] is not None : options [ "echo" ] = not self . options [ "no_echo" ] connect_opts = json . dumps ( options , sort_keys = True ) return b'' . join ( [ CONNECT_OP + _SPC_ + connect_opts . encode ( ) + _CRLF_ ] )

Generates a JSON string with the params to be used when sending CONNECT to the server .


def _process_pong(self):
        """
        Process PONG sent by server.
        """
        if len(self._pongs) > 0:
            future = self._pongs.pop(0)
            future.set_result(True)
            self._pongs_received += 1
            self._pings_outstanding -= 1

def _process_pong ( self ) : if len ( self . _pongs ) > 0 : future = self . _pongs . pop ( 0 ) future . set_result ( True ) self . _pongs_received += 1 self . _pings_outstanding -= 1


def _process_msg(self, sid, subject, reply, data):
        """
        Process MSG sent by server.
        """
        payload_size = len(data)
        self.stats['in_msgs'] += 1
        self.stats['in_bytes'] += payload_size

        sub = self._subs.get(sid)
        if sub is None:
            # Skip in case no subscription present.
            return

        sub.received += 1
        if sub.max_msgs > 0 and sub.received >= sub.max_msgs:
            # Enough messages so can throwaway subscription now.
            self._subs.pop(sid, None)
        msg = self._build_message(subject, reply, data)

        # Check if it is an old style request.
        if sub.future is not None:
            if sub.future.cancelled():
                # Already gave up, nothing to do.
                return
            sub.future.set_result(msg)
            return

        # Let subscription wait_for_msgs coroutine process the messages,
        # but in case sending to the subscription task would block,
        # then consider it to be an slow consumer and drop the message.
        try:
            sub.pending_size += payload_size
            if sub.pending_size >= sub.pending_bytes_limit:
                # Substract again the bytes since throwing away
                # the message so would not be pending data.
                sub.pending_size -= payload_size

                if self._error_cb is not None:
                    yield from self._error_cb(
                        ErrSlowConsumer(subject=subject, sid=sid))
                return
            sub.pending_queue.put_nowait(msg)
        except asyncio.QueueFull:
            if self._error_cb is not None:
                yield from self._error_cb(
                    ErrSlowConsumer(subject=subject, sid=sid))

def _process_msg ( self , sid , subject , reply , data ) : payload_size = len ( data ) self . stats [ 'in_msgs' ] += 1 self . stats [ 'in_bytes' ] += payload_size sub = self . _subs . get ( sid ) if sub is None : return sub . received += 1 if sub . max_msgs > 0 and sub . received >= sub . max_msgs : self . _subs . pop ( sid , None ) msg = self . _build_message ( subject , reply , data ) if sub . future is not None : if sub . future . cancelled ( ) : return sub . future . set_result ( msg ) return try : sub . pending_size += payload_size if sub . pending_size >= sub . pending_bytes_limit : sub . pending_size -= payload_size if self . _error_cb is not None : yield from self . _error_cb ( ErrSlowConsumer ( subject = subject , sid = sid ) ) return sub . pending_queue . put_nowait ( msg ) except asyncio . QueueFull : if self . _error_cb is not None : yield from self . _error_cb ( ErrSlowConsumer ( subject = subject , sid = sid ) )


def _process_info(self, info):
        """
        Process INFO lines sent by the server to reconfigure client
        with latest updates from cluster to enable server discovery.
        """
        if 'connect_urls' in info:
            if info['connect_urls']:
                connect_urls = []
                for connect_url in info['connect_urls']:
                    uri = urlparse("nats://%s" % connect_url)
                    srv = Srv(uri)
                    srv.discovered = True

                    # Filter for any similar server in the server pool already.
                    should_add = True
                    for s in self._server_pool:
                        if uri.netloc == s.uri.netloc:
                            should_add = False
                    if should_add:
                        connect_urls.append(srv)

                if self.options["dont_randomize"] is not True:
                    shuffle(connect_urls)
                for srv in connect_urls:
                    self._server_pool.append(srv)

def _process_info ( self , info ) : if 'connect_urls' in info : if info [ 'connect_urls' ] : connect_urls = [ ] for connect_url in info [ 'connect_urls' ] : uri = urlparse ( "nats://%s" % connect_url ) srv = Srv ( uri ) srv . discovered = True should_add = True for s in self . _server_pool : if uri . netloc == s . uri . netloc : should_add = False if should_add : connect_urls . append ( srv ) if self . options [ "dont_randomize" ] is not True : shuffle ( connect_urls ) for srv in connect_urls : self . _server_pool . append ( srv )

Process INFO lines sent by the server to reconfigure client with latest updates from cluster to enable server discovery .


def _process_connect_init(self):
        """
        Process INFO received from the server and CONNECT to the server
        with authentication.  It is also responsible of setting up the
        reading and ping interval tasks from the client.
        """
        self._status = Client.CONNECTING

        connection_completed = self._io_reader.readline()
        info_line = yield from asyncio.wait_for(connection_completed, self.options["connect_timeout"])
        if INFO_OP not in info_line:
            raise NatsError("nats: empty response from server when expecting INFO message")

        _, info = info_line.split(INFO_OP + _SPC_, 1)

        try:
            srv_info = json.loads(info.decode())
        except:
            raise NatsError("nats: info message, json parse error")

        self._process_info(srv_info)
        self._server_info = srv_info

        if 'max_payload' in self._server_info:
            self._max_payload = self._server_info["max_payload"]

        if 'tls_required' in self._server_info and self._server_info['tls_required']:
            ssl_context = None
            if "tls" in self.options:
                ssl_context = self.options.get('tls')
            elif self._current_server.uri.scheme == 'tls':
                ssl_context = ssl.create_default_context()
            else:
                raise NatsError('nats: no ssl context provided')

            transport = self._io_writer.transport
            sock = transport.get_extra_info('socket')
            if not sock:
                # This shouldn't happen
                raise NatsError('nats: unable to get socket')

            yield from self._io_writer.drain()  # just in case something is left

            self._io_reader, self._io_writer = \
                yield from asyncio.open_connection(
                    loop=self._loop,
                    limit=DEFAULT_BUFFER_SIZE,
                    sock=sock,
                    ssl=ssl_context,
                    server_hostname=self._current_server.uri.hostname,
                )

        # Refresh state of parser upon reconnect.
        if self.is_reconnecting:
            self._ps.reset()

        connect_cmd = self._connect_command()
        self._io_writer.write(connect_cmd)
        self._io_writer.write(PING_PROTO)
        yield from self._io_writer.drain()

        # FIXME: Add readline timeout
        next_op = yield from self._io_reader.readline()
        if self.options["verbose"] and OK_OP in next_op:
            next_op = yield from self._io_reader.readline()

        if ERR_OP in next_op:
            err_line = next_op.decode()
            _, err_msg = err_line.split(" ", 1)
            # FIXME: Maybe handling could be more special here,
            # checking for ErrAuthorization for example.
            # yield from self._process_err(err_msg)
            raise NatsError("nats: " + err_msg.rstrip('\r\n'))

        if PONG_PROTO in next_op:
            self._status = Client.CONNECTED

        self._reading_task = self._loop.create_task(self._read_loop())
        self._pongs = []
        self._pings_outstanding = 0
        self._ping_interval_task = self._loop.create_task(
            self._ping_interval())

        # Task for kicking the flusher queue
        self._flusher_task = self._loop.create_task(self._flusher())

Process INFO received from the server and CONNECT to the server with authentication . It is also responsible of setting up the reading and ping interval tasks from the client .


def _flusher(self):
        """
        Coroutine which continuously tries to consume pending commands
        and then flushes them to the socket.
        """
        while True:
            if not self.is_connected or self.is_connecting:
                break

            try:
                yield from self._flush_queue.get()

                if self._pending_data_size > 0:
                    self._io_writer.writelines(self._pending[:])
                    self._pending = []
                    self._pending_data_size = 0
                    yield from self._io_writer.drain()
            except OSError as e:
                if self._error_cb is not None:
                    yield from self._error_cb(e)
                yield from self._process_op_err(e)
                break
            except asyncio.CancelledError:
                break

def _flusher ( self ) : while True : if not self . is_connected or self . is_connecting : break try : yield from self . _flush_queue . get ( ) if self . _pending_data_size > 0 : self . _io_writer . writelines ( self . _pending [ : ] ) self . _pending = [ ] self . _pending_data_size = 0 yield from self . _io_writer . drain ( ) except OSError as e : if self . _error_cb is not None : yield from self . _error_cb ( e ) yield from self . _process_op_err ( e ) break except asyncio . CancelledError : break

Coroutine which continuously tries to consume pending commands and then flushes them to the socket .


def _read_loop(self):
        """
        Coroutine which gathers bytes sent by the server
        and feeds them to the protocol parser.
        In case of error while reading, it will stop running
        and its task has to be rescheduled.
        """
        while True:
            try:
                should_bail = self.is_closed or self.is_reconnecting
                if should_bail or self._io_reader is None:
                    break
                if self.is_connected and self._io_reader.at_eof():
                    if self._error_cb is not None:
                        yield from self._error_cb(ErrStaleConnection)
                    yield from self._process_op_err(ErrStaleConnection)
                    break

                b = yield from self._io_reader.read(DEFAULT_BUFFER_SIZE)
                yield from self._ps.parse(b)
            except ErrProtocol:
                yield from self._process_op_err(ErrProtocol)
                break
            except OSError as e:
                yield from self._process_op_err(e)
                break
            except asyncio.CancelledError:
                break

def _read_loop ( self ) : while True : try : should_bail = self . is_closed or self . is_reconnecting if should_bail or self . _io_reader is None : break if self . is_connected and self . _io_reader . at_eof ( ) : if self . _error_cb is not None : yield from self . _error_cb ( ErrStaleConnection ) yield from self . _process_op_err ( ErrStaleConnection ) break b = yield from self . _io_reader . read ( DEFAULT_BUFFER_SIZE ) yield from self . _ps . parse ( b ) except ErrProtocol : yield from self . _process_op_err ( ErrProtocol ) break except OSError as e : yield from self . _process_op_err ( e ) break except asyncio . CancelledError : break

Coroutine which gathers bytes sent by the server and feeds them to the protocol parser . In case of error while reading it will stop running and its task has to be rescheduled .


def coactivation(dataset, seed, threshold=0.0, output_dir='.', prefix='', r=6):
    """ Compute and save coactivation map given input image as seed.

    This is essentially just a wrapper for a meta-analysis defined
    by the contrast between those studies that activate within the seed
    and those that don't.

    Args:
      dataset: a Dataset instance containing study and activation data.
      seed: either a Nifti or Analyze image defining the boundaries of the
        seed, or a list of triples (x/y/z) defining the seed(s). Note that
        voxels do not need to be contiguous to define a seed--all supra-
        threshold voxels will be lumped together.
      threshold: optional float indicating the threshold above which voxels
        are considered to be part of the seed ROI (default = 0)
      r: optional integer indicating radius (in mm) of spheres to grow
        (only used if seed is a list of coordinates).
      output_dir: output directory to write to. Defaults to current.
        If none, defaults to using the first part of the seed filename.
      prefix: optional string to prepend to all coactivation images.

    Output:
      A set of meta-analysis images identical to that generated by
      meta.MetaAnalysis.
    """

    if isinstance(seed, string_types):
        ids = dataset.get_studies(mask=seed, activation_threshold=threshold)
    else:
        ids = dataset.get_studies(peaks=seed, r=r,
                                  activation_threshold=threshold)

    ma = meta.MetaAnalysis(dataset, ids)
    ma.save_results(output_dir, prefix)

def coactivation ( dataset , seed , threshold = 0.0 , output_dir = '.' , prefix = '' , r = 6 ) : if isinstance ( seed , string_types ) : ids = dataset . get_studies ( mask = seed , activation_threshold = threshold ) else : ids = dataset . get_studies ( peaks = seed , r = r , activation_threshold = threshold ) ma = meta . MetaAnalysis ( dataset , ids ) ma . save_results ( output_dir , prefix )

Compute and save coactivation map given input image as seed .


def decode(self, images, save=None, round=4, names=None, **kwargs):
        """ Decodes a set of images.

        Args:
          images: The images to decode. Can be:
            - A single String specifying the filename of the image to decode
            - A list of filenames
            - A single NumPy array containing the image data
          save: Optional filename to save results to. If None (default), returns
            all results as an array.
          round: Optional integer indicating number of decimals to round result
            to. Defaults to 4.
          names: Optional list of names corresponding to the images in filenames.
            If passed, must be of same length and in same order as filenames.
            By default, the columns in the output will be named using the image
            filenames.

        Returns:
          An n_features x n_files numpy array, where each feature is a row and
          each image is a column. The meaning of the values depends on the
          decoding method used. """

        if isinstance(images, string_types):
            images = [images]

        if isinstance(images, list):
            imgs_to_decode = imageutils.load_imgs(images, self.masker)
        else:
            imgs_to_decode = images

        methods = {
            'pearson': self._pearson_correlation,
            'dot': self._dot_product,
            'roi': self._roi_association
        }

        result = np.around(
            methods[self.method](imgs_to_decode, **kwargs), round)

        # if save is not None:

        if names is None:
            if type(images).__module__ == np.__name__:
                names = ['image_%d' % i for i in range(images.shape[1])]
            elif self.method == 'roi':
                names = ['cluster_%d' % i for i in range(result.shape[1])]
            else:
                names = images

        result = pd.DataFrame(result, columns=names, index=self.feature_names)

        if save is not None:
            result.to_csv(save, index_label='Feature')
        return result

def decode ( self , images , save = None , round = 4 , names = None , ** kwargs ) : if isinstance ( images , string_types ) : images = [ images ] if isinstance ( images , list ) : imgs_to_decode = imageutils . load_imgs ( images , self . masker ) else : imgs_to_decode = images methods = { 'pearson' : self . _pearson_correlation , 'dot' : self . _dot_product , 'roi' : self . _roi_association } result = np . around ( methods [ self . method ] ( imgs_to_decode , ** kwargs ) , round ) if names is None : if type ( images ) . __module__ == np . __name__ : names = [ 'image_%d' % i for i in range ( images . shape [ 1 ] ) ] elif self . method == 'roi' : names = [ 'cluster_%d' % i for i in range ( result . shape [ 1 ] ) ] else : names = images result = pd . DataFrame ( result , columns = names , index = self . feature_names ) if save is not None : result . to_csv ( save , index_label = 'Feature' ) return result


def _load_features_from_array(self, features):
        """ Load feature data from a 2D ndarray on disk. """
        self.feature_images = np.load(features)
        self.feature_names = range(self.feature_images.shape[1])

def _load_features_from_array ( self , features ) : self . feature_images = np . load ( features ) self . feature_names = range ( self . feature_images . shape [ 1 ] )

Load feature data from a 2D ndarray on disk .


def _load_features_from_images(self, images, names=None):
        """ Load feature image data from image files.

        Args:
          images: A list of image filenames.
          names: An optional list of strings to use as the feature names. Must
            be in the same order as the images.
        """
        if names is not None and len(names) != len(images):
            raise Exception(
                "Lists of feature names and images must be of same length!")
        self.feature_names = names if names is not None else images
        self.feature_images = imageutils.load_imgs(images, self.masker)

Load feature image data from image files .


def _pearson_correlation(self, imgs_to_decode):
        """ Decode images using Pearson's r.

        Computes the correlation between each input image and each feature
        image across voxels.

        Args:
            imgs_to_decode: An ndarray of images to decode, with voxels in rows
                and images in columns.

        Returns:
            An n_features x n_images 2D array, with each cell representing the
            pearson correlation between the i'th feature and the j'th image
            across all voxels.
        """
        x, y = imgs_to_decode.astype(float), self.feature_images.astype(float)
        return self._xy_corr(x, y)

def _pearson_correlation ( self , imgs_to_decode ) : x , y = imgs_to_decode . astype ( float ) , self . feature_images . astype ( float ) return self . _xy_corr ( x , y )


def _dot_product(self, imgs_to_decode):
        """ Decoding using the dot product.
        """
        return np.dot(imgs_to_decode.T, self.feature_images).T

def _dot_product ( self , imgs_to_decode ) : return np . dot ( imgs_to_decode . T , self . feature_images ) . T


def feature_selection(feat_select, X, y):
    """" Implements various kinds of feature selection """
    # K-best
    if re.match('.*-best', feat_select) is not None:
        n = int(feat_select.split('-')[0])

        selector = SelectKBest(k=n)

        import warnings
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=UserWarning)
            features_selected = np.where(
                selector.fit(X, y).get_support() is True)[0]

    elif re.match('.*-randombest', feat_select) is not None:
        n = int(feat_select.split('-')[0])

        from random import shuffle
        features = range(0, X.shape[1])
        shuffle(features)

        features_selected = features[:n]

    return features_selected

def feature_selection ( feat_select , X , y ) : if re . match ( '.*-best' , feat_select ) is not None : n = int ( feat_select . split ( '-' ) [ 0 ] ) selector = SelectKBest ( k = n ) import warnings with warnings . catch_warnings ( ) : warnings . simplefilter ( 'ignore' , category = UserWarning ) features_selected = np . where ( selector . fit ( X , y ) . get_support ( ) is True ) [ 0 ] elif re . match ( '.*-randombest' , feat_select ) is not None : n = int ( feat_select . split ( '-' ) [ 0 ] ) from random import shuffle features = range ( 0 , X . shape [ 1 ] ) shuffle ( features ) features_selected = features [ : n ] return features_selected


def get_studies_by_regions(dataset, masks, threshold=0.08, remove_overlap=True,
                           studies=None, features=None,
                           regularization="scale"):
    """ Set up data for a classification task given a set of masks

        Given a set of masks, this function retrieves studies associated with
        each mask at the specified threshold, optionally removes overlap and
        filters by studies and features, and returns studies by feature matrix
        (X) and class labels (y)

        Args:
            dataset: a Neurosynth dataset
            maks: a list of paths to Nifti masks
            threshold: percentage of voxels active within the mask for study
                to be included
            remove_overlap: A boolean indicating if studies studies that
                appear in more than one mask should be excluded
            studies: An optional list of study names used to constrain the set
                used in classification. If None, will use all features in the
                dataset.
            features: An optional list of feature names used to constrain the
                set used in classification. If None, will use all features in
                the dataset.
            regularize: Optional boolean indicating if X should be regularized

        Returns:
            A tuple (X, y) of np arrays.
            X is a feature by studies matrix and y is a vector of class labels
    """

    import nibabel as nib
    import os

    # Load masks using NiBabel

    try:
        loaded_masks = [nib.load(os.path.relpath(m)) for m in masks]
    except OSError:
        print('Error loading masks. Check the path')

    # Get a list of studies that activate for each mask file--i.e.,  a list of
    # lists

    grouped_ids = [dataset.get_studies(mask=m, activation_threshold=threshold)
                   for m in loaded_masks]

    # Flattened ids

    flat_ids = reduce(lambda a, b: a + b, grouped_ids)

    # Remove duplicates

    if remove_overlap:
        import collections
        flat_ids = [id for (id, count) in
                    collections.Counter(flat_ids).items() if count == 1]
        grouped_ids = [[x for x in m if x in flat_ids] for m in
                       grouped_ids]  # Remove

    # Create class label(y)
    y = [[idx] * len(ids) for (idx, ids) in enumerate(grouped_ids)]
    y = reduce(lambda a, b: a + b, y)  # Flatten
    y = np.array(y)

    # Extract feature set for each class separately
    X = [dataset.get_feature_data(ids=group_ids, features=features)
         for group_ids in grouped_ids]

    X = np.vstack(tuple(X))

    if regularization:
        X = regularize(X, method=regularization)

    return (X, y)

def
get_studies_by_regions
(
dataset
,
masks
,
threshold
=
0.08
,
remove_overlap
=
True
,
studies
=
,
features
=
,
regularization
=
"scale"
)
:
import
nibabel
as
nib
import
os
try
:
loaded_masks
=
[
nib
.
load
(
os
.
path
.
relpath
(
m
)
)
for
m
in
masks
]
except
OSError
:
print
(
'Error loading masks. Check the path'
)
grouped_ids
=
[
dataset
.
get_studies
(
mask
=
m
,
activation_threshold
=
threshold
)
for
m
in
loaded_masks
]
flat_ids
=
reduce
(
lambda
a
,
b
:
a
+
b
,
grouped_ids
)
if
remove_overlap
:
import
collections
flat_ids
=
[
id
for
(
id
,
count
)
in
collections
.
Counter
(
flat_ids
)
.
items
(
)
if
count
==
1
]
grouped_ids
=
[
[
x
for
x
in
m
if
x
in
flat_ids
]
for
m
in
grouped_ids
]
y
=
[
[
idx
]
*
len
(
ids
)
for
(
idx
,
ids
)
in
enumerate
(
grouped_ids
)
]
y
=
reduce
(
lambda
a
,
b
:
a
+
b
,
y
)
y
=
np
.
array
(
y
)
X
=
[
dataset
.
get_feature_data
(
ids
=
group_ids
,
features
=
features
)
for
group_ids
in
grouped_ids
]
X
=
np
.
vstack
(
tuple
(
X
)
)
if
regularization
:
X
=
regularize
(
X
,
method
=
regularization
)
return
(
X
,
y
)


def get_feature_order(dataset, features):
    """ Returns a list with the order that features requested appear in
    dataset """
    all_features = dataset.get_feature_names()

    i = [all_features.index(f) for f in features]

    return i

def get_feature_order ( dataset , features ) : all_features = dataset . get_feature_names ( ) i = [ all_features . index ( f ) for f in features ] return i


def classify_regions(dataset, masks, method='ERF', threshold=0.08,
                     remove_overlap=True, regularization='scale',
                     output='summary', studies=None, features=None,
                     class_weight='auto', classifier=None,
                     cross_val='4-Fold', param_grid=None, scoring='accuracy'):
    """ Perform classification on specified regions

        Given a set of masks, this function retrieves studies associated with
        each mask at the specified threshold, optionally removes overlap and
        filters by studies and features. Then it trains an algorithm to
        classify studies based on features and tests performance.

        Args:
            dataset: a Neurosynth dataset
            maks: a list of paths to Nifti masks
            method: a string indicating which method to used.
                'SVM': Support Vector Classifier with rbf kernel
                'ERF': Extremely Randomized Forest classifier
                'Dummy': A dummy classifier using stratified classes as
                    predictor
            threshold: percentage of voxels active within the mask for study
                to be included
            remove_overlap: A boolean indicating if studies studies that
                appear in more than one mask should be excluded
            regularization: A string indicating type of regularization to use.
                If None, performs no regularization.
                'scale': Unit scale without demeaning
            output: A string indicating output type
                'summary': Dictionary with summary statistics including score
                    and n
                'summary_clf': Same as above but also includes classifier
                'clf': Only returns classifier
                Warning: using cv without grid will return an untrained
                classifier
            studies: An optional list of study names used to constrain the set
                used in classification. If None, will use all features in the
                dataset.
            features: An optional list of feature names used to constrain the
                set used in classification. If None, will use all features in
                the dataset.
            class_weight: Parameter to pass to classifier determining how to
                weight classes
            classifier: An optional sci-kit learn classifier to use instead of
                pre-set up classifiers set up using 'method'
            cross_val: A string indicating type of cross validation to use.
                Can also pass a scikit_classifier
            param_grid: A dictionary indicating which parameters to optimize
                using GridSearchCV. If None, no GridSearch will be used

        Returns:
            A tuple (X, y) of np arrays.
            X is a feature by studies matrix and y is a vector of class labels
    """

    (X, y) = get_studies_by_regions(dataset, masks, threshold, remove_overlap,
                                    studies, features,
                                    regularization=regularization)

    return classify(X, y, method, classifier, output, cross_val,
                    class_weight, scoring=scoring, param_grid=param_grid)

def classify_regions ( dataset , masks , method = 'ERF' , threshold = 0.08 , remove_overlap = True , regularization = 'scale' , output = 'summary' , studies = None , features = None , class_weight = 'auto' , classifier = None , cross_val = '4-Fold' , param_grid = None , scoring = 'accuracy' ) : ( X , y ) = get_studies_by_regions ( dataset , masks , threshold , remove_overlap , studies , features , regularization = regularization ) return classify ( X , y , method , classifier , output , cross_val , class_weight , scoring = scoring , param_grid = param_grid )


def classify(X, y, clf_method='ERF', classifier=None, output='summary_clf',
             cross_val=None, class_weight=None, regularization=None,
             param_grid=None, scoring='accuracy', refit_all=True,
             feat_select=None):
    """ Wrapper for scikit-learn classification functions
    Imlements various types of classification and cross validation """

    # Build classifier
    clf = Classifier(clf_method, classifier, param_grid)

    # Fit & test model with or without cross-validation
    if cross_val is not None:
        score = clf.cross_val_fit(X, y, cross_val, scoring=scoring,
                                  feat_select=feat_select,
                                  class_weight=class_weight)
    else:
        # Does not support scoring function
        score = clf.fit(X, y, class_weight=class_weight).score(X, y)

    # Return some stuff...
    from collections import Counter

    if output == 'clf':
        return clf
    else:
        if output == 'summary':
            output = {'score': score, 'n': dict(Counter(y))}
        elif output == 'summary_clf':
            output = {
                'score': score,
                'n': dict(Counter(y)),
                'clf': clf,
                'features_selected': clf.features_selected,
                'predictions': clf.predictions
            }

        return output

def classify ( X , y , clf_method = 'ERF' , classifier = None , output = 'summary_clf' , cross_val = None , class_weight = None , regularization = None , param_grid = None , scoring = 'accuracy' , refit_all = True , feat_select = None ) : clf = Classifier ( clf_method , classifier , param_grid ) if cross_val is not None : score = clf . cross_val_fit ( X , y , cross_val , scoring = scoring , feat_select = feat_select , class_weight = class_weight ) else : score = clf . fit ( X , y , class_weight = class_weight ) . score ( X , y ) from collections import Counter if output == 'clf' : return clf else : if output == 'summary' : output = { 'score' : score , 'n' : dict ( Counter ( y ) ) } elif output == 'summary_clf' : output = { 'score' : score , 'n' : dict ( Counter ( y ) ) , 'clf' : clf , 'features_selected' : clf . features_selected , 'predictions' : clf . predictions } return output

Wrapper for scikit - learn classification functions Imlements various types of classification and cross validation


def fit(self, X, y, cv=None, class_weight='auto'):
        """ Fits X to outcomes y, using clf """

        # Incorporate error checking such as :
        # if isinstance(self.classifier, ScikitClassifier):
        #     do one thingNone
        # otherwiseNone.

        self.X = X
        self.y = y

        self.set_class_weight(class_weight=class_weight, y=y)

        self.clf = self.clf.fit(X, y)

        return self.clf

def fit ( self , X , y , cv = None , class_weight = 'auto' ) : self . X = X self . y = y self . set_class_weight ( class_weight = class_weight , y = y ) self . clf = self . clf . fit ( X , y ) return self . clf


def set_class_weight(self, class_weight='auto', y=None):
        """ Sets the class_weight of the classifier to match y """

        if class_weight is None:
            cw = None

            try:
                self.clf.set_params(class_weight=cw)
            except ValueError:
                pass

        elif class_weight == 'auto':
            c = np.bincount(y)
            ii = np.nonzero(c)[0]
            c = c / float(c.sum())
            cw = dict(zip(ii[::-1], c[ii]))

            try:
                self.clf.set_params(class_weight=cw)
            except ValueError:
                import warnings
                warnings.warn(
                    "Tried to set class_weight, but failed. The classifier "
                    "probably doesn't support it")


def cross_val_fit(self, X, y, cross_val='4-Fold', scoring='accuracy',
                      feat_select=None, class_weight='auto'):
        """ Fits X to outcomes y, using clf and cv_method """

        from sklearn import cross_validation

        self.X = X
        self.y = y

        self.set_class_weight(class_weight=class_weight, y=y)

        # Set cross validator
        if isinstance(cross_val, string_types):
            if re.match('.*-Fold', cross_val) is not None:
                n = int(cross_val.split('-')[0])
                self.cver = cross_validation.StratifiedKFold(self.y, n)
            else:
                raise Exception('Unrecognized cross validation method')
        else:
            self.cver = cross_val

        if feat_select is not None:
            self.features_selected = []

        # Perform cross-validated classification
        from sklearn.grid_search import GridSearchCV
        if isinstance(self.clf, GridSearchCV):
            import warnings

            if feat_select is not None:
                warnings.warn(
                    "Cross-validated feature selection not supported with "
                    "GridSearchCV")
            self.clf.set_params(cv=self.cver, scoring=scoring)

            with warnings.catch_warnings():
                warnings.simplefilter('ignore', category=UserWarning)
                self.clf = self.clf.fit(X, y)

            self.cvs = self.clf.best_score_
        else:
            self.cvs = self.feat_select_cvs(
                feat_select=feat_select, scoring=scoring)

        if feat_select is not None:
            fs = feature_selection(
                feat_select, X, y)
            self.features_selected.append(fs)

            X = X[:, fs]

        self.clf.fit(X, y)

        return self.cvs.mean()

Fits X to outcomes y using clf and cv_method


def fit_dataset(self, dataset, y, features=None,
                    feature_type='features'):
        """ Given a dataset, fits either features or voxels to y """

        # Get data from dataset

        if feature_type == 'features':
            X = np.rot90(dataset.feature_table.data.toarray())
        elif feature_type == 'voxels':
            X = np.rot90(dataset.image_table.data.toarray())

        self.sk_classifier.fit(X, y)

def fit_dataset ( self , dataset , y , features = None , feature_type = 'features' ) : if feature_type == 'features' : X = np . rot90 ( dataset . feature_table . data . toarray ( ) ) elif feature_type == 'voxels' : X = np . rot90 ( dataset . image_table . data . toarray ( ) ) self . sk_classifier . fit ( X , y )

Given a dataset fits either features or voxels to y


def average_within_regions(dataset, regions, masker=None, threshold=None,
                           remove_zero=True):
    """ Aggregates over all voxels within each ROI in the input image.

    Takes a Dataset and a Nifti image that defines distinct regions, and
    returns a numpy matrix of  ROIs x mappables, where the value at each
    ROI is the proportion of active voxels in that ROI. Each distinct ROI
    must have a unique value in the image; non-contiguous voxels with the
    same value will be assigned to the same ROI.

    Args:
        dataset: Either a Dataset instance from which image data are
            extracted, or a Numpy array containing image data to use. If
            the latter, the array contains voxels in rows and
            features/studies in columns. The number of voxels must be equal
            to the length of the vectorized image mask in the regions
            image.
        regions: An image defining the boundaries of the regions to use.
            Can be one of:
            1) A string name of the NIFTI or Analyze-format image
            2) A NiBabel SpatialImage
            3) A list of NiBabel images
            4) A 1D numpy array of the same length as the mask vector in
                the Dataset's current Masker.
        masker: Optional masker used to load image if regions is not a
            numpy array. Must be passed if dataset is a numpy array.
        threshold: An optional float in the range of 0 - 1 or integer. If
            passed, the array will be binarized, with ROI values above the
            threshold assigned to True and values below the threshold
            assigned to False. (E.g., if threshold = 0.05, only ROIs in
            which more than 5% of voxels are active will be considered
            active.) If threshold is integer, studies will only be
            considered active if they activate more than that number of
            voxels in the ROI.
        remove_zero: An optional boolean; when True, assume that voxels
            with value of 0 should not be considered as a separate ROI, and
            will be ignored.

    Returns:
        A 2D numpy array with ROIs in rows and mappables in columns.
    """

    if masker is not None:
        masker = masker
    else:
        if isinstance(dataset, Dataset):
            masker = dataset.masker
        else:
            if not type(regions).__module__.startswith('numpy'):
                raise ValueError(
                    "If dataset is a numpy array and regions is not a numpy "
                    "array, a masker must be provided.")

    if not type(regions).__module__.startswith('numpy'):
        regions = masker.mask(regions)

    if isinstance(dataset, Dataset):
        dataset = dataset.get_image_data(dense=False)

    # If multiple images are passed, give each one a unique value
    if regions.ndim == 2:
        m = regions
        for i in range(regions.shape[1]):
            _nz = np.nonzero(m[:, i])[0]
            if isinstance(threshold, int):
                m[_nz, i] = 1.0
            else:
                m[_nz, i] = 1.0 / np.count_nonzero(m[:, i])

    # Otherwise create an ROI-coding matrix
    else:
        labels = np.unique(regions)

        if remove_zero:
            labels = labels[np.nonzero(labels)]

        n_regions = labels.size

        m = np.zeros((regions.size, n_regions))
        for i in range(n_regions):
            if isinstance(threshold, int):
                m[regions == labels[i], i] = 1.0
            else:
                m[regions == labels[i], i] = 1.0 / \
                    np.sum(regions == labels[i])

    # Call dot() on the array itself as this will use sparse matrix
    # multiplication if possible.
    result = dataset.T.dot(m).T

    if threshold is not None:
        result[result < threshold] = 0.0
        result = result.astype(bool)

    return result

Aggregates over all voxels within each ROI in the input image .


def get_random_voxels(dataset, n_voxels):
    """ Returns mappable data for a random subset of voxels.

    May be useful as a baseline in predictive analyses--e.g., to compare
    performance of a more principled feature selection method with simple
    random selection.

    Args:
        dataset: A Dataset instance
        n_voxels: An integer specifying the number of random voxels to select.

    Returns:
        A 2D numpy array with (randomly-selected) voxels in rows and mappables
        in columns.
    """
    voxels = np.arange(dataset.masker.n_vox_in_vol)
    np.random.shuffle(voxels)
    selected = voxels[0:n_voxels]
    return dataset.get_image_data(voxels=selected)

def get_random_voxels ( dataset , n_voxels ) : voxels = np . arange ( dataset . masker . n_vox_in_vol ) np . random . shuffle ( voxels ) selected = voxels [ 0 : n_voxels ] return dataset . get_image_data ( voxels = selected )

Returns mappable data for a random subset of voxels .


def _get_top_words(model, feature_names, n_top_words=40):
    """ Return top forty words from each topic in trained topic model.
    """
    topic_words = []
    for topic in model.components_:
        top_words = [feature_names[i] for i in topic.argsort()[:-n_top_words-1:-1]]
        topic_words += [top_words]
    return topic_words

def _get_top_words ( model , feature_names , n_top_words = 40 ) : topic_words = [ ] for topic in model . components_ : top_words = [ feature_names [ i ] for i in topic . argsort ( ) [ : - n_top_words - 1 : - 1 ] ] topic_words += [ top_words ] return topic_words

Return top forty words from each topic in trained topic model .


def pearson(x, y):
    """ Correlates row vector x with each row vector in 2D array y. """
    data = np.vstack((x, y))
    ms = data.mean(axis=1)[(slice(None, None, None), None)]
    datam = data - ms
    datass = np.sqrt(np.sum(datam**2, axis=1))
    temp = np.dot(datam[1:], datam[0].T)
    rs = temp / (datass[1:] * datass[0])
    return rs

def pearson ( x , y ) : data = np . vstack ( ( x , y ) ) ms = data . mean ( axis = 1 ) [ ( slice ( None , None , None ) , None ) ] datam = data - ms datass = np . sqrt ( np . sum ( datam ** 2 , axis = 1 ) ) temp = np . dot ( datam [ 1 : ] , datam [ 0 ] . T ) rs = temp / ( datass [ 1 : ] * datass [ 0 ] ) return rs

Correlates row vector x with each row vector in 2D array y .


def fdr(p, q=.05):
    """ Determine FDR threshold given a p value array and desired false
    discovery rate q. """
    s = np.sort(p)
    nvox = p.shape[0]
    null = np.array(range(1, nvox + 1), dtype='float') * q / nvox
    below = np.where(s <= null)[0]
    return s[max(below)] if len(below) else -1

def fdr ( p , q = .05 ) : s = np . sort ( p ) nvox = p . shape [ 0 ] null = np . array ( range ( 1 , nvox + 1 ) , dtype = 'float' ) * q / nvox below = np . where ( s <= null ) [ 0 ] return s [ max ( below ) ] if len ( below ) else - 1

Determine FDR threshold given a p value array and desired false discovery rate q .


def _load_activations(self, filename):
        """ Load activation data from a text file.

        Args:
            filename (str): a string pointing to the location of the txt file
                to read from.
        """
        logger.info("Loading activation data from %s..." % filename)

        activations = pd.read_csv(filename, sep='\t')
        activations.columns = [col.lower()
                               for col in list(activations.columns)]

        # Make sure all mandatory columns exist
        mc = ['x', 'y', 'z', 'id', 'space']
        if (set(mc) - set(list(activations.columns))):
            logger.error(
                "At least one of mandatory columns (x, y, z, id, and space) "
                "is missing from input file.")
            return

        # Transform to target space where needed
        spaces = activations['space'].unique()
        xyz = activations[['x', 'y', 'z']].values
        for s in spaces:
            if s != self.transformer.target:
                inds = activations['space'] == s
                xyz[inds] = self.transformer.apply(s, xyz[inds])
        activations[['x', 'y', 'z']] = xyz

        # xyz --> ijk
        ijk = pd.DataFrame(
            transformations.xyz_to_mat(xyz), columns=['i', 'j', 'k'])
        activations = pd.concat([activations, ijk], axis=1)
        return activations


def create_image_table(self, r=None):
        """ Create and store a new ImageTable instance based on the current
        Dataset. Will generally be called privately, but may be useful as a
        convenience method in cases where the user wants to re-generate the
        table with a new smoothing kernel of different radius.

        Args:
            r (int): An optional integer indicating the radius of the smoothing
                kernel. By default, this is None, which will keep whatever
                value is currently set in the Dataset instance.
        """
        logger.info("Creating image table...")
        if r is not None:
            self.r = r
        self.image_table = ImageTable(self)

def
create_image_table
(
self
,
r
=
)
:
logger
.
info
(
"Creating image table..."
)
if
r
is
not
:
self
.
r
=
r
self
.
image_table
=
ImageTable
(
self
)

Create and store a new ImageTable instance based on the current Dataset . Will generally be called privately but may be useful as a convenience method in cases where the user wants to re - generate the table with a new smoothing kernel of different radius .


def get_studies(self, features=None, expression=None, mask=None,
                    peaks=None, frequency_threshold=0.001,
                    activation_threshold=0.0, func=np.sum, return_type='ids',
                    r=6
                    ):
        """ Get IDs or data for studies that meet specific criteria.

        If multiple criteria are passed, the set intersection is returned. For
        example, passing expression='emotion' and mask='my_mask.nii.gz' would
        return only those studies that are associated with emotion AND report
        activation within the voxels indicated in the passed image.

        Args:
            ids (list): A list of IDs of studies to retrieve.
            features (list or str): The name of a feature, or a list of
                features, to use for selecting studies.
            expression (str): A string expression to pass to the PEG for study
                retrieval.
            mask: the mask image (see Masker documentation for valid data
                types).
            peaks (ndarray or list): Either an n x 3 numpy array, or a list of
                lists or tuples (e.g., [(-10, 22, 14)]) specifying the world
                (x/y/z) coordinates of the target location(s).
            frequency_threshold (float): For feature-based or expression-based
                selection, the threshold for selecting studies--i.e., the
                cut-off for a study to be included. Must be a float in range
                [0, 1].
            activation_threshold (int or float): For mask-based selection,
                threshold for a study to be included based on amount of
                activation displayed. If an integer, represents the absolute
                number of voxels that must be active within the mask in order
                for a study to be selected. If a float, it represents the
                proportion of voxels that must be active.
            func (Callable): The function to use when aggregating over the list
                of features. See documentation in FeatureTable.get_ids() for a
                full explanation. Only used for feature- or expression-based
                selection.
            return_type (str): A string specifying what data to return. Valid
                options are:
                'ids': returns a list of IDs of selected studies.
                'images': returns a voxel x study matrix of data for all
                selected studies.
                'weights': returns a dict where the keys are study IDs and the
                values are the computed weights. Only valid when performing
                feature-based selection.
            r (int): For peak-based selection, the distance cut-off (in mm)
                for inclusion (i.e., only studies with one or more activations
                within r mm of one of the passed foci will be returned).

        Returns:
            When return_type is 'ids' (default), returns a list of IDs of the
            selected studies. When return_type is 'data', returns a 2D numpy
            array, with voxels in rows and studies in columns. When return_type
            is 'weights' (valid only for expression-based selection), returns
            a dict, where the keys are study IDs, and the values are the
            computed weights.

        Examples
        --------
        Select all studies tagged with the feature 'emotion':

            >>> ids = dataset.get_studies(features='emotion')

        Select all studies that activate at least 20% of voxels in an amygdala
        mask, and retrieve activation data rather than IDs:

            >>> data = dataset.get_studies(mask='amygdala_mask.nii.gz',
                threshold=0.2, return_type='images')

        Select studies that report at least one activation within 12 mm of at
        least one of three specific foci:

            >>> ids = dataset.get_studies(peaks=[[12, -20, 30], [-26, 22, 22],
                                                [0, 36, -20]], r=12)

        """
        results = []

        # Feature-based selection
        if features is not None:
            # Need to handle weights as a special case, because we can't
            # retrieve the weights later using just the IDs.
            if return_type == 'weights':
                if expression is not None or mask is not None or \
                        peaks is not None:
                    raise ValueError(
                        "return_type cannot be 'weights' when feature-based "
                        "search is used in conjunction with other search "
                        "modes.")
                return self.feature_table.get_ids(
                    features, frequency_threshold, func, get_weights=True)
            else:
                results.append(self.feature_table.get_ids(
                    features, frequency_threshold, func))

        # Logical expression-based selection
        if expression is not None:
            _ids = self.feature_table.get_ids_by_expression(
                expression, frequency_threshold, func)
            results.append(list(_ids))

        # Mask-based selection
        if mask is not None:
            mask = self.masker.mask(mask, in_global_mask=True).astype(bool)
            num_vox = np.sum(mask)
            prop_mask_active = self.image_table.data.T.dot(mask).astype(float)
            if isinstance(activation_threshold, float):
                prop_mask_active /= num_vox
            indices = np.where(prop_mask_active > activation_threshold)[0]
            results.append([self.image_table.ids[ind] for ind in indices])

        # Peak-based selection
        if peaks is not None:
            r = float(r)
            found = set()
            for p in peaks:
                xyz = np.array(p, dtype=float)
                x = self.activations['x']
                y = self.activations['y']
                z = self.activations['z']
                dists = np.sqrt(np.square(x - xyz[0]) + np.square(y - xyz[1]) +
                                np.square(z - xyz[2]))
                inds = np.where((dists > 5.5) & (dists < 6.5))[0]
                tmp = dists[inds]
                found |= set(self.activations[dists <= r]['id'].unique())
            results.append(found)

        # Get intersection of all sets
        ids = list(reduce(lambda x, y: set(x) & set(y), results))

        if return_type == 'ids':
            return ids
        elif return_type == 'data':
            return self.get_image_data(ids)

Get IDs or data for studies that meet specific criteria .


def add_features(self, features, append=True, merge='outer',
                     duplicates='ignore', min_studies=0.0, threshold=0.001):
        """ Construct a new FeatureTable from file.

        Args:
            features: Feature data to add. Can be:
                (a) A text file containing the feature data, where each row is
                a study in the database, with features in columns. The first
                column must contain the IDs of the studies to match up with the
                image data.
                (b) A pandas DataFrame, where studies are in rows, features are
                in columns, and the index provides the study IDs.
            append (bool): If True, adds new features to existing ones
                incrementally. If False, replaces old features.
            merge, duplicates, min_studies, threshold: Additional arguments
                passed to FeatureTable.add_features().
         """
        if (not append) or not hasattr(self, 'feature_table'):
            self.feature_table = FeatureTable(self)

        self.feature_table.add_features(features, merge=merge,
                                        duplicates=duplicates,
                                        min_studies=min_studies,
                                        threshold=threshold)

def add_features ( self , features , append = True , merge = 'outer' , duplicates = 'ignore' , min_studies = 0.0 , threshold = 0.001 ) : if ( not append ) or not hasattr ( self , 'feature_table' ) : self . feature_table = FeatureTable ( self ) self . feature_table . add_features ( features , merge = merge , duplicates = duplicates , min_studies = min_studies , threshold = threshold )

Construct a new FeatureTable from file .


def get_feature_names(self, features=None):
        """ Returns names of features. If features is None, returns all
        features. Otherwise assumes the user is trying to find the order of the
        features.  """
        if features:
            return self.feature_table.get_ordered_names(features)
        else:
            return self.feature_table.feature_names

def get_feature_names ( self , features = None ) : if features : return self . feature_table . get_ordered_names ( features ) else : return self . feature_table . feature_names

Returns names of features . If features is None returns all features . Otherwise assumes the user is trying to find the order of the features .


def get_feature_counts(self, threshold=0.001):
        """ Returns a dictionary, where the keys are the feature names
        and the values are the number of studies tagged with the feature. """
        counts = np.sum(self.get_feature_data() >= threshold, 0)
        return dict(zip(self.get_feature_names(), list(counts)))

def get_feature_counts ( self , threshold = 0.001 ) : counts = np . sum ( self . get_feature_data ( ) >= threshold , 0 ) return dict ( zip ( self . get_feature_names ( ) , list ( counts ) ) )

Returns a dictionary where the keys are the feature names and the values are the number of studies tagged with the feature .


def load(cls, filename):
        """ Load a pickled Dataset instance from file. """
        try:
            dataset = pickle.load(open(filename, 'rb'))
        except UnicodeDecodeError:
            # Need to try this for python3
            dataset = pickle.load(open(filename, 'rb'), encoding='latin')

        if hasattr(dataset, 'feature_table'):
            dataset.feature_table._csr_to_sdf()
        return dataset

def load ( cls , filename ) : try : dataset = pickle . load ( open ( filename , 'rb' ) ) except UnicodeDecodeError : dataset = pickle . load ( open ( filename , 'rb' ) , encoding = 'latin' ) if hasattr ( dataset , 'feature_table' ) : dataset . feature_table . _csr_to_sdf ( ) return dataset

Load a pickled Dataset instance from file .


def save(self, filename):
        """ Pickle the Dataset instance to the provided file.
        """
        if hasattr(self, 'feature_table'):
            self.feature_table._sdf_to_csr()

        pickle.dump(self, open(filename, 'wb'), -1)

        if hasattr(self, 'feature_table'):
            self.feature_table._csr_to_sdf()

def save ( self , filename ) : if hasattr ( self , 'feature_table' ) : self . feature_table . _sdf_to_csr ( ) pickle . dump ( self , open ( filename , 'wb' ) , - 1 ) if hasattr ( self , 'feature_table' ) : self . feature_table . _csr_to_sdf ( )

Pickle the Dataset instance to the provided file .


def get_image_data(self, ids=None, voxels=None, dense=True):
        """ Slices and returns a subset of image data.

        Args:
            ids (list, array): A list or 1D numpy array of study ids to
                return. If None, returns data for all studies.
            voxels (list, array): A list or 1D numpy array of voxel indices
                (i.e., rows) to return. If None, returns data for all voxels.
            dense (bool): Optional boolean. When True (default), convert the
                result to a dense array before returning. When False, keep as
                sparse matrix.

        Returns:
          A 2D numpy array with voxels in rows and studies in columns.
        """
        if dense and ids is None and voxels is None:
            logger.warning(
                "Warning: get_image_data() is being called without specifying "
                "a subset of studies or voxels to retrieve. This may result in"
                " a very large amount of data (several GB) being read into "
                "memory. If you experience any problems, consider returning a "
                "sparse matrix by passing dense=False, or pass in a list of "
                "ids of voxels to retrieve only a portion of the data.")

        result = self.data
        if ids is not None:
            idxs = np.where(np.in1d(np.array(self.ids), np.array(ids)))[0]
            result = result[:, idxs]
        if voxels is not None:
            result = result[voxels, :]
        return result.toarray() if dense else result

Slices and returns a subset of image data .


def get_feature_data(self, ids=None, features=None, dense=True):
        """ Slices and returns a subset of feature data.

        Args:
            ids (list, array): A list or 1D numpy array of study ids to
                return rows for. If None, returns data for all studies
                (i.e., all rows in array).
            features (list, array): A list or 1D numpy array of named features
                to return. If None, returns data for all features (i.e., all
                columns in array).
            dense (bool): Optional boolean. When True (default), convert the
                result to a dense array before returning. When False, keep as
                sparse matrix. Note that if ids is not None, the returned array
                will always be dense.
        Returns:
          A pandas DataFrame with study IDs in rows and features incolumns.
        """
        result = self.data

        if ids is not None:
            result = result.ix[ids]

        if features is not None:
            result = result.ix[:, features]

        return result.to_dense() if dense else result

def get_feature_data ( self , ids = None , features = None , dense = True ) : result = self . data if ids is not None : result = result . ix [ ids ] if features is not None : result = result . ix [ : , features ] return result . to_dense ( ) if dense else result

Slices and returns a subset of feature data .


def get_ordered_names(self, features):
        """ Given a list of features, returns features in order that they
        appear in database.

        Args:
            features (list): A list or 1D numpy array of named features to
            return.

        Returns:
            A list of features in order they appear in database.
        """

        idxs = np.where(
            np.in1d(self.data.columns.values, np.array(features)))[0]
        return list(self.data.columns[idxs].values)

def get_ordered_names ( self , features ) : idxs = np . where ( np . in1d ( self . data . columns . values , np . array ( features ) ) ) [ 0 ] return list ( self . data . columns [ idxs ] . values )

Given a list of features returns features in order that they appear in database .


def get_ids(self, features, threshold=0.0, func=np.sum, get_weights=False):
        """ Returns a list of all studies in the table that meet the desired
        feature-based criteria.

        Will most commonly be used to retrieve studies that use one or more
        features with some minimum frequency; e.g.,:

            get_ids(['fear', 'anxiety'], threshold=0.001)

        Args:
            features (lists): a list of feature names to search on.
            threshold (float): optional float indicating threshold features
                must pass to be included.
            func (Callable): any numpy function to use for thresholding
                (default: sum). The function will be applied to the list of
                features and the result compared to the threshold. This can be
                used to change the meaning of the query in powerful ways. E.g,:
                    max: any of the features have to pass threshold
                        (i.e., max > thresh)
                    min: all features must each individually pass threshold
                        (i.e., min > thresh)
                    sum: the summed weight of all features must pass threshold
                        (i.e., sum > thresh)
            get_weights (bool): if True, returns a dict with ids => weights.

        Returns:
            When get_weights is false (default), returns a list of study
                names. When true, returns a dict, with study names as keys
                and feature weights as values.
        """
        if isinstance(features, str):
            features = [features]
        features = self.search_features(features)  # Expand wild cards
        feature_weights = self.data.ix[:, features]
        weights = feature_weights.apply(func, 1)
        above_thresh = weights[weights >= threshold]
        # ids_to_keep = self.ids[above_thresh]
        return above_thresh if get_weights else list(above_thresh.index)

def get_ids ( self , features , threshold = 0.0 , func = np . sum , get_weights = False ) : if isinstance ( features , str ) : features = [ features ] features = self . search_features ( features ) feature_weights = self . data . ix [ : , features ] weights = feature_weights . apply ( func , 1 ) above_thresh = weights [ weights >= threshold ] return above_thresh if get_weights else list ( above_thresh . index )

Returns a list of all studies in the table that meet the desired feature - based criteria .


def search_features(self, search):
        ''' Returns all features that match any of the elements in the input
        list.

        Args:
            search (str, list): A string or list of strings defining the query.

        Returns:
            A list of matching feature names.
        '''
        if isinstance(search, string_types):
            search = [search]
        search = [s.replace('*', '.*') for s in search]
        cols = list(self.data.columns)
        results = []
        for s in search:
            results.extend([f for f in cols if re.match(s + '$', f)])
        return list(set(results))

def search_features ( self , search ) : if isinstance ( search , string_types ) : search = [ search ] search = [ s . replace ( '*' , '.*' ) for s in search ] cols = list ( self . data . columns ) results = [ ] for s in search : results . extend ( [ f for f in cols if re . match ( s + '$' , f ) ] ) return list ( set ( results ) )

Returns all features that match any of the elements in the input list .


def get_ids_by_expression(self, expression, threshold=0.001, func=np.sum):
        """ Use a PEG to parse expression and return study IDs."""
        lexer = lp.Lexer()
        lexer.build()
        parser = lp.Parser(
            lexer, self.dataset, threshold=threshold, func=func)
        parser.build()
        return parser.parse(expression).keys().values

def get_ids_by_expression ( self , expression , threshold = 0.001 , func = np . sum ) : lexer = lp . Lexer ( ) lexer . build ( ) parser = lp . Parser ( lexer , self . dataset , threshold = threshold , func = func ) parser . build ( ) return parser . parse ( expression ) . keys ( ) . values

Use a PEG to parse expression and return study IDs .


def _sdf_to_csr(self):
        """ Convert FeatureTable to SciPy CSR matrix. """
        data = self.data.to_dense()
        self.data = {
            'columns': list(data.columns),
            'index': list(data.index),
            'values': sparse.csr_matrix(data.values)
        }

def _sdf_to_csr ( self ) : data = self . data . to_dense ( ) self . data = { 'columns' : list ( data . columns ) , 'index' : list ( data . index ) , 'values' : sparse . csr_matrix ( data . values ) }

Convert FeatureTable to SciPy CSR matrix .


def deprecated(*args):
    """ Deprecation warning decorator. Takes optional deprecation message,
    otherwise will use a generic warning. """
    def wrap(func):
        def wrapped_func(*args, **kwargs):
            warnings.warn(msg, category=DeprecationWarning)
            return func(*args, **kwargs)
        return wrapped_func

    if len(args) == 1 and callable(args[0]):
        msg = "Function '%s' will be deprecated in future versions of " \
            "Neurosynth." % args[0].__name__
        return wrap(args[0])
    else:
        msg = args[0]
        return wrap

Deprecation warning decorator . Takes optional deprecation message otherwise will use a generic warning .


def transform(foci, mat):
    """ Convert coordinates from one space to another using provided
    transformation matrix. """
    t = linalg.pinv(mat)
    foci = np.hstack((foci, np.ones((foci.shape[0], 1))))
    return np.dot(foci, t)[:, 0:3]

def transform ( foci , mat ) : t = linalg . pinv ( mat ) foci = np . hstack ( ( foci , np . ones ( ( foci . shape [ 0 ] , 1 ) ) ) ) return np . dot ( foci , t ) [ : , 0 : 3 ]

Convert coordinates from one space to another using provided transformation matrix .


def xyz_to_mat(foci, xyz_dims=None, mat_dims=None):
    """ Convert an N x 3 array of XYZ coordinates to matrix indices. """
    foci = np.hstack((foci, np.ones((foci.shape[0], 1))))
    mat = np.array([[-0.5, 0, 0, 45], [0, 0.5, 0, 63], [0, 0, 0.5, 36]]).T
    result = np.dot(foci, mat)[:, ::-1]  # multiply and reverse column order
    return np.round_(result).astype(int)

def xyz_to_mat ( foci , xyz_dims = None , mat_dims = None ) : foci = np . hstack ( ( foci , np . ones ( ( foci . shape [ 0 ] , 1 ) ) ) ) mat = np . array ( [ [ - 0.5 , 0 , 0 , 45 ] , [ 0 , 0.5 , 0 , 63 ] , [ 0 , 0 , 0.5 , 36 ] ] ) . T result = np . dot ( foci , mat ) [ : , : : - 1 ] return np . round_ ( result ) . astype ( int )

Convert an N x 3 array of XYZ coordinates to matrix indices .


def apply(self, name, foci):
        """ Apply a named transformation to a set of foci.

        If the named transformation doesn't exist, return foci untransformed.
        """
        if name in self.transformations:
            return transform(foci, self.transformations[name])
        else:
            logger.info(
                "No transformation named '%s' found; coordinates left "
                "untransformed." % name)
            return foci

Apply a named transformation to a set of foci .


def mask(self, image, nan_to_num=True, layers=None, in_global_mask=False):
        """ Vectorize an image and mask out all invalid voxels.

        Args:
            images: The image to vectorize and mask. Input can be any object
                handled by get_image().
            layers: Which mask layers to use (specified as int, string, or
                list of ints and strings). When None, applies the conjunction
                of all layers.
            nan_to_num: boolean indicating whether to convert NaNs to 0.
            in_global_mask: Whether to return the resulting masked vector in
                the globally masked space (i.e., n_voxels =
                len(self.global_mask)). If False (default), returns in the full
                image space (i.e., n_voxels = len(self.volume)).
        Returns:
          A 1D NumPy array of in-mask voxels.
        """
        self.set_mask(layers)
        image = self.get_image(image, output='vector')

        if in_global_mask:
            masked_data = image[self.global_mask]
            masked_data[~self.get_mask(in_global_mask=True)] = 0
        else:
            masked_data = image[self.current_mask]

        if nan_to_num:
            masked_data = np.nan_to_num(masked_data)

        return masked_data

def mask ( self , image , nan_to_num = True , layers = None , in_global_mask = False ) : self . set_mask ( layers ) image = self . get_image ( image , output = 'vector' ) if in_global_mask : masked_data = image [ self . global_mask ] masked_data [ ~ self . get_mask ( in_global_mask = True ) ] = 0 else : masked_data = image [ self . current_mask ] if nan_to_num : masked_data = np . nan_to_num ( masked_data ) return masked_data

Vectorize an image and mask out all invalid voxels .


def get_mask(self, layers=None, output='vector', in_global_mask=True):
        """ Set the current mask by taking the conjunction of all specified
        layers.

        Args:
            layers: Which layers to include. See documentation for add() for
                format.
            include_global_mask: Whether or not to automatically include the
                global mask (i.e., self.volume) in the conjunction.
        """
        if in_global_mask:
            output = 'vector'

        if layers is None:
            layers = self.layers.keys()
        elif not isinstance(layers, list):
            layers = [layers]

        layers = map(lambda x: x if isinstance(x, string_types)
                     else self.stack[x], layers)
        layers = [self.layers[l] for l in layers if l in self.layers]

        # Always include the original volume
        layers.append(self.full)
        layers = np.vstack(layers).T.astype(bool)
        mask = layers.all(axis=1)
        mask = self.get_image(mask, output)
        return mask[self.global_mask] if in_global_mask else mask

def get_mask ( self , layers = None , output = 'vector' , in_global_mask = True ) : if in_global_mask : output = 'vector' if layers is None : layers = self . layers . keys ( ) elif not isinstance ( layers , list ) : layers = [ layers ] layers = map ( lambda x : x if isinstance ( x , string_types ) else self . stack [ x ] , layers ) layers = [ self . layers [ l ] for l in layers if l in self . layers ] layers . append ( self . full ) layers = np . vstack ( layers ) . T . astype ( bool ) mask = layers . all ( axis = 1 ) mask = self . get_image ( mask , output ) return mask [ self . global_mask ] if in_global_mask else mask

Set the current mask by taking the conjunction of all specified layers .


def load_imgs(filenames, masker, nan_to_num=True):
    """ Load multiple images from file into an ndarray.

    Args:
      filenames: A single filename or list of filenames pointing to valid
        images.
      masker: A Masker instance.
      nan_to_num: Optional boolean indicating whether to convert NaNs to zero.

    Returns:
      An m x n 2D numpy array, where m = number of voxels in mask and
      n = number of images passed.
    """
    if isinstance(filenames, string_types):
        filenames = [filenames]
    data = np.zeros((masker.n_vox_in_mask, len(filenames)))
    for i, f in enumerate(filenames):
        data[:, i] = masker.mask(f, nan_to_num)
    return data

def load_imgs ( filenames , masker , nan_to_num = True ) : if isinstance ( filenames , string_types ) : filenames = [ filenames ] data = np . zeros ( ( masker . n_vox_in_mask , len ( filenames ) ) ) for i , f in enumerate ( filenames ) : data [ : , i ] = masker . mask ( f , nan_to_num ) return data

Load multiple images from file into an ndarray .


def save_img(data, filename, masker, header=None):
    """ Save a vectorized image to file. """
    if not header:
        header = masker.get_header()
    header.set_data_dtype(data.dtype)  # Avoids loss of precision
    # Update min/max -- this should happen on save, but doesn't seem to
    header['cal_max'] = data.max()
    header['cal_min'] = data.min()
    img = nifti1.Nifti1Image(masker.unmask(data), None, header)
    img.to_filename(filename)

def save_img ( data , filename , masker , header = None ) : if not header : header = masker . get_header ( ) header . set_data_dtype ( data . dtype ) header [ 'cal_max' ] = data . max ( ) header [ 'cal_min' ] = data . min ( ) img = nifti1 . Nifti1Image ( masker . unmask ( data ) , None , header ) img . to_filename ( filename )


def set_logging_level(level=None):
    """Set neurosynth's logging level

    Args
      level : str
        Name of the logging level (warning, error, info, etc) known
        to logging module.  If no level provided, it would get that one
        from environment variable NEUROSYNTH_LOGLEVEL
    """
    if level is None:
        level = os.environ.get('NEUROSYNTH_LOGLEVEL', 'warn')
    if level is not None:
        logger.setLevel(getattr(logging, level.upper()))
    return logger.getEffectiveLevel()

def set_logging_level ( level = None ) : if level is None : level = os . environ . get ( 'NEUROSYNTH_LOGLEVEL' , 'warn' ) if level is not None : logger . setLevel ( getattr ( logging , level . upper ( ) ) ) return logger . getEffectiveLevel ( )