def get_direct(self):
        """
        Allows direct access to the buffer element.
        Blocks until there is data that can be read.

        :return: A guard object that returns the buffer element.
        """

        read_idx = self.__get_idx()

        if read_idx is QueueClosed:
            return QueueClosed

        # Once the guard is released, read_idx will be placed into write_queue.
        return self.Guard(self.write_queue, self.arys, lambda: read_idx)

def get_direct ( self ) : read_idx = self . __get_idx ( ) if read_idx is QueueClosed : return QueueClosed return self . Guard ( self . write_queue , self . arys , lambda : read_idx )

Allows direct access to the buffer element . Blocks until there is data that can be read .


def close(self):
        """Close the queue, signalling that no more data can be put into the queue."""
        self.read_queue.put(QueueClosed)
        self.write_queue.put(QueueClosed)

def close ( self ) : self . read_queue . put ( QueueClosed ) self . write_queue . put ( QueueClosed )

Close the queue signalling that no more data can be put into the queue .


def __get_batch(self, path, length, last=False):
        """
        Get a block of data from the node at path.

        :param path: The path to the node to read from.
        :param length: The length along the outer dimension to read.
        :param last: True if the remainder elements should be read.
        :return: A copy of the requested block of data as a numpy array.
        """
        import tables
        h5_file = tables.open_file(self.filename, 'r')
        h5_node = h5_file.get_node(path)

        if len(h5_node) == 0:
            raise Exception("Cannot read from empty dataset.")

        # If the length isn't specified, then fall back to default values.
        if length is None:
            chunkshape = h5_node.chunkshape
            # If the array isn't chunked, then try to make the block close to 128KB.
            if chunkshape is None:
                default_length = 128*2**10//h5_node[0].nbytes  # Divides by one row of the dataset.
                length = min(h5_node.shape[0], default_length)
            # If it is chunked, then use the chunkshape for best performance.
            else:
                length = chunkshape[0]

        if last:
            example = h5_node[length*(len(h5_node)//length):].copy()
        else:
            example = h5_node[:length].copy()

        h5_file.close()
        return example

def
__get_batch
(
self
,
path
,
length
,
last
=
False
)
:
import
tables
h5_file
=
tables
.
open_file
(
self
.
filename
,
'r'
)
h5_node
=
h5_file
.
get_node
(
path
)
if
len
(
h5_node
)
==
0
:
raise
Exception
(
"Cannot read from empty dataset."
)
if
length
is
:
chunkshape
=
h5_node
.
chunkshape
if
chunkshape
is
:
default_length
=
128
*
2
**
10
//
h5_node
[
0
]
.
nbytes
length
=
min
(
h5_node
.
shape
[
0
]
,
default_length
)
else
:
length
=
chunkshape
[
0
]
if
last
:
example
=
h5_node
[
length
*
(
len
(
h5_node
)
//
length
)
:
]
.
copy
(
)
else
:
example
=
h5_node
[
:
length
]
.
copy
(
)
h5_file
.
close
(
)
return
example

Get a block of data from the node at path .


def get_remainder(self, path, block_size):
        """
        Get the remainder elements. These elements will not be read in the direct queue access cyclic=False mode.

        :param path: The HDF5 path to the dataset to be read.
        :param block_size: The block size is used to calculate which elements will remain.
        :return: A copy of the remainder elements as a numpy array.
        """
        return self.__get_batch(path, length=block_size, last=True)

def get_remainder ( self , path , block_size ) : return self . __get_batch ( path , length = block_size , last = True )

Get the remainder elements . These elements will not be read in the direct queue access cyclic = False mode .


def get_queue(self, path, n_procs=4, read_ahead=None, cyclic=False, block_size=None, ordered=False):
        """
        Get a queue that allows direct access to the internal buffer. If the dataset to be read is chunked, the
        block_size should be a multiple of the chunk size to maximise performance. In this case it is best to leave it
        to the default. When cyclic=False, and block_size does not divide the dataset evenly, the remainder elements
        will not be returned by the queue. When cyclic=True, the remainder elements will be part of a block that wraps
        around the end and includes element from the beginning of the dataset. By default, blocks are returned in the
        order in which they become available. The ordered option will force blocks to be returned in on-disk order.

        :param path: The HDF5 path to the dataset that should be read.
        :param n_procs: The number of background processes used to read the datset in parallel.
        :param read_ahead: The number of blocks to allocate in the internal buffer.
        :param cyclic: True if the queue should wrap at the end of the dataset.
        :param block_size: The size along the outer dimension of the blocks to be read. Defaults to a multiple of
            the chunk size, or to a 128KB sized block if the dataset is not chunked.
        :param ordered: Force the reader return data in on-disk order. May result in performance penalty.
        :return: A queue object that allows access to the internal buffer.
        """
        # Get a block_size length of elements from the dataset to serve as a template for creating the buffer.
        # If block_size=None, then get_batch calculates an appropriate block size.
        example = self.__get_batch(path, block_size)
        block_size = example.shape[0]

        if read_ahead is None:
            # 2x No. of processes for writing, 1 extra for reading.
            read_ahead = 2*n_procs + 1

        cbuf = SharedCircBuf(read_ahead, example)
        stop = multiprocessing.Event()
        barrier = Barrier(n_procs)

        # If ordering has been requested, create a synchronizer.
        sync = GuardSynchronizer() if ordered else None

        procs = []
        for i in range(n_procs):
            # Each process is offset in the dataset by i*block_size
            # The skip length is set to n_procs*block_size so that no block is read by 2 processes.
            process = multiprocessing.Process(target=_Streamer__read_process, args=(
                self, path, block_size, cbuf, stop, barrier, cyclic,
                i * block_size, n_procs * block_size, sync
            ))
            process.daemon = True
            process.start()
            procs.append(process)

        # If the queue is not cyclic, then the cessation of reading data needs to be monitored.
        if not cyclic:

            # This closure defines a background thread that waits until all processes have finished.
            # At this point, all data from the dataset has been read, and the buffer is closed.
            def monitor():
                for p in procs:
                    p.join()
                cbuf.close()

            monitor_thread = threading.Thread(target=monitor)
            monitor_thread.daemon = True
            monitor_thread.start()

        return Streamer.Queue(cbuf, stop, block_size)

def get_queue ( self , path , n_procs = 4 , read_ahead = None , cyclic = False , block_size = None , ordered = False ) : example = self . __get_batch ( path , block_size ) block_size = example . shape [ 0 ] if read_ahead is None : read_ahead = 2 * n_procs + 1 cbuf = SharedCircBuf ( read_ahead , example ) stop = multiprocessing . Event ( ) barrier = Barrier ( n_procs ) sync = GuardSynchronizer ( ) if ordered else None procs = [ ] for i in range ( n_procs ) : process = multiprocessing . Process ( target = _Streamer__read_process , args = ( self , path , block_size , cbuf , stop , barrier , cyclic , i * block_size , n_procs * block_size , sync ) ) process . daemon = True process . start ( ) procs . append ( process ) if not cyclic : def monitor ( ) : for p in procs : p . join ( ) cbuf . close ( ) monitor_thread = threading . Thread ( target = monitor ) monitor_thread . daemon = True monitor_thread . start ( ) return Streamer . Queue ( cbuf , stop , block_size )

Get a queue that allows direct access to the internal buffer . If the dataset to be read is chunked the block_size should be a multiple of the chunk size to maximise performance . In this case it is best to leave it to the default . When cyclic = False and block_size does not divide the dataset evenly the remainder elements will not be returned by the queue . When cyclic = True the remainder elements will be part of a block that wraps around the end and includes element from the beginning of the dataset . By default blocks are returned in the order in which they become available . The ordered option will force blocks to be returned in on - disk order .


def get_generator(self, path, *args, **kw_args):
        """
        Get a generator that allows convenient access to the streamed data.
        Elements from the dataset are returned from the generator one row at a time.
        Unlike the direct access queue, this generator also returns the remainder elements.
        Additional arguments are forwarded to get_queue.
        See the get_queue method for documentation of these parameters.

        :param path:
        :return: A generator that iterates over the rows in the dataset.
        """
        q = self.get_queue(path=path, *args, **kw_args)

        try:
            # This generator just implements a standard access pattern for the direct access queue.
            for guard in q.iter():
                with guard as batch:
                    batch_copy = batch.copy()

                for row in batch_copy:
                    yield row

            last_batch = self.get_remainder(path, q.block_size)
            for row in last_batch:
                yield row

        finally:
            q.close()

def get_generator ( self , path , * args , ** kw_args ) : q = self . get_queue ( path = path , * args , ** kw_args ) try : for guard in q . iter ( ) : with guard as batch : batch_copy = batch . copy ( ) for row in batch_copy : yield row last_batch = self . get_remainder ( path , q . block_size ) for row in last_batch : yield row finally : q . close ( )

Get a generator that allows convenient access to the streamed data . Elements from the dataset are returned from the generator one row at a time . Unlike the direct access queue this generator also returns the remainder elements . Additional arguments are forwarded to get_queue . See the get_queue method for documentation of these parameters .


def parse(ifp, pb_cls, **kwargs):
    """Parse a stream.

    Args:
        ifp (string or file-like object): input stream.
        pb_cls (protobuf.message.Message.__class__): The class object of
            the protobuf message type encoded in the stream.
    """
    mode = 'rb'
    if isinstance(ifp, str):
        istream = open(ifp, mode=mode, **kwargs)
    else:
        istream = open(fileobj=ifp, mode=mode, **kwargs)
    with istream:
        for data in istream:
            pb_obj = pb_cls()
            pb_obj.ParseFromString(data)
            yield pb_obj

def parse ( ifp , pb_cls , ** kwargs ) : mode = 'rb' if isinstance ( ifp , str ) : istream = open ( ifp , mode = mode , ** kwargs ) else : istream = open ( fileobj = ifp , mode = mode , ** kwargs ) with istream : for data in istream : pb_obj = pb_cls ( ) pb_obj . ParseFromString ( data ) yield pb_obj


def dump(ofp, *pb_objs, **kwargs):
    """Write to a stream.

    Args:
        ofp (string or file-like object): output stream.
        pb_objs (*protobuf.message.Message): list of protobuf message objects
            to be written.
    """
    mode = 'wb'
    if isinstance(ofp, str):
        ostream = open(ofp, mode=mode, **kwargs)
    else:
        ostream = open(fileobj=ofp, mode=mode, **kwargs)
    with ostream:
        ostream.write(*pb_objs)

def dump ( ofp , * pb_objs , ** kwargs ) : mode = 'wb' if isinstance ( ofp , str ) : ostream = open ( ofp , mode = mode , ** kwargs ) else : ostream = open ( fileobj = ofp , mode = mode , ** kwargs ) with ostream : ostream . write ( * pb_objs )


def _read_varint(self):
        """Read a varint from file, parse it, and return the decoded integer.
        """
        buff = self._fd.read(1)
        if buff == b'':
            return 0

        while (bytearray(buff)[-1] & 0x80) >> 7 == 1:  # while the MSB is 1
            new_byte = self._fd.read(1)
            if new_byte == b'':
                raise EOFError('unexpected EOF.')
            buff += new_byte

        varint, _ = decodeVarint(buff, 0)

        return varint

def
_read_varint
(
self
)
:
buff
=
self
.
_fd
.
read
(
1
)
if
buff
==
b''
:
return
0
while
(
bytearray
(
buff
)
[
1
]
&
0x80
)
>>
7
==
1
:
new_byte
=
self
.
_fd
.
read
(
1
)
if
new_byte
==
b''
:
raise
EOFError
(
'unexpected EOF.'
)
buff
+=
new_byte
varint
,
_
=
decodeVarint
(
buff
,
0
)
return
varint

Read a varint from file parse it and return the decoded integer .


def _get_objs(self):
        """A generator yielding all protobuf object data in the file. It is the
        main parser of the stream encoding.
        """
        while True:
            count = self._read_varint()
            if count == 0:
                break
            # Read a group containing `count` number of objects.
            for _ in range(count):
                size = self._read_varint()
                if size == 0:
                    raise EOFError('unexpected EOF.')
                # Read an object from the object group.
                yield self._fd.read(size)

            if self._group_delim:
                yield self._delimiter() if self._delimiter is not None \
                                        else None

def
_get_objs
(
self
)
:
while
True
:
count
=
self
.
_read_varint
(
)
if
count
==
0
:
break
for
_
in
range
(
count
)
:
size
=
self
.
_read_varint
(
)
if
size
==
0
:
raise
EOFError
(
'unexpected EOF.'
)
yield
self
.
_fd
.
read
(
size
)
if
self
.
_group_delim
:
yield
self
.
_delimiter
(
)
if
self
.
_delimiter
is
not
else

A generator yielding all protobuf object data in the file . It is the main parser of the stream encoding .


def close(self):
        """Close the stream."""
        self.flush()
        if self._myfd is not None:
            self._myfd.close()
            self._myfd = None

def close ( self ) : self . flush ( ) if self . _myfd is not None : self . _myfd . close ( ) self . _myfd = None


def write(self, *pb2_obj):
        """Write a group of one or more protobuf objects to the file. Multiple
        object groups can be written by calling this method several times
        before closing stream or exiting the runtime context.

        The input protobuf objects get buffered and will be written down when
        the number of buffered objects exceed the `self._buffer_size`.

        Args:
            pb2_obj (*protobuf.message.Message): list of protobuf messages.
        """
        base = len(self._write_buff)

        for idx, obj in enumerate(pb2_obj):
            if self._buffer_size > 0 and \
                    (idx + base) != 0 and \
                    (idx + base) % self._buffer_size == 0:
                self.flush()
            self._write_buff.append(obj)

        if self._buffer_size == 0:
            self.flush()

def write ( self , * pb2_obj ) : base = len ( self . _write_buff ) for idx , obj in enumerate ( pb2_obj ) : if self . _buffer_size > 0 and ( idx + base ) != 0 and ( idx + base ) % self . _buffer_size == 0 : self . flush ( ) self . _write_buff . append ( obj ) if self . _buffer_size == 0 : self . flush ( )

Write a group of one or more protobuf objects to the file . Multiple object groups can be written by calling this method several times before closing stream or exiting the runtime context .


def flush(self):
        """Write down buffer to the file."""
        if not self.is_output():
            return

        count = len(self._write_buff)
        if count == 0:
            return

        encodeVarint(self._fd.write, count, True)

        for obj in self._write_buff:
            obj_str = obj.SerializeToString()
            encodeVarint(self._fd.write, len(obj_str), True)
            self._fd.write(obj_str)

        self._write_buff = []

def flush ( self ) : if not self . is_output ( ) : return count = len ( self . _write_buff ) if count == 0 : return encodeVarint ( self . _fd . write , count , True ) for obj in self . _write_buff : obj_str = obj . SerializeToString ( ) encodeVarint ( self . _fd . write , len ( obj_str ) , True ) self . _fd . write ( obj_str ) self . _write_buff = [ ]


def get_game_dir(self, username=False):
        """Returns joined game directory path relative to Steamapps"""
        if not self.common and not username:
            raise RuntimeError("Can't determine this game's directory without username")
        if self.common:
            subdir = "common"
        else:
            subdir = "username"
        subsubdir = self.dir
        if WIN32 or CYGWIN:
            subsubdir = subsubdir.lower()
        return os.path.join(subdir, subsubdir)


def input_text_with_keyboard_emulation(self, text):
        """
            Works around the problem of emulating user interactions with text inputs.
            Emulates a key-down action on the first char of the input. This way, implementations which
            require key-down event to trigger auto-suggest are testable.
            Then the chains sends the rest of the text and releases the key.
        """
        ActionChains(self.driver).key_down(text).key_up(Keys.CONTROL).perform()

def input_text_with_keyboard_emulation ( self , text ) : ActionChains ( self . driver ) . key_down ( text ) . key_up ( Keys . CONTROL ) . perform ( )

Works around the problem of emulating user interactions with text inputs . Emulates a key - down action on the first char of the input . This way implementations which require key - down event to trigger auto - suggest are testable . Then the chains sends the rest of the text and releases the key .


def make_fake_movie(nframes, mask_shape=(64, 64), mask_center=None,
                    bg_intensity=0.1, mask_sigma=10, dt=0.02, rate=1.0,
                    tau=1., sigma=0.001, seed=None):
    """
    Generate 2D fake fluorescence movie

    Arguments:
    ---------------------------------------------------------------------------
        nframes:        number of timebins to simulate
        mask_shape:     tuple (nrows, ncols), shape of a single movie frame
        mask_center:    tuple (x, y), pixel coords of cell center
        bg_intensity:   scalar, amplitude of (static) baseline fluorescence
        mask_sigma:     scalar, standard deviation of Gaussian mask
        dt:             timestep (s)
        rate:           mean spike rate (Hz)
        tau:            time constant of decay in calcium concentration (s)
        sigma:          SD of additive noise on fluorescence
        seed:           Seed for RNG

    Returns:
    ---------------------------------------------------------------------------
        F:          fluorescence [npixels, nframes]
        c:          calcium concentration [nframes,]
        n:          spike train [nframes,]
        theta:      tuple of true model parameters:
                    (sigma, alpha, beta, lambda, gamma)

    """

    gen = np.random.RandomState(seed)

    # poisson spikes
    n = gen.poisson(rate * dt, size=nframes)

    # internal calcium dynamics
    gamma = np.exp(-dt / tau)
    c = signal.lfilter(np.r_[1], np.r_[1, -gamma], n, axis=0)

    # pixel weights (sum == 1)
    nr, nc = mask_shape
    npix = nr * nc
    if mask_center is None:
        mask_center = (nc // 2., nr // 2.)
    a, b = mask_center
    y, x = np.ogrid[:nr, :nc]
    xs = (x - a) ** 2.
    ys = (y - b) ** 2.
    twoss = 2. * mask_sigma ** 2.
    alpha = np.exp(-1 * ((xs / twoss) + (ys / twoss))).ravel()
    alpha /= alpha.sum()

    # background fluorescence
    beta = gen.randn(npix) * bg_intensity

    # firing rate (spike probability per sec)
    lamb = rate

    # spatially & temporally white noise
    epsilon = gen.randn(npix, nframes) * sigma

    # simulated fluorescence
    F = c[None, :] * alpha[:, None] + beta[:, None] + epsilon

    theta = (sigma, alpha, beta, lamb, gamma)

    return F, c, n, theta

def make_fake_movie ( nframes , mask_shape = ( 64 , 64 ) , mask_center = None , bg_intensity = 0.1 , mask_sigma = 10 , dt = 0.02 , rate = 1.0 , tau = 1. , sigma = 0.001 , seed = None ) : gen = np . random . RandomState ( seed ) n = gen . poisson ( rate * dt , size = nframes ) gamma = np . exp ( - dt / tau ) c = signal . lfilter ( np . r_ [ 1 ] , np . r_ [ 1 , - gamma ] , n , axis = 0 ) nr , nc = mask_shape npix = nr * nc if mask_center is None : mask_center = ( nc // 2. , nr // 2. ) a , b = mask_center y , x = np . ogrid [ : nr , : nc ] xs = ( x - a ) ** 2. ys = ( y - b ) ** 2. twoss = 2. * mask_sigma ** 2. alpha = np . exp ( - 1 * ( ( xs / twoss ) + ( ys / twoss ) ) ) . ravel ( ) alpha /= alpha . sum ( ) beta = gen . randn ( npix ) * bg_intensity lamb = rate epsilon = gen . randn ( npix , nframes ) * sigma F = c [ None , : ] * alpha [ : , None ] + beta [ : , None ] + epsilon theta = ( sigma , alpha , beta , lamb , gamma ) return F , c , n , theta


def evaluate_traits(self):
        """
        Evaluates traits and returns a list containing the description of traits which are not true.
        Notice that if LAZY_EVALUATION is set to False all traits are evaluated before returning. Use this option
        only for debugging purposes.
        """
        return_value = []
        for trait in self.traits:
            if not trait.condition():
                if not self.traits_eager_evaluation:
                    return [trait.description]
                else:
                    return_value.append(trait.description)
        return return_value

def evaluate_traits ( self ) : return_value = [ ] for trait in self . traits : if not trait . condition ( ) : if not self . traits_eager_evaluation : return [ trait . description ] else : return_value . append ( trait . description ) return return_value

Evaluates traits and returns a list containing the description of traits which are not true . Notice that if LAZY_EVALUATION is set to False all traits are evaluated before returning . Use this option only for debugging purposes .


def until_condition(self, condition, condition_description):
        """
        Waits until conditions is True or returns a non-None value.
        If any of the trait is still not present after timeout, raises a TimeoutException.
        """
        end_time = time.time() + self._timeout
        count = 1
        while True:
            try:
                if not hasattr(condition, '__call__'):
                    raise TypeError("condition is not callable")
                value = condition()
                if type(value) is bool and value is not False:
                    return value
                elif type(value) is not bool and value is not None:
                    return value
                else:
                    logger.debug("#" + str(count) + " - wait until " + condition_description)  # pragma: no cover
            except self._ignored_exceptions as ex:
                logger.debug("Captured {0} : {1}".format(str(ex.__class__).replace("<type '", "").replace("'>", ""),
                                                         str(ex)))  # pragma: no cover
            time.sleep(self._poll)
            count += 1
            if time.time() > end_time:  # pragma: no cover
                break
        raise TimeoutException(
            msg="condition <" + condition_description + "> was not true after " + str(self._timeout) + " seconds.")

def
until_condition
(
self
,
condition
,
condition_description
)
:
end_time
=
time
.
time
(
)
+
self
.
_timeout
count
=
1
while
True
:
try
:
if
not
hasattr
(
condition
,
'__call__'
)
:
raise
TypeError
(
"condition is not callable"
)
value
=
condition
(
)
if
type
(
value
)
is
bool
and
value
is
not
False
:
return
value
elif
type
(
value
)
is
not
bool
and
value
is
not
:
return
value
else
:
logger
.
debug
(
"#"
+
str
(
count
)
+
" - wait until "
+
condition_description
)
except
self
.
_ignored_exceptions
as
ex
:
logger
.
debug
(
"Captured {0} : {1}"
.
format
(
str
(
ex
.
__class__
)
.
replace
(
"<type '"
,
""
)
.
replace
(
"'>"
,
""
)
,
str
(
ex
)
)
)
time
.
sleep
(
self
.
_poll
)
count
+=
1
if
time
.
time
(
)
>
end_time
:
break
raise
TimeoutException
(
msg
=
"condition <"
+
condition_description
+
"> was not true after "
+
str
(
self
.
_timeout
)
+
" seconds."
)

Waits until conditions is True or returns a non - None value . If any of the trait is still not present after timeout raises a TimeoutException .


def until_traits_are_present(self, element_with_traits):
        """
        Waits until all traits are present.
        If any of the traits is still not present after timeout, raises a TimeoutException.
        """
        end_time = time.time() + self._timeout
        count = 1
        missing_traits_descriptions = None
        while True:
            missing_traits_descriptions = []
            try:
                missing_traits_descriptions = element_with_traits.evaluate_traits()
                if len(missing_traits_descriptions) == 0:
                    return True
                else:
                    logger.debug("#{0} - wait until all traits are present: <{1}>".format(str(count), '> <'.join(
                        missing_traits_descriptions)))
            except self._ignored_exceptions as ex:  # pragma: no cover
                logger.debug("Captured {0}: {1}".format(str(ex.__class__).replace("<type '", "").replace("'>", ""),
                                                        str(ex)))  # pragma: no cover
                pass  # pragma: no cover
            time.sleep(self._poll)
            count += 1
            if time.time() > end_time:
                break
        raise TimeoutException(
            msg="conditions " + '<' + '> <'.join(missing_traits_descriptions) + '>' + " not true after " + str(
                self._timeout) + " seconds.")

Waits until all traits are present . If any of the traits is still not present after timeout raises a TimeoutException .


def with_ignored_exceptions(self, *ignored_exceptions):
        """
        Set a list of exceptions that should be ignored inside the wait loop.
        """
        for exception in ignored_exceptions:
            self._ignored_exceptions = self._ignored_exceptions + (exception,)
        return self

def with_ignored_exceptions ( self , * ignored_exceptions ) : for exception in ignored_exceptions : self . _ignored_exceptions = self . _ignored_exceptions + ( exception , ) return self

Set a list of exceptions that should be ignored inside the wait loop .


def main_volume(self, operator, value=None):
        """
        Execute Main.Volume.

        Returns int
        """
        try:
            res = int(self.exec_command('main', 'volume', operator, value))
            return res

        except (ValueError, TypeError):
            pass

        return None

def main_volume ( self , operator , value = None ) : try : res = int ( self . exec_command ( 'main' , 'volume' , operator , value ) ) return res except ( ValueError , TypeError ) : pass return None

Execute Main.Volume.

        Returns int


def main_source(self, operator, value=None):
        """
        Execute Main.Source.

        Returns int
        """
        try:
            source = int(self.exec_command('main', 'source', operator, value))
            return source
        except (ValueError, TypeError):
            pass

        return None

def main_source ( self , operator , value = None ) : try : source = int ( self . exec_command ( 'main' , 'source' , operator , value ) ) return source except ( ValueError , TypeError ) : pass return None

Execute Main.Source.

        Returns int


def _send(self, message, read_reply=False):
        """Send a command string to the amplifier."""
        sock = None
        for tries in range(0, 3):
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.connect((self._host, self.PORT))
                break
            except (ConnectionError, BrokenPipeError):
                if tries == 3:
                    print("socket connect failed.")
                    return
                sleep(0.1)
        sock.send(codecs.decode(message, 'hex_codec'))
        if read_reply:
            sleep(0.1)
            reply = ''
            tries = 0
            max_tries = 20
            while len(reply) < len(message) and tries < max_tries:
                try:
                    reply += codecs.encode(sock.recv(self.BUFFERSIZE), 'hex')\
                        .decode("utf-8")
                except (ConnectionError, BrokenPipeError):
                    pass
                tries += 1
            sock.close()
            if tries >= max_tries:
                return
            return reply
        sock.close()

def
_send
(
self
,
message
,
read_reply
=
False
)
:
sock
=
for
tries
in
range
(
0
,
3
)
:
try
:
sock
=
socket
.
socket
(
socket
.
AF_INET
,
socket
.
SOCK_STREAM
)
sock
.
connect
(
(
self
.
_host
,
self
.
PORT
)
)
break
except
(
ConnectionError
,
BrokenPipeError
)
:
if
tries
==
3
:
print
(
"socket connect failed."
)
return
sleep
(
0.1
)
sock
.
send
(
codecs
.
decode
(
message
,
'hex_codec'
)
)
if
read_reply
:
sleep
(
0.1
)
reply
=
''
tries
=
0
max_tries
=
20
while
len
(
reply
)
<
len
(
message
)
and
tries
<
max_tries
:
try
:
reply
+=
codecs
.
encode
(
sock
.
recv
(
self
.
BUFFERSIZE
)
,
'hex'
)
.
decode
(
"utf-8"
)
except
(
ConnectionError
,
BrokenPipeError
)
:
pass
tries
+=
1
sock
.
close
(
)
if
tries
>=
max_tries
:
return
return
reply
sock
.
close
(
)

Send a command string to the amplifier .


def status(self):
        """
        Return the status of the device.

        Returns a dictionary with keys 'volume' (int 0-200) , 'power' (bool),
         'muted' (bool) and 'source' (str).
        """
        nad_reply = self._send(self.POLL_VOLUME +
                               self.POLL_POWER +
                               self.POLL_MUTED +
                               self.POLL_SOURCE, read_reply=True)
        if nad_reply is None:
            return

        # split reply into parts of 10 characters
        num_chars = 10
        nad_status = [nad_reply[i:i + num_chars]
                      for i in range(0, len(nad_reply), num_chars)]

        return {'volume': int(nad_status[0][-2:], 16),
                'power': nad_status[1][-2:] == '01',
                'muted': nad_status[2][-2:] == '01',
                'source': self.SOURCES_REVERSED[nad_status[3][-2:]]}

def status ( self ) : nad_reply = self . _send ( self . POLL_VOLUME + self . POLL_POWER + self . POLL_MUTED + self . POLL_SOURCE , read_reply = True ) if nad_reply is None : return num_chars = 10 nad_status = [ nad_reply [ i : i + num_chars ] for i in range ( 0 , len ( nad_reply ) , num_chars ) ] return { 'volume' : int ( nad_status [ 0 ] [ - 2 : ] , 16 ) , 'power' : nad_status [ 1 ] [ - 2 : ] == '01' , 'muted' : nad_status [ 2 ] [ - 2 : ] == '01' , 'source' : self . SOURCES_REVERSED [ nad_status [ 3 ] [ - 2 : ] ] }


def power_off(self):
        """Power the device off."""
        status = self.status()
        if status['power']:  # Setting power off when it is already off can cause hangs
            self._send(self.CMD_POWERSAVE + self.CMD_OFF)

def power_off ( self ) : status = self . status ( ) if status [ 'power' ] : self . _send ( self . CMD_POWERSAVE + self . CMD_OFF )


def power_on(self):
        """Power the device on."""
        status = self.status()
        if not status['power']:
            self._send(self.CMD_ON, read_reply=True)
            sleep(0.5)

def power_on ( self ) : status = self . status ( ) if not status [ 'power' ] : self . _send ( self . CMD_ON , read_reply = True ) sleep ( 0.5 )


def set_volume(self, volume):
        """Set volume level of the device. Accepts integer values 0-200."""
        if 0 <= volume <= 200:
            volume = format(volume, "02x")  # Convert to hex
            self._send(self.CMD_VOLUME + volume)

def set_volume ( self , volume ) : if 0 <= volume <= 200 : volume = format ( volume , "02x" ) self . _send ( self . CMD_VOLUME + volume )

Set volume level of the device . Accepts integer values 0 - 200 .


def select_source(self, source):
        """Select a source from the list of sources."""
        status = self.status()
        if status['power']:  # Changing source when off may hang NAD7050
            if status['source'] != source:  # Setting the source to the current source will hang the NAD7050
                if source in self.SOURCES:
                    self._send(self.CMD_SOURCE + self.SOURCES[source], read_reply=True)

def select_source ( self , source ) : status = self . status ( ) if status [ 'power' ] : if status [ 'source' ] != source : if source in self . SOURCES : self . _send ( self . CMD_SOURCE + self . SOURCES [ source ] , read_reply = True )

Select a source from the list of sources .


def deobfuscate(request, key, juice=None):
    """
    Deobfuscates the URL and returns HttpResponse from source view.
    SEO juice is mostly ignored as it is intended for display purposes only.
    """
    try:
        url = decrypt(str(key),
                      settings.UNFRIENDLY_SECRET,
                      settings.UNFRIENDLY_IV,
                      checksum=settings.UNFRIENDLY_ENFORCE_CHECKSUM)
    except (CheckSumError, InvalidKeyError):
        return HttpResponseNotFound()

    try:
        url = url.decode('utf-8')
    except UnicodeDecodeError:
        return HttpResponseNotFound()

    url_parts = urlparse(unquote(url))
    path = url_parts.path
    query = url_parts.query

    try:
        view, args, kwargs = resolve(path)
    except Resolver404:
        return HttpResponseNotFound()

    # fix-up the environ object
    environ = request.environ.copy()
    environ['PATH_INFO'] = path[len(environ['SCRIPT_NAME']):]
    environ['QUERY_STRING'] = query

    # init a new request
    patched_request = request.__class__(environ)

    # copy over any missing request attributes - this feels hackish
    missing_items = set(dir(request)) - set(dir(patched_request))
    while missing_items:
        missing_item = missing_items.pop()
        patched_request.__setattr__(missing_item,
                                    request.__getattribute__(missing_item))

    # mark this request as obfuscated
    patched_request.META['obfuscated'] = True

    response = view(patched_request, *args, **kwargs)

    # offer up a friendlier juice-powered filename if downloaded
    if juice and not response.has_header('Content-Disposition'):
        response['Content-Disposition'] = 'inline; filename=%s' % juice

    return response

def
deobfuscate
(
request
,
key
,
juice
=
)
:
try
:
url
=
decrypt
(
str
(
key
)
,
settings
.
UNFRIENDLY_SECRET
,
settings
.
UNFRIENDLY_IV
,
checksum
=
settings
.
UNFRIENDLY_ENFORCE_CHECKSUM
)
except
(
CheckSumError
,
InvalidKeyError
)
:
return
HttpResponseNotFound
(
)
try
:
url
=
url
.
decode
(
'utf-8'
)
except
UnicodeDecodeError
:
return
HttpResponseNotFound
(
)
url_parts
=
urlparse
(
unquote
(
url
)
)
path
=
url_parts
.
path
query
=
url_parts
.
query
try
:
view
,
args
,
kwargs
=
resolve
(
path
)
except
Resolver404
:
return
HttpResponseNotFound
(
)
environ
=
request
.
environ
.
copy
(
)
environ
[
'PATH_INFO'
]
=
path
[
len
(
environ
[
'SCRIPT_NAME'
]
)
:
]
environ
[
'QUERY_STRING'
]
=
query
patched_request
=
request
.
__class__
(
environ
)
missing_items
=
set
(
dir
(
request
)
)
set
(
dir
(
patched_request
)
)
while
missing_items
:
missing_item
=
missing_items
.
pop
(
)
patched_request
.
__setattr__
(
missing_item
,
request
.
__getattribute__
(
missing_item
)
)
patched_request
.
META
[
'obfuscated'
]
=
True
response
=
view
(
patched_request
,
*
args
,
**
kwargs
)
if
juice
and
not
response
.
has_header
(
'Content-Disposition'
)
:
response
[
'Content-Disposition'
]
=
'inline; filename=%s'
%
juice
return
response

Deobfuscates the URL and returns HttpResponse from source view . SEO juice is mostly ignored as it is intended for display purposes only .


def _crc(plaintext):
    """Generates crc32. Modulo keep the value within int range."""
    if not isinstance(plaintext, six.binary_type):
        plaintext = six.b(plaintext)
    return (zlib.crc32(plaintext) % 2147483647) & 0xffffffff

def _crc ( plaintext ) : if not isinstance ( plaintext , six . binary_type ) : plaintext = six . b ( plaintext ) return ( zlib . crc32 ( plaintext ) % 2147483647 ) & 0xffffffff

Generates crc32 . Modulo keep the value within int range .


def obfuscate(value, juice=None):
    """
    Template filter that obfuscates whatever text it is applied to. The text is
    supposed to be a URL, but it will obfuscate anything.

    Usage:
        Extremely unfriendly URL:
        {{ "/my-secret-path/"|obfuscate }}

        Include some SEO juice:
        {{ "/my-secret-path/"|obfuscate:"some SEO friendly text" }}
    """
    if not settings.UNFRIENDLY_ENABLE_FILTER:
        return value
    kwargs = {
        'key': encrypt(value,
                       settings.UNFRIENDLY_SECRET,
                       settings.UNFRIENDLY_IV,
                       checksum=settings.UNFRIENDLY_ENFORCE_CHECKSUM),
    }
    if juice:
        kwargs['juice'] = slugify(juice)
    return reverse('unfriendly-deobfuscate', kwargs=kwargs)

def obfuscate ( value , juice = None ) : if not settings . UNFRIENDLY_ENABLE_FILTER : return value kwargs = { 'key' : encrypt ( value , settings . UNFRIENDLY_SECRET , settings . UNFRIENDLY_IV , checksum = settings . UNFRIENDLY_ENFORCE_CHECKSUM ) , } if juice : kwargs [ 'juice' ] = slugify ( juice ) return reverse ( 'unfriendly-deobfuscate' , kwargs = kwargs )

Template filter that obfuscates whatever text it is applied to . The text is supposed to be a URL but it will obfuscate anything .


def missing_schema(self,html,song_name):
		'''
		It will print the list of songs that can be downloaded
		'''
		#html=self.get_html_response(url)
		soup=BeautifulSoup(html)
		name=' '.join(song_name)
		print '%s not found'%name
		print "But you can download any of the following songs :"
		a_list=soup.findAll('a','touch')
		for x in xrange(len(a_list)-1):
			r=a_list[x]
			p=str(r)
			q=re.sub(r'||||||','',p)
			print q


def list_of_all_href(self,html):
		'''
		It will return all hyper links found in the mr-jatt page for download
		'''
		soup=BeautifulSoup(html)
		links=[]
		a_list=soup.findAll('a','touch')
		for x in xrange(len(a_list)-1):
			link = a_list[x].get('href')
			name = a_list[x]
			name = str(name)
			name=re.sub(r'||||||','',name)
			name=re.sub(r'^[0-9]+\.','',name)
			links.append([link,name])

		#quit()
		return links

It will return all hyper links found in the mr - jatt page for download


def check_if_song_name(self,html):
		'''
		Returns true if user entered artist or movie name
		'''
		soup=BeautifulSoup(html)
		a_list=soup.findAll('a','touch')
		#print a_list
		text=[str(x) for x in a_list]
		text=''.join(text)
		text=text.lower()
		string1='download in 48 kbps'
		string2='download in 128 kbps'
		string3='download in 320 kbps'

		href=''
		if string3 in text:
			#print 'Downloading in 320 kbps'
			href=a_list[2].get('href')

		elif string2 in text:
			#print 'Downloading in 128 kbps'
			href=a_list[1].get('href')

		elif string1 in text:
			#print 'Downloading in 48 kbps'
			href=a_list[0].get('href')
		else:
			return (True,'nothing')

		return (False,href)

def
check_if_song_name
(
self
,
html
)
:
soup
=
BeautifulSoup
(
html
)
a_list
=
soup
.
findAll
(
'a'
,
'touch'
)
text
=
[
str
(
x
)
for
x
in
a_list
]
text
=
''
.
join
(
text
)
text
=
text
.
lower
(
)
string1
=
'download in 48 kbps'
string2
=
'download in 128 kbps'
string3
=
'download in 320 kbps'
href
=
''
if
string3
in
text
:
href
=
a_list
[
2
]
.
get
(
'href'
)
elif
string2
in
text
:
href
=
a_list
[
1
]
.
get
(
'href'
)
elif
string1
in
text
:
href
=
a_list
[
0
]
.
get
(
'href'
)
else
:
return
(
True
,
'nothing'
)
return
(
False
,
href
)


def Parse(self,url,song_name,flag):
		'''
		It will the resource URL if song is found,
		Otherwise it will return the list of songs that can be downloaded
		'''
		file_download=FileDownload()
		html=file_download.get_html_response(url)
		if flag == False:
			soup=BeautifulSoup(html)
			a_list=soup.findAll('a','touch')
			#print a_list
			text=[str(x) for x in a_list]
			text=''.join(text)
			text=text.lower()
			string1='download in 48 kbps'
			string2='download in 128 kbps'
			string3='download in 320 kbps'

			href=''
			if string3 in text:
				print 'Downloading in 320 kbps'
				href=a_list[2].get('href')

			elif string2 in text:
				print 'Downloading in 128 kbps'
				href=a_list[1].get('href')

			elif string1 in text:
				print 'Downloading in 48 kbps'
				href=a_list[0].get('href')
			else:
				self.missing_schema(html,song_name)
				quit()

			return href
		else:
			x,href=self.check_if_song_name(html)
			links = []
			if x==True:
				links=self.list_of_all_href(html)
			else:
				file_download=FileDownload()
				file_download.file_download_cross_platform(href)
				quit()

			return links

def
Parse
(
self
,
url
,
song_name
,
flag
)
:
file_download
=
FileDownload
(
)
html
=
file_download
.
get_html_response
(
url
)
if
flag
==
False
:
soup
=
BeautifulSoup
(
html
)
a_list
=
soup
.
findAll
(
'a'
,
'touch'
)
text
=
[
str
(
x
)
for
x
in
a_list
]
text
=
''
.
join
(
text
)
text
=
text
.
lower
(
)
string1
=
'download in 48 kbps'
string2
=
'download in 128 kbps'
string3
=
'download in 320 kbps'
href
=
''
if
string3
in
text
:
print
'Downloading in 320 kbps'
href
=
a_list
[
2
]
.
get
(
'href'
)
elif
string2
in
text
:
print
'Downloading in 128 kbps'
href
=
a_list
[
1
]
.
get
(
'href'
)
elif
string1
in
text
:
print
'Downloading in 48 kbps'
href
=
a_list
[
0
]
.
get
(
'href'
)
else
:
self
.
missing_schema
(
html
,
song_name
)
quit
(
)
return
href
else
:
x
,
href
=
self
.
check_if_song_name
(
html
)
links
=
[
]
if
x
==
True
:
links
=
self
.
list_of_all_href
(
html
)
else
:
file_download
=
FileDownload
(
)
file_download
.
file_download_cross_platform
(
href
)
quit
(
)
return
links

It will the resource URL if song is found Otherwise it will return the list of songs that can be downloaded


def google_url(self,song_name,website):
		''' It will return the google url to be searched'''
		name='+'.join(song_name)
		prefix='https://www.google.co.in/search?q='
		website=website.split(" ")
		suffix='+'.join(website)
		url=prefix+name+suffix
		#print url
		return url

def
google_url
(
self
,
song_name
,
website
)
:
name
=
'+'
.
join
(
song_name
)
prefix
=
'https://www.google.co.in/search?q='
website
=
website
.
split
(
" "
)
suffix
=
'+'
.
join
(
website
)
url
=
prefix
+
name
+
suffix
return
url


def parse_google(self,html):
		'''It will parse google html response
			and return the first url
		 '''
		soup = BeautifulSoup(html)
		href=soup.find('div','g').find('a').get('href')
		href_list=href.split('&')
		download_url=href_list[0]
		download_url=download_url.strip()
		download_url=download_url.replace('/url?q=','')
		return download_url

def parse_google ( self , html ) : soup = BeautifulSoup ( html ) href = soup . find ( 'div' , 'g' ) . find ( 'a' ) . get ( 'href' ) href_list = href . split ( '&' ) download_url = href_list [ 0 ] download_url = download_url . strip ( ) download_url = download_url . replace ( '/url?q=' , '' ) return download_url


def Parse(self,song_name,website):
		'''
		song_name is a list of strings
		website is a string
		It will return the url from where music file needs to be downloaded
		'''
		url_to_be_parsed=self.google_url(song_name,website)
		file_download=FileDownload()
		html=file_download.get_html_response(url_to_be_parsed)
		website_url=self.parse_google(html)
		return website_url

def Parse ( self , song_name , website ) : url_to_be_parsed = self . google_url ( song_name , website ) file_download = FileDownload ( ) html = file_download . get_html_response ( url_to_be_parsed ) website_url = self . parse_google ( html ) return website_url


def get_html_response(self,url):
		'''It will download the html page specified by url and return the html response '''
		print "Downloading page %s .."%url
		try:
			response=requests.get(url,timeout=50)
		except requests.exceptions.SSLError:
			try:
				response=requests.get(url,verify=False,timeout=50)
			except requests.exceptions.RequestException as e:
				print e
				quit()
		except requests.exceptions.RequestException as e:
			print e
			quit()

		return response.content

def
get_html_response
(
self
,
url
)
:
print
"Downloading page %s .."
%
url
try
:
response
=
requests
.
get
(
url
,
timeout
=
50
)
except
requests
.
exceptions
.
SSLError
:
try
:
response
=
requests
.
get
(
url
,
verify
=
False
,
timeout
=
50
)
except
requests
.
exceptions
.
RequestException
as
e
:
print
e
quit
(
)
except
requests
.
exceptions
.
RequestException
as
e
:
print
e
quit
(
)
return
response
.
content


def file_download_using_requests(self,url):
		'''It will download file specified by url using requests module'''
		file_name=url.split('/')[-1]

		if os.path.exists(os.path.join(os.getcwd(),file_name)):
			print 'File already exists'
			return
		#print 'Downloading file %s '%file_name
		#print 'Downloading from %s'%url

		try:
			r=requests.get(url,stream=True,timeout=200)
		except requests.exceptions.SSLError:
			try:
				response=requests.get(url,stream=True,verify=False,timeout=200)
			except requests.exceptions.RequestException as e:
				print e
				quit()
		except requests.exceptions.RequestException as e:
				print e
				quit()
		chunk_size = 1024
		total_size = int(r.headers['Content-Length'])
		total_chunks = total_size/chunk_size

		file_iterable = r.iter_content(chunk_size = chunk_size)
		tqdm_iter = tqdm(iterable = file_iterable,total = total_chunks,unit = 'KB',
			leave = False
			)
		with open(file_name,'wb') as f:
			for data in tqdm_iter:
				f.write(data)

		#total_size=float(r.headers['Content-Length'])/(1024*1024)
		'''print 'Total size of file to be downloaded %.2f MB '%total_size
		total_downloaded_size=0.0
		with open(file_name,'wb') as f:
			for chunk in r.iter_content(chunk_size=1*1024*1024):
				if chunk:
					size_of_chunk=float(len(chunk))/(1024*1024)
					total_downloaded_size+=size_of_chunk
					print '{0:.0%} Downloaded'.format(total_downloaded_size/total_size)
					f.write(chunk)'''

		print 'Downloaded file %s '%file_name

def
file_download_using_requests
(
self
,
url
)
:
file_name
=
url
.
split
(
'/'
)
[
1
]
if
os
.
path
.
exists
(
os
.
path
.
join
(
os
.
getcwd
(
)
,
file_name
)
)
:
print
'File already exists'
return
try
:
r
=
requests
.
get
(
url
,
stream
=
True
,
timeout
=
200
)
except
requests
.
exceptions
.
SSLError
:
try
:
response
=
requests
.
get
(
url
,
stream
=
True
,
verify
=
False
,
timeout
=
200
)
except
requests
.
exceptions
.
RequestException
as
e
:
print
e
quit
(
)
except
requests
.
exceptions
.
RequestException
as
e
:
print
e
quit
(
)
chunk_size
=
1024
total_size
=
int
(
r
.
headers
[
'Content-Length'
]
)
total_chunks
=
total_size
/
chunk_size
file_iterable
=
r
.
iter_content
(
chunk_size
=
chunk_size
)
tqdm_iter
=
tqdm
(
iterable
=
file_iterable
,
total
=
total_chunks
,
unit
=
'KB'
,
leave
=
False
)
with
open
(
file_name
,
'wb'
)
as
f
:
for
data
in
tqdm_iter
:
f
.
write
(
data
)
print
'Downloaded file %s '
%
file_name


def file_download_using_wget(self,url):
		'''It will download file specified by url using wget utility of linux '''
		file_name=url.split('/')[-1]
		print 'Downloading file %s '%file_name
		command='wget -c --read-timeout=50 --tries=3 -q --show-progress --no-check-certificate '
		url='"'+url+'"'
		command=command+url
		os.system(command)


def findStationCodesByCity(city_name, token):
    """Lookup AQI database for station codes in a given city."""
    req = requests.get(
        API_ENDPOINT_SEARCH,
        params={
            'token': token,
            'keyword': city_name
        })

    if req.status_code == 200 and req.json()["status"] == "ok":
        return [result["uid"] for result in req.json()["data"]]
    else:
        return []

def findStationCodesByCity ( city_name , token ) : req = requests . get ( API_ENDPOINT_SEARCH , params = { 'token' : token , 'keyword' : city_name } ) if req . status_code == 200 and req . json ( ) [ "status" ] == "ok" : return [ result [ "uid" ] for result in req . json ( ) [ "data" ] ] else : return [ ]

Lookup AQI database for station codes in a given city .


def get_location_observation(lat, lng, token):
    """Lookup observations by geo coordinates."""
    req = requests.get(
        API_ENDPOINT_GEO % (lat, lng),
        params={
            'token': token
        })

    if req.status_code == 200 and req.json()["status"] == "ok":
        return parse_observation_response(req.json()["data"])
    return {}

def get_location_observation ( lat , lng , token ) : req = requests . get ( API_ENDPOINT_GEO % ( lat , lng ) , params = { 'token' : token } ) if req . status_code == 200 and req . json ( ) [ "status" ] == "ok" : return parse_observation_response ( req . json ( ) [ "data" ] ) return { }

Lookup observations by geo coordinates .


def parse_observation_response(json):
    """Decode AQICN observation response JSON into python object."""
    logging.debug(json)

    iaqi = json['iaqi']
    result = {
        'idx': json['idx'],
        'city': json.get('city', ''),
        'aqi': json['aqi'],
        'dominentpol': json.get("dominentpol", ''),
        'time': json['time']['s'],
        'iaqi': [{'p': item, 'v': iaqi[item]['v']} for item in iaqi]
    }

    return result

def parse_observation_response ( json ) : logging . debug ( json ) iaqi = json [ 'iaqi' ] result = { 'idx' : json [ 'idx' ] , 'city' : json . get ( 'city' , '' ) , 'aqi' : json [ 'aqi' ] , 'dominentpol' : json . get ( "dominentpol" , '' ) , 'time' : json [ 'time' ] [ 's' ] , 'iaqi' : [ { 'p' : item , 'v' : iaqi [ item ] [ 'v' ] } for item in iaqi ] } return result

Decode AQICN observation response JSON into python object .


def get_station_observation(station_code, token):
    """Request station data for a specific station identified by code.

    A language parameter can also be specified to translate location
    information (default: "en")
    """
    req = requests.get(
        API_ENDPOINT_OBS % (station_code),
        params={
            'token': token
        })

    if req.status_code == 200 and req.json()['status'] == "ok":
        return parse_observation_response(req.json()['data'])
    else:
        return {}

def get_station_observation ( station_code , token ) : req = requests . get ( API_ENDPOINT_OBS % ( station_code ) , params = { 'token' : token } ) if req . status_code == 200 and req . json ( ) [ 'status' ] == "ok" : return parse_observation_response ( req . json ( ) [ 'data' ] ) else : return { }

Request station data for a specific station identified by code .


def search_paths(self):
        """The list of logical paths which are used to search for an asset.
        This property makes sense only if the attributes was created with
        logical path.

        It is assumed that the logical path can be a directory containing a
        file named ``index`` with the same suffix.

        Example::

            >>> attrs = AssetAttributes(environment, 'js/app.js')
            >>> attrs.search_paths
            ['js/app.js', 'js/app/index.js']

            >>> attrs = AssetAttributes(environment, 'js/app/index.js')
            >>> attrs.search_paths
            ['js/models/index.js']
        """
        paths = [self.path]
        if os.path.basename(self.path_without_suffix) != 'index':
            path = os.path.join(self.path_without_suffix, 'index')
            paths.append(path + ''.join(self.suffix))
        return paths

def search_paths ( self ) : paths = [ self . path ] if os . path . basename ( self . path_without_suffix ) != 'index' : path = os . path . join ( self . path_without_suffix , 'index' ) paths . append ( path + '' . join ( self . suffix ) ) return paths

The list of logical paths which are used to search for an asset . This property makes sense only if the attributes was created with logical path .


def compilers(self):
        """The list of compilers used to build asset."""
        return [self.environment.compilers.get(e) for e in self.compiler_extensions]

def compilers ( self ) : return [ self . environment . compilers . get ( e ) for e in self . compiler_extensions ]

The list of compilers used to build asset .


def mimetype(self):
        """MIME type of the asset."""
        return (self.environment.mimetypes.get(self.format_extension) or
                self.compiler_mimetype or 'application/octet-stream')

def mimetype ( self ) : return ( self . environment . mimetypes . get ( self . format_extension ) or self . compiler_mimetype or 'application/octet-stream' )


def compiler_mimetype(self):
        """Implicit MIME type of the asset by its compilers."""
        for compiler in reversed(self.compilers):
            if compiler.result_mimetype:
                return compiler.result_mimetype
        return None

def compiler_mimetype ( self ) : for compiler in reversed ( self . compilers ) : if compiler . result_mimetype : return compiler . result_mimetype return None

Implicit MIME type of the asset by its compilers .


def compiler_format_extension(self):
        """Implicit format extension on the asset by its compilers."""
        for extension, mimetype in self.environment.mimetypes.items():
            if mimetype == self.compiler_mimetype:
                return extension
        return None

def compiler_format_extension ( self ) : for extension , mimetype in self . environment . mimetypes . items ( ) : if mimetype == self . compiler_mimetype : return extension return None

Implicit format extension on the asset by its compilers .


def register(self, mimetype, processor):
        """Register passed `processor` for passed `mimetype`."""
        if mimetype not in self or processor not in self[mimetype]:
            self.setdefault(mimetype, []).append(processor)

def register ( self , mimetype , processor ) : if mimetype not in self or processor not in self [ mimetype ] : self . setdefault ( mimetype , [ ] ) . append ( processor )

Register passed processor for passed mimetype .


def unregister(self, mimetype, processor):
        """Remove passed `processor` for passed `mimetype`. If processor for
        this MIME type does not found in the registry, nothing happens.
        """
        if mimetype in self and processor in self[mimetype]:
            self[mimetype].remove(processor)

def unregister ( self , mimetype , processor ) : if mimetype in self and processor in self [ mimetype ] : self [ mimetype ] . remove ( processor )

Remove passed processor for passed mimetype . If processor for this MIME type does not found in the registry nothing happens .


def paths(self):
        """The list of search paths. It is built from registered finders, which
        has ``paths`` property. Can be useful for compilers to resolve internal
        dependencies.
        """
        if not hasattr(self, '_paths'):
            paths = []
            for finder in self.finders:
                if hasattr(finder, 'paths'):
                    paths.extend(finder.paths)
            self._paths = paths
        return self._paths

def paths ( self ) : if not hasattr ( self , '_paths' ) : paths = [ ] for finder in self . finders : if hasattr ( finder , 'paths' ) : paths . extend ( finder . paths ) self . _paths = paths return self . _paths

The list of search paths . It is built from registered finders which has paths property . Can be useful for compilers to resolve internal dependencies .


def register_defaults(self):
        """Register default compilers, preprocessors and MIME types."""
        self.mimetypes.register_defaults()
        self.preprocessors.register_defaults()
        self.postprocessors.register_defaults()

def register_defaults ( self ) : self . mimetypes . register_defaults ( ) self . preprocessors . register_defaults ( ) self . postprocessors . register_defaults ( )

Register default compilers preprocessors and MIME types .


def import_qtcore():
    """
    This nasty piece of code is here to force the loading of IDA's
     Qt bindings.
    Without it, Python attempts to load PySide from the site-packages
     directory, and failing, as it does not play nicely with IDA.

    via: github.com/tmr232/Cute
    """
    has_ida = False
    try:
        # if we're running under IDA,
        # then we'll use IDA's Qt bindings
        import idaapi
        has_ida = True
    except ImportError:
        # not running under IDA,
        # so use default Qt installation
        has_ida = False

    if has_ida:
        old_path = sys.path[:]
        try:
            ida_python_path = os.path.dirname(idaapi.__file__)
            sys.path.insert(0, ida_python_path)
            if idaapi.IDA_SDK_VERSION >= 690:
                from PyQt5 import QtCore
                return QtCore
            else:
                from PySide import QtCore
                return QtCore
        finally:
            sys.path = old_path
    else:
        try:
            from PyQt5 import QtCore
            return QtCore
        except ImportError:
            pass

        try:
            from PySide import QtCore
            return QtCore
        except ImportError:
            pass

        raise ImportError("No module named PySide or PyQt")

def
import_qtcore
(
)
:
has_ida
=
False
try
:
import
idaapi
has_ida
=
True
except
ImportError
:
has_ida
=
False
if
has_ida
:
old_path
=
sys
.
path
[
:
]
try
:
ida_python_path
=
os
.
path
.
dirname
(
idaapi
.
__file__
)
sys
.
path
.
insert
(
0
,
ida_python_path
)
if
idaapi
.
IDA_SDK_VERSION
>=
690
:
from
PyQt5
import
QtCore
return
QtCore
else
:
from
PySide
import
QtCore
return
QtCore
finally
:
sys
.
path
=
old_path
else
:
try
:
from
PyQt5
import
QtCore
return
QtCore
except
ImportError
:
pass
try
:
from
PySide
import
QtCore
return
QtCore
except
ImportError
:
pass
raise
ImportError
(
"No module named PySide or PyQt"
)

This nasty piece of code is here to force the loading of IDA s Qt bindings . Without it Python attempts to load PySide from the site - packages directory and failing as it does not play nicely with IDA .


def get_meta_netnode():
    """
    Get the netnode used to store settings metadata in the current IDB.
    Note that this implicitly uses the open IDB via the idc iterface.
    """
    node_name = "$ {org:s}.{application:s}".format(
        org=IDA_SETTINGS_ORGANIZATION,
        application=IDA_SETTINGS_APPLICATION)
    return netnode.Netnode(node_name)

def
get_meta_netnode
(
)
:
node_name
=
"$ {org:s}.{application:s}"
.
format
(
org
=
IDA_SETTINGS_ORGANIZATION
,
application
=
IDA_SETTINGS_APPLICATION
)
return
netnode
.
Netnode
(
node_name
)

Get the netnode used to store settings metadata in the current IDB . Note that this implicitly uses the open IDB via the idc iterface .


def add_netnode_plugin_name(plugin_name):
    """
    Add the given plugin name to the list of plugin names registered in
      the current IDB.
    Note that this implicitly uses the open IDB via the idc iterface.
    """
    current_names = set(get_netnode_plugin_names())
    if plugin_name in current_names:
        return

    current_names.add(plugin_name)

    get_meta_netnode()[PLUGIN_NAMES_KEY] = json.dumps(list(current_names))

def add_netnode_plugin_name ( plugin_name ) : current_names = set ( get_netnode_plugin_names ( ) ) if plugin_name in current_names : return current_names . add ( plugin_name ) get_meta_netnode ( ) [ PLUGIN_NAMES_KEY ] = json . dumps ( list ( current_names ) )

Add the given plugin name to the list of plugin names registered in the current IDB . Note that this implicitly uses the open IDB via the idc iterface .


def del_netnode_plugin_name(plugin_name):
    """
    Remove the given plugin name to the list of plugin names registered in
      the current IDB.
    Note that this implicitly uses the open IDB via the idc iterface.
    """
    current_names = set(get_netnode_plugin_names())
    if plugin_name not in current_names:
        return

    try:
        current_names.remove(plugin_name)
    except KeyError:
        return

    get_meta_netnode()[PLUGIN_NAMES_KEY] = json.dumps(list(current_names))

def del_netnode_plugin_name ( plugin_name ) : current_names = set ( get_netnode_plugin_names ( ) ) if plugin_name not in current_names : return try : current_names . remove ( plugin_name ) except KeyError : return get_meta_netnode ( ) [ PLUGIN_NAMES_KEY ] = json . dumps ( list ( current_names ) )

Remove the given plugin name to the list of plugin names registered in the current IDB . Note that this implicitly uses the open IDB via the idc iterface .


def import_settings(settings, config_path):
    """
    Import settings from the given file system path to given settings instance.

    type settings: IDASettingsInterface
    type config_path: str
    """
    other = QtCore.QSettings(config_path, QtCore.QSettings.IniFormat)
    for k in other.allKeys():
        settings[k] = other.value(k)

def import_settings ( settings , config_path ) : other = QtCore . QSettings ( config_path , QtCore . QSettings . IniFormat ) for k in other . allKeys ( ) : settings [ k ] = other . value ( k )

Import settings from the given file system path to given settings instance .


def export_settings(settings, config_path):
    """
    Export the given settings instance to the given file system path.

    type settings: IDASettingsInterface
    type config_path: str
    """
    other = QtCore.QSettings(config_path, QtCore.QSettings.IniFormat)
    for k, v in settings.iteritems():
        other.setValue(k, v)

def export_settings ( settings , config_path ) : other = QtCore . QSettings ( config_path , QtCore . QSettings . IniFormat ) for k , v in settings . iteritems ( ) : other . setValue ( k , v )

Export the given settings instance to the given file system path .


def directory(self):
        """
        Fetch the IDASettings instance for the curren plugin with directory scope.

        rtype: IDASettingsInterface
        """
        if self._config_directory is None:
            ensure_ida_loaded()
        return DirectoryIDASettings(self._plugin_name, directory=self._config_directory)

def directory ( self ) : if self . _config_directory is None : ensure_ida_loaded ( ) return DirectoryIDASettings ( self . _plugin_name , directory = self . _config_directory )

Fetch the IDASettings instance for the curren plugin with directory scope .


def iterkeys(self):
        """
        Enumerate the keys found at any scope for the current plugin.

        rtype: Generator[str]
        """
        visited_keys = set()
        try:
            for key in self.idb.iterkeys():
                if key not in visited_keys:
                    yield key
                    visited_keys.add(key)
        except (PermissionError, EnvironmentError):
            pass

        try:
            for key in self.directory.iterkeys():
                if key not in visited_keys:
                    yield key
                    visited_keys.add(key)
        except (PermissionError, EnvironmentError):
            pass

        try:
            for key in self.user.iterkeys():
                if key not in visited_keys:
                    yield key
                    visited_keys.add(key)
        except (PermissionError, EnvironmentError):
            pass

        try:
            for key in self.system.iterkeys():
                if key not in visited_keys:
                    yield key
                    visited_keys.add(key)
        except (PermissionError, EnvironmentError):
            pass

def iterkeys ( self ) : visited_keys = set ( ) try : for key in self . idb . iterkeys ( ) : if key not in visited_keys : yield key visited_keys . add ( key ) except ( PermissionError , EnvironmentError ) : pass try : for key in self . directory . iterkeys ( ) : if key not in visited_keys : yield key visited_keys . add ( key ) except ( PermissionError , EnvironmentError ) : pass try : for key in self . user . iterkeys ( ) : if key not in visited_keys : yield key visited_keys . add ( key ) except ( PermissionError , EnvironmentError ) : pass try : for key in self . system . iterkeys ( ) : if key not in visited_keys : yield key visited_keys . add ( key ) except ( PermissionError , EnvironmentError ) : pass

Enumerate the keys found at any scope for the current plugin .


def simple_error_handler(exc, *args):
    """
    Returns the response that should be used for any given exception.

    By default we handle the REST framework `APIException`, and also
    Django's builtin `Http404` and `PermissionDenied` exceptions.

    Any unhandled exceptions may return `None`, which will cause a 500 error
    to be raised.
    """
    if isinstance(exc, exceptions.APIException):
        headers = {}
        if getattr(exc, 'auth_header', None):
            headers['WWW-Authenticate'] = exc.auth_header
        if getattr(exc, 'wait', None):
            headers['X-Throttle-Wait-Seconds'] = '%d' % exc.wait

        return Response({'error': exc.detail},
                        status=exc.status_code,
                        headers=headers)

    elif isinstance(exc, Http404):
        return Response({'error': 'Not found'},
                        status=status.HTTP_404_NOT_FOUND)

    elif isinstance(exc, PermissionDenied):
        return Response({'error': 'Permission denied'},
                        status=status.HTTP_403_FORBIDDEN)

    # Note: Unhandled exceptions will raise a 500 error.
    return None

def
simple_error_handler
(
exc
,
*
args
)
:
if
isinstance
(
exc
,
exceptions
.
APIException
)
:
headers
=
{
}
if
getattr
(
exc
,
'auth_header'
,
)
:
headers
[
'WWW-Authenticate'
]
=
exc
.
auth_header
if
getattr
(
exc
,
'wait'
,
)
:
headers
[
'X-Throttle-Wait-Seconds'
]
=
'%d'
%
exc
.
wait
return
Response
(
{
'error'
:
exc
.
detail
}
,
status
=
exc
.
status_code
,
headers
=
headers
)
elif
isinstance
(
exc
,
Http404
)
:
return
Response
(
{
'error'
:
'Not found'
}
,
status
=
status
.
HTTP_404_NOT_FOUND
)
elif
isinstance
(
exc
,
PermissionDenied
)
:
return
Response
(
{
'error'
:
'Permission denied'
}
,
status
=
status
.
HTTP_403_FORBIDDEN
)
return

Returns the response that should be used for any given exception .


def table(name, auth=None, eager=True):
    """Returns a given table for the given user."""
    auth = auth or []
    dynamodb = boto.connect_dynamodb(*auth)

    table = dynamodb.get_table(name)
    return Table(table=table, eager=eager)

def table ( name , auth = None , eager = True ) : auth = auth or [ ] dynamodb = boto . connect_dynamodb ( * auth ) table = dynamodb . get_table ( name ) return Table ( table = table , eager = eager )

Returns a given table for the given user .


def tables(auth=None, eager=True):
    """Returns a list of tables for the given user."""
    auth = auth or []
    dynamodb = boto.connect_dynamodb(*auth)

    return [table(t, auth, eager=eager) for t in dynamodb.list_tables()]

def tables ( auth = None , eager = True ) : auth = auth or [ ] dynamodb = boto . connect_dynamodb ( * auth ) return [ table ( t , auth , eager = eager ) for t in dynamodb . list_tables ( ) ]

Returns a list of tables for the given user .


def fetch_items(self, category, **kwargs):
        """Fetch packages and summary from Crates.io

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']

        if category == CATEGORY_CRATES:
            return self.__fetch_crates(from_date)
        else:
            return self.__fetch_summary()

def fetch_items ( self , category , ** kwargs ) : from_date = kwargs [ 'from_date' ] if category == CATEGORY_CRATES : return self . __fetch_crates ( from_date ) else : return self . __fetch_summary ( )

Fetch packages and summary from Crates . io


def metadata_id(item):
        """Extracts the identifier from an item depending on its type."""

        if Crates.metadata_category(item) == CATEGORY_CRATES:
            return str(item['id'])
        else:
            ts = item['fetched_on']
            ts = str_to_datetime(ts)
            return str(ts.timestamp())

def metadata_id ( item ) : if Crates . metadata_category ( item ) == CATEGORY_CRATES : return str ( item [ 'id' ] ) else : ts = item [ 'fetched_on' ] ts = str_to_datetime ( ts ) return str ( ts . timestamp ( ) )

Extracts the identifier from an item depending on its type .


def metadata_updated_on(item):
        """Extracts the update time from an item.

        Depending on the item, the timestamp is extracted from the
        'updated_at' or 'fetched_on' fields.
        This date is converted to UNIX timestamp format.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        if Crates.metadata_category(item) == CATEGORY_CRATES:
            ts = item['updated_at']
        else:
            ts = item['fetched_on']

        ts = str_to_datetime(ts)

        return ts.timestamp()

def metadata_updated_on ( item ) : if Crates . metadata_category ( item ) == CATEGORY_CRATES : ts = item [ 'updated_at' ] else : ts = item [ 'fetched_on' ] ts = str_to_datetime ( ts ) return ts . timestamp ( )


def __fetch_crate_owner_team(self, crate_id):
        """Get crate team owner"""

        raw_owner_team = self.client.crate_attribute(crate_id, 'owner_team')

        owner_team = json.loads(raw_owner_team)

        return owner_team

def __fetch_crate_owner_team ( self , crate_id ) : raw_owner_team = self . client . crate_attribute ( crate_id , 'owner_team' ) owner_team = json . loads ( raw_owner_team ) return owner_team


def __fetch_crate_owner_user(self, crate_id):
        """Get crate user owners"""

        raw_owner_user = self.client.crate_attribute(crate_id, 'owner_user')

        owner_user = json.loads(raw_owner_user)

        return owner_user

def __fetch_crate_owner_user ( self , crate_id ) : raw_owner_user = self . client . crate_attribute ( crate_id , 'owner_user' ) owner_user = json . loads ( raw_owner_user ) return owner_user


def __fetch_crate_versions(self, crate_id):
        """Get crate versions data"""

        raw_versions = self.client.crate_attribute(crate_id, "versions")

        version_downloads = json.loads(raw_versions)

        return version_downloads

def __fetch_crate_versions ( self , crate_id ) : raw_versions = self . client . crate_attribute ( crate_id , "versions" ) version_downloads = json . loads ( raw_versions ) return version_downloads


def __fetch_crate_version_downloads(self, crate_id):
        """Get crate version downloads"""

        raw_version_downloads = self.client.crate_attribute(crate_id, "downloads")

        version_downloads = json.loads(raw_version_downloads)

        return version_downloads

def __fetch_crate_version_downloads ( self , crate_id ) : raw_version_downloads = self . client . crate_attribute ( crate_id , "downloads" ) version_downloads = json . loads ( raw_version_downloads ) return version_downloads


def __fetch_crate_data(self, crate_id):
        """Get crate data"""

        raw_crate = self.client.crate(crate_id)

        crate = json.loads(raw_crate)
        return crate['crate']

def __fetch_crate_data ( self , crate_id ) : raw_crate = self . client . crate ( crate_id ) crate = json . loads ( raw_crate ) return crate [ 'crate' ]


def summary(self):
        """Get Crates.io summary"""

        path = urijoin(CRATES_API_URL, CATEGORY_SUMMARY)
        raw_content = self.fetch(path)

        return raw_content

def summary ( self ) : path = urijoin ( CRATES_API_URL , CATEGORY_SUMMARY ) raw_content = self . fetch ( path ) return raw_content


def crates(self, from_page=1):
        """Get crates in alphabetical order"""

        path = urijoin(CRATES_API_URL, CATEGORY_CRATES)
        raw_crates = self.__fetch_items(path, from_page)

        return raw_crates

def crates ( self , from_page = 1 ) : path = urijoin ( CRATES_API_URL , CATEGORY_CRATES ) raw_crates = self . __fetch_items ( path , from_page ) return raw_crates


def crate(self, crate_id):
        """Get a crate by its ID"""

        path = urijoin(CRATES_API_URL, CATEGORY_CRATES, crate_id)
        raw_crate = self.fetch(path)

        return raw_crate

def crate ( self , crate_id ) : path = urijoin ( CRATES_API_URL , CATEGORY_CRATES , crate_id ) raw_crate = self . fetch ( path ) return raw_crate


def crate_attribute(self, crate_id, attribute):
        """Get crate attribute"""

        path = urijoin(CRATES_API_URL, CATEGORY_CRATES, crate_id, attribute)
        raw_attribute_data = self.fetch(path)

        return raw_attribute_data

def crate_attribute ( self , crate_id , attribute ) : path = urijoin ( CRATES_API_URL , CATEGORY_CRATES , crate_id , attribute ) raw_attribute_data = self . fetch ( path ) return raw_attribute_data


def __fetch_items(self, path, page=1):
        """Return the items from Crates.io API using pagination"""

        fetch_data = True
        parsed_crates = 0
        total_crates = 0

        while fetch_data:
            logger.debug("Fetching page: %i", page)

            try:
                payload = {'sort': 'alphabetical', 'page': page}
                raw_content = self.fetch(path, payload=payload)
                content = json.loads(raw_content)

                parsed_crates += len(content['crates'])

                if not total_crates:
                    total_crates = content['meta']['total']

            except requests.exceptions.HTTPError as e:
                logger.error("HTTP exception raised - %s", e.response.text)
                raise e

            yield raw_content
            page += 1

            if parsed_crates >= total_crates:
                fetch_data = False

def
__fetch_items
(
self
,
path
,
page
=
1
)
:
fetch_data
=
True
parsed_crates
=
0
total_crates
=
0
while
fetch_data
:
logger
.
debug
(
"Fetching page: %i"
,
page
)
try
:
payload
=
{
'sort'
:
'alphabetical'
,
'page'
:
page
}
raw_content
=
self
.
fetch
(
path
,
payload
=
payload
)
content
=
json
.
loads
(
raw_content
)
parsed_crates
+=
len
(
content
[
'crates'
]
)
if
not
total_crates
:
total_crates
=
content
[
'meta'
]
[
'total'
]
except
requests
.
exceptions
.
HTTPError
as
e
:
logger
.
error
(
"HTTP exception raised - %s"
,
e
.
response
.
text
)
raise
e
yield
raw_content
page
+=
1
if
parsed_crates
>=
total_crates
:
fetch_data
=
False

Return the items from Crates . io API using pagination


def fetch(self, category=CATEGORY_QUESTION, offset=DEFAULT_OFFSET):
        """Fetch questions from the Kitsune url.

        :param category: the category of items to fetch
        :offset: obtain questions after offset
        :returns: a generator of questions
        """
        if not offset:
            offset = DEFAULT_OFFSET

        kwargs = {"offset": offset}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_QUESTION , offset = DEFAULT_OFFSET ) : if not offset : offset = DEFAULT_OFFSET kwargs = { "offset" : offset } items = super ( ) . fetch ( category , ** kwargs ) return items


def fetch_items(self, category, **kwargs):
        """Fetch questions from the Kitsune url

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        offset = kwargs['offset']

        logger.info("Looking for questions at url '%s' using offset %s",
                    self.url, str(offset))

        nquestions = 0  # number of questions processed
        tquestions = 0  # number of questions from API data
        equestions = 0  # number of questions dropped by errors

        # Always get complete pages so the first item is always
        # the first one in the page
        page = int(offset / KitsuneClient.ITEMS_PER_PAGE)
        page_offset = page * KitsuneClient.ITEMS_PER_PAGE
        # drop questions from page before the offset
        drop_questions = offset - page_offset
        current_offset = offset

        questions_page = self.client.get_questions(offset)

        while True:
            try:
                raw_questions = next(questions_page)
            except StopIteration:
                break
            except requests.exceptions.HTTPError as e:
                # Continue with the next page if it is a 500 error
                if e.response.status_code == 500:
                    logger.exception(e)
                    logger.error("Problem getting Kitsune questions. "
                                 "Loosing %i questions. Going to the next page.",
                                 KitsuneClient.ITEMS_PER_PAGE)
                    equestions += KitsuneClient.ITEMS_PER_PAGE
                    current_offset += KitsuneClient.ITEMS_PER_PAGE
                    questions_page = self.client.get_questions(current_offset)
                    continue
                else:
                    # If it is another error just propagate the exception
                    raise e

            try:
                questions_data = json.loads(raw_questions)
                tquestions = questions_data['count']
                questions = questions_data['results']
            except (ValueError, KeyError) as ex:
                logger.error(ex)
                cause = ("Bad JSON format for mozilla_questions: %s" % (raw_questions))
                raise ParseError(cause=cause)

            for question in questions:
                if drop_questions > 0:
                    # Remove extra questions due to page base retrieval
                    drop_questions -= 1
                    continue
                question['offset'] = current_offset
                current_offset += 1
                question['answers_data'] = []
                for raw_answers in self.client.get_question_answers(question['id']):
                    answers = json.loads(raw_answers)['results']
                    question['answers_data'] += answers
                yield question
                nquestions += 1

            logger.debug("Questions: %i/%i", nquestions + offset, tquestions)

        logger.info("Total number of questions: %i (%i total)", nquestions, tquestions)
        logger.info("Questions with errors dropped: %i", equestions)


def get_questions(self, offset=None):
        """Retrieve questions from older to newer updated starting offset"""

        page = KitsuneClient.FIRST_PAGE

        if offset:
            page += int(offset / KitsuneClient.ITEMS_PER_PAGE)

        while True:
            api_questions_url = urijoin(self.base_url, '/question') + '/'

            params = {
                "page": page,
                "ordering": "updated"
            }

            questions = self.fetch(api_questions_url, params)
            yield questions

            questions_json = json.loads(questions)
            next_uri = questions_json['next']
            if not next_uri:
                break
            page += 1

def get_questions ( self , offset = None ) : page = KitsuneClient . FIRST_PAGE if offset : page += int ( offset / KitsuneClient . ITEMS_PER_PAGE ) while True : api_questions_url = urijoin ( self . base_url , '/question' ) + '/' params = { "page" : page , "ordering" : "updated" } questions = self . fetch ( api_questions_url , params ) yield questions questions_json = json . loads ( questions ) next_uri = questions_json [ 'next' ] if not next_uri : break page += 1


def fetch(self, category=CATEGORY_EVENT, offset=REMO_DEFAULT_OFFSET):
        """Fetch items from the ReMo url.

        The method retrieves, from a ReMo URL, the set of items
        of the given `category`.

        :param category: the category of items to fetch
        :param offset: obtain items after offset
        :returns: a generator of items
        """
        if not offset:
            offset = REMO_DEFAULT_OFFSET

        kwargs = {"offset": offset}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_EVENT , offset = REMO_DEFAULT_OFFSET ) : if not offset : offset = REMO_DEFAULT_OFFSET kwargs = { "offset" : offset } items = super ( ) . fetch ( category , ** kwargs ) return items


def metadata_updated_on(item):
        """Extracts the update time from a ReMo item.

        The timestamp is extracted from 'end' field.
        This date is converted to a perceval format using a float value.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        if 'end' in item:
            # events updated field
            updated = item['end']
        elif 'date_joined_program' in item:
            # users updated field that always appear
            updated = item['date_joined_program']
        elif 'report_date' in item:
            # activities updated field
            updated = item['report_date']
        else:
            raise ValueError("Can't find updated field for item " + str(item))

        return float(str_to_datetime(updated).timestamp())

def
metadata_updated_on
(
item
)
:
if
'end'
in
item
:
updated
=
item
[
'end'
]
elif
'date_joined_program'
in
item
:
updated
=
item
[
'date_joined_program'
]
elif
'report_date'
in
item
:
updated
=
item
[
'report_date'
]
else
:
raise
ValueError
(
"Can't find updated field for item "
+
str
(
item
)
)
return
float
(
str_to_datetime
(
updated
)
.
timestamp
(
)
)

Extracts the update time from a ReMo item .


def metadata_category(item):
        """Extracts the category from a ReMo item.

        This backend generates items types 'event', 'activity'
        or 'user'. To guess the type of item, the code will look
        for unique fields.
        """
        if 'estimated_attendance' in item:
            category = CATEGORY_EVENT
        elif 'activity' in item:
            category = CATEGORY_ACTIVITY
        elif 'first_name' in item:
            category = CATEGORY_USER
        else:
            raise TypeError("Could not define the category of item " + str(item))

        return category

Extracts the category from a ReMo item .


def get_items(self, category=CATEGORY_EVENT, offset=REMO_DEFAULT_OFFSET):
        """Retrieve all items for category using pagination """

        more = True  # There are more items to be processed
        next_uri = None  # URI for the next items page query
        page = ReMoClient.FIRST_PAGE
        page += int(offset / ReMoClient.ITEMS_PER_PAGE)

        if category == CATEGORY_EVENT:
            api = self.api_events_url
        elif category == CATEGORY_ACTIVITY:
            api = self.api_activities_url
        elif category == CATEGORY_USER:
            api = self.api_users_url
        else:
            raise ValueError(category + ' not supported in ReMo')

        while more:
            params = {
                "page": page,
                "orderby": "ASC"
            }

            logger.debug("ReMo client calls APIv2: %s params: %s",
                         api, str(params))

            raw_items = self.fetch(api, payload=params)
            yield raw_items

            items_data = json.loads(raw_items)
            next_uri = items_data['next']

            if not next_uri:
                more = False
            else:
                # https://reps.mozilla.org/remo/api/remo/v1/events/?orderby=ASC&page=269
                parsed_uri = urllib.parse.urlparse(next_uri)
                parsed_params = urllib.parse.parse_qs(parsed_uri.query)
                page = parsed_params['page'][0]


def buffer_list(self):
        """
        The buffer list this instance operates on.

        Only available in mode != AIOBLOCK_MODE_POLL.

        Changes on a submitted transfer are not fully applied until its
        next submission: kernel will still be using original buffer list.
        """
        if self._iocb.aio_lio_opcode == libaio.IO_CMD_POLL:
            raise AttributeError
        return self._buffer_list

def buffer_list ( self ) : if self . _iocb . aio_lio_opcode == libaio . IO_CMD_POLL : raise AttributeError return self . _buffer_list

The buffer list this instance operates on .


def io_priority(self):
        """
        IO priority for this instance.
        """
        return (
            self._iocb.aio_reqprio
            if self._iocb.u.c.flags & libaio.IOCB_FLAG_IOPRIO else
            None
        )

def io_priority ( self ) : return ( self . _iocb . aio_reqprio if self . _iocb . u . c . flags & libaio . IOCB_FLAG_IOPRIO else None )


def close(self):
        """
        Cancels all pending IO blocks.
        Waits until all non-cancellable IO blocks finish.
        De-initialises AIO context.
        """
        if self._ctx is not None:
            # Note: same as io_destroy
            self._io_queue_release(self._ctx)
            del self._ctx

def close ( self ) : if self . _ctx is not None : self . _io_queue_release ( self . _ctx ) del self . _ctx

Cancels all pending IO blocks . Waits until all non - cancellable IO blocks finish . De - initialises AIO context .


def submit(self, block_list):
        """
        Submits transfers.

        block_list (list of AIOBlock)
            The IO blocks to hand off to kernel.

        Returns the number of successfully submitted blocks.
        """
        # io_submit ioctl will only return an error for issues with the first
        # transfer block. If there are issues with a later block, it will stop
        # submission and return the number of submitted blocks. So it is safe
        # to only update self._submitted once io_submit returned.
        submitted_count = libaio.io_submit(
            self._ctx,
            len(block_list),
            (libaio.iocb_p * len(block_list))(*[
                # pylint: disable=protected-access
                pointer(x._iocb)
                # pylint: enable=protected-access
                for x in block_list
            ]),
        )
        submitted = self._submitted
        for block in block_list[:submitted_count]:
            # pylint: disable=protected-access
            submitted[addressof(block._iocb)] = (block, block._getSubmissionState())
            # pylint: enable=protected-access
        return submitted_count

def submit ( self , block_list ) : submitted_count = libaio . io_submit ( self . _ctx , len ( block_list ) , ( libaio . iocb_p * len ( block_list ) ) ( * [ pointer ( x . _iocb ) for x in block_list ] ) , ) submitted = self . _submitted for block in block_list [ : submitted_count ] : submitted [ addressof ( block . _iocb ) ] = ( block , block . _getSubmissionState ( ) ) return submitted_count


def cancel(self, block):
        """
        Cancel an IO block.

        block (AIOBlock)
            The IO block to cancel.

        Returns cancelled block's event data (see getEvents), or None if the
        kernel returned EINPROGRESS. In the latter case, event completion will
        happen on a later getEvents call.
        """
        event = libaio.io_event()
        try:
            # pylint: disable=protected-access
            libaio.io_cancel(self._ctx, byref(block._iocb), byref(event))
            # pylint: enable=protected-access
        except OSError as exc:
            if exc.errno == errno.EINPROGRESS:
                return None
            raise
        return self._eventToPython(event)

def cancel ( self , block ) : event = libaio . io_event ( ) try : libaio . io_cancel ( self . _ctx , byref ( block . _iocb ) , byref ( event ) ) except OSError as exc : if exc . errno == errno . EINPROGRESS : return None raise return self . _eventToPython ( event )


def cancelAll(self):
        """
        Cancel all submitted IO blocks.

        Blocks until all submitted transfers have been finalised.
        Submitting more transfers or processing completion events while this
        method is running produces undefined behaviour.
        Returns the list of values returned by individual cancellations.
        See "cancel" documentation.
        """
        cancel = self.cancel
        result = []
        for block, _ in self._submitted.itervalues():
            try:
                result.append(cancel(block))
            except OSError as exc:
                # EINVAL should mean we requested to cancel a not-in-flight
                # transfer - maybe it was just completed and we just did
                # not process its completion event yet.
                if exc.errno != errno.EINVAL:
                    raise
        return result

def cancelAll ( self ) : cancel = self . cancel result = [ ] for block , _ in self . _submitted . itervalues ( ) : try : result . append ( cancel ( block ) ) except OSError as exc : if exc . errno != errno . EINVAL : raise return result


def getEvents(self, min_nr=1, nr=None, timeout=None):
        """
        Returns a list of event data from submitted IO blocks.

        min_nr (int, None)
            When timeout is None, minimum number of events to collect before
            returning.
            If None, waits for all submitted events.
        nr (int, None)
            Maximum number of events to return.
            If None, set to maxevents given at construction or to the number of
            currently submitted events, whichever is larger.
        timeout (float, None):
            Time to wait for events.
            If None, become blocking.

        Returns a list of 3-tuples, containing:
        - completed AIOBlock instance
        - res, file-object-type-dependent value
        - res2, another file-object-type-dependent value
        """
        if min_nr is None:
            min_nr = len(self._submitted)
        if nr is None:
            nr = max(len(self._submitted), self._maxevents)
        if timeout is None:
            timeoutp = None
        else:
            sec = int(timeout)
            timeout = libaio.timespec(sec, int((timeout - sec) * 1e9))
            timeoutp = byref(timeout)
        event_buffer = (libaio.io_event * nr)()
        actual_nr = libaio.io_getevents(
            self._ctx,
            min_nr,
            nr,
            event_buffer,
            timeoutp,
        )
        return [
            self._eventToPython(event_buffer[x])
            for x in xrange(actual_nr)
        ]

def getEvents ( self , min_nr = 1 , nr = None , timeout = None ) : if min_nr is None : min_nr = len ( self . _submitted ) if nr is None : nr = max ( len ( self . _submitted ) , self . _maxevents ) if timeout is None : timeoutp = None else : sec = int ( timeout ) timeout = libaio . timespec ( sec , int ( ( timeout - sec ) * 1e9 ) ) timeoutp = byref ( timeout ) event_buffer = ( libaio . io_event * nr ) ( ) actual_nr = libaio . io_getevents ( self . _ctx , min_nr , nr , event_buffer , timeoutp , ) return [ self . _eventToPython ( event_buffer [ x ] ) for x in xrange ( actual_nr ) ]

Returns a list of event data from submitted IO blocks .


def fetch(self, category=CATEGORY_EVENT):
        """Fetch events from the MozillaClub URL.

        The method retrieves, from a MozillaClub URL, the
        events. The data is a Google spreadsheet retrieved using
        the feed API REST.

        :param category: the category of items to fetch

        :returns: a generator of events
        """
        kwargs = {}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_EVENT ) : kwargs = { } items = super ( ) . fetch ( category , ** kwargs ) return items


def get_cells(self):
        """Retrieve all cells from the spreadsheet."""

        logger.info("Retrieving all cells spreadsheet data ...")
        logger.debug("MozillaClub client calls API: %s", self.base_url)
        raw_cells = self.fetch(self.base_url)

        return raw_cells.text

Retrieve all cells from the spreadsheet .


def parse(self):
        """Parse the MozillaClub spreadsheet feed cells json."""

        nevents_wrong = 0

        feed_json = json.loads(self.feed)

        if 'entry' not in feed_json['feed']:
            return

        self.cells = feed_json['feed']['entry']
        self.ncell = 0

        event_fields = self.__get_event_fields()

        # Process all events reading the rows according to the event template
        # The only way to detect the end of row is looking to the
        # number of column. When the max number is reached (cell_cols) the next
        # cell is from the next row.
        while self.ncell < len(self.cells):
            # Process the next row (event) getting all cols to build the event
            event = self.__get_next_event(event_fields)

            if event['Date of Event'] is None or event['Club Name'] is None:
                logger.warning("Wrong event data: %s", event)
                nevents_wrong += 1
                continue
            yield event

        logger.info("Total number of wrong events: %i", nevents_wrong)

def
parse
(
self
)
:
nevents_wrong
=
0
feed_json
=
json
.
loads
(
self
.
feed
)
if
'entry'
not
in
feed_json
[
'feed'
]
:
return
self
.
cells
=
feed_json
[
'feed'
]
[
'entry'
]
self
.
ncell
=
0
event_fields
=
self
.
__get_event_fields
(
)
while
self
.
ncell
<
len
(
self
.
cells
)
:
event
=
self
.
__get_next_event
(
event_fields
)
if
event
[
'Date of Event'
]
is
or
event
[
'Club Name'
]
is
:
logger
.
warning
(
"Wrong event data: %s"
,
event
)
nevents_wrong
+=
1
continue
yield
event
logger
.
info
(
"Total number of wrong events: %i"
,
nevents_wrong
)

Parse the MozillaClub spreadsheet feed cells json .


def export_formats(self, pid_type):
        """List of export formats."""
        if pid_type not in self._export_formats:
            fmts = self.app.config.get('RECORDS_UI_EXPORT_FORMATS', {}).get(
                pid_type, {})
            self._export_formats[pid_type] = sorted(
                [(k, v) for k, v in fmts.items() if v],
                key=lambda x: x[1]['order'],
            )
        return self._export_formats[pid_type]

def export_formats ( self , pid_type ) : if pid_type not in self . _export_formats : fmts = self . app . config . get ( 'RECORDS_UI_EXPORT_FORMATS' , { } ) . get ( pid_type , { } ) self . _export_formats [ pid_type ] = sorted ( [ ( k , v ) for k , v in fmts . items ( ) if v ] , key = lambda x : x [ 1 ] [ 'order' ] , ) return self . _export_formats [ pid_type ]


def permission_factory(self):
        """Load default permission factory."""
        if self._permission_factory is None:
            imp = self.app.config['RECORDS_UI_DEFAULT_PERMISSION_FACTORY']
            self._permission_factory = obj_or_import_string(imp)
        return self._permission_factory

def permission_factory ( self ) : if self . _permission_factory is None : imp = self . app . config [ 'RECORDS_UI_DEFAULT_PERMISSION_FACTORY' ] self . _permission_factory = obj_or_import_string ( imp ) return self . _permission_factory


def create_blueprint(endpoints):
    """Create Invenio-Records-UI blueprint.

    The factory installs one URL route per endpoint defined, and adds an
    error handler for rendering tombstones.

    :param endpoints: Dictionary of endpoints to be installed. See usage
        documentation for further details.
    :returns: The initialized blueprint.
    """
    blueprint = Blueprint(
        'invenio_records_ui',
        __name__,
        url_prefix='',
        template_folder='templates',
        static_folder='static',
    )

    @blueprint.errorhandler(PIDDeletedError)
    def tombstone_errorhandler(error):
        return render_template(
            current_app.config['RECORDS_UI_TOMBSTONE_TEMPLATE'],
            pid=error.pid,
            record=error.record or {},
        ), 410

    @blueprint.context_processor
    def inject_export_formats():
        return dict(
            export_formats=(
                current_app.extensions['invenio-records-ui'].export_formats)
            )

    for endpoint, options in (endpoints or {}).items():
        blueprint.add_url_rule(**create_url_rule(endpoint, **options))

    return blueprint

def create_blueprint ( endpoints ) : blueprint = Blueprint ( 'invenio_records_ui' , __name__ , url_prefix = '' , template_folder = 'templates' , static_folder = 'static' , ) @ blueprint . errorhandler ( PIDDeletedError ) def tombstone_errorhandler ( error ) : return render_template ( current_app . config [ 'RECORDS_UI_TOMBSTONE_TEMPLATE' ] , pid = error . pid , record = error . record or { } , ) , 410 @ blueprint . context_processor def inject_export_formats ( ) : return dict ( export_formats = ( current_app . extensions [ 'invenio-records-ui' ] . export_formats ) ) for endpoint , options in ( endpoints or { } ) . items ( ) : blueprint . add_url_rule ( ** create_url_rule ( endpoint , ** options ) ) return blueprint

Create Invenio - Records - UI blueprint .