def register_range_type(pgrange, pyrange, conn):
    """
    Register a new range type as a PostgreSQL range.

        >>> register_range_type("int4range", intrange, conn)

    The above will make sure intrange is regarded as an int4range for queries
    and that int4ranges will be cast into intrange when fetching rows.

    pgrange should be the full name including schema for the custom range type.

    Note that adaption is global, meaning if a range type is passed to a regular
    psycopg2 connection it will adapt it to its proper range type. Parsing of
    rows from the database however is not global and just set on a per connection
    basis.
    """

    register_adapter(pyrange, partial(adapt_range, pgrange))
    register_range_caster(
        pgrange, pyrange, *query_range_oids(pgrange, conn), scope=conn)

def register_range_type ( pgrange , pyrange , conn ) : register_adapter ( pyrange , partial ( adapt_range , pgrange ) ) register_range_caster ( pgrange , pyrange , * query_range_oids ( pgrange , conn ) , scope = conn )

Register a new range type as a PostgreSQL range .


def get_api_error(response):
  """Acquires the correct error for a given response.

  :param requests.Response response: HTTP error response
  :returns: the appropriate error for a given response
  :rtype: APIError

  """
  error_class = _status_code_to_class.get(response.status_code, APIError)
  return error_class(response)

def get_api_error ( response ) : error_class = _status_code_to_class . get ( response . status_code , APIError ) return error_class ( response )

Acquires the correct error for a given response .


def get_param_values(request, model=None):
    """
    Converts the request parameters to Python.

    :param request: <pyramid.request.Request> || <dict>

    :return: <dict>
    """
    if type(request) == dict:
        return request

    params = get_payload(request)

    # support in-place editing formatted request
    try:
        del params['pk']
        params[params.pop('name')] = params.pop('value')
    except KeyError:
        pass

    return {
        k.rstrip('[]'): safe_eval(v) if not type(v) == list else [safe_eval(sv) for sv in v]
        for k, v in params.items()
    }

def get_param_values ( request , model = None ) : if type ( request ) == dict : return request params = get_payload ( request ) try : del params [ 'pk' ] params [ params . pop ( 'name' ) ] = params . pop ( 'value' ) except KeyError : pass return { k . rstrip ( '[]' ) : safe_eval ( v ) if not type ( v ) == list else [ safe_eval ( sv ) for sv in v ] for k , v in params . items ( ) }

Converts the request parameters to Python .


def get_context(request, model=None):
    """
    Extracts ORB context information from the request.

    :param request: <pyramid.request.Request>
    :param model: <orb.Model> || None

    :return: {<str> key: <variant> value} values, <orb.Context>
    """
    # convert request parameters to python
    param_values = get_param_values(request, model=model)

    # extract the full orb context if provided
    context = param_values.pop('orb_context', {})
    if isinstance(context, (unicode, str)):
        context = projex.rest.unjsonify(context)

    # otherwise, extract the limit information
    has_limit = 'limit' in context or 'limit' in param_values

    # create the new orb context
    orb_context = orb.Context(**context)

    # build up context information from the request params
    used = set()
    query_context = {}
    for key in orb.Context.Defaults:
        if key in param_values:
            used.add(key)
            query_context[key] = param_values.get(key)

    # generate a simple query object
    schema_values = {}
    if model:
        # extract match dict items
        for key, value in request.matchdict.items():
            if model.schema().column(key, raise_=False):
                schema_values[key] = value

        # extract payload items
        for key, value in param_values.items():
            root_key = key.split('.')[0]
            schema_object = model.schema().column(root_key, raise_=False) or model.schema().collector(root_key)
            if schema_object:
                value = param_values.pop(key)
                if isinstance(schema_object, orb.Collector) and type(value) not in (tuple, list):
                    value = [value]
                schema_values[key] = value

    # generate the base context information
    query_context['scope'] = {
        'request': request
    }

    # include any request specific scoping or information from the request
    # first, look for default ORB context for all requests
    try:
        default_context = request.orb_default_context

    # then, look for scope specific information for all requests
    except AttributeError:
        try:
            query_context['scope'].update(request.orb_scope)
        except AttributeError:
            pass

    # if request specific context defaults exist, then
    # merge them with the rest of the query context
    else:
        if 'scope' in default_context:
            query_context['scope'].update(default_context.pop('scope'))

        # setup defaults based on the request
        for k, v in default_context.items():
            query_context.setdefault(k, v)

    orb_context.update(query_context)
    return schema_values, orb_context

def get_context ( request , model = None ) : param_values = get_param_values ( request , model = model ) context = param_values . pop ( 'orb_context' , { } ) if isinstance ( context , ( unicode , str ) ) : context = projex . rest . unjsonify ( context ) has_limit = 'limit' in context or 'limit' in param_values orb_context = orb . Context ( ** context ) used = set ( ) query_context = { } for key in orb . Context . Defaults : if key in param_values : used . add ( key ) query_context [ key ] = param_values . get ( key ) schema_values = { } if model : for key , value in request . matchdict . items ( ) : if model . schema ( ) . column ( key , raise_ = False ) : schema_values [ key ] = value for key , value in param_values . items ( ) : root_key = key . split ( '.' ) [ 0 ] schema_object = model . schema ( ) . column ( root_key , raise_ = False ) or model . schema ( ) . collector ( root_key ) if schema_object : value = param_values . pop ( key ) if isinstance ( schema_object , orb . Collector ) and type ( value ) not in ( tuple , list ) : value = [ value ] schema_values [ key ] = value query_context [ 'scope' ] = { 'request' : request } try : default_context = request . orb_default_context except AttributeError : try : query_context [ 'scope' ] . update ( request . orb_scope ) except AttributeError : pass else : if 'scope' in default_context : query_context [ 'scope' ] . update ( default_context . pop ( 'scope' ) ) for k , v in default_context . items ( ) : query_context . setdefault ( k , v ) orb_context . update ( query_context ) return schema_values , orb_context

Extracts ORB context information from the request .


def _real_time_thread(self):
    """Handles real-time updates to the order book."""
    while self.ws_client.connected():
      if self.die:
        break

      if self.pause:
        sleep(5)
        continue

      message = self.ws_client.receive()

      if message is None:
        break

      message_type = message['type']

      if message_type  == 'error':
        continue
      if message['sequence'] <= self.sequence:
        continue

      if message_type == 'open':
        self._handle_open(message)
      elif message_type == 'match':
        self._handle_match(message)
      elif message_type == 'done':
        self._handle_done(message)
      elif message_type == 'change':
        self._handle_change(message)
      else:
        continue

    self.ws_client.disconnect()

def _real_time_thread ( self ) : while self . ws_client . connected ( ) : if self . die : break if self . pause : sleep ( 5 ) continue message = self . ws_client . receive ( ) if message is None : break message_type = message [ 'type' ] if message_type == 'error' : continue if message [ 'sequence' ] <= self . sequence : continue if message_type == 'open' : self . _handle_open ( message ) elif message_type == 'match' : self . _handle_match ( message ) elif message_type == 'done' : self . _handle_done ( message ) elif message_type == 'change' : self . _handle_change ( message ) else : continue self . ws_client . disconnect ( )

Handles real - time updates to the order book .


def _keep_alive_thread(self):
    """Used exclusively as a thread which keeps the WebSocket alive."""
    while True:
      with self._lock:
        if self.connected():
          self._ws.ping()
        else:
          self.disconnect()
          self._thread = None
          return
      sleep(30)

def _keep_alive_thread ( self ) : while True : with self . _lock : if self . connected ( ) : self . _ws . ping ( ) else : self . disconnect ( ) self . _thread = None return sleep ( 30 )

Used exclusively as a thread which keeps the WebSocket alive .


def connect(self):
    """Connects and subscribes to the WebSocket Feed."""
    if not self.connected():
      self._ws = create_connection(self.WS_URI)
      message = {
        'type':self.WS_TYPE,
        'product_id':self.WS_PRODUCT_ID
      }
      self._ws.send(dumps(message))

      # There will be only one keep alive thread per client instance
      with self._lock:
        if not self._thread:
          thread = Thread(target=self._keep_alive_thread, args=[])
          thread.start()

def connect ( self ) : if not self . connected ( ) : self . _ws = create_connection ( self . WS_URI ) message = { 'type' : self . WS_TYPE , 'product_id' : self . WS_PRODUCT_ID } self . _ws . send ( dumps ( message ) ) with self . _lock : if not self . _thread : thread = Thread ( target = self . _keep_alive_thread , args = [ ] ) thread . start ( )

Connects and subscribes to the WebSocket Feed .


def cached_httpbl_exempt(view_func):
    """
    Marks a view function as being exempt from the cached httpbl view protection.
    """
    # We could just do view_func.cached_httpbl_exempt = True, but decorators
    # are nicer if they don't have side-effects, so we return a new
    # function.
    def wrapped_view(*args, **kwargs):
        return view_func(*args, **kwargs)
    wrapped_view.cached_httpbl_exempt = True
    return wraps(view_func, assigned=available_attrs(view_func))(wrapped_view)

def cached_httpbl_exempt ( view_func ) : def wrapped_view ( * args , ** kwargs ) : return view_func ( * args , ** kwargs ) wrapped_view . cached_httpbl_exempt = True return wraps ( view_func , assigned = available_attrs ( view_func ) ) ( wrapped_view )

Marks a view function as being exempt from the cached httpbl view protection .


def get_conn(self, aws_access_key=None, aws_secret_key=None):
        '''
        Hook point for overriding how the CounterPool gets its connection to
        AWS.
        '''
        return boto.connect_dynamodb(
            aws_access_key_id=aws_access_key,
            aws_secret_access_key=aws_secret_key,
        )

def get_conn ( self , aws_access_key = None , aws_secret_key = None ) : return boto . connect_dynamodb ( aws_access_key_id = aws_access_key , aws_secret_access_key = aws_secret_key , )

Hook point for overriding how the CounterPool gets its connection to AWS .


def get_schema(self):
        '''
        Hook point for overriding how the CounterPool determines the schema
        to be used when creating a missing table.
        '''
        if not self.schema:
            raise NotImplementedError(
                'You must provide a schema value or override the get_schema method'
            )

        return self.conn.create_schema(**self.schema)

Hook point for overriding how the CounterPool determines the schema to be used when creating a missing table .


def create_table(self):
        '''
        Hook point for overriding how the CounterPool creates a new table
        in DynamooDB
        '''
        table = self.conn.create_table(
            name=self.get_table_name(),
            schema=self.get_schema(),
            read_units=self.get_read_units(),
            write_units=self.get_write_units(),
        )

        if table.status != 'ACTIVE':
            table.refresh(wait_for_active=True, retry_seconds=1)

        return table

def create_table ( self ) : table = self . conn . create_table ( name = self . get_table_name ( ) , schema = self . get_schema ( ) , read_units = self . get_read_units ( ) , write_units = self . get_write_units ( ) , ) if table . status != 'ACTIVE' : table . refresh ( wait_for_active = True , retry_seconds = 1 ) return table


def get_table(self):
        '''
        Hook point for overriding how the CounterPool transforms table_name
        into a boto DynamoDB Table object.
        '''
        if hasattr(self, '_table'):
            table = self._table
        else:
            try:
                table = self.conn.get_table(self.get_table_name())
            except boto.exception.DynamoDBResponseError:
                if self.auto_create_table:
                    table = self.create_table()
                else:
                    raise

            self._table = table

        return table

def get_table ( self ) : if hasattr ( self , '_table' ) : table = self . _table else : try : table = self . conn . get_table ( self . get_table_name ( ) ) except boto . exception . DynamoDBResponseError : if self . auto_create_table : table = self . create_table ( ) else : raise self . _table = table return table

Hook point for overriding how the CounterPool transforms table_name into a boto DynamoDB Table object .


def create_item(self, hash_key, start=0, extra_attrs=None):
        '''
        Hook point for overriding how the CouterPool creates a DynamoDB item
        for a given counter when an existing item can't be found.
        '''
        table = self.get_table()
        now = datetime.utcnow().replace(microsecond=0).isoformat()
        attrs = {
            'created_on': now,
            'modified_on': now,
            'count': start,
        }

        if extra_attrs:
            attrs.update(extra_attrs)

        item = table.new_item(
            hash_key=hash_key,
            attrs=attrs,
        )

        return item

def create_item ( self , hash_key , start = 0 , extra_attrs = None ) : table = self . get_table ( ) now = datetime . utcnow ( ) . replace ( microsecond = 0 ) . isoformat ( ) attrs = { 'created_on' : now , 'modified_on' : now , 'count' : start , } if extra_attrs : attrs . update ( extra_attrs ) item = table . new_item ( hash_key = hash_key , attrs = attrs , ) return item

Hook point for overriding how the CouterPool creates a DynamoDB item for a given counter when an existing item can t be found .


def get_item(self, hash_key, start=0, extra_attrs=None):
        '''
        Hook point for overriding how the CouterPool fetches a DynamoDB item
        for a given counter.
        '''
        table = self.get_table()

        try:
            item = table.get_item(hash_key=hash_key)
        except DynamoDBKeyNotFoundError:
            item = None

        if item is None:
            item = self.create_item(
                hash_key=hash_key,
                start=start,
                extra_attrs=extra_attrs,
            )

        return item

def get_item ( self , hash_key , start = 0 , extra_attrs = None ) : table = self . get_table ( ) try : item = table . get_item ( hash_key = hash_key ) except DynamoDBKeyNotFoundError : item = None if item is None : item = self . create_item ( hash_key = hash_key , start = start , extra_attrs = extra_attrs , ) return item

Hook point for overriding how the CouterPool fetches a DynamoDB item for a given counter .


def get_counter(self, name, start=0):
        '''
        Gets the DynamoDB item behind a counter and ties it to a Counter
        instace.
        '''
        item = self.get_item(hash_key=name, start=start)
        counter = Counter(dynamo_item=item, pool=self)

        return counter

def get_counter ( self , name , start = 0 ) : item = self . get_item ( hash_key = name , start = start ) counter = Counter ( dynamo_item = item , pool = self ) return counter

Gets the DynamoDB item behind a counter and ties it to a Counter instace .


def many_to_one(clsname, **kw):
    """Use an event to build a many-to-one relationship on a class.

    This makes use of the :meth:`.References._reference_table` method
    to generate a full foreign key relationship to the remote table.

    """
    @declared_attr
    def m2o(cls):
        cls._references((cls.__name__, clsname))
        return relationship(clsname, **kw)
    return m2o

def many_to_one ( clsname , ** kw ) : @ declared_attr def m2o ( cls ) : cls . _references ( ( cls . __name__ , clsname ) ) return relationship ( clsname , ** kw ) return m2o

Use an event to build a many - to - one relationship on a class .


def one_to_many(clsname, **kw):
    """Use an event to build a one-to-many relationship on a class.

    This makes use of the :meth:`.References._reference_table` method
    to generate a full foreign key relationship from the remote table.

    """
    @declared_attr
    def o2m(cls):
        cls._references((clsname, cls.__name__))
        return relationship(clsname, **kw)
    return o2m

def one_to_many ( clsname , ** kw ) : @ declared_attr def o2m ( cls ) : cls . _references ( ( clsname , cls . __name__ ) ) return relationship ( clsname , ** kw ) return o2m

Use an event to build a one - to - many relationship on a class .


def handle_data(self, data):
        """
        Djeffify data between tags
        """
        if data.strip():
            data = djeffify_string(data)
        self.djhtml += data

def handle_data ( self , data ) : if data . strip ( ) : data = djeffify_string ( data ) self . djhtml += data


def _reference_table(cls, ref_table):
        """Create a foreign key reference from the local class to the given remote
        table.

        Adds column references to the declarative class and adds a
        ForeignKeyConstraint.

        """
        # create pairs of (Foreign key column, primary key column)
        cols = [(sa.Column(), refcol) for refcol in ref_table.primary_key]

        # set "tablename_colname = Foreign key Column" on the local class
        for col, refcol in cols:
            setattr(cls, "%s_%s" % (ref_table.name, refcol.name), col)

        # add a ForeignKeyConstraint([local columns], [remote columns])
        cls.__table__.append_constraint(sa.ForeignKeyConstraint(*zip(*cols)))

def _reference_table ( cls , ref_table ) : cols = [ ( sa . Column ( ) , refcol ) for refcol in ref_table . primary_key ] for col , refcol in cols : setattr ( cls , "%s_%s" % ( ref_table . name , refcol . name ) , col ) cls . __table__ . append_constraint ( sa . ForeignKeyConstraint ( * zip ( * cols ) ) )

Create a foreign key reference from the local class to the given remote table .


def prepare_path(path):
    """
    Path join helper method
    Join paths if list passed

    :type path: str|unicode|list
    :rtype: str|unicode
    """
    if type(path) == list:
        return os.path.join(*path)
    return path

def prepare_path ( path ) : if type ( path ) == list : return os . path . join ( * path ) return path


def read_from_file(file_path, encoding="utf-8"):
    """
    Read helper method

    :type file_path: str|unicode
    :type encoding: str|unicode
    :rtype: str|unicode
    """
    with codecs.open(file_path, "r", encoding) as f:
        return f.read()

def read_from_file ( file_path , encoding = "utf-8" ) : with codecs . open ( file_path , "r" , encoding ) as f : return f . read ( )


def write_to_file(file_path, contents, encoding="utf-8"):
    """
    Write helper method

    :type file_path: str|unicode
    :type contents: str|unicode
    :type encoding: str|unicode
    """
    with codecs.open(file_path, "w", encoding) as f:
        f.write(contents)

def write_to_file ( file_path , contents , encoding = "utf-8" ) : with codecs . open ( file_path , "w" , encoding ) as f : f . write ( contents )


def copy_file(src, dest):
    """
    Copy file helper method

    :type src: str|unicode
    :type dest: str|unicode
    """
    dir_path = os.path.dirname(dest)
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)
    shutil.copy2(src, dest)

def copy_file ( src , dest ) : dir_path = os . path . dirname ( dest ) if not os . path . exists ( dir_path ) : os . makedirs ( dir_path ) shutil . copy2 ( src , dest )


def get_path_extension(path):
    """
    Split file name and extension

    :type path: str|unicode
    :rtype: one str|unicode
    """
    file_path, file_ext = os.path.splitext(path)
    return file_ext.lstrip('.')

def get_path_extension ( path ) : file_path , file_ext = os . path . splitext ( path ) return file_ext . lstrip ( '.' )


def split_path(path):
        """
        Helper method for absolute and relative paths resolution
        Split passed path and return each directory parts

        example: "/usr/share/dir"
        return: ["usr", "share", "dir"]

        @type path: one of (unicode, str)
        @rtype: list
        """
        result_parts = []
        #todo: check loops
        while path != "/":
            parts = os.path.split(path)
            if parts[1] == path:
                result_parts.insert(0, parts[1])
                break
            elif parts[0] == path:
                result_parts.insert(0, parts[0])
                break
            else:
                path = parts[0]
                result_parts.insert(0, parts[1])
        return result_parts

def split_path ( path ) : result_parts = [ ] while path != "/" : parts = os . path . split ( path ) if parts [ 1 ] == path : result_parts . insert ( 0 , parts [ 1 ] ) break elif parts [ 0 ] == path : result_parts . insert ( 0 , parts [ 0 ] ) break else : path = parts [ 0 ] result_parts . insert ( 0 , parts [ 1 ] ) return result_parts


def _create_api_uri(self, *parts):
    """Creates fully qualified endpoint URIs.

    :param parts: the string parts that form the request URI

    """
    return urljoin(self.API_URI, '/'.join(map(quote, parts)))

def _create_api_uri ( self , * parts ) : return urljoin ( self . API_URI , '/' . join ( map ( quote , parts ) ) )


def _format_iso_time(self, time):
    """Makes sure we have proper ISO 8601 time.

    :param time: either already ISO 8601 a string or datetime.datetime
    :returns: ISO 8601 time
    :rtype: str

    """
    if isinstance(time, str):
      return time
    elif isinstance(time, datetime):
      return time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
    else:
      return None

def _format_iso_time ( self , time ) : if isinstance ( time , str ) : return time elif isinstance ( time , datetime ) : return time . strftime ( '%Y-%m-%dT%H:%M:%S.%fZ' ) else : return None

Makes sure we have proper ISO 8601 time .


def _handle_response(self, response):
    """Returns the given response or raises an APIError for non-2xx responses.

    :param requests.Response response: HTTP response
    :returns: requested data
    :rtype: requests.Response
    :raises APIError: for non-2xx responses

    """
    if not str(response.status_code).startswith('2'):
      raise get_api_error(response)
    return response

def _handle_response ( self , response ) : if not str ( response . status_code ) . startswith ( '2' ) : raise get_api_error ( response ) return response

Returns the given response or raises an APIError for non - 2xx responses .


def _check_next(self):
    """Checks if a next message is possible.

    :returns: True if a next message is possible, otherwise False
    :rtype: bool

    """
    if self.is_initial:
      return True
    if self.before:
      if self.before_cursor:
        return True
      else:
        return False
    else:
      if self.after_cursor:
        return True
      else:
        return False

def _check_next ( self ) : if self . is_initial : return True if self . before : if self . before_cursor : return True else : return False else : if self . after_cursor : return True else : return False


def _wrap_color(self, code, text, format=None, style=None):
        """ Colors text with code and given format """
        color = None
        if code[:3] == self.bg.PREFIX:
            color = self.bg.COLORS.get(code, None)
        if not color:
            color = self.fg.COLORS.get(code, None)

        if not color:
            raise Exception('Color code not found')

        if format and format not in self.formats:
            raise Exception('Color format not found')

        fmt = "0;"
        if format == 'bold':
            fmt = "1;"
        elif format == 'underline':
            fmt = "4;"

        # Manage the format
        parts = color.split('[')
        color = '{0}[{1}{2}'.format(parts[0], fmt, parts[1])

        if self.has_colors and self.colors_enabled:
            # Set brightness
            st = ''
            if style:
                st = self.st.COLORS.get(style, '')
            return "{0}{1}{2}{3}".format(st, color, text, self.st.COLORS['reset_all'])
        else:
            return text

def
_wrap_color
(
self
,
code
,
text
,
format
=
,
style
=
)
:
color
=
if
code
[
:
3
]
==
self
.
bg
.
PREFIX
:
color
=
self
.
bg
.
COLORS
.
get
(
code
,
)
if
not
color
:
color
=
self
.
fg
.
COLORS
.
get
(
code
,
)
if
not
color
:
raise
Exception
(
'Color code not found'
)
if
format
and
format
not
in
self
.
formats
:
raise
Exception
(
'Color format not found'
)
fmt
=
"0;"
if
format
==
'bold'
:
fmt
=
"1;"
elif
format
==
'underline'
:
fmt
=
"4;"
parts
=
color
.
split
(
'['
)
color
=
'{0}[{1}{2}'
.
format
(
parts
[
0
]
,
fmt
,
parts
[
1
]
)
if
self
.
has_colors
and
self
.
colors_enabled
:
st
=
''
if
style
:
st
=
self
.
st
.
COLORS
.
get
(
style
,
''
)
return
"{0}{1}{2}{3}"
.
format
(
st
,
color
,
text
,
self
.
st
.
COLORS
[
'reset_all'
]
)
else
:
return
text


def RegisterMessage(self, message):
    """Registers the given message type in the local database.

    Args:
      message: a message.Message, to be registered.

    Returns:
      The provided message.
    """

    desc = message.DESCRIPTOR
    self._symbols[desc.full_name] = message
    if desc.file.name not in self._symbols_by_file:
      self._symbols_by_file[desc.file.name] = {}
    self._symbols_by_file[desc.file.name][desc.full_name] = message
    self.pool.AddDescriptor(desc)
    return message

def RegisterMessage ( self , message ) : desc = message . DESCRIPTOR self . _symbols [ desc . full_name ] = message if desc . file . name not in self . _symbols_by_file : self . _symbols_by_file [ desc . file . name ] = { } self . _symbols_by_file [ desc . file . name ] [ desc . full_name ] = message self . pool . AddDescriptor ( desc ) return message

Registers the given message type in the local database .


def insert(self, index, value):
        """
        Insert object before index.

        :param int index: index to insert in
        :param string value: path to insert
        """
        self._list.insert(index, value)
        self._sync()

def insert ( self , index , value ) : self . _list . insert ( index , value ) self . _sync ( )


def parse(self, string):
        """
        Parse runtime path representation to list.

        :param string string: runtime path string
        :return: list of runtime paths
        :rtype: list of string
        """
        var, eq, values = string.strip().partition('=')
        assert var == 'runtimepath'
        assert eq == '='
        return values.split(',')

def parse ( self , string ) : var , eq , values = string . strip ( ) . partition ( '=' ) assert var == 'runtimepath' assert eq == '=' return values . split ( ',' )

Parse runtime path representation to list .


def add_bundle(self, *args):
        """
        Add some bundle to build group

        :type bundle: static_bundle.bundles.AbstractBundle
        @rtype: BuildGroup
        """
        for bundle in args:
            if not self.multitype and self.has_bundles():
                first_bundle = self.get_first_bundle()
                if first_bundle.get_type() != bundle.get_type():
                    raise Exception(
                        'Different bundle types for one Asset: %s[%s -> %s]'
                        'check types or set multitype parameter to True'
                        % (self.name, first_bundle.get_type(), bundle.get_type())
                    )
            self.bundles.append(bundle)
        return self


def collect_files(self):
        """
        Return collected files links

        :rtype: list[static_bundle.files.StaticFileResult]
        """
        self.files = []
        for bundle in self.bundles:
            bundle.init_build(self, self.builder)
            bundle_files = bundle.prepare()
            self.files.extend(bundle_files)
        return self

def collect_files ( self ) : self . files = [ ] for bundle in self . bundles : bundle . init_build ( self , self . builder ) bundle_files = bundle . prepare ( ) self . files . extend ( bundle_files ) return self


def get_minifier(self):
        """
        Asset minifier
        Uses default minifier in bundle if it's not defined

        :rtype: static_bundle.minifiers.DefaultMinifier|None
        """
        if self.minifier is None:
            if not self.has_bundles():
                raise Exception("Unable to get default minifier, no bundles in build group")
            minifier = self.get_first_bundle().get_default_minifier()
        else:
            minifier = self.minifier
        if minifier:
            minifier.init_asset(self)
        return minifier

Asset minifier Uses default minifier in bundle if it s not defined


def render_asset(self, name):
        """
        Render all includes in asset by names

        :type name: str|unicode
        :rtype: str|unicode
        """
        result = ""
        if self.has_asset(name):
            asset = self.get_asset(name)
            if asset.files:
                for f in asset.files:
                    result += f.render_include() + "\r\n"
        return result

def render_asset ( self , name ) : result = "" if self . has_asset ( name ) : asset = self . get_asset ( name ) if asset . files : for f in asset . files : result += f . render_include ( ) + "\r\n" return result


def collect_links(self, env=None):
        """
        Return links without build files
        """
        for asset in self.assets.values():
            if asset.has_bundles():
                asset.collect_files()
        if env is None:
            env = self.config.env
        if env == static_bundle.ENV_PRODUCTION:
            self._minify(emulate=True)
        self._add_url_prefix()

def collect_links ( self , env = None ) : for asset in self . assets . values ( ) : if asset . has_bundles ( ) : asset . collect_files ( ) if env is None : env = self . config . env if env == static_bundle . ENV_PRODUCTION : self . _minify ( emulate = True ) self . _add_url_prefix ( )


def _default_json_default(obj):
    """ Coerce everything to strings.
    All objects representing time get output according to default_date_fmt.
    """
    if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
        return obj.strftime(default_date_fmt)
    else:
        return str(obj)

def _default_json_default ( obj ) : if isinstance ( obj , ( datetime . datetime , datetime . date , datetime . time ) ) : return obj . strftime ( default_date_fmt ) else : return str ( obj )

Coerce everything to strings . All objects representing time get output according to default_date_fmt .


def init_logs(path=None,
              target=None,
              logger_name='root',
              level=logging.DEBUG,
              maxBytes=1*1024*1024,
              backupCount=5,
              application_name='default',
              server_hostname=None,
              fields=None):
    """Initialize the zlogger.

    Sets up a rotating file handler to the specified path and file with
    the given size and backup count limits, sets the default
    application_name, server_hostname, and default/whitelist fields.

    :param path: path to write the log file
    :param target: name of the log file
    :param logger_name: name of the logger (defaults to root)
    :param level: log level for this logger (defaults to logging.DEBUG)
    :param maxBytes: size of the file before rotation (default 1MB)
    :param application_name: app name to add to each log entry
    :param server_hostname: hostname to add to each log entry
    :param fields: default/whitelist fields.
    :type path: string
    :type target: string
    :type logger_name: string
    :type level: int
    :type maxBytes: int
    :type backupCount: int
    :type application_name: string
    :type server_hostname: string
    :type fields: dict
    """
    log_file = os.path.abspath(
        os.path.join(path, target))
    logger = logging.getLogger(logger_name)
    logger.setLevel(level)

    handler = logging.handlers.RotatingFileHandler(
        log_file, maxBytes=maxBytes, backupCount=backupCount)
    handler.setLevel(level)

    handler.setFormatter(
        JsonFormatter(
            application_name=application_name,
            server_hostname=server_hostname,
            fields=fields))

    logger.addHandler(handler)

def init_logs ( path = None , target = None , logger_name = 'root' , level = logging . DEBUG , maxBytes = 1 * 1024 * 1024 , backupCount = 5 , application_name = 'default' , server_hostname = None , fields = None ) : log_file = os . path . abspath ( os . path . join ( path , target ) ) logger = logging . getLogger ( logger_name ) logger . setLevel ( level ) handler = logging . handlers . RotatingFileHandler ( log_file , maxBytes = maxBytes , backupCount = backupCount ) handler . setLevel ( level ) handler . setFormatter ( JsonFormatter ( application_name = application_name , server_hostname = server_hostname , fields = fields ) ) logger . addHandler ( handler )


def format(self, record):
        """formats a logging.Record into a standard json log entry

        :param record: record to be formatted
        :type record: logging.Record
        :return: the formatted json string
        :rtype: string
        """

        record_fields = record.__dict__.copy()
        self._set_exc_info(record_fields)

        event_name = 'default'
        if record_fields.get('event_name'):
            event_name = record_fields.pop('event_name')

        log_level = 'INFO'
        if record_fields.get('log_level'):
            log_level = record_fields.pop('log_level')

        [record_fields.pop(k) for k in record_fields.keys()
         if k not in self.fields]

        defaults = self.defaults.copy()
        fields = self.fields.copy()
        fields.update(record_fields)
        filtered_fields = {}
        for k, v in fields.iteritems():
            if v is not None:
                filtered_fields[k] = v

        defaults.update({
            'event_timestamp': self._get_now(),
            'event_name': event_name,
            'log_level': log_level,
            'fields': filtered_fields})

        return json.dumps(defaults, default=self.json_default)

def format ( self , record ) : record_fields = record . __dict__ . copy ( ) self . _set_exc_info ( record_fields ) event_name = 'default' if record_fields . get ( 'event_name' ) : event_name = record_fields . pop ( 'event_name' ) log_level = 'INFO' if record_fields . get ( 'log_level' ) : log_level = record_fields . pop ( 'log_level' ) [ record_fields . pop ( k ) for k in record_fields . keys ( ) if k not in self . fields ] defaults = self . defaults . copy ( ) fields = self . fields . copy ( ) fields . update ( record_fields ) filtered_fields = { } for k , v in fields . iteritems ( ) : if v is not None : filtered_fields [ k ] = v defaults . update ( { 'event_timestamp' : self . _get_now ( ) , 'event_name' : event_name , 'log_level' : log_level , 'fields' : filtered_fields } ) return json . dumps ( defaults , default = self . json_default )

formats a logging . Record into a standard json log entry


def includeme(config):
    """
    Initialize the model for a Pyramid app.

    Activate this setup using ``config.include('baka_model')``.

    """
    settings = config.get_settings()
    should_create = asbool(settings.get('baka_model.should_create_all', False))
    should_drop = asbool(settings.get('baka_model.should_drop_all', False))

    # Configure the transaction manager to support retrying retryable
    # exceptions. We also register the session factory with the thread-local
    # transaction manager, so that all sessions it creates are registered.
    #    "tm.attempts": 3,
    config.add_settings({
        "retry.attempts": 3,
        "tm.activate_hook": tm_activate_hook,
        "tm.annotate_user": False,
    })

    # use pyramid_retry couse pyramid_tm disabled it
    config.include('pyramid_retry')
    # use pyramid_tm to hook the transaction lifecycle to the request
    config.include('pyramid_tm')

    engine = get_engine(settings)
    session_factory = get_session_factory(engine)

    config.registry['db_session_factory'] = session_factory

    # make request.db available for use in Pyramid
    config.add_request_method(
        # r.tm is the transaction manager used by pyramid_tm
        lambda r: get_tm_session(session_factory, r.tm),
        'db',
        reify=True
    )

    # service model factory
    config.include('.service')

    # Register a deferred action to bind the engine when the configuration is
    # committed. Deferring the action means that this module can be included
    # before model modules without ill effect.
    config.action(None, bind_engine, (engine,), {
        'should_create': should_create,
        'should_drop': should_drop
    }, order=10)

def includeme ( config ) : settings = config . get_settings ( ) should_create = asbool ( settings . get ( 'baka_model.should_create_all' , False ) ) should_drop = asbool ( settings . get ( 'baka_model.should_drop_all' , False ) ) config . add_settings ( { "retry.attempts" : 3 , "tm.activate_hook" : tm_activate_hook , "tm.annotate_user" : False , } ) config . include ( 'pyramid_retry' ) config . include ( 'pyramid_tm' ) engine = get_engine ( settings ) session_factory = get_session_factory ( engine ) config . registry [ 'db_session_factory' ] = session_factory config . add_request_method ( lambda r : get_tm_session ( session_factory , r . tm ) , 'db' , reify = True ) config . include ( '.service' ) config . action ( None , bind_engine , ( engine , ) , { 'should_create' : should_create , 'should_drop' : should_drop } , order = 10 )

Initialize the model for a Pyramid app .


def get_abs_and_rel_paths(self, root_path, file_name, input_dir):
        """
        Return absolute and relative path for file

        :type root_path: str|unicode
        :type file_name: str|unicode
        :type input_dir: str|unicode
        :rtype: tuple

        """
        # todo: change relative path resolving [bug on duplicate dir names in path]
        relative_dir = root_path.replace(input_dir, '')
        return os.path.join(root_path, file_name), relative_dir + '/' + file_name

def get_abs_and_rel_paths ( self , root_path , file_name , input_dir ) : relative_dir = root_path . replace ( input_dir , '' ) return os . path . join ( root_path , file_name ) , relative_dir + '/' + file_name


def AddEnumDescriptor(self, enum_desc):
    """Adds an EnumDescriptor to the pool.

    This method also registers the FileDescriptor associated with the message.

    Args:
      enum_desc: An EnumDescriptor.
    """

    if not isinstance(enum_desc, descriptor.EnumDescriptor):
      raise TypeError('Expected instance of descriptor.EnumDescriptor.')

    self._enum_descriptors[enum_desc.full_name] = enum_desc
    self.AddFileDescriptor(enum_desc.file)


def FindFileContainingSymbol(self, symbol):
    """Gets the FileDescriptor for the file containing the specified symbol.

    Args:
      symbol: The name of the symbol to search for.

    Returns:
      A FileDescriptor that contains the specified symbol.

    Raises:
      KeyError: if the file can not be found in the pool.
    """

    symbol = _NormalizeFullyQualifiedName(symbol)
    try:
      return self._descriptors[symbol].file
    except KeyError:
      pass

    try:
      return self._enum_descriptors[symbol].file
    except KeyError:
      pass

    try:
      file_proto = self._internal_db.FindFileContainingSymbol(symbol)
    except KeyError as error:
      if self._descriptor_db:
        file_proto = self._descriptor_db.FindFileContainingSymbol(symbol)
      else:
        raise error
    if not file_proto:
      raise KeyError('Cannot find a file containing %s' % symbol)
    return self._ConvertFileProtoToFileDescriptor(file_proto)

def
FindFileContainingSymbol
(
self
,
symbol
)
:
symbol
=
_NormalizeFullyQualifiedName
(
symbol
)
try
:
return
self
.
_descriptors
[
symbol
]
.
file
except
KeyError
:
pass
try
:
return
self
.
_enum_descriptors
[
symbol
]
.
file
except
KeyError
:
pass
try
:
file_proto
=
self
.
_internal_db
.
FindFileContainingSymbol
(
symbol
)
except
KeyError
as
error
:
if
self
.
_descriptor_db
:
file_proto
=
self
.
_descriptor_db
.
FindFileContainingSymbol
(
symbol
)
else
:
raise
error
if
not
file_proto
:
raise
KeyError
(
'Cannot find a file containing %s'
%
symbol
)
return
self
.
_ConvertFileProtoToFileDescriptor
(
file_proto
)

Gets the FileDescriptor for the file containing the specified symbol .


def FindMessageTypeByName(self, full_name):
    """Loads the named descriptor from the pool.

    Args:
      full_name: The full name of the descriptor to load.

    Returns:
      The descriptor for the named type.
    """

    full_name = _NormalizeFullyQualifiedName(full_name)
    if full_name not in self._descriptors:
      self.FindFileContainingSymbol(full_name)
    return self._descriptors[full_name]

def FindMessageTypeByName ( self , full_name ) : full_name = _NormalizeFullyQualifiedName ( full_name ) if full_name not in self . _descriptors : self . FindFileContainingSymbol ( full_name ) return self . _descriptors [ full_name ]

Loads the named descriptor from the pool .


def FindEnumTypeByName(self, full_name):
    """Loads the named enum descriptor from the pool.

    Args:
      full_name: The full name of the enum descriptor to load.

    Returns:
      The enum descriptor for the named type.
    """

    full_name = _NormalizeFullyQualifiedName(full_name)
    if full_name not in self._enum_descriptors:
      self.FindFileContainingSymbol(full_name)
    return self._enum_descriptors[full_name]

def FindEnumTypeByName ( self , full_name ) : full_name = _NormalizeFullyQualifiedName ( full_name ) if full_name not in self . _enum_descriptors : self . FindFileContainingSymbol ( full_name ) return self . _enum_descriptors [ full_name ]

Loads the named enum descriptor from the pool .


def FindExtensionByName(self, full_name):
    """Loads the named extension descriptor from the pool.

    Args:
      full_name: The full name of the extension descriptor to load.

    Returns:
      A FieldDescriptor, describing the named extension.
    """
    full_name = _NormalizeFullyQualifiedName(full_name)
    message_name, _, extension_name = full_name.rpartition('.')
    try:
      # Most extensions are nested inside a message.
      scope = self.FindMessageTypeByName(message_name)
    except KeyError:
      # Some extensions are defined at file scope.
      scope = self.FindFileContainingSymbol(full_name)
    return scope.extensions_by_name[extension_name]

def FindExtensionByName ( self , full_name ) : full_name = _NormalizeFullyQualifiedName ( full_name ) message_name , _ , extension_name = full_name . rpartition ( '.' ) try : scope = self . FindMessageTypeByName ( message_name ) except KeyError : scope = self . FindFileContainingSymbol ( full_name ) return scope . extensions_by_name [ extension_name ]

Loads the named extension descriptor from the pool .


def _ConvertEnumDescriptor(self, enum_proto, package=None, file_desc=None,
                             containing_type=None, scope=None):
    """Make a protobuf EnumDescriptor given an EnumDescriptorProto protobuf.

    Args:
      enum_proto: The descriptor_pb2.EnumDescriptorProto protobuf message.
      package: Optional package name for the new message EnumDescriptor.
      file_desc: The file containing the enum descriptor.
      containing_type: The type containing this enum.
      scope: Scope containing available types.

    Returns:
      The added descriptor
    """

    if package:
      enum_name = '.'.join((package, enum_proto.name))
    else:
      enum_name = enum_proto.name

    if file_desc is None:
      file_name = None
    else:
      file_name = file_desc.name

    values = [self._MakeEnumValueDescriptor(value, index)
              for index, value in enumerate(enum_proto.value)]
    desc = descriptor.EnumDescriptor(name=enum_proto.name,
                                     full_name=enum_name,
                                     filename=file_name,
                                     file=file_desc,
                                     values=values,
                                     containing_type=containing_type,
                                     options=enum_proto.options)
    scope['.%s' % enum_name] = desc
    self._enum_descriptors[enum_name] = desc
    return desc

def _ConvertEnumDescriptor ( self , enum_proto , package = None , file_desc = None , containing_type = None , scope = None ) : if package : enum_name = '.' . join ( ( package , enum_proto . name ) ) else : enum_name = enum_proto . name if file_desc is None : file_name = None else : file_name = file_desc . name values = [ self . _MakeEnumValueDescriptor ( value , index ) for index , value in enumerate ( enum_proto . value ) ] desc = descriptor . EnumDescriptor ( name = enum_proto . name , full_name = enum_name , filename = file_name , file = file_desc , values = values , containing_type = containing_type , options = enum_proto . options ) scope [ '.%s' % enum_name ] = desc self . _enum_descriptors [ enum_name ] = desc return desc

Make a protobuf EnumDescriptor given an EnumDescriptorProto protobuf .


def _MakeFieldDescriptor(self, field_proto, message_name, index,
                           is_extension=False):
    """Creates a field descriptor from a FieldDescriptorProto.

    For message and enum type fields, this method will do a look up
    in the pool for the appropriate descriptor for that type. If it
    is unavailable, it will fall back to the _source function to
    create it. If this type is still unavailable, construction will
    fail.

    Args:
      field_proto: The proto describing the field.
      message_name: The name of the containing message.
      index: Index of the field
      is_extension: Indication that this field is for an extension.

    Returns:
      An initialized FieldDescriptor object
    """

    if message_name:
      full_name = '.'.join((message_name, field_proto.name))
    else:
      full_name = field_proto.name

    return descriptor.FieldDescriptor(
        name=field_proto.name,
        full_name=full_name,
        index=index,
        number=field_proto.number,
        type=field_proto.type,
        cpp_type=None,
        message_type=None,
        enum_type=None,
        containing_type=None,
        label=field_proto.label,
        has_default_value=False,
        default_value=None,
        is_extension=is_extension,
        extension_scope=None,
        options=field_proto.options)

def _MakeFieldDescriptor ( self , field_proto , message_name , index , is_extension = False ) : if message_name : full_name = '.' . join ( ( message_name , field_proto . name ) ) else : full_name = field_proto . name return descriptor . FieldDescriptor ( name = field_proto . name , full_name = full_name , index = index , number = field_proto . number , type = field_proto . type , cpp_type = None , message_type = None , enum_type = None , containing_type = None , label = field_proto . label , has_default_value = False , default_value = None , is_extension = is_extension , extension_scope = None , options = field_proto . options )

Creates a field descriptor from a FieldDescriptorProto .


def get_tm_session(session_factory, transaction_manager):
    """
    Get a ``sqlalchemy.orm.Session`` instance backed by a transaction.

    This function will hook the session to the transaction manager which
    will take care of committing any changes.

    - When using pyramid_tm it will automatically be committed or aborted
      depending on whether an exception is raised.

    - When using scripts you should wrap the session in a manager yourself.
      For example::

          import transaction

          engine = get_engine(settings)
          session_factory = get_session_factory(engine)
          with transaction.manager:
              dbsession = get_tm_session(session_factory, transaction.manager)

    """
    dbsession = session_factory()
    zope.sqlalchemy.register(
        dbsession, transaction_manager=transaction_manager)
    return dbsession

def get_tm_session ( session_factory , transaction_manager ) : dbsession = session_factory ( ) zope . sqlalchemy . register ( dbsession , transaction_manager = transaction_manager ) return dbsession

Get a sqlalchemy . orm . Session instance backed by a transaction .


def generate(length=DEFAULT_LENGTH):
    """
    Generate a random string of the specified length.

    The returned string is composed of an alphabet that shouldn't include any
    characters that are easily mistakeable for one another (I, 1, O, 0), and
    hopefully won't accidentally contain any English-language curse words.
    """
    return ''.join(random.SystemRandom().choice(ALPHABET)
                   for _ in range(length))

def generate ( length = DEFAULT_LENGTH ) : return '' . join ( random . SystemRandom ( ) . choice ( ALPHABET ) for _ in range ( length ) )

Generate a random string of the specified length .


def require(name, field, data_type):
    """Require that the named `field` has the right `data_type`"""
    if not isinstance(field, data_type):
        msg = '{0} must have {1}, got: {2}'.format(name, data_type, field)
        raise AssertionError(msg)

def
require
(
name
,
field
,
data_type
)
:
if
not
isinstance
(
field
,
data_type
)
:
msg
=
'{0} must have {1}, got: {2}'
.
format
(
name
,
data_type
,
field
)
raise
AssertionError
(
msg
)

Require that the named field has the right data_type


def flush(self):
        """Forces a flush from the internal queue to the server"""
        queue = self.queue
        size = queue.qsize()
        queue.join()
        self.log.debug('successfully flushed %s items.', size)

def
flush
(
self
)
:
queue
=
self
.
queue
size
=
queue
.
qsize
(
)
queue
.
join
(
)
self
.
log
.
debug
(
'successfully flushed %s items.'
,
size
)


def open(name=None, fileobj=None, closefd=True):
    """
    Use all decompressor possible to make the stream
    """
    return Guesser().open(name=name, fileobj=fileobj, closefd=closefd)

def open ( name = None , fileobj = None , closefd = True ) : return Guesser ( ) . open ( name = name , fileobj = fileobj , closefd = closefd )


def marv(ctx, config, loglevel, logfilter, verbosity):
    """Manage a Marv site"""
    if config is None:
        cwd = os.path.abspath(os.path.curdir)
        while cwd != os.path.sep:
            config = os.path.join(cwd, 'marv.conf')
            if os.path.exists(config):
                break
            cwd = os.path.dirname(cwd)
        else:
            config = '/etc/marv/marv.conf'
            if not os.path.exists(config):
                config = None
    ctx.obj = config
    setup_logging(loglevel, verbosity, logfilter)

def marv ( ctx , config , loglevel , logfilter , verbosity ) : if config is None : cwd = os . path . abspath ( os . path . curdir ) while cwd != os . path . sep : config = os . path . join ( cwd , 'marv.conf' ) if os . path . exists ( config ) : break cwd = os . path . dirname ( cwd ) else : config = '/etc/marv/marv.conf' if not os . path . exists ( config ) : config = None ctx . obj = config setup_logging ( loglevel , verbosity , logfilter )


def MessageSetItemDecoder(extensions_by_number):
  """Returns a decoder for a MessageSet item.

  The parameter is the _extensions_by_number map for the message class.

  The message set message looks like this:
    message MessageSet {
      repeated group Item = 1 {
        required int32 type_id = 2;
        required string message = 3;
      }
    }
  """

  type_id_tag_bytes = encoder.TagBytes(2, wire_format.WIRETYPE_VARINT)
  message_tag_bytes = encoder.TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)
  item_end_tag_bytes = encoder.TagBytes(1, wire_format.WIRETYPE_END_GROUP)

  local_ReadTag = ReadTag
  local_DecodeVarint = _DecodeVarint
  local_SkipField = SkipField

  def DecodeItem(buffer, pos, end, message, field_dict):
    message_set_item_start = pos
    type_id = -1
    message_start = -1
    message_end = -1

    # Technically, type_id and message can appear in any order, so we need
    # a little loop here.
    while 1:
      (tag_bytes, pos) = local_ReadTag(buffer, pos)
      if tag_bytes == type_id_tag_bytes:
        (type_id, pos) = local_DecodeVarint(buffer, pos)
      elif tag_bytes == message_tag_bytes:
        (size, message_start) = local_DecodeVarint(buffer, pos)
        pos = message_end = message_start + size
      elif tag_bytes == item_end_tag_bytes:
        break
      else:
        pos = SkipField(buffer, pos, end, tag_bytes)
        if pos == -1:
          raise _DecodeError('Missing group end tag.')

    if pos > end:
      raise _DecodeError('Truncated message.')

    if type_id == -1:
      raise _DecodeError('MessageSet item missing type_id.')
    if message_start == -1:
      raise _DecodeError('MessageSet item missing message.')

    extension = extensions_by_number.get(type_id)
    if extension is not None:
      value = field_dict.get(extension)
      if value is None:
        value = field_dict.setdefault(
            extension, extension.message_type._concrete_class())
      if value._InternalParse(buffer, message_start,message_end) != message_end:
        # The only reason _InternalParse would return early is if it encountered
        # an end-group tag.
        raise _DecodeError('Unexpected end-group tag.')
    else:
      if not message._unknown_fields:
        message._unknown_fields = []
      message._unknown_fields.append((MESSAGE_SET_ITEM_TAG,
                                      buffer[message_set_item_start:pos]))

    return pos

  return DecodeItem

def
MessageSetItemDecoder
(
extensions_by_number
)
:
type_id_tag_bytes
=
encoder
.
TagBytes
(
2
,
wire_format
.
WIRETYPE_VARINT
)
message_tag_bytes
=
encoder
.
TagBytes
(
3
,
wire_format
.
WIRETYPE_LENGTH_DELIMITED
)
item_end_tag_bytes
=
encoder
.
TagBytes
(
1
,
wire_format
.
WIRETYPE_END_GROUP
)
local_ReadTag
=
ReadTag
local_DecodeVarint
=
_DecodeVarint
local_SkipField
=
SkipField
def
DecodeItem
(
buffer
,
pos
,
end
,
message
,
field_dict
)
:
message_set_item_start
=
pos
type_id
=
1
message_start
=
1
message_end
=
1
while
1
:
(
tag_bytes
,
pos
)
=
local_ReadTag
(
buffer
,
pos
)
if
tag_bytes
==
type_id_tag_bytes
:
(
type_id
,
pos
)
=
local_DecodeVarint
(
buffer
,
pos
)
elif
tag_bytes
==
message_tag_bytes
:
(
size
,
message_start
)
=
local_DecodeVarint
(
buffer
,
pos
)
pos
=
message_end
=
message_start
+
size
elif
tag_bytes
==
item_end_tag_bytes
:
break
else
:
pos
=
SkipField
(
buffer
,
pos
,
end
,
tag_bytes
)
if
pos
==
1
:
raise
_DecodeError
(
'Missing group end tag.'
)
if
pos
>
end
:
raise
_DecodeError
(
'Truncated message.'
)
if
type_id
==
1
:
raise
_DecodeError
(
'MessageSet item missing type_id.'
)
if
message_start
==
1
:
raise
_DecodeError
(
'MessageSet item missing message.'
)
extension
=
extensions_by_number
.
get
(
type_id
)
if
extension
is
not
:
value
=
field_dict
.
get
(
extension
)
if
value
is
:
value
=
field_dict
.
setdefault
(
extension
,
extension
.
message_type
.
_concrete_class
(
)
)
if
value
.
_InternalParse
(
buffer
,
message_start
,
message_end
)
!=
message_end
:
raise
_DecodeError
(
'Unexpected end-group tag.'
)
else
:
if
not
message
.
_unknown_fields
:
message
.
_unknown_fields
=
[
]
message
.
_unknown_fields
.
append
(
(
MESSAGE_SET_ITEM_TAG
,
buffer
[
message_set_item_start
:
pos
]
)
)
return
pos
return
DecodeItem

Returns a decoder for a MessageSet item .


def get_app_name():
    """Flask like implementation of getting the applicaiton name via
    the filename of the including file

    """
    fn = getattr(sys.modules['__main__'], '__file__', None)
    if fn is None:
        return '__main__'
    return os.path.splitext(os.path.basename(fn))[0]

def get_app_name ( ) : fn = getattr ( sys . modules [ '__main__' ] , '__file__' , None ) if fn is None : return '__main__' return os . path . splitext ( os . path . basename ( fn ) ) [ 0 ]


def get_function(function_name):
    """
    Given a Python function name, return the function it refers to.
    """
    module, basename = str(function_name).rsplit('.', 1)
    try:
        return getattr(__import__(module, fromlist=[basename]), basename)
    except (ImportError, AttributeError):
        raise FunctionNotFound(function_name)

def get_function ( function_name ) : module , basename = str ( function_name ) . rsplit ( '.' , 1 ) try : return getattr ( __import__ ( module , fromlist = [ basename ] ) , basename ) except ( ImportError , AttributeError ) : raise FunctionNotFound ( function_name )

Given a Python function name return the function it refers to .


def handle_add_fun(self, function_name):
        """Add a function to the function list, in order."""
        function_name = function_name.strip()
        try:
            function = get_function(function_name)
        except Exception, exc:
            self.wfile.write(js_error(exc) + NEWLINE)
            return
        # This tests to see if the function has been decorated with the view
        # server synchronisation decorator (``decorate_view``).
        if not getattr(function, 'view_decorated', None):
            self.functions[function_name] = (self.function_counter, function)
        # The decorator gets called with the logger function.
        else:
            self.functions[function_name] = (self.function_counter,
                function(self.log))
        self.function_counter += 1
        return True

def handle_add_fun ( self , function_name ) : function_name = function_name . strip ( ) try : function = get_function ( function_name ) except Exception , exc : self . wfile . write ( js_error ( exc ) + NEWLINE ) return if not getattr ( function , 'view_decorated' , None ) : self . functions [ function_name ] = ( self . function_counter , function ) else : self . functions [ function_name ] = ( self . function_counter , function ( self . log ) ) self . function_counter += 1 return True

Add a function to the function list in order .


def handle_map_doc(self, document):
        """Return the mapping of a document according to the function list."""
        # This uses the stored set of functions, sorted by order of addition.
        for function in sorted(self.functions.values(), key=lambda x: x[0]):
            try:
                # It has to be run through ``list``, because it may be a
                # generator function.
                yield [list(function(document))]
            except Exception, exc:
                # Otherwise, return an empty list and log the event.
                yield []
                self.log(repr(exc))

def handle_map_doc ( self , document ) : for function in sorted ( self . functions . values ( ) , key = lambda x : x [ 0 ] ) : try : yield [ list ( function ( document ) ) ] except Exception , exc : yield [ ] self . log ( repr ( exc ) )

Return the mapping of a document according to the function list .


def handle_reduce(self, reduce_function_names, mapped_docs):
        """Reduce several mapped documents by several reduction functions."""
        reduce_functions = []
        # This gets a large list of reduction functions, given their names.
        for reduce_function_name in reduce_function_names:
            try:
                reduce_function = get_function(reduce_function_name)
                if getattr(reduce_function, 'view_decorated', None):
                    reduce_function = reduce_function(self.log)
                reduce_functions.append(reduce_function)
            except Exception, exc:
                self.log(repr(exc))
                reduce_functions.append(lambda *args, **kwargs: None)
        # Transform lots of (key, value) pairs into one (keys, values) pair.
        keys, values = zip(
            (key, value) for ((key, doc_id), value) in mapped_docs)
        # This gets the list of results from the reduction functions.
        results = []
        for reduce_function in reduce_functions:
            try:
                results.append(reduce_function(keys, values, rereduce=False))
            except Exception, exc:
                self.log(repr(exc))
                results.append(None)
        return [True, results]

def handle_reduce ( self , reduce_function_names , mapped_docs ) : reduce_functions = [ ] for reduce_function_name in reduce_function_names : try : reduce_function = get_function ( reduce_function_name ) if getattr ( reduce_function , 'view_decorated' , None ) : reduce_function = reduce_function ( self . log ) reduce_functions . append ( reduce_function ) except Exception , exc : self . log ( repr ( exc ) ) reduce_functions . append ( lambda * args , ** kwargs : None ) keys , values = zip ( ( key , value ) for ( ( key , doc_id ) , value ) in mapped_docs ) results = [ ] for reduce_function in reduce_functions : try : results . append ( reduce_function ( keys , values , rereduce = False ) ) except Exception , exc : self . log ( repr ( exc ) ) results . append ( None ) return [ True , results ]

Reduce several mapped documents by several reduction functions .


def handle_rereduce(self, reduce_function_names, values):
        """Re-reduce a set of values, with a list of rereduction functions."""
        # This gets a large list of reduction functions, given their names.
        reduce_functions = []
        for reduce_function_name in reduce_function_names:
            try:
                reduce_function = get_function(reduce_function_name)
                if getattr(reduce_function, 'view_decorated', None):
                    reduce_function = reduce_function(self.log)
                reduce_functions.append(reduce_function)
            except Exception, exc:
                self.log(repr(exc))
                reduce_functions.append(lambda *args, **kwargs: None)
        # This gets the list of results from those functions.
        results = []
        for reduce_function in reduce_functions:
            try:
                results.append(reduce_function(None, values, rereduce=True))
            except Exception, exc:
                self.log(repr(exc))
                results.append(None)
        return [True, results]

def handle_rereduce ( self , reduce_function_names , values ) : reduce_functions = [ ] for reduce_function_name in reduce_function_names : try : reduce_function = get_function ( reduce_function_name ) if getattr ( reduce_function , 'view_decorated' , None ) : reduce_function = reduce_function ( self . log ) reduce_functions . append ( reduce_function ) except Exception , exc : self . log ( repr ( exc ) ) reduce_functions . append ( lambda * args , ** kwargs : None ) results = [ ] for reduce_function in reduce_functions : try : results . append ( reduce_function ( None , values , rereduce = True ) ) except Exception , exc : self . log ( repr ( exc ) ) results . append ( None ) return [ True , results ]

Re - reduce a set of values with a list of rereduction functions .


def handle_validate(self, function_name, new_doc, old_doc, user_ctx):
        """Validate...this function is undocumented, but still in CouchDB."""
        try:
            function = get_function(function_name)
        except Exception, exc:
            self.log(repr(exc))
            return False
        try:
            return function(new_doc, old_doc, user_ctx)
        except Exception, exc:
            self.log(repr(exc))
            return repr(exc)

def handle_validate ( self , function_name , new_doc , old_doc , user_ctx ) : try : function = get_function ( function_name ) except Exception , exc : self . log ( repr ( exc ) ) return False try : return function ( new_doc , old_doc , user_ctx ) except Exception , exc : self . log ( repr ( exc ) ) return repr ( exc )

Validate ... this function is undocumented but still in CouchDB .


def handle(self):
        """The main function called to handle a request."""
        while True:
            try:
                line = self.rfile.readline()
                try:
                    # All input data are lines of JSON like the following:
                    #   ["<cmd_name>" "<cmd_arg1>" "<cmd_arg2>" ...]
                    # So I handle this by dispatching to various methods.
                    cmd = json.loads(line)
                except Exception, exc:
                    # Sometimes errors come up. Once again, I can't predict
                    # anything, but can at least tell CouchDB about the error.
                    self.wfile.write(repr(exc) + NEWLINE)
                    continue
                else:
                    # Automagically get the command handler.
                    handler = getattr(self, 'handle_' + cmd[0], None)
                    if not handler:
                        # We are ready to not find commands. It probably won't
                        # happen, but fortune favours the prepared.
                        self.wfile.write(
                            repr(CommandNotFound(cmd[0])) + NEWLINE)
                        continue
                    return_value = handler(*cmd[1:])
                    if not return_value:
                        continue
                    # We write the output back to CouchDB.
                    self.wfile.write(
                        one_lineify(json.dumps(return_value)) + NEWLINE)
            except Exception, exc:
                self.wfile.write(repr(exc) + NEWLINE)
                continue

def handle ( self ) : while True : try : line = self . rfile . readline ( ) try : cmd = json . loads ( line ) except Exception , exc : self . wfile . write ( repr ( exc ) + NEWLINE ) continue else : handler = getattr ( self , 'handle_' + cmd [ 0 ] , None ) if not handler : self . wfile . write ( repr ( CommandNotFound ( cmd [ 0 ] ) ) + NEWLINE ) continue return_value = handler ( * cmd [ 1 : ] ) if not return_value : continue self . wfile . write ( one_lineify ( json . dumps ( return_value ) ) + NEWLINE ) except Exception , exc : self . wfile . write ( repr ( exc ) + NEWLINE ) continue

The main function called to handle a request .


def log(self, string):
        """Log an event on the CouchDB server."""
        self.wfile.write(json.dumps({'log': string}) + NEWLINE)

def log ( self , string ) : self . wfile . write ( json . dumps ( { 'log' : string } ) + NEWLINE )


def guid(*args):
    """
    Generates a universally unique ID.
    Any arguments only create more randomness.
    """
    t = float(time.time() * 1000)
    r = float(random.random()*10000000000000)

    a = random.random() * 10000000000000
    data = str(t) + ' ' + str(r) + ' ' + str(a) + ' ' + str(args)
    data = hashlib.md5(data.encode()).hexdigest()[:10]

    return data

def
guid
(
*
args
)
:
t
=
float
(
time
.
time
(
)
*
1000
)
r
=
float
(
random
.
random
(
)
*
10000000000000
)
a
=
random
.
random
(
)
*
10000000000000
data
=
str
(
t
)
+
' '
+
str
(
r
)
+
' '
+
str
(
a
)
+
' '
+
str
(
args
)
data
=
hashlib
.
md5
(
data
.
encode
(
)
)
.
hexdigest
(
)
[
:
10
]
return
data

Generates a universally unique ID . Any arguments only create more randomness .


def revoke_token(self, token, callback):
        '''
        revoke_token removes the access token from the data_store
        '''
        yield Task(self.data_store.remove, 'tokens', token=token)
        callback()

def revoke_token ( self , token , callback ) : yield Task ( self . data_store . remove , 'tokens' , token = token ) callback ( )


def _auth(self, client_id, key, method, callback):
        '''
        _auth - internal method to ensure the client_id and client_secret passed with
        the nonce match
        '''
        available = auth_methods.keys()
        if method not in available:
            raise Proauth2Error('invalid_request',
                                'unsupported authentication method: %s'
                                'available methods: %s' % \
                                (method, '\n'.join(available)))
        client = yield Task(self.data_store.fetch, 'applications',
                            client_id=client_id)
        if not client: raise Proauth2Error('access_denied')
        if not auth_methods[method](key, client['client_secret']):
            raise Proauth2Error('access_denied')
        callback()

def
_auth
(
self
,
client_id
,
key
,
method
,
callback
)
:
available
=
auth_methods
.
keys
(
)
if
method
not
in
available
:
raise
Proauth2Error
(
'invalid_request'
,
'unsupported authentication method: %s'
'available methods: %s'
%
(
method
,
'\n'
.
join
(
available
)
)
)
client
=
yield
Task
(
self
.
data_store
.
fetch
,
'applications'
,
client_id
=
client_id
)
if
not
client
:
raise
Proauth2Error
(
'access_denied'
)
if
not
auth_methods
[
method
]
(
key
,
client
[
'client_secret'
]
)
:
raise
Proauth2Error
(
'access_denied'
)
callback
(
)


def _validate_request_code(self, code, client_id, callback):
        '''
        _validate_request_code - internal method for verifying the the given nonce.
        also removes the nonce from the data_store, as they are intended for
        one-time use.
        '''
        nonce = yield Task(self.data_store.fetch, 'nonce_codes', code=code)
        if not nonce:
            raise Proauth2Error('access_denied', 'invalid request code: %s' % code)
        if client_id != nonce['client_id']:
            raise Proauth2Error('access_denied', 'invalid request code: %s' % code)
        user_id = nonce['user_id']
        expires = nonce['expires']
        yield Task(self.data_store.remove, 'nonce_codes', code=code,
                   client_id=client_id, user_id=user_id)

        if time() > expires:
            raise Proauth2Error('access_denied', 'request code %s expired' % code)

        callback(user_id)

def
_validate_request_code
(
self
,
code
,
client_id
,
callback
)
:
nonce
=
yield
Task
(
self
.
data_store
.
fetch
,
'nonce_codes'
,
code
=
code
)
if
not
nonce
:
raise
Proauth2Error
(
'access_denied'
,
'invalid request code: %s'
%
code
)
if
client_id
!=
nonce
[
'client_id'
]
:
raise
Proauth2Error
(
'access_denied'
,
'invalid request code: %s'
%
code
)
user_id
=
nonce
[
'user_id'
]
expires
=
nonce
[
'expires'
]
yield
Task
(
self
.
data_store
.
remove
,
'nonce_codes'
,
code
=
code
,
client_id
=
client_id
,
user_id
=
user_id
)
if
time
(
)
>
expires
:
raise
Proauth2Error
(
'access_denied'
,
'request code %s expired'
%
code
)
callback
(
user_id
)

_validate_request_code - internal method for verifying the the given nonce . also removes the nonce from the data_store as they are intended for one - time use .


def _generate_token(self, length=32):
        '''
        _generate_token - internal function for generating randomized alphanumberic
        strings of a given length
        '''
        return ''.join(choice(ascii_letters + digits) for x in range(length))

def _generate_token ( self , length = 32 ) : return '' . join ( choice ( ascii_letters + digits ) for x in range ( length ) )


def merge_ordered(ordereds: typing.Iterable[typing.Any]) -> typing.Iterable[typing.Any]:
    """Merge multiple ordered so that within-ordered order is preserved
    """
    seen_set = set()
    add_seen = seen_set.add
    return reversed(tuple(map(
        lambda obj: add_seen(obj) or obj,
        filterfalse(
            seen_set.__contains__,
            chain.from_iterable(map(reversed, reversed(ordereds))),
        ),
    )))

def merge_ordered ( ordereds : typing . Iterable [ typing . Any ] ) -> typing . Iterable [ typing . Any ] : seen_set = set ( ) add_seen = seen_set . add return reversed ( tuple ( map ( lambda obj : add_seen ( obj ) or obj , filterfalse ( seen_set . __contains__ , chain . from_iterable ( map ( reversed , reversed ( ordereds ) ) ) , ) , ) ) )

Merge multiple ordered so that within - ordered order is preserved


def validate_params(required, optional, params):
    """
    Helps us validate the parameters for the request

    :param valid_options: a list of strings of valid options for the
                          api request
    :param params: a dict, the key-value store which we really only care about
                   the key which has tells us what the user is using for the
                   API request

    :returns: None or throws an exception if the validation fails
    """

    missing_fields = [x for x in required if x not in params]
    if missing_fields:
        field_strings = ", ".join(missing_fields)
        raise Exception("Missing fields: %s" % field_strings)

    disallowed_fields = [x for x in params if x not in optional and x not in required]
    if disallowed_fields:
        field_strings = ", ".join(disallowed_fields)
        raise Exception("Disallowed fields: %s" % field_strings)

def
validate_params
(
required
,
optional
,
params
)
:
missing_fields
=
[
x
for
x
in
required
if
x
not
in
params
]
if
missing_fields
:
field_strings
=
", "
.
join
(
missing_fields
)
raise
Exception
(
"Missing fields: %s"
%
field_strings
)
disallowed_fields
=
[
x
for
x
in
params
if
x
not
in
optional
and
x
not
in
required
]
if
disallowed_fields
:
field_strings
=
", "
.
join
(
disallowed_fields
)
raise
Exception
(
"Disallowed fields: %s"
%
field_strings
)


def __get_current_datetime(self):
        """Get current datetime for every file."""
        self.wql_time = "SELECT LocalDateTime FROM Win32_OperatingSystem"
        self.current_time = self.query(self.wql_time)
        # [{'LocalDateTime': '20160824161431.977000+480'}]'
        self.current_time_string = str(
            self.current_time[0].get('LocalDateTime').split('.')[0])
        # '20160824161431'
        self.current_time_format = datetime.datetime.strptime(
            self.current_time_string, '%Y%m%d%H%M%S')
        # param: datetime.datetime(2016, 8, 24, 16, 14, 31) -> type:
        # datetime.datetime
        return self.current_time_format


def run(self):
        """run your main spider here
        as for branch spider result data, you can return everything or do whatever with it
        in your own code

        :return: None
        """
        config = config_creator()
        debug = config.debug
        branch_thread_sleep = config.branch_thread_sleep
        while 1:
            url = self.branch_queue.get()
            if debug:
                print('branch thread-{} start'.format(url))
            branch_spider = self.branch_spider(url)
            sleep(random.randrange(*branch_thread_sleep))
            branch_spider.request_page()
            if debug:
                print('branch thread-{} end'.format(url))
            self.branch_queue.task_done()

def
run
(
self
)
:
config
=
config_creator
(
)
debug
=
config
.
debug
branch_thread_sleep
=
config
.
branch_thread_sleep
while
1
:
url
=
self
.
branch_queue
.
get
(
)
if
debug
:
print
(
'branch thread-{} start'
.
format
(
url
)
)
branch_spider
=
self
.
branch_spider
(
url
)
sleep
(
random
.
randrange
(
*
branch_thread_sleep
)
)
branch_spider
.
request_page
(
)
if
debug
:
print
(
'branch thread-{} end'
.
format
(
url
)
)
self
.
branch_queue
.
task_done
(
)

run your main spider here as for branch spider result data you can return everything or do whatever with it in your own code


def get_version(relpath):
    """Read version info from a file without importing it"""
    from os.path import dirname, join

    if '__file__' not in globals():
        # Allow to use function interactively
        root = '.'
    else:
        root = dirname(__file__)

    # The code below reads text file with unknown encoding in
    # in Python2/3 compatible way. Reading this text file
    # without specifying encoding will fail in Python 3 on some
    # systems (see http://goo.gl/5XmOH). Specifying encoding as
    # open() parameter is incompatible with Python 2

    # cp437 is the encoding without missing points, safe against:
    #   UnicodeDecodeError: 'charmap' codec can't decode byte...

    for line in open(join(root, relpath), 'rb'):
        line = line.decode('cp437')
        if '__version__' in line:
            if '"' in line:
                # __version__ = "0.9"
                return line.split('"')[1]
            elif "'" in line:
                return line.split("'")[1]

def get_version ( relpath ) : from os . path import dirname , join if '__file__' not in globals ( ) : root = '.' else : root = dirname ( __file__ ) for line in open ( join ( root , relpath ) , 'rb' ) : line = line . decode ( 'cp437' ) if '__version__' in line : if '"' in line : return line . split ( '"' ) [ 1 ] elif "'" in line : return line . split ( "'" ) [ 1 ]


def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
                   syntax=None):
  """Make a protobuf Descriptor given a DescriptorProto protobuf.

  Handles nested descriptors. Note that this is limited to the scope of defining
  a message inside of another message. Composite fields can currently only be
  resolved if the message is defined in the same scope as the field.

  Args:
    desc_proto: The descriptor_pb2.DescriptorProto protobuf message.
    package: Optional package name for the new message Descriptor (string).
    build_file_if_cpp: Update the C++ descriptor pool if api matches.
                       Set to False on recursion, so no duplicates are created.
    syntax: The syntax/semantics that should be used.  Set to "proto3" to get
            proto3 field presence semantics.
  Returns:
    A Descriptor for protobuf messages.
  """
  if api_implementation.Type() == 'cpp' and build_file_if_cpp:
    # The C++ implementation requires all descriptors to be backed by the same
    # definition in the C++ descriptor pool. To do this, we build a
    # FileDescriptorProto with the same definition as this descriptor and build
    # it into the pool.
    from typy.google.protobuf import descriptor_pb2
    file_descriptor_proto = descriptor_pb2.FileDescriptorProto()
    file_descriptor_proto.message_type.add().MergeFrom(desc_proto)

    # Generate a random name for this proto file to prevent conflicts with any
    # imported ones. We need to specify a file name so the descriptor pool
    # accepts our FileDescriptorProto, but it is not important what that file
    # name is actually set to.
    proto_name = str(uuid.uuid4())

    if package:
      file_descriptor_proto.name = os.path.join(package.replace('.', '/'),
                                                proto_name + '.proto')
      file_descriptor_proto.package = package
    else:
      file_descriptor_proto.name = proto_name + '.proto'

    _message.default_pool.Add(file_descriptor_proto)
    result = _message.default_pool.FindFileByName(file_descriptor_proto.name)

    if _USE_C_DESCRIPTORS:
      return result.message_types_by_name[desc_proto.name]

  full_message_name = [desc_proto.name]
  if package: full_message_name.insert(0, package)

  # Create Descriptors for enum types
  enum_types = {}
  for enum_proto in desc_proto.enum_type:
    full_name = '.'.join(full_message_name + [enum_proto.name])
    enum_desc = EnumDescriptor(
      enum_proto.name, full_name, None, [
          EnumValueDescriptor(enum_val.name, ii, enum_val.number)
          for ii, enum_val in enumerate(enum_proto.value)])
    enum_types[full_name] = enum_desc

  # Create Descriptors for nested types
  nested_types = {}
  for nested_proto in desc_proto.nested_type:
    full_name = '.'.join(full_message_name + [nested_proto.name])
    # Nested types are just those defined inside of the message, not all types
    # used by fields in the message, so no loops are possible here.
    nested_desc = MakeDescriptor(nested_proto,
                                 package='.'.join(full_message_name),
                                 build_file_if_cpp=False,
                                 syntax=syntax)
    nested_types[full_name] = nested_desc

  fields = []
  for field_proto in desc_proto.field:
    full_name = '.'.join(full_message_name + [field_proto.name])
    enum_desc = None
    nested_desc = None
    if field_proto.HasField('type_name'):
      type_name = field_proto.type_name
      full_type_name = '.'.join(full_message_name +
                                [type_name[type_name.rfind('.')+1:]])
      if full_type_name in nested_types:
        nested_desc = nested_types[full_type_name]
      elif full_type_name in enum_types:
        enum_desc = enum_types[full_type_name]
      # Else type_name references a non-local type, which isn't implemented
    field = FieldDescriptor(
        field_proto.name, full_name, field_proto.number - 1,
        field_proto.number, field_proto.type,
        FieldDescriptor.ProtoTypeToCppProtoType(field_proto.type),
        field_proto.label, None, nested_desc, enum_desc, None, False, None,
        options=field_proto.options, has_default_value=False)
    fields.append(field)

  desc_name = '.'.join(full_message_name)
  return Descriptor(desc_proto.name, desc_name, None, None, fields,
                    list(nested_types.values()), list(enum_types.values()), [],
                    options=desc_proto.options)

def MakeDescriptor ( desc_proto , package = '' , build_file_if_cpp = True , syntax = None ) : if api_implementation . Type ( ) == 'cpp' and build_file_if_cpp : from typy . google . protobuf import descriptor_pb2 file_descriptor_proto = descriptor_pb2 . FileDescriptorProto ( ) file_descriptor_proto . message_type . add ( ) . MergeFrom ( desc_proto ) proto_name = str ( uuid . uuid4 ( ) ) if package : file_descriptor_proto . name = os . path . join ( package . replace ( '.' , '/' ) , proto_name + '.proto' ) file_descriptor_proto . package = package else : file_descriptor_proto . name = proto_name + '.proto' _message . default_pool . Add ( file_descriptor_proto ) result = _message . default_pool . FindFileByName ( file_descriptor_proto . name ) if _USE_C_DESCRIPTORS : return result . message_types_by_name [ desc_proto . name ] full_message_name = [ desc_proto . name ] if package : full_message_name . insert ( 0 , package ) enum_types = { } for enum_proto in desc_proto . enum_type : full_name = '.' . join ( full_message_name + [ enum_proto . name ] ) enum_desc = EnumDescriptor ( enum_proto . name , full_name , None , [ EnumValueDescriptor ( enum_val . name , ii , enum_val . number ) for ii , enum_val in enumerate ( enum_proto . value ) ] ) enum_types [ full_name ] = enum_desc nested_types = { } for nested_proto in desc_proto . nested_type : full_name = '.' . join ( full_message_name + [ nested_proto . name ] ) nested_desc = MakeDescriptor ( nested_proto , package = '.' . join ( full_message_name ) , build_file_if_cpp = False , syntax = syntax ) nested_types [ full_name ] = nested_desc fields = [ ] for field_proto in desc_proto . field : full_name = '.' . join ( full_message_name + [ field_proto . name ] ) enum_desc = None nested_desc = None if field_proto . HasField ( 'type_name' ) : type_name = field_proto . type_name full_type_name = '.' . join ( full_message_name + [ type_name [ type_name . rfind ( '.' ) + 1 : ] ] ) if full_type_name in nested_types : nested_desc = nested_types [ full_type_name ] elif full_type_name in enum_types : enum_desc = enum_types [ full_type_name ] field = FieldDescriptor ( field_proto . name , full_name , field_proto . number - 1 , field_proto . number , field_proto . type , FieldDescriptor . ProtoTypeToCppProtoType ( field_proto . type ) , field_proto . label , None , nested_desc , enum_desc , None , False , None , options = field_proto . options , has_default_value = False ) fields . append ( field ) desc_name = '.' . join ( full_message_name ) return Descriptor ( desc_proto . name , desc_name , None , None , fields , list ( nested_types . values ( ) ) , list ( enum_types . values ( ) ) , [ ] , options = desc_proto . options )

Make a protobuf Descriptor given a DescriptorProto protobuf .


def GetTopLevelContainingType(self):
    """Returns the root if this is a nested type, or itself if its the root."""
    desc = self
    while desc.containing_type is not None:
      desc = desc.containing_type
    return desc

def GetTopLevelContainingType ( self ) : desc = self while desc . containing_type is not None : desc = desc . containing_type return desc

Returns the root if this is a nested type or itself if its the root .


def FindMethodByName(self, name):
    """Searches for the specified method, and returns its descriptor."""
    for method in self.methods:
      if name == method.name:
        return method
    return None

def FindMethodByName ( self , name ) : for method in self . methods : if name == method . name : return method return None

Searches for the specified method and returns its descriptor .


def MessageToJson(message, including_default_value_fields=False):
  """Converts protobuf message to JSON format.

  Args:
    message: The protocol buffers message instance to serialize.
    including_default_value_fields: If True, singular primitive fields,
        repeated fields, and map fields will always be serialized.  If
        False, only serialize non-empty fields.  Singular message fields
        and oneof fields are not affected by this option.

  Returns:
    A string containing the JSON formatted protocol buffer message.
  """
  js = _MessageToJsonObject(message, including_default_value_fields)
  return json.dumps(js, indent=2)

def MessageToJson ( message , including_default_value_fields = False ) : js = _MessageToJsonObject ( message , including_default_value_fields ) return json . dumps ( js , indent = 2 )

Converts protobuf message to JSON format .


def _MessageToJsonObject(message, including_default_value_fields):
  """Converts message to an object according to Proto3 JSON Specification."""
  message_descriptor = message.DESCRIPTOR
  full_name = message_descriptor.full_name
  if _IsWrapperMessage(message_descriptor):
    return _WrapperMessageToJsonObject(message)
  if full_name in _WKTJSONMETHODS:
    return _WKTJSONMETHODS[full_name][0](
        message, including_default_value_fields)
  js = {}
  return _RegularMessageToJsonObject(
      message, js, including_default_value_fields)

def _MessageToJsonObject ( message , including_default_value_fields ) : message_descriptor = message . DESCRIPTOR full_name = message_descriptor . full_name if _IsWrapperMessage ( message_descriptor ) : return _WrapperMessageToJsonObject ( message ) if full_name in _WKTJSONMETHODS : return _WKTJSONMETHODS [ full_name ] [ 0 ] ( message , including_default_value_fields ) js = { } return _RegularMessageToJsonObject ( message , js , including_default_value_fields )

Converts message to an object according to Proto3 JSON Specification .


def _StructMessageToJsonObject(message, unused_including_default=False):
  """Converts Struct message according to Proto3 JSON Specification."""
  fields = message.fields
  ret = {}
  for key in fields:
    ret[key] = _ValueMessageToJsonObject(fields[key])
  return ret

def _StructMessageToJsonObject ( message , unused_including_default = False ) : fields = message . fields ret = { } for key in fields : ret [ key ] = _ValueMessageToJsonObject ( fields [ key ] ) return ret

Converts Struct message according to Proto3 JSON Specification .


def Parse(text, message):
  """Parses a JSON representation of a protocol message into a message.

  Args:
    text: Message JSON representation.
    message: A protocol beffer message to merge into.

  Returns:
    The same message passed as argument.

  Raises::
    ParseError: On JSON parsing problems.
  """
  if not isinstance(text, six.text_type): text = text.decode('utf-8')
  try:
    if sys.version_info < (2, 7):
      # object_pair_hook is not supported before python2.7
      js = json.loads(text)
    else:
      js = json.loads(text, object_pairs_hook=_DuplicateChecker)
  except ValueError as e:
    raise ParseError('Failed to load JSON: {0}.'.format(str(e)))
  _ConvertMessage(js, message)
  return message

def
Parse
(
text
,
message
)
:
if
not
isinstance
(
text
,
six
.
text_type
)
:
text
=
text
.
decode
(
'utf-8'
)
try
:
if
sys
.
version_info
<
(
2
,
7
)
:
js
=
json
.
loads
(
text
)
else
:
js
=
json
.
loads
(
text
,
object_pairs_hook
=
_DuplicateChecker
)
except
ValueError
as
e
:
raise
ParseError
(
'Failed to load JSON: {0}.'
.
format
(
str
(
e
)
)
)
_ConvertMessage
(
js
,
message
)
return
message

Parses a JSON representation of a protocol message into a message .


def _ConvertFieldValuePair(js, message):
  """Convert field value pairs into regular message.

  Args:
    js: A JSON object to convert the field value pairs.
    message: A regular protocol message to record the data.

  Raises:
    ParseError: In case of problems converting.
  """
  names = []
  message_descriptor = message.DESCRIPTOR
  for name in js:
    try:
      field = message_descriptor.fields_by_camelcase_name.get(name, None)
      if not field:
        raise ParseError(
            'Message type "{0}" has no field named "{1}".'.format(
                message_descriptor.full_name, name))
      if name in names:
        raise ParseError(
            'Message type "{0}" should not have multiple "{1}" fields.'.format(
                message.DESCRIPTOR.full_name, name))
      names.append(name)
      # Check no other oneof field is parsed.
      if field.containing_oneof is not None:
        oneof_name = field.containing_oneof.name
        if oneof_name in names:
          raise ParseError('Message type "{0}" should not have multiple "{1}" '
                           'oneof fields.'.format(
                               message.DESCRIPTOR.full_name, oneof_name))
        names.append(oneof_name)

      value = js[name]
      if value is None:
        message.ClearField(field.name)
        continue

      # Parse field value.
      if _IsMapEntry(field):
        message.ClearField(field.name)
        _ConvertMapFieldValue(value, message, field)
      elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
        message.ClearField(field.name)
        if not isinstance(value, list):
          raise ParseError('repeated field {0} must be in [] which is '
                           '{1}.'.format(name, value))
        if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
          # Repeated message field.
          for item in value:
            sub_message = getattr(message, field.name).add()
            # None is a null_value in Value.
            if (item is None and
                sub_message.DESCRIPTOR.full_name != 'google.protobuf.Value'):
              raise ParseError('null is not allowed to be used as an element'
                               ' in a repeated field.')
            _ConvertMessage(item, sub_message)
        else:
          # Repeated scalar field.
          for item in value:
            if item is None:
              raise ParseError('null is not allowed to be used as an element'
                               ' in a repeated field.')
            getattr(message, field.name).append(
                _ConvertScalarFieldValue(item, field))
      elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
        sub_message = getattr(message, field.name)
        _ConvertMessage(value, sub_message)
      else:
        setattr(message, field.name, _ConvertScalarFieldValue(value, field))
    except ParseError as e:
      if field and field.containing_oneof is None:
        raise ParseError('Failed to parse {0} field: {1}'.format(name, e))
      else:
        raise ParseError(str(e))
    except ValueError as e:
      raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))
    except TypeError as e:
      raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))

Convert field value pairs into regular message .


def _ConvertMessage(value, message):
  """Convert a JSON object into a message.

  Args:
    value: A JSON object.
    message: A WKT or regular protocol message to record the data.

  Raises:
    ParseError: In case of convert problems.
  """
  message_descriptor = message.DESCRIPTOR
  full_name = message_descriptor.full_name
  if _IsWrapperMessage(message_descriptor):
    _ConvertWrapperMessage(value, message)
  elif full_name in _WKTJSONMETHODS:
    _WKTJSONMETHODS[full_name][1](value, message)
  else:
    _ConvertFieldValuePair(value, message)

def _ConvertMessage ( value , message ) : message_descriptor = message . DESCRIPTOR full_name = message_descriptor . full_name if _IsWrapperMessage ( message_descriptor ) : _ConvertWrapperMessage ( value , message ) elif full_name in _WKTJSONMETHODS : _WKTJSONMETHODS [ full_name ] [ 1 ] ( value , message ) else : _ConvertFieldValuePair ( value , message )


def _ConvertValueMessage(value, message):
  """Convert a JSON representation into Value message."""
  if isinstance(value, dict):
    _ConvertStructMessage(value, message.struct_value)
  elif isinstance(value, list):
    _ConvertListValueMessage(value, message.list_value)
  elif value is None:
    message.null_value = 0
  elif isinstance(value, bool):
    message.bool_value = value
  elif isinstance(value, six.string_types):
    message.string_value = value
  elif isinstance(value, _INT_OR_FLOAT):
    message.number_value = value
  else:
    raise ParseError('Unexpected type for Value message.')

def
_ConvertValueMessage
(
value
,
message
)
:
if
isinstance
(
value
,
dict
)
:
_ConvertStructMessage
(
value
,
message
.
struct_value
)
elif
isinstance
(
value
,
list
)
:
_ConvertListValueMessage
(
value
,
message
.
list_value
)
elif
value
is
:
message
.
null_value
=
0
elif
isinstance
(
value
,
bool
)
:
message
.
bool_value
=
value
elif
isinstance
(
value
,
six
.
string_types
)
:
message
.
string_value
=
value
elif
isinstance
(
value
,
_INT_OR_FLOAT
)
:
message
.
number_value
=
value
else
:
raise
ParseError
(
'Unexpected type for Value message.'
)

Convert a JSON representation into Value message .


def _ConvertListValueMessage(value, message):
  """Convert a JSON representation into ListValue message."""
  if not isinstance(value, list):
    raise ParseError(
        'ListValue must be in [] which is {0}.'.format(value))
  message.ClearField('values')
  for item in value:
    _ConvertValueMessage(item, message.values.add())

def
_ConvertListValueMessage
(
value
,
message
)
:
if
not
isinstance
(
value
,
list
)
:
raise
ParseError
(
'ListValue must be in [] which is {0}.'
.
format
(
value
)
)
message
.
ClearField
(
'values'
)
for
item
in
value
:
_ConvertValueMessage
(
item
,
message
.
values
.
add
(
)
)

Convert a JSON representation into ListValue message .


def _ConvertStructMessage(value, message):
  """Convert a JSON representation into Struct message."""
  if not isinstance(value, dict):
    raise ParseError(
        'Struct must be in a dict which is {0}.'.format(value))
  for key in value:
    _ConvertValueMessage(value[key], message.fields[key])
  return

Convert a JSON representation into Struct message .


def update_config(new_config):
    """ Update config options with the provided dictionary of options.
    """
    flask_app.base_config.update(new_config)

    # Check for changed working directory.
    if new_config.has_key('working_directory'):
        wd = os.path.abspath(new_config['working_directory'])
        if nbmanager.notebook_dir != wd:
            if not os.path.exists(wd):
                raise IOError('Path not found: %s' % wd)
            nbmanager.notebook_dir = wd

def
update_config
(
new_config
)
:
flask_app
.
base_config
.
update
(
new_config
)
if
new_config
.
has_key
(
'working_directory'
)
:
wd
=
os
.
path
.
abspath
(
new_config
[
'working_directory'
]
)
if
nbmanager
.
notebook_dir
!=
wd
:
if
not
os
.
path
.
exists
(
wd
)
:
raise
IOError
(
'Path not found: %s'
%
wd
)
nbmanager
.
notebook_dir
=
wd

Update config options with the provided dictionary of options .


def end_timing(self):
        """
        Completes measuring time interval and updates counter.
        """

        if self._callback != None:
            elapsed = time.clock() * 1000 - self._start
            self._callback.end_timing(self._counter, elapsed)

def end_timing ( self ) : if self . _callback != None : elapsed = time . clock ( ) * 1000 - self . _start self . _callback . end_timing ( self . _counter , elapsed )

Completes measuring time interval and updates counter .


def ToJsonString(self):
    """Converts Duration to string format.

    Returns:
      A string converted from self. The string format will contains
      3, 6, or 9 fractional digits depending on the precision required to
      represent the exact Duration value. For example: "1s", "1.010s",
      "1.000000100s", "-3.100s"
    """
    if self.seconds < 0 or self.nanos < 0:
      result = '-'
      seconds = - self.seconds + int((0 - self.nanos) // 1e9)
      nanos = (0 - self.nanos) % 1e9
    else:
      result = ''
      seconds = self.seconds + int(self.nanos // 1e9)
      nanos = self.nanos % 1e9
    result += '%d' % seconds
    if (nanos % 1e9) == 0:
      # If there are 0 fractional digits, the fractional
      # point '.' should be omitted when serializing.
      return result + 's'
    if (nanos % 1e6) == 0:
      # Serialize 3 fractional digits.
      return result + '.%03ds' % (nanos / 1e6)
    if (nanos % 1e3) == 0:
      # Serialize 6 fractional digits.
      return result + '.%06ds' % (nanos / 1e3)
    # Serialize 9 fractional digits.
    return result + '.%09ds' % nanos

def ToJsonString ( self ) : if self . seconds < 0 or self . nanos < 0 : result = '-' seconds = - self . seconds + int ( ( 0 - self . nanos ) // 1e9 ) nanos = ( 0 - self . nanos ) % 1e9 else : result = '' seconds = self . seconds + int ( self . nanos // 1e9 ) nanos = self . nanos % 1e9 result += '%d' % seconds if ( nanos % 1e9 ) == 0 : return result + 's' if ( nanos % 1e6 ) == 0 : return result + '.%03ds' % ( nanos / 1e6 ) if ( nanos % 1e3 ) == 0 : return result + '.%06ds' % ( nanos / 1e3 ) return result + '.%09ds' % nanos


def FromJsonString(self, value):
    """Converts a string to Duration.

    Args:
      value: A string to be converted. The string must end with 's'. Any
          fractional digits (or none) are accepted as long as they fit into
          precision. For example: "1s", "1.01s", "1.0000001s", "-3.100s

    Raises:
      ParseError: On parsing problems.
    """
    if len(value) < 1 or value[-1] != 's':
      raise ParseError(
          'Duration must end with letter "s": {0}.'.format(value))
    try:
      pos = value.find('.')
      if pos == -1:
        self.seconds = int(value[:-1])
        self.nanos = 0
      else:
        self.seconds = int(value[:pos])
        if value[0] == '-':
          self.nanos = int(round(float('-0{0}'.format(value[pos: -1])) *1e9))
        else:
          self.nanos = int(round(float('0{0}'.format(value[pos: -1])) *1e9))
    except ValueError:
      raise ParseError(
          'Couldn\'t parse duration: {0}.'.format(value))