def submit_form(self, form):
        """Make `Register` object for submitting the registration form.

        Convert form data to legacy fields if `self.form` is `None`.

        :Parameters:
            - `form`: The form to submit. Its type doesn't have to be "submit"
              (a "submit" form will be created here), so it could be the form
              obtained from `get_form` just with the data entered.

        :return: new registration element
        :returntype: `Register`"""

        result = Register()
        if self.form:
            result.form = form.make_submit()
            return result

        if "FORM_TYPE" not in form or "jabber:iq:register" not in form["FORM_TYPE"].values:
            raise ValueError("FORM_TYPE is not jabber:iq:register")

        for field in legacy_fields:
            self.__logger.debug(u"submitted field %r" % (field, ))
            value = getattr(self, field)
            try:
                form_value = form[field].value
            except KeyError:
                if value:
                    raise ValueError("Required field with no value!")
                continue
            setattr(result, field, form_value)

        return result

def
submit_form
(
self
,
form
)
:
result
=
Register
(
)
if
self
.
form
:
result
.
form
=
form
.
make_submit
(
)
return
result
if
"FORM_TYPE"
not
in
form
or
"jabber:iq:register"
not
in
form
[
"FORM_TYPE"
]
.
values
:
raise
ValueError
(
"FORM_TYPE is not jabber:iq:register"
)
for
field
in
legacy_fields
:
self
.
__logger
.
debug
(
u"submitted field %r"
%
(
field
,
)
)
value
=
getattr
(
self
,
field
)
try
:
form_value
=
form
[
field
]
.
value
except
KeyError
:
if
value
:
raise
ValueError
(
"Required field with no value!"
)
continue
setattr
(
result
,
field
,
form_value
)
return
result

Make Register object for submitting the registration form .


def from_xml(self,xmlnode):
        """Initialize Delay object from an XML node.

        :Parameters:
            - `xmlnode`: the jabber:x:delay XML element.
        :Types:
            - `xmlnode`: `libxml2.xmlNode`"""
        if xmlnode.type!="element":
            raise ValueError("XML node is not a jabber:x:delay element (not an element)")
        ns=get_node_ns_uri(xmlnode)
        if ns and ns!=DELAY_NS or xmlnode.name!="x":
            raise ValueError("XML node is not a jabber:x:delay element")
        stamp=xmlnode.prop("stamp")
        if stamp.endswith("Z"):
            stamp=stamp[:-1]
        if "-" in stamp:
            stamp=stamp.split("-",1)[0]
        try:
            tm = time.strptime(stamp, "%Y%m%dT%H:%M:%S")
        except ValueError:
            raise BadRequestProtocolError("Bad timestamp")
        tm=tm[0:8]+(0,)
        self.timestamp=datetime.datetime.fromtimestamp(time.mktime(tm))
        delay_from=from_utf8(xmlnode.prop("from"))
        if delay_from:
            try:
                self.delay_from = JID(delay_from)
            except JIDError:
                raise JIDMalformedProtocolError("Bad JID in the jabber:x:delay 'from' attribute")
        else:
            self.delay_from = None
        self.reason = from_utf8(xmlnode.getContent())

Initialize Delay object from an XML node .


def handle_read(self):
        """
        Accept any incoming connections.
        """
        with self._lock:
            logger.debug("handle_read()")
            if self._socket is None:
                return
            while True:
                try:
                    sock, address = self._socket.accept()
                except socket.error, err:
                    if err.args[0] in BLOCKING_ERRORS:
                        break
                    else:
                        raise
                logger.debug("Accepted connection from: {0!r}".format(address))
                self._target(sock, address)

def
handle_read
(
self
)
:
with
self
.
_lock
:
logger
.
debug
(
"handle_read()"
)
if
self
.
_socket
is
:
return
while
True
:
try
:
sock
,
address
=
self
.
_socket
.
accept
(
)
except
socket
.
error
,
err
:
if
err
.
args
[
0
]
in
BLOCKING_ERRORS
:
break
else
:
raise
logger
.
debug
(
"Accepted connection from: {0!r}"
.
format
(
address
)
)
self
.
_target
(
sock
,
address
)


def make_error_response(self, cond):
        """Create error response for the any non-error presence stanza.

        :Parameters:
            - `cond`: error condition name, as defined in XMPP specification.
        :Types:
            - `cond`: `unicode`

        :return: new presence stanza.
        :returntype: `Presence`
        """

        if self.stanza_type == "error":
            raise ValueError("Errors may not be generated in response"
                                                                " to errors")

        stanza = Presence(stanza_type = "error", from_jid = self.from_jid,
                            to_jid = self.to_jid, stanza_id = self.stanza_id,
                            status = self._status, show = self._show,
                            priority = self._priority, error_cond = cond)

        if self._payload is None:
            self.decode_payload()

        for payload in self._payload:
            stanza.add_payload(payload)

        return stanza

Create error response for the any non - error presence stanza .


def activate(self):
		"""
		Activate an plan in a CREATED state.
		"""
		obj = self.find_paypal_object()
		if obj.state == enums.BillingPlanState.CREATED:
			success = obj.activate()
			if not success:
				raise PaypalApiError("Failed to activate plan: %r" % (obj.error))
		# Resync the updated data to the database
		self.get_or_update_from_api_data(obj, always_sync=True)
		return obj

def
activate
(
self
)
:
obj
=
self
.
find_paypal_object
(
)
if
obj
.
state
==
enums
.
BillingPlanState
.
CREATED
:
success
=
obj
.
activate
(
)
if
not
success
:
raise
PaypalApiError
(
"Failed to activate plan: %r"
%
(
obj
.
error
)
)
self
.
get_or_update_from_api_data
(
obj
,
always_sync
=
True
)
return
obj


def execute(self):
		"""
		Execute the PreparedBillingAgreement by creating and executing a
		matching BillingAgreement.
		"""
		# Save the execution time first.
		# If execute() fails, executed_at will be set, with no executed_agreement set.
		self.executed_at = now()
		self.save()

		with transaction.atomic():
			ret = BillingAgreement.execute(self.id)
			ret.user = self.user
			ret.save()
			self.executed_agreement = ret
			self.save()

		return ret

def execute ( self ) : self . executed_at = now ( ) self . save ( ) with transaction . atomic ( ) : ret = BillingAgreement . execute ( self . id ) ret . user = self . user ret . save ( ) self . executed_agreement = ret self . save ( ) return ret

Execute the PreparedBillingAgreement by creating and executing a matching BillingAgreement .


def webhook_handler(*event_types):
	"""
	Decorator that registers a function as a webhook handler.

	Usage examples:

	>>> # Hook a single event
	>>> @webhook_handler("payment.sale.completed")
	>>> def on_payment_received(event):
	>>>     payment = event.get_resource()
	>>>     print("Received payment:", payment)

	>>> # Multiple events supported
	>>> @webhook_handler("billing.subscription.suspended", "billing.subscription.cancelled")
	>>> def on_subscription_stop(event):
	>>>     subscription = event.get_resource()
	>>>     print("Stopping subscription:", subscription)

	>>> # Using a wildcard works as well
	>>> @webhook_handler("billing.subscription.*")
	>>> def on_subscription_update(event):
	>>>     subscription = event.get_resource()
	>>>     print("Updated subscription:", subscription)
	"""

	# First expand all wildcards and verify the event types are valid
	event_types_to_register = set()
	for event_type in event_types:
		# Always convert to lowercase
		event_type = event_type.lower()
		if "*" in event_type:
			# expand it
			for t in WEBHOOK_EVENT_TYPES:
				if fnmatch(t, event_type):
					event_types_to_register.add(t)
		elif event_type not in WEBHOOK_EVENT_TYPES:
			raise ValueError("Unknown webhook event: %r" % (event_type))
		else:
			event_types_to_register.add(event_type)

	# Now register them
	def decorator(func):
		for event_type in event_types_to_register:
			WEBHOOK_SIGNALS[event_type].connect(func)
		return func

	return decorator

def
webhook_handler
(
*
event_types
)
:
event_types_to_register
=
set
(
)
for
event_type
in
event_types
:
event_type
=
event_type
.
lower
(
)
if
"*"
in
event_type
:
for
t
in
WEBHOOK_EVENT_TYPES
:
if
fnmatch
(
t
,
event_type
)
:
event_types_to_register
.
add
(
t
)
elif
event_type
not
in
WEBHOOK_EVENT_TYPES
:
raise
ValueError
(
"Unknown webhook event: %r"
%
(
event_type
)
)
else
:
event_types_to_register
.
add
(
event_type
)
def
decorator
(
func
)
:
for
event_type
in
event_types_to_register
:
WEBHOOK_SIGNALS
[
event_type
]
.
connect
(
func
)
return
func
return
decorator

Decorator that registers a function as a webhook handler .


def from_request(cls, request, webhook_id=PAYPAL_WEBHOOK_ID):
		"""
		Create, validate and process a WebhookEventTrigger given a Django
		request object.

		The webhook_id parameter expects the ID of the Webhook that was
		triggered (defaults to settings.PAYPAL_WEBHOOK_ID). This is required
		for Webhook verification.

		The process is three-fold:
		1. Create a WebhookEventTrigger object from a Django request.
		2. Verify the WebhookEventTrigger as a Paypal webhook using the SDK.
		3. If valid, process it into a WebhookEvent object (and child resource).
		"""

		headers = fix_django_headers(request.META)
		assert headers
		try:
			body = request.body.decode(request.encoding or "utf-8")
		except Exception:
			body = "(error decoding body)"

		ip = request.META["REMOTE_ADDR"]
		obj = cls.objects.create(headers=headers, body=body, remote_ip=ip)

		try:
			obj.valid = obj.verify(PAYPAL_WEBHOOK_ID)
			if obj.valid:
				# Process the item (do not save it, it'll get saved below)
				obj.process(save=False)
		except Exception as e:
			max_length = WebhookEventTrigger._meta.get_field("exception").max_length
			obj.exception = str(e)[:max_length]
			obj.traceback = format_exc()
		finally:
			obj.save()

		return obj

def
from_request
(
cls
,
request
,
webhook_id
=
PAYPAL_WEBHOOK_ID
)
:
headers
=
fix_django_headers
(
request
.
META
)
assert
headers
try
:
body
=
request
.
body
.
decode
(
request
.
encoding
or
"utf-8"
)
except
Exception
:
body
=
"(error decoding body)"
ip
=
request
.
META
[
"REMOTE_ADDR"
]
obj
=
cls
.
objects
.
create
(
headers
=
headers
,
body
=
body
,
remote_ip
=
ip
)
try
:
obj
.
valid
=
obj
.
verify
(
PAYPAL_WEBHOOK_ID
)
if
obj
.
valid
:
obj
.
process
(
save
=
False
)
except
Exception
as
e
:
max_length
=
WebhookEventTrigger
.
_meta
.
get_field
(
"exception"
)
.
max_length
obj
.
exception
=
str
(
e
)
[
:
max_length
]
obj
.
traceback
=
format_exc
(
)
finally
:
obj
.
save
(
)
return
obj

Create validate and process a WebhookEventTrigger given a Django request object .


def check_paypal_api_key(app_configs=None, **kwargs):
	"""Check that the Paypal API keys are configured correctly"""
	messages = []

	mode = getattr(djpaypal_settings, "PAYPAL_MODE", None)
	if mode not in VALID_MODES:
		msg = "Invalid PAYPAL_MODE specified: {}.".format(repr(mode))
		hint = "PAYPAL_MODE must be one of {}".format(", ".join(repr(k) for k in VALID_MODES))
		messages.append(checks.Critical(msg, hint=hint, id="djpaypal.C001"))

	for setting in "PAYPAL_CLIENT_ID", "PAYPAL_CLIENT_SECRET":
		if not getattr(djpaypal_settings, setting, None):
			msg = "Invalid value specified for {}".format(setting)
			hint = "Add PAYPAL_CLIENT_ID and PAYPAL_CLIENT_SECRET to your settings."
			messages.append(checks.Critical(msg, hint=hint, id="djpaypal.C002"))

	return messages


async def _create_upstream_applications(self):
        """
        Create the upstream applications.
        """
        loop = asyncio.get_event_loop()
        for steam_name, ApplicationsCls in self.applications.items():
            application = ApplicationsCls(self.scope)
            upstream_queue = asyncio.Queue()
            self.application_streams[steam_name] = upstream_queue
            self.application_futures[steam_name] = loop.create_task(
                application(
                    upstream_queue.get,
                    partial(self.dispatch_downstream, steam_name=steam_name)
                )
            )

async def _create_upstream_applications ( self ) : loop = asyncio . get_event_loop ( ) for steam_name , ApplicationsCls in self . applications . items ( ) : application = ApplicationsCls ( self . scope ) upstream_queue = asyncio . Queue ( ) self . application_streams [ steam_name ] = upstream_queue self . application_futures [ steam_name ] = loop . create_task ( application ( upstream_queue . get , partial ( self . dispatch_downstream , steam_name = steam_name ) ) )


async def send_upstream(self, message, stream_name=None):
        """
        Send a message upstream to a de-multiplexed application.

        If stream_name is includes will send just to that upstream steam, if not included will send ot all upstream
        steams.
        """
        if stream_name is None:
            for steam_queue in self.application_streams.values():
                await steam_queue.put(message)
            return
        steam_queue = self.application_streams.get(stream_name)
        if steam_queue is None:
            raise ValueError("Invalid multiplexed frame received (stream not mapped)")
        await steam_queue.put(message)

Send a message upstream to a de - multiplexed application .


async def dispatch_downstream(self, message, steam_name):
        """
        Handle a downstream message coming from an upstream steam.

        if there is not handling method set for this method type it will propagate the message further downstream.

        This is called as part of the co-routine of an upstream steam, not the same loop as used for upstream messages
        in the de-multiplexer.
        """
        handler = getattr(self, get_handler_name(message), None)
        if handler:
            await handler(message, stream_name=steam_name)
        else:
            # if there is not handler then just pass the message further downstream.
            await self.base_send(message)

async def dispatch_downstream ( self , message , steam_name ) : handler = getattr ( self , get_handler_name ( message ) , None ) if handler : await handler ( message , stream_name = steam_name ) else : await self . base_send ( message )

Handle a downstream message coming from an upstream steam .


async def receive_json(self, content, **kwargs):
        """
        Rout the message down the correct stream.
        """
        # Check the frame looks good
        if isinstance(content, dict) and "stream" in content and "payload" in content:
            # Match it to a channel
            steam_name = content["stream"]
            payload = content["payload"]
            # block upstream frames
            if steam_name not in self.applications_accepting_frames:
                raise ValueError("Invalid multiplexed frame received (stream not mapped)")
            # send it on to the application that handles this stream
            await self.send_upstream(
                message={
                    "type": "websocket.receive",
                    "text": await self.encode_json(payload)
                },
                stream_name=steam_name
            )
            return
        else:
            raise ValueError("Invalid multiplexed **frame received (no channel/payload key)")

Rout the message down the correct stream .


async def websocket_disconnect(self, message):
        """
        Handle the disconnect message.

        This is propagated to all upstream applications.
        """
        # set this flag so as to ensure we don't send a downstream `websocket.close` message due to all
        # child applications closing.
        self.closing = True
        # inform all children
        await self.send_upstream(message)
        await super().websocket_disconnect(message)

async def websocket_disconnect ( self , message ) : self . closing = True await self . send_upstream ( message ) await super ( ) . websocket_disconnect ( message )


async def disconnect(self, code):
        """
        default is to wait for the child applications to close.
        """
        try:
            await asyncio.wait(
                self.application_futures.values(),
                return_when=asyncio.ALL_COMPLETED,
                timeout=self.application_close_timeout
            )
        except asyncio.TimeoutError:
            pass

async def disconnect ( self , code ) : try : await asyncio . wait ( self . application_futures . values ( ) , return_when = asyncio . ALL_COMPLETED , timeout = self . application_close_timeout ) except asyncio . TimeoutError : pass

default is to wait for the child applications to close .


async def websocket_send(self, message, stream_name):
        """
        Capture downstream websocket.send messages from the upstream applications.
        """
        text = message.get("text")
        # todo what to do on binary!
        json = await self.decode_json(text)
        data = {
            "stream": stream_name,
            "payload": json
        }
        await self.send_json(data)

async def websocket_send ( self , message , stream_name ) : text = message . get ( "text" ) json = await self . decode_json ( text ) data = { "stream" : stream_name , "payload" : json } await self . send_json ( data )

Capture downstream websocket . send messages from the upstream applications .


async def websocket_accept(self, message, stream_name):
        """
        Intercept downstream `websocket.accept` message and thus allow this upsteam application to accept websocket
        frames.
        """
        is_first = not self.applications_accepting_frames
        self.applications_accepting_frames.add(stream_name)
        # accept the connection after the first upstream application accepts.
        if is_first:
            await self.accept()

async def websocket_accept ( self , message , stream_name ) : is_first = not self . applications_accepting_frames self . applications_accepting_frames . add ( stream_name ) if is_first : await self . accept ( )

Intercept downstream websocket . accept message and thus allow this upsteam application to accept websocket frames .


async def websocket_close(self, message, stream_name):
        """
        Handle downstream `websocket.close` message.

        Will disconnect this upstream application from receiving any new frames.

        If there are not more upstream applications accepting messages it will then call `close`.
        """
        if stream_name in self.applications_accepting_frames:
            # remove from set of upsteams steams than can receive new messages
            self.applications_accepting_frames.remove(stream_name)

        # we are already closing due to an upstream websocket.disconnect command

        if self.closing:
            return
        # if none of the upstream applications are listing we need to close.
        if not self.applications_accepting_frames:
            await self.close(message.get("code"))

async def websocket_close ( self , message , stream_name ) : if stream_name in self . applications_accepting_frames : self . applications_accepting_frames . remove ( stream_name ) if self . closing : return if not self . applications_accepting_frames : await self . close ( message . get ( "code" ) )

Handle downstream websocket . close message .


def cast(cls, fx_spot, domestic_curve=None, foreign_curve=None):
        """
        creator method to build FxCurve

        :param float fx_spot: fx spot rate
        :param RateCurve domestic_curve: domestic discount curve
        :param RateCurve foreign_curve: foreign discount curve
        :return:
        """
        assert domestic_curve.origin == foreign_curve.origin
        return cls(fx_spot, domestic_curve=domestic_curve, foreign_curve=foreign_curve)

def cast ( cls , fx_spot , domestic_curve = None , foreign_curve = None ) : assert domestic_curve . origin == foreign_curve . origin return cls ( fx_spot , domestic_curve = domestic_curve , foreign_curve = foreign_curve )


def retry(
        exceptions=(Exception,), interval=0, max_retries=10, success=None,
        timeout=-1):
    """Decorator to retry a function 'max_retries' amount of times

    :param tuple exceptions: Exceptions to be caught for retries
    :param int interval: Interval between retries in seconds
    :param int max_retries: Maximum number of retries to have, if
        set to -1 the decorator will loop forever
    :param function success: Function to indicate success criteria
    :param int timeout: Timeout interval in seconds, if -1 will retry forever
    :raises MaximumRetriesExceeded: Maximum number of retries hit without
        reaching the success criteria
    :raises TypeError: Both exceptions and success were left None causing the
        decorator to have no valid exit criteria.

    Example:
        Use it to decorate a function!

        .. sourcecode:: python

            from retry import retry

            @retry(exceptions=(ArithmeticError,), success=lambda x: x > 0)
            def foo(bar):
                if bar < 0:
                    raise ArithmeticError('testing this')
                return bar
            foo(5)
            # Should return 5
            foo(-1)
            # Should raise ArithmeticError
            foo(0)
            # Should raise MaximumRetriesExceeded
    """
    if not exceptions and success is None:
        raise TypeError(
            '`exceptions` and `success` parameter can not both be None')
    # For python 3 compatability
    exceptions = exceptions or (_DummyException,)
    _retries_error_msg = ('Exceeded maximum number of retries {} at '
                          'an interval of {}s for function {}')

    _timeout_error_msg = 'Maximum timeout of {}s reached for function {}'

    @decorator
    def wrapper(func, *args, **kwargs):
        signal.signal(
            signal.SIGALRM, _timeout(
                _timeout_error_msg.format(timeout, func.__name__)))
        run_func = functools.partial(func, *args, **kwargs)
        logger = logging.getLogger(func.__module__)
        if max_retries < 0:
            iterator = itertools.count()
        else:
            iterator = range(max_retries)
        if timeout > 0:
            signal.alarm(timeout)
        for num, _ in enumerate(iterator, 1):
            try:
                result = run_func()
                if success is None or success(result):
                    signal.alarm(0)
                    return result
            except exceptions:
                logger.exception(
                    'Exception experienced when trying function {}'.format(
                        func.__name__))
                if num == max_retries:
                    raise
            logger.warning(
                'Retrying {} in {}s...'.format(
                    func.__name__, interval))
            time.sleep(interval)
        else:
            raise MaximumRetriesExceeded(
                _retries_error_msg.format(
                    max_retries, interval, func.__name__))
    return wrapper

Decorator to retry a function max_retries amount of times


def __button_action(self, data=None):
        """Button action event"""
        if any(not x for x in (self._ename.value, self._p1.value, self._p2.value, self._file.value)):
            print("Missing one of the required fields (event name, player names, file name)")
            return
        self.__p1chars = []
        self.__p2chars = []
        options = Namespace()
        self.__history.append(self.__save_form())
        options.ename = self._ename.value
        if self._ename_min.value:
            options.ename_min = self._ename_min.value
        else:
            options.ename_min = options.ename
        options.pID = self._pID.value
        options.mtype = self._mtype.value
        options.mmid = options.mtype
        options.p1 = self._p1.value
        options.p2 = self._p2.value
        options.p1char = self._p1char.value
        options.p2char = self._p2char.value
        options.bracket = self._bracket.value
        isadir = os.path.isdir(self._file.value)
        if isadir:
            options.file = max([os.path.join(self._file.value, f) for f in os.listdir(self._file.value) if os.path.isfile(os.path.join(self._file.value, f))], key=os.path.getmtime)
        else:
            options.file = self._file.value
        options.tags = self._tags.value
        options.msuffix = self._msuffix.value
        options.mprefix = self._mprefix.value
        options.privacy = self._privacy.value
        options.descrip = self._description.value
        options.titleformat = self._titleformat.value
        if self._p1sponsor.value:
            options.p1 = " | ".join((self._p1sponsor.value, options.p1))
        if self._p2sponsor.value:
            options.p2 = " | ".join((self._p2sponsor.value, options.p2))
        options.ignore = False
        self.__reset_match(False, isadir)
        self.__add_to_qview(options)
        self._queueref.append(options)
        if consts.firstrun:
            thr = threading.Thread(target=self.__worker)
            thr.daemon = True
            thr.start()
            consts.firstrun = False


def multiglob_compile(globs, prefix=False):
    """Generate a single "A or B or C" regex from a list of shell globs.

    :param globs: Patterns to be processed by :mod:`fnmatch`.
    :type globs: iterable of :class:`~__builtins__.str`

    :param prefix: If ``True``, then :meth:`~re.RegexObject.match` will
        perform prefix matching rather than exact string matching.
    :type prefix: :class:`~__builtins__.bool`

    :rtype: :class:`re.RegexObject`
    """
    if not globs:
        # An empty globs list should only match empty strings
        return re.compile('^$')
    elif prefix:
        globs = [x + '*' for x in globs]
    return re.compile('|'.join(fnmatch.translate(x) for x in globs))

def multiglob_compile ( globs , prefix = False ) : if not globs : return re . compile ( '^$' ) elif prefix : globs = [ x + '*' for x in globs ] return re . compile ( '|' . join ( fnmatch . translate ( x ) for x in globs ) )

Generate a single A or B or C regex from a list of shell globs .


def getPaths(roots, ignores=None):
    """
    Recursively walk a set of paths and return a listing of contained files.

    :param roots: Relative or absolute paths to files or folders.
    :type roots: :class:`~__builtins__.list` of :class:`~__builtins__.str`

    :param ignores: A list of :py:mod:`fnmatch` globs to avoid walking and
        omit from results
    :type ignores: :class:`~__builtins__.list` of :class:`~__builtins__.str`

    :returns: Absolute paths to only files.
    :rtype: :class:`~__builtins__.list` of :class:`~__builtins__.str`

    .. todo:: Try to optimize the ignores matching. Running a regex on every
       filename is a fairly significant percentage of the time taken according
       to the profiler.
    """
    paths, count, ignores = [], 0, ignores or []

    # Prepare the ignores list for most efficient use
    ignore_re = multiglob_compile(ignores, prefix=False)

    for root in roots:
        # For safety, only use absolute, real paths.
        root = os.path.realpath(root)

        # Handle directly-referenced filenames properly
        # (And override ignores to "do as I mean, not as I say")
        if os.path.isfile(root):
            paths.append(root)
            continue

        for fldr in os.walk(root):
            out.write("Gathering file paths to compare... (%d files examined)"
                      % count)

            # Don't even descend into IGNOREd directories.
            for subdir in fldr[1]:
                dirpath = os.path.join(fldr[0], subdir)
                if ignore_re.match(dirpath):
                    fldr[1].remove(subdir)

            for filename in fldr[2]:
                filepath = os.path.join(fldr[0], filename)
                if ignore_re.match(filepath):
                    continue  # Skip IGNOREd files.

                paths.append(filepath)
                count += 1

    out.write("Found %s files to be compared for duplication." % (len(paths)),
              newline=True)
    return paths

Recursively walk a set of paths and return a listing of contained files .


def groupBy(groups_in, classifier, fun_desc='?', keep_uniques=False,
            *args, **kwargs):
    """Subdivide groups of paths according to a function.

    :param groups_in: Grouped sets of paths.
    :type groups_in: :class:`~__builtins__.dict` of iterables

    :param classifier: Function to group a list of paths by some attribute.
    :type classifier: ``function(list, *args, **kwargs) -> str``

    :param fun_desc: Human-readable term for what the classifier operates on.
        (Used in log messages)
    :type fun_desc: :class:`~__builtins__.str`

    :param keep_uniques: If ``False``, discard groups with only one member.
    :type keep_uniques: :class:`~__builtins__.bool`

    :returns: A dict mapping classifier keys to groups of matches.
    :rtype: :class:`~__builtins__.dict`

    :attention: Grouping functions generally use a :class:`~__builtins__.set`
        ``groups`` as extra protection against accidentally counting a given
        file twice. (Complimentary to use of :func:`os.path.realpath` in
        :func:`~fastdupes.getPaths`)

    .. todo:: Find some way to bring back the file-by-file status text
    """
    groups, count, group_count = {}, 0, len(groups_in)
    for pos, paths in enumerate(groups_in.values()):
        out.write("Subdividing group %d of %d by %s... (%d files examined, %d "
                  "in current group)" % (
                      pos + 1, group_count, fun_desc, count, len(paths)
                  ))

        for key, group in classifier(paths, *args, **kwargs).items():
            groups.setdefault(key, set()).update(group)
            count += len(group)

    if not keep_uniques:
        # Return only the groups with more than one file.
        groups = dict([(x, groups[x]) for x in groups if len(groups[x]) > 1])

    out.write("Found %s sets of files with identical %s. (%d files examined)"
              % (len(groups), fun_desc, count), newline=True)
    return groups

Subdivide groups of paths according to a function .


def groupify(function):
    """Decorator to convert a function which takes a single value and returns
    a key into one which takes a list of values and returns a dict of key-group
    mappings.

    :param function: A function which takes a value and returns a hash key.
    :type function: ``function(value) -> key``

    :rtype:
        .. parsed-literal::
            function(iterable) ->
                {key: :class:`~__builtins__.set` ([value, ...]), ...}
    """

    @wraps(function)
    def wrapper(paths, *args, **kwargs):  # pylint: disable=missing-docstring
        groups = {}

        for path in paths:
            key = function(path, *args, **kwargs)
            if key is not None:
                groups.setdefault(key, set()).add(path)

        return groups
    return wrapper

def groupify ( function ) : @ wraps ( function ) def wrapper ( paths , * args , ** kwargs ) : groups = { } for path in paths : key = function ( path , * args , ** kwargs ) if key is not None : groups . setdefault ( key , set ( ) ) . add ( path ) return groups return wrapper

Decorator to convert a function which takes a single value and returns a key into one which takes a list of values and returns a dict of key - group mappings .


def sizeClassifier(path, min_size=DEFAULTS['min_size']):
    """Sort a file into a group based on on-disk size.

    :param paths: See :func:`fastdupes.groupify`

    :param min_size: Files smaller than this size (in bytes) will be ignored.
    :type min_size: :class:`__builtins__.int`

    :returns: See :func:`fastdupes.groupify`

    .. todo:: Rework the calling of :func:`~os.stat` to minimize the number of
        calls. It's a fairly significant percentage of the time taken according
        to the profiler.
    """
    filestat = _stat(path)
    if stat.S_ISLNK(filestat.st_mode):
        return  # Skip symlinks.

    if filestat.st_size < min_size:
        return  # Skip files below the size limit

    return filestat.st_size

def sizeClassifier ( path , min_size = DEFAULTS [ 'min_size' ] ) : filestat = _stat ( path ) if stat . S_ISLNK ( filestat . st_mode ) : return if filestat . st_size < min_size : return return filestat . st_size

Sort a file into a group based on on - disk size .


def groupByContent(paths):
    """Byte-for-byte comparison on an arbitrary number of files in parallel.

    This operates by opening all files in parallel and comparing
    chunk-by-chunk. This has the following implications:

        - Reads the same total amount of data as hash comparison.
        - Performs a *lot* of disk seeks. (Best suited for SSDs)
        - Vulnerable to file handle exhaustion if used on its own.

    :param paths: List of potentially identical files.
    :type paths: iterable

    :returns: A dict mapping one path to a list of all paths (self included)
              with the same contents.

    .. todo:: Start examining the ``while handles:`` block to figure out how to
        minimize thrashing in situations where read-ahead caching is active.
        Compare savings by read-ahead to savings due to eliminating false
        positives as quickly as possible. This is a 2-variable min/max problem.

    .. todo:: Look into possible solutions for pathological cases of thousands
        of files with the same size and same pre-filter results. (File handle
        exhaustion)
    """
    handles, results = [], []

    # Silently ignore files we don't have permission to read.
    hList = []
    for path in paths:
        try:
            hList.append((path, open(path, 'rb'), ''))
        except IOError:
            pass  # TODO: Verbose-mode output here.
    handles.append(hList)

    while handles:
        # Process more blocks.
        more, done = compareChunks(handles.pop(0))

        # Add the results to the top-level lists.
        handles.extend(more)
        results.extend(done)

    # Keep the same API as the others.
    return dict((x[0], x) for x in results)

def groupByContent ( paths ) : handles , results = [ ] , [ ] hList = [ ] for path in paths : try : hList . append ( ( path , open ( path , 'rb' ) , '' ) ) except IOError : pass handles . append ( hList ) while handles : more , done = compareChunks ( handles . pop ( 0 ) ) handles . extend ( more ) results . extend ( done ) return dict ( ( x [ 0 ] , x ) for x in results )

Byte - for - byte comparison on an arbitrary number of files in parallel .


def compareChunks(handles, chunk_size=CHUNK_SIZE):
    """Group a list of file handles based on equality of the next chunk of
    data read from them.

    :param handles: A list of open handles for file-like objects with
        otentially-identical contents.
    :param chunk_size: The amount of data to read from each handle every time
        this function is called.

    :returns: Two lists of lists:

        * Lists to be fed back into this function individually
        * Finished groups of duplicate paths. (including unique files as
          single-file lists)

    :rtype: ``(list, list)``

    .. attention:: File handles will be closed when no longer needed
    .. todo:: Discard chunk contents immediately once they're no longer needed
    """
    chunks = [(path, fh, fh.read(chunk_size)) for path, fh, _ in handles]
    more, done = [], []

    # While there are combinations not yet tried...
    while chunks:
        # Compare the first chunk to all successive chunks
        matches, non_matches = [chunks[0]], []
        for chunk in chunks[1:]:
            if matches[0][2] == chunk[2]:
                matches.append(chunk)
            else:
                non_matches.append(chunk)
        # Check for EOF or obviously unique files
        if len(matches) == 1 or matches[0][2] == "":
            for x in matches:
                x[1].close()
            done.append([x[0] for x in matches])
        else:
            more.append(matches)
        chunks = non_matches

    return more, done

def compareChunks ( handles , chunk_size = CHUNK_SIZE ) : chunks = [ ( path , fh , fh . read ( chunk_size ) ) for path , fh , _ in handles ] more , done = [ ] , [ ] while chunks : matches , non_matches = [ chunks [ 0 ] ] , [ ] for chunk in chunks [ 1 : ] : if matches [ 0 ] [ 2 ] == chunk [ 2 ] : matches . append ( chunk ) else : non_matches . append ( chunk ) if len ( matches ) == 1 or matches [ 0 ] [ 2 ] == "" : for x in matches : x [ 1 ] . close ( ) done . append ( [ x [ 0 ] for x in matches ] ) else : more . append ( matches ) chunks = non_matches return more , done

Group a list of file handles based on equality of the next chunk of data read from them .


def pruneUI(dupeList, mainPos=1, mainLen=1):
    """Display a list of files and prompt for ones to be kept.

    The user may enter ``all`` or one or more numbers separated by spaces
    and/or commas.

    .. note:: It is impossible to accidentally choose to keep none of the
        displayed files.

    :param dupeList: A list duplicate file paths
    :param mainPos: Used to display "set X of Y"
    :param mainLen: Used to display "set X of Y"
    :type dupeList: :class:`~__builtins__.list`
    :type mainPos: :class:`~__builtins__.int`
    :type mainLen: :class:`~__builtins__.int`

    :returns: A list of files to be deleted.
    :rtype: :class:`~__builtins__.int`
    """
    dupeList = sorted(dupeList)
    print
    for pos, val in enumerate(dupeList):
        print "%d) %s" % (pos + 1, val)
    while True:
        choice = raw_input("[%s/%s] Keepers: " % (mainPos, mainLen)).strip()
        if not choice:
            print ("Please enter a space/comma-separated list of numbers or "
                   "'all'.")
            continue
        elif choice.lower() == 'all':
            return []
        try:
            out = [int(x) - 1 for x in choice.replace(',', ' ').split()]
            return [val for pos, val in enumerate(dupeList) if pos not in out]
        except ValueError:
            print("Invalid choice. Please enter a space/comma-separated list"
                  "of numbers or 'all'.")

Display a list of files and prompt for ones to be kept .


def find_dupes(paths, exact=False, ignores=None, min_size=0):
    """High-level code to walk a set of paths and find duplicate groups.

    :param exact: Whether to compare file contents by hash or by reading
                  chunks in parallel.
    :type exact: :class:`~__builtins__.bool`

    :param paths: See :meth:`~fastdupes.getPaths`
    :param ignores: See :meth:`~fastdupes.getPaths`
    :param min_size: See :meth:`~fastdupes.sizeClassifier`

    :returns: A list of groups of files with identical contents
    :rtype: ``[[path, ...], [path, ...]]``
    """
    groups = {'': getPaths(paths, ignores)}
    groups = groupBy(groups, sizeClassifier, 'sizes', min_size=min_size)

    # This serves one of two purposes depending on run-mode:
    # - Minimize number of files checked by full-content comparison (hash)
    # - Minimize chances of file handle exhaustion and limit seeking (exact)
    groups = groupBy(groups, hashClassifier, 'header hashes', limit=HEAD_SIZE)

    if exact:
        groups = groupBy(groups, groupByContent, fun_desc='contents')
    else:
        groups = groupBy(groups, hashClassifier, fun_desc='hashes')

    return groups

def
find_dupes
(
paths
,
exact
=
False
,
ignores
=
,
min_size
=
0
)
:
groups
=
{
''
:
getPaths
(
paths
,
ignores
)
}
groups
=
groupBy
(
groups
,
sizeClassifier
,
'sizes'
,
min_size
=
min_size
)
groups
=
groupBy
(
groups
,
hashClassifier
,
'header hashes'
,
limit
=
HEAD_SIZE
)
if
exact
:
groups
=
groupBy
(
groups
,
groupByContent
,
fun_desc
=
'contents'
)
else
:
groups
=
groupBy
(
groups
,
hashClassifier
,
fun_desc
=
'hashes'
)
return
groups

High - level code to walk a set of paths and find duplicate groups .


def write(self, text, newline=False):
        """Use ``\\r`` to overdraw the current line with the given text.

        This function transparently handles tracking how much overdrawing is
        necessary to erase the previous line when used consistently.

        :param text: The text to be outputted
        :param newline: Whether to start a new line and reset the length count.
        :type text: :class:`~__builtins__.str`
        :type newline: :class:`~__builtins__.bool`
        """
        if not self.isatty:
            self.fobj.write('%s\n' % text)
            return

        msg_len = len(text)
        self.max_len = max(self.max_len, msg_len)

        self.fobj.write("\r%-*s" % (self.max_len, text))
        if newline or not self.isatty:
            self.fobj.write('\n')
            self.max_len = 0

def write ( self , text , newline = False ) : if not self . isatty : self . fobj . write ( '%s\n' % text ) return msg_len = len ( text ) self . max_len = max ( self . max_len , msg_len ) self . fobj . write ( "\r%-*s" % ( self . max_len , text ) ) if newline or not self . isatty : self . fobj . write ( '\n' ) self . max_len = 0

Use \\ r to overdraw the current line with the given text .


def summarize(text, char_limit, sentence_filter=None, debug=False):
    '''
    select sentences in terms of maximum coverage problem

    Args:
      text: text to be summarized (unicode string)
      char_limit: summary length (the number of characters)

    Returns:
      list of extracted sentences

    Reference:
      Hiroya Takamura, Manabu Okumura.
      Text summarization model based on maximum coverage problem and its
      variant. (section 3)
      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.222.6945
    '''
    debug_info = {}

    sents = list(tools.sent_splitter_ja(text))
    words_list = [
        # pulp variables should be utf-8 encoded
        w.encode('utf-8') for s in sents for w in tools.word_segmenter_ja(s)
    ]

    tf = collections.Counter()
    for words in words_list:
        for w in words:
            tf[w] += 1.0

    if sentence_filter is not None:
        valid_indices = [i for i, s in enumerate(sents) if sentence_filter(s)]
        sents = [sents[i] for i in valid_indices]
        words_list = [words_list[i] for i in valid_indices]

    sent_ids = [str(i) for i in range(len(sents))]  # sentence id
    sent_id2len = dict((id_, len(s)) for id_, s in zip(sent_ids, sents))  # c

    word_contain = dict()  # a
    for id_, words in zip(sent_ids, words_list):
        word_contain[id_] = collections.defaultdict(lambda: 0)
        for w in words:
            word_contain[id_][w] = 1

    prob = pulp.LpProblem('summarize', pulp.LpMaximize)

    # x
    sent_vars = pulp.LpVariable.dicts('sents', sent_ids, 0, 1, pulp.LpBinary)
    # z
    word_vars = pulp.LpVariable.dicts('words', tf.keys(), 0, 1, pulp.LpBinary)

    # first, set objective function: sum(w*z)
    prob += pulp.lpSum([tf[w] * word_vars[w] for w in tf])

    # next, add constraints
    # limit summary length: sum(c*x) <= K
    prob += pulp.lpSum(
        [sent_id2len[id_] * sent_vars[id_] for id_ in sent_ids]
    ) <= char_limit, 'lengthRequirement'
    # for each term, sum(a*x) <= z
    for w in tf:
        prob += pulp.lpSum(
            [word_contain[id_][w] * sent_vars[id_] for id_ in sent_ids]
        ) >= word_vars[w], 'z:{}'.format(w)

    prob.solve()
    # print("Status:", pulp.LpStatus[prob.status])

    sent_indices = []
    for v in prob.variables():
        # print v.name, "=", v.varValue
        if v.name.startswith('sents') and v.varValue == 1:
            sent_indices.append(int(v.name.split('_')[-1]))

    return [sents[i] for i in sent_indices], debug_info

def summarize ( text , char_limit , sentence_filter = None , debug = False ) : debug_info = { } sents = list ( tools . sent_splitter_ja ( text ) ) words_list = [ w . encode ( 'utf-8' ) for s in sents for w in tools . word_segmenter_ja ( s ) ] tf = collections . Counter ( ) for words in words_list : for w in words : tf [ w ] += 1.0 if sentence_filter is not None : valid_indices = [ i for i , s in enumerate ( sents ) if sentence_filter ( s ) ] sents = [ sents [ i ] for i in valid_indices ] words_list = [ words_list [ i ] for i in valid_indices ] sent_ids = [ str ( i ) for i in range ( len ( sents ) ) ] sent_id2len = dict ( ( id_ , len ( s ) ) for id_ , s in zip ( sent_ids , sents ) ) word_contain = dict ( ) for id_ , words in zip ( sent_ids , words_list ) : word_contain [ id_ ] = collections . defaultdict ( lambda : 0 ) for w in words : word_contain [ id_ ] [ w ] = 1 prob = pulp . LpProblem ( 'summarize' , pulp . LpMaximize ) sent_vars = pulp . LpVariable . dicts ( 'sents' , sent_ids , 0 , 1 , pulp . LpBinary ) word_vars = pulp . LpVariable . dicts ( 'words' , tf . keys ( ) , 0 , 1 , pulp . LpBinary ) prob += pulp . lpSum ( [ tf [ w ] * word_vars [ w ] for w in tf ] ) prob += pulp . lpSum ( [ sent_id2len [ id_ ] * sent_vars [ id_ ] for id_ in sent_ids ] ) <= char_limit , 'lengthRequirement' for w in tf : prob += pulp . lpSum ( [ word_contain [ id_ ] [ w ] * sent_vars [ id_ ] for id_ in sent_ids ] ) >= word_vars [ w ] , 'z:{}' . format ( w ) prob . solve ( ) sent_indices = [ ] for v in prob . variables ( ) : if v . name . startswith ( 'sents' ) and v . varValue == 1 : sent_indices . append ( int ( v . name . split ( '_' ) [ - 1 ] ) ) return [ sents [ i ] for i in sent_indices ] , debug_info


def lexrank(sentences, continuous=False, sim_threshold=0.1, alpha=0.9,
            use_divrank=False, divrank_alpha=0.25):
    '''
    compute centrality score of sentences.

    Args:
      sentences: [u'こんにちは．', u'私の名前は飯沼です．', ... ]
      continuous: if True, apply continuous LexRank. (see reference)
      sim_threshold: if continuous is False and smilarity is greater or
        equal to sim_threshold, link the sentences.
      alpha: the damping factor of PageRank and DivRank
      divrank: if True, apply DivRank instead of PageRank
      divrank_alpha: strength of self-link [0.0-1.0]
        (it's not the damping factor, see divrank.py)

    Returns: tuple
      (
        {
          # sentence index -> score
          0: 0.003,
          1: 0.002,
          ...
        },
        similarity_matrix
      )

    Reference:
      Günes Erkan and Dragomir R. Radev.
      LexRank: graph-based lexical centrality as salience in text
      summarization. (section 3)
      http://www.cs.cmu.edu/afs/cs/project/jair/pub/volume22/erkan04a-html/erkan04a.html
    '''
    # configure ranker
    ranker_params = {'max_iter': 1000}
    if use_divrank:
        ranker = divrank_scipy
        ranker_params['alpha'] = divrank_alpha
        ranker_params['d'] = alpha
    else:
        ranker = networkx.pagerank_scipy
        ranker_params['alpha'] = alpha

    graph = networkx.DiGraph()

    # sentence -> tf
    sent_tf_list = []
    for sent in sentences:
        words = tools.word_segmenter_ja(sent)
        tf = collections.Counter(words)
        sent_tf_list.append(tf)

    sent_vectorizer = DictVectorizer(sparse=True)
    sent_vecs = sent_vectorizer.fit_transform(sent_tf_list)

    # compute similarities between senteces
    sim_mat = 1 - pairwise_distances(sent_vecs, sent_vecs, metric='cosine')

    if continuous:
        linked_rows, linked_cols = numpy.where(sim_mat > 0)
    else:
        linked_rows, linked_cols = numpy.where(sim_mat >= sim_threshold)

    # create similarity graph
    graph.add_nodes_from(range(sent_vecs.shape[0]))
    for i, j in zip(linked_rows, linked_cols):
        if i == j:
            continue
        weight = sim_mat[i,j] if continuous else 1.0
        graph.add_edge(i, j, {'weight': weight})

    scores = ranker(graph, **ranker_params)
    return scores, sim_mat

def lexrank ( sentences , continuous = False , sim_threshold = 0.1 , alpha = 0.9 , use_divrank = False , divrank_alpha = 0.25 ) : ranker_params = { 'max_iter' : 1000 } if use_divrank : ranker = divrank_scipy ranker_params [ 'alpha' ] = divrank_alpha ranker_params [ 'd' ] = alpha else : ranker = networkx . pagerank_scipy ranker_params [ 'alpha' ] = alpha graph = networkx . DiGraph ( ) sent_tf_list = [ ] for sent in sentences : words = tools . word_segmenter_ja ( sent ) tf = collections . Counter ( words ) sent_tf_list . append ( tf ) sent_vectorizer = DictVectorizer ( sparse = True ) sent_vecs = sent_vectorizer . fit_transform ( sent_tf_list ) sim_mat = 1 - pairwise_distances ( sent_vecs , sent_vecs , metric = 'cosine' ) if continuous : linked_rows , linked_cols = numpy . where ( sim_mat > 0 ) else : linked_rows , linked_cols = numpy . where ( sim_mat >= sim_threshold ) graph . add_nodes_from ( range ( sent_vecs . shape [ 0 ] ) ) for i , j in zip ( linked_rows , linked_cols ) : if i == j : continue weight = sim_mat [ i , j ] if continuous else 1.0 graph . add_edge ( i , j , { 'weight' : weight } ) scores = ranker ( graph , ** ranker_params ) return scores , sim_mat


def get_summarizer(self, name):
        '''
        import summarizers on-demand
        '''
        if name in self.summarizers:
            pass
        elif name == 'lexrank':
            from . import lexrank
            self.summarizers[name] = lexrank.summarize
        elif name == 'mcp':
            from . import mcp_summ
            self.summarizers[name] = mcp_summ.summarize

        return self.summarizers[name]

def get_summarizer ( self , name ) : if name in self . summarizers : pass elif name == 'lexrank' : from . import lexrank self . summarizers [ name ] = lexrank . summarize elif name == 'mcp' : from . import mcp_summ self . summarizers [ name ] = mcp_summ . summarize return self . summarizers [ name ]


def code_mapping(level, msg, default=99):
    """Return an error code between 0 and 99."""
    try:
        return code_mappings_by_level[level][msg]
    except KeyError:
        pass
    # Following assumes any variable messages take the format
    # of 'Fixed text "variable text".' only:
    # e.g. 'Unknown directive type "req".'
    # ---> 'Unknown directive type'
    # e.g. 'Unknown interpreted text role "need".'
    # ---> 'Unknown interpreted text role'
    if msg.count('"') == 2 and ' "' in msg and msg.endswith('".'):
        txt = msg[: msg.index(' "')]
        return code_mappings_by_level[level].get(txt, default)
    return default

def
code_mapping
(
level
,
msg
,
default
=
99
)
:
try
:
return
code_mappings_by_level
[
level
]
[
msg
]
except
KeyError
:
pass
if
msg
.
count
(
'"'
)
==
2
and
' "'
in
msg
and
msg
.
endswith
(
'".'
)
:
txt
=
msg
[
:
msg
.
index
(
' "'
)
]
return
code_mappings_by_level
[
level
]
.
get
(
txt
,
default
)
return
default


def is_public(self):
        """Return True iff this function should be considered public."""
        if self.all is not None:
            return self.name in self.all
        else:
            return not self.name.startswith("_")

def is_public ( self ) : if self . all is not None : return self . name in self . all else : return not self . name . startswith ( "_" )

Return True iff this function should be considered public .


def is_public(self):
        """Return True iff this method should be considered public."""
        # Check if we are a setter/deleter method, and mark as private if so.
        for decorator in self.decorators:
            # Given 'foo', match 'foo.bar' but not 'foobar' or 'sfoo'
            if re.compile(r"^{}\.".format(self.name)).match(decorator.name):
                return False
        name_is_public = (
            not self.name.startswith("_")
            or self.name in VARIADIC_MAGIC_METHODS
            or self.is_magic
        )
        return self.parent.is_public and name_is_public

def is_public ( self ) : for decorator in self . decorators : if re . compile ( r"^{}\." . format ( self . name ) ) . match ( decorator . name ) : return False name_is_public = ( not self . name . startswith ( "_" ) or self . name in VARIADIC_MAGIC_METHODS or self . is_magic ) return self . parent . is_public and name_is_public

Return True iff this method should be considered public .


def is_public(self):
        """Return True iff this class should be considered public."""
        return (
            not self.name.startswith("_")
            and self.parent.is_class
            and self.parent.is_public
        )

def is_public ( self ) : return ( not self . name . startswith ( "_" ) and self . parent . is_class and self . parent . is_public )

Return True iff this class should be considered public .


def parse(self, filelike, filename):
        """Parse the given file-like object and return its Module object."""
        self.log = log
        self.source = filelike.readlines()
        src = "".join(self.source)
        # This may raise a SyntaxError:
        compile(src, filename, "exec")
        self.stream = TokenStream(StringIO(src))
        self.filename = filename
        self.all = None
        self.future_imports = set()
        self._accumulated_decorators = []
        return self.parse_module()

def parse ( self , filelike , filename ) : self . log = log self . source = filelike . readlines ( ) src = "" . join ( self . source ) compile ( src , filename , "exec" ) self . stream = TokenStream ( StringIO ( src ) ) self . filename = filename self . all = None self . future_imports = set ( ) self . _accumulated_decorators = [ ] return self . parse_module ( )

Parse the given file - like object and return its Module object .


def consume(self, kind):
        """Consume one token and verify it is of the expected kind."""
        next_token = self.stream.move()
        assert next_token.kind == kind

def consume ( self , kind ) : next_token = self . stream . move ( ) assert next_token . kind == kind

Consume one token and verify it is of the expected kind .


def leapfrog(self, kind, value=None):
        """Skip tokens in the stream until a certain token kind is reached.

        If `value` is specified, tokens whose values are different will also
        be skipped.
        """
        while self.current is not None:
            if self.current.kind == kind and (
                value is None or self.current.value == value
            ):
                self.consume(kind)
                return
            self.stream.move()

def leapfrog ( self , kind , value = None ) : while self . current is not None : if self . current . kind == kind and ( value is None or self . current . value == value ) : self . consume ( kind ) return self . stream . move ( )

Skip tokens in the stream until a certain token kind is reached .


def parse_docstring(self):
        """Parse a single docstring and return its value."""
        self.log.debug(
            "parsing docstring, token is %r (%s)", self.current.kind, self.current.value
        )
        while self.current.kind in (tk.COMMENT, tk.NEWLINE, tk.NL):
            self.stream.move()
            self.log.debug(
                "parsing docstring, token is %r (%s)",
                self.current.kind,
                self.current.value,
            )
        if self.current.kind == tk.STRING:
            docstring = self.current.value
            self.stream.move()
            return docstring
        return None

def
parse_docstring
(
self
)
:
self
.
log
.
debug
(
"parsing docstring, token is %r (%s)"
,
self
.
current
.
kind
,
self
.
current
.
value
)
while
self
.
current
.
kind
in
(
tk
.
COMMENT
,
tk
.
NEWLINE
,
tk
.
NL
)
:
self
.
stream
.
move
(
)
self
.
log
.
debug
(
"parsing docstring, token is %r (%s)"
,
self
.
current
.
kind
,
self
.
current
.
value
,
)
if
self
.
current
.
kind
==
tk
.
STRING
:
docstring
=
self
.
current
.
value
self
.
stream
.
move
(
)
return
docstring
return

Parse a single docstring and return its value .


def parse_definitions(self, class_, all=False):
        """Parse multiple definitions and yield them."""
        while self.current is not None:
            self.log.debug(
                "parsing definition list, current token is %r (%s)",
                self.current.kind,
                self.current.value,
            )
            self.log.debug("got_newline: %s", self.stream.got_logical_newline)
            if all and self.current.value == "__all__":
                self.parse_all()
            elif (
                self.current.kind == tk.OP
                and self.current.value == "@"
                and self.stream.got_logical_newline
            ):
                self.consume(tk.OP)
                self.parse_decorators()
            elif self.current.value in ["def", "class"]:
                yield self.parse_definition(class_._nest(self.current.value))
            elif self.current.kind == tk.INDENT:
                self.consume(tk.INDENT)
                for definition in self.parse_definitions(class_):
                    yield definition
            elif self.current.kind == tk.DEDENT:
                self.consume(tk.DEDENT)
                return
            elif self.current.value == "from":
                self.parse_from_import_statement()
            else:
                self.stream.move()

Parse multiple definitions and yield them .


def parse_from_import_statement(self):
        """Parse a 'from x import y' statement.

        The purpose is to find __future__ statements.
        """
        self.log.debug("parsing from/import statement.")
        is_future_import = self._parse_from_import_source()
        self._parse_from_import_names(is_future_import)

def
parse_from_import_statement
(
self
)
:
self
.
log
.
debug
(
"parsing from/import statement."
)
is_future_import
=
self
.
_parse_from_import_source
(
)
self
.
_parse_from_import_names
(
is_future_import
)


def _parse_from_import_names(self, is_future_import):
        """Parse the 'y' part in a 'from x import y' statement."""
        if self.current.value == "(":
            self.consume(tk.OP)
            expected_end_kinds = (tk.OP,)
        else:
            expected_end_kinds = (tk.NEWLINE, tk.ENDMARKER)
        while self.current.kind not in expected_end_kinds and not (
            self.current.kind == tk.OP and self.current.value == ";"
        ):
            if self.current.kind != tk.NAME:
                self.stream.move()
                continue
            self.log.debug(
                "parsing import, token is %r (%s)",
                self.current.kind,
                self.current.value,
            )
            if is_future_import:
                self.log.debug("found future import: %s", self.current.value)
                self.future_imports.add(self.current.value)
            self.consume(tk.NAME)
            self.log.debug(
                "parsing import, token is %r (%s)",
                self.current.kind,
                self.current.value,
            )
            if self.current.kind == tk.NAME and self.current.value == "as":
                self.consume(tk.NAME)  # as
                if self.current.kind == tk.NAME:
                    self.consume(tk.NAME)  # new name, irrelevant
            if self.current.value == ",":
                self.consume(tk.OP)
            self.log.debug(
                "parsing import, token is %r (%s)",
                self.current.kind,
                self.current.value,
            )

def
_parse_from_import_names
(
self
,
is_future_import
)
:
if
self
.
current
.
value
==
"("
:
self
.
consume
(
tk
.
OP
)
expected_end_kinds
=
(
tk
.
OP
,
)
else
:
expected_end_kinds
=
(
tk
.
NEWLINE
,
tk
.
ENDMARKER
)
while
self
.
current
.
kind
not
in
expected_end_kinds
and
not
(
self
.
current
.
kind
==
tk
.
OP
and
self
.
current
.
value
==
";"
)
:
if
self
.
current
.
kind
!=
tk
.
NAME
:
self
.
stream
.
move
(
)
continue
self
.
log
.
debug
(
"parsing import, token is %r (%s)"
,
self
.
current
.
kind
,
self
.
current
.
value
,
)
if
is_future_import
:
self
.
log
.
debug
(
"found future import: %s"
,
self
.
current
.
value
)
self
.
future_imports
.
add
(
self
.
current
.
value
)
self
.
consume
(
tk
.
NAME
)
self
.
log
.
debug
(
"parsing import, token is %r (%s)"
,
self
.
current
.
kind
,
self
.
current
.
value
,
)
if
self
.
current
.
kind
==
tk
.
NAME
and
self
.
current
.
value
==
"as"
:
self
.
consume
(
tk
.
NAME
)
if
self
.
current
.
kind
==
tk
.
NAME
:
self
.
consume
(
tk
.
NAME
)
if
self
.
current
.
value
==
","
:
self
.
consume
(
tk
.
OP
)
self
.
log
.
debug
(
"parsing import, token is %r (%s)"
,
self
.
current
.
kind
,
self
.
current
.
value
,
)

Parse the y part in a from x import y statement .


def run(self):
        """Use docutils to check docstrings are valid RST."""
        # Is there any reason not to call load_source here?
        if self.err is not None:
            assert self.source is None
            msg = "%s%03i %s" % (
                rst_prefix,
                rst_fail_load,
                "Failed to load file: %s" % self.err,
            )
            yield 0, 0, msg, type(self)
            module = []
        try:
            module = parse(StringIO(self.source), self.filename)
        except SyntaxError as err:
            msg = "%s%03i %s" % (
                rst_prefix,
                rst_fail_parse,
                "Failed to parse file: %s" % err,
            )
            yield 0, 0, msg, type(self)
            module = []
        except AllError:
            msg = "%s%03i %s" % (
                rst_prefix,
                rst_fail_all,
                "Failed to parse __all__ entry.",
            )
            yield 0, 0, msg, type(self)
            module = []
        for definition in module:
            if not definition.docstring:
                # People can use flake8-docstrings to report missing
                # docstrings
                continue
            try:
                # Note we use the PEP257 trim algorithm to remove the
                # leading whitespace from each line - this avoids false
                # positive severe error "Unexpected section title."
                unindented = trim(dequote_docstring(definition.docstring))
                # Off load RST validation to reStructuredText-lint
                # which calls docutils internally.
                # TODO: Should we pass the Python filename as filepath?
                rst_errors = list(rst_lint.lint(unindented))
            except Exception as err:
                # e.g. UnicodeDecodeError
                msg = "%s%03i %s" % (
                    rst_prefix,
                    rst_fail_lint,
                    "Failed to lint docstring: %s - %s" % (definition.name, err),
                )
                yield definition.start, 0, msg, type(self)
                continue
            for rst_error in rst_errors:
                # TODO - make this a configuration option?
                if rst_error.level <= 1:
                    continue
                # Levels:
                #
                # 0 - debug   --> we don't receive these
                # 1 - info    --> RST1## codes
                # 2 - warning --> RST2## codes
                # 3 - error   --> RST3## codes
                # 4 - severe  --> RST4## codes
                #
                # Map the string to a unique code:
                msg = rst_error.message.split("\n", 1)[0]
                code = code_mapping(rst_error.level, msg)
                assert code < 100, code
                code += 100 * rst_error.level
                msg = "%s%03i %s" % (rst_prefix, code, msg)

                # This will return the line number by combining the
                # start of the docstring with the offet within it.
                # We don't know the column number, leaving as zero.
                yield definition.start + rst_error.line, 0, msg, type(self)

def
run
(
self
)
:
if
self
.
err
is
not
:
assert
self
.
source
is
msg
=
"%s%03i %s"
%
(
rst_prefix
,
rst_fail_load
,
"Failed to load file: %s"
%
self
.
err
,
)
yield
0
,
0
,
msg
,
type
(
self
)
module
=
[
]
try
:
module
=
parse
(
StringIO
(
self
.
source
)
,
self
.
filename
)
except
SyntaxError
as
err
:
msg
=
"%s%03i %s"
%
(
rst_prefix
,
rst_fail_parse
,
"Failed to parse file: %s"
%
err
,
)
yield
0
,
0
,
msg
,
type
(
self
)
module
=
[
]
except
AllError
:
msg
=
"%s%03i %s"
%
(
rst_prefix
,
rst_fail_all
,
"Failed to parse __all__ entry."
,
)
yield
0
,
0
,
msg
,
type
(
self
)
module
=
[
]
for
definition
in
module
:
if
not
definition
.
docstring
:
continue
try
:
unindented
=
trim
(
dequote_docstring
(
definition
.
docstring
)
)
rst_errors
=
list
(
rst_lint
.
lint
(
unindented
)
)
except
Exception
as
err
:
msg
=
"%s%03i %s"
%
(
rst_prefix
,
rst_fail_lint
,
"Failed to lint docstring: %s - %s"
%
(
definition
.
name
,
err
)
,
)
yield
definition
.
start
,
0
,
msg
,
type
(
self
)
continue
for
rst_error
in
rst_errors
:
if
rst_error
.
level
<=
1
:
continue
msg
=
rst_error
.
message
.
split
(
"\n"
,
1
)
[
0
]
code
=
code_mapping
(
rst_error
.
level
,
msg
)
assert
code
<
100
,
code
code
+=
100
*
rst_error
.
level
msg
=
"%s%03i %s"
%
(
rst_prefix
,
code
,
msg
)
yield
definition
.
start
+
rst_error
.
line
,
0
,
msg
,
type
(
self
)

Use docutils to check docstrings are valid RST .


def load_source(self):
        """Load the source for the specified file."""
        if self.filename in self.STDIN_NAMES:
            self.filename = "stdin"
            if sys.version_info[0] < 3:
                self.source = sys.stdin.read()
            else:
                self.source = TextIOWrapper(sys.stdin.buffer, errors="ignore").read()
        else:
            # Could be a Python 2.7 StringIO with no context manager, sigh.
            # with tokenize_open(self.filename) as fd:
            #     self.source = fd.read()
            handle = tokenize_open(self.filename)
            self.source = handle.read()
            handle.close()

def load_source ( self ) : if self . filename in self . STDIN_NAMES : self . filename = "stdin" if sys . version_info [ 0 ] < 3 : self . source = sys . stdin . read ( ) else : self . source = TextIOWrapper ( sys . stdin . buffer , errors = "ignore" ) . read ( ) else : handle = tokenize_open ( self . filename ) self . source = handle . read ( ) handle . close ( )

Load the source for the specified file .


def _darkest(self):

        """ Returns the darkest swatch.

        Knowing the contract between a light and a dark swatch
        can help us decide how to display readable typography.

        """

        rgb, n = (1.0, 1.0, 1.0), 3.0
        for r,g,b in self:
            if r+g+b < n:
                rgb, n = (r,g,b), r+g+b

        return rgb

def _darkest ( self ) : rgb , n = ( 1.0 , 1.0 , 1.0 ) , 3.0 for r , g , b in self : if r + g + b < n : rgb , n = ( r , g , b ) , r + g + b return rgb

Returns the darkest swatch . Knowing the contract between a light and a dark swatch can help us decide how to display readable typography .


def parse_theme(self, xml):

        """ Parses a theme from XML returned by Kuler.

        Gets the theme's id, label and swatches.
        All of the swatches are converted to RGB.
        If we have a full description for a theme id in cache,
        parse that to get tags associated with the theme.

        """

        kt = KulerTheme()
        kt.author = xml.getElementsByTagName("author")[0]
        kt.author = kt.author.childNodes[1].childNodes[0].nodeValue
        kt.id = int(self.parse_tag(xml, "id"))
        kt.label = self.parse_tag(xml, "label")
        mode = self.parse_tag(xml, "mode")

        for swatch in xml.getElementsByTagName("swatch"):

            c1 = float(self.parse_tag(swatch, "c1"))
            c2 = float(self.parse_tag(swatch, "c2"))
            c3 = float(self.parse_tag(swatch, "c3"))
            c4 = float(self.parse_tag(swatch, "c4"))

            if mode == "rgb":
                kt.append((c1,c2,c3))
            if mode == "cmyk":
                kt.append(cmyk_to_rgb(c1,c2,c3,c4))
            if mode == "hsv":
                kt.append(colorsys.hsv_to_rgb(c1,c2,c3))
            if mode == "hex":
                kt.append(hex_to_rgb(c1))
            if mode == "lab":
                kt.append(lab_to_rgb(c1,c2,c3))

        # If we have the full theme in cache,
        # parse tags from it.
        if self._cache.exists(self.id_string + str(kt.id)):
            xml = self._cache.read(self.id_string + str(kt.id))
            xml = minidom.parseString(xml)
        for tags in xml.getElementsByTagName("tag"):
            tags = self.parse_tag(tags, "label")
            tags = tags.split(" ")
            kt.tags.extend(tags)

        return kt

def
parse_theme
(
self
,
xml
)
:
kt
=
KulerTheme
(
)
kt
.
author
=
xml
.
getElementsByTagName
(
"author"
)
[
0
]
kt
.
author
=
kt
.
author
.
childNodes
[
1
]
.
childNodes
[
0
]
.
nodeValue
kt
.
id
=
int
(
self
.
parse_tag
(
xml
,
"id"
)
)
kt
.
label
=
self
.
parse_tag
(
xml
,
"label"
)
mode
=
self
.
parse_tag
(
xml
,
"mode"
)
for
swatch
in
xml
.
getElementsByTagName
(
"swatch"
)
:
c1
=
float
(
self
.
parse_tag
(
swatch
,
"c1"
)
)
c2
=
float
(
self
.
parse_tag
(
swatch
,
"c2"
)
)
c3
=
float
(
self
.
parse_tag
(
swatch
,
"c3"
)
)
c4
=
float
(
self
.
parse_tag
(
swatch
,
"c4"
)
)
if
mode
==
"rgb"
:
kt
.
append
(
(
c1
,
c2
,
c3
)
)
if
mode
==
"cmyk"
:
kt
.
append
(
cmyk_to_rgb
(
c1
,
c2
,
c3
,
c4
)
)
if
mode
==
"hsv"
:
kt
.
append
(
colorsys
.
hsv_to_rgb
(
c1
,
c2
,
c3
)
)
if
mode
==
"hex"
:
kt
.
append
(
hex_to_rgb
(
c1
)
)
if
mode
==
"lab"
:
kt
.
append
(
lab_to_rgb
(
c1
,
c2
,
c3
)
)
if
self
.
_cache
.
exists
(
self
.
id_string
+
str
(
kt
.
id
)
)
:
xml
=
self
.
_cache
.
read
(
self
.
id_string
+
str
(
kt
.
id
)
)
xml
=
minidom
.
parseString
(
xml
)
for
tags
in
xml
.
getElementsByTagName
(
"tag"
)
:
tags
=
self
.
parse_tag
(
tags
,
"label"
)
tags
=
tags
.
split
(
" "
)
kt
.
tags
.
extend
(
tags
)
return
kt

Parses a theme from XML returned by Kuler . Gets the theme s id label and swatches . All of the swatches are converted to RGB . If we have a full description for a theme id in cache parse that to get tags associated with the theme .


def listener(self, sock, *args):
        '''Asynchronous connection listener. Starts a handler for each connection.'''
        conn, addr = sock.accept()
        f = conn.makefile(conn)
        self.shell = ShoebotCmd(self.bot, stdin=f, stdout=f, intro=INTRO)

        print(_("Connected"))
        GObject.io_add_watch(conn, GObject.IO_IN, self.handler)
        if self.shell.intro:
            self.shell.stdout.write(str(self.shell.intro)+"\n")
            self.shell.stdout.flush()
        return True

def listener ( self , sock , * args ) : conn , addr = sock . accept ( ) f = conn . makefile ( conn ) self . shell = ShoebotCmd ( self . bot , stdin = f , stdout = f , intro = INTRO ) print ( _ ( "Connected" ) ) GObject . io_add_watch ( conn , GObject . IO_IN , self . handler ) if self . shell . intro : self . shell . stdout . write ( str ( self . shell . intro ) + "\n" ) self . shell . stdout . flush ( ) return True

Asynchronous connection listener . Starts a handler for each connection .


def handler(self, conn, *args):
        '''
        Asynchronous connection handler. Processes each line from the socket.
        '''
        # lines from cmd.Cmd
        self.shell.stdout.write(self.shell.prompt)
        line = self.shell.stdin.readline()
        if not len(line):
            line = 'EOF'
            return False
        else:
            line = line.rstrip('\r\n')
            line = self.shell.precmd(line)
            stop = self.shell.onecmd(line)
            stop = self.shell.postcmd(stop, line)
            self.shell.stdout.flush()
            self.shell.postloop()
            # end lines from cmd.Cmd
            if stop:
                self.shell = None
                conn.close()
            return not stop

def handler ( self , conn , * args ) : self . shell . stdout . write ( self . shell . prompt ) line = self . shell . stdin . readline ( ) if not len ( line ) : line = 'EOF' return False else : line = line . rstrip ( '\r\n' ) line = self . shell . precmd ( line ) stop = self . shell . onecmd ( line ) stop = self . shell . postcmd ( stop , line ) self . shell . stdout . flush ( ) self . shell . postloop ( ) if stop : self . shell = None conn . close ( ) return not stop

Asynchronous connection handler . Processes each line from the socket .


def trusted_cmd(f):
    """
    Trusted commands cannot be run remotely

    :param f:
    :return:
    """
    def run_cmd(self, line):
        if self.trusted:
            f(self, line)
        else:
            print("Sorry cannot do %s here." % f.__name__[3:])

    global trusted_cmds
    trusted_cmds.add(f.__name__)
    run_cmd.__doc__ = f.__doc__
    return run_cmd

def
trusted_cmd
(
f
)
:
def
run_cmd
(
self
,
line
)
:
if
self
.
trusted
:
f
(
self
,
line
)
else
:
print
(
"Sorry cannot do %s here."
%
f
.
__name__
[
3
:
]
)
global
trusted_cmds
trusted_cmds
.
add
(
f
.
__name__
)
run_cmd
.
__doc__
=
f
.
__doc__
return
run_cmd


def do_escape_nl(self, arg):
        """
        Escape newlines in any responses
        """
        if arg.lower() == 'off':
            self.escape_nl = False
        else:
            self.escape_nl = True

def do_escape_nl ( self , arg ) : if arg . lower ( ) == 'off' : self . escape_nl = False else : self . escape_nl = True


def do_restart(self, line):
        """
        Attempt to restart the bot.
        """
        self.bot._frame = 0
        self.bot._namespace.clear()
        self.bot._namespace.update(self.bot._initial_namespace)

def do_restart ( self , line ) : self . bot . _frame = 0 self . bot . _namespace . clear ( ) self . bot . _namespace . update ( self . bot . _initial_namespace )


def do_play(self, line):
        """
        Resume playback if bot is paused
        """
        if self.pause_speed is None:
            self.bot._speed = self.pause_speed
            self.pause_speed = None
        self.print_response("Play")

def do_play ( self , line ) : if self . pause_speed is None : self . bot . _speed = self . pause_speed self . pause_speed = None self . print_response ( "Play" )


def do_vars(self, line):
        """
        List bot variables and values
        """
        if self.bot._vars:
            max_name_len = max([len(name) for name in self.bot._vars])
            for i, (name, v) in enumerate(self.bot._vars.items()):
                keep = i < len(self.bot._vars) - 1
                self.print_response("%s = %s" % (name.ljust(max_name_len), v.value), keep=keep)
        else:
            self.print_response("No vars")

def
do_vars
(
self
,
line
)
:
if
self
.
bot
.
_vars
:
max_name_len
=
max
(
[
len
(
name
)
for
name
in
self
.
bot
.
_vars
]
)
for
i
,
(
name
,
v
)
in
enumerate
(
self
.
bot
.
_vars
.
items
(
)
)
:
keep
=
i
<
len
(
self
.
bot
.
_vars
)
1
self
.
print_response
(
"%s = %s"
%
(
name
.
ljust
(
max_name_len
)
,
v
.
value
)
,
keep
=
keep
)
else
:
self
.
print_response
(
"No vars"
)


def do_fullscreen(self, line):
        """
        Make the current window fullscreen
        """
        self.bot.canvas.sink.trigger_fullscreen_action(True)
        print(self.response_prompt, file=self.stdout)

def do_fullscreen ( self , line ) : self . bot . canvas . sink . trigger_fullscreen_action ( True ) print ( self . response_prompt , file = self . stdout )


def do_windowed(self, line):
        """
        Un-fullscreen the current window
        """
        self.bot.canvas.sink.trigger_fullscreen_action(False)
        print(self.response_prompt, file=self.stdout)

def do_windowed ( self , line ) : self . bot . canvas . sink . trigger_fullscreen_action ( False ) print ( self . response_prompt , file = self . stdout )


def do_help(self, arg):
        """
        Show help on all commands.
        """
        print(self.response_prompt, file=self.stdout)
        return cmd.Cmd.do_help(self, arg)

def do_help ( self , arg ) : print ( self . response_prompt , file = self . stdout ) return cmd . Cmd . do_help ( self , arg )


def do_set(self, line):
        """
        Set a variable.
        """
        try:
            name, value = [part.strip() for part in line.split('=')]
            if name not in self.bot._vars:
                self.print_response('No such variable %s enter vars to see available vars' % name)
                return
            variable = self.bot._vars[name]
            variable.value = variable.sanitize(value.strip(';'))

            success, msg = self.bot.canvas.sink.var_changed(name, variable.value)
            if success:
                print('{}={}'.format(name, variable.value), file=self.stdout)
            else:
                print('{}\n'.format(msg), file=self.stdout)
        except Exception as e:
            print('Invalid Syntax.', e)
            return


def precmd(self, line):
        """
        Allow commands to have a last parameter of 'cookie=somevalue'

        TODO somevalue will be prepended onto any output lines so
        that editors can distinguish output from certain kinds
        of events they have sent.

        :param line:
        :return:
        """
        args = shlex.split(line or "")
        if args and 'cookie=' in args[-1]:
            cookie_index = line.index('cookie=')
            cookie = line[cookie_index + 7:]
            line = line[:cookie_index].strip()
            self.cookie = cookie
        if line.startswith('#'):
            return ''
        elif '=' in line:
            # allow  somevar=somevalue

            # first check if we really mean a command
            cmdname = line.partition(" ")[0]
            if hasattr(self, "do_%s" % cmdname):
                return line

            if not line.startswith("set "):
                return "set " + line
            else:
                return line
        if len(args) and args[0] in self.shortcuts:
            return "%s %s" % (self.shortcuts[args[0]], " ".join(args[1:]))
        else:
            return line

def
precmd
(
self
,
line
)
:
args
=
shlex
.
split
(
line
or
""
)
if
args
and
'cookie='
in
args
[
1
]
:
cookie_index
=
line
.
index
(
'cookie='
)
cookie
=
line
[
cookie_index
+
7
:
]
line
=
line
[
:
cookie_index
]
.
strip
(
)
self
.
cookie
=
cookie
if
line
.
startswith
(
'#'
)
:
return
''
elif
'='
in
line
:
cmdname
=
line
.
partition
(
" "
)
[
0
]
if
hasattr
(
self
,
"do_%s"
%
cmdname
)
:
return
line
if
not
line
.
startswith
(
"set "
)
:
return
"set "
+
line
else
:
return
line
if
len
(
args
)
and
args
[
0
]
in
self
.
shortcuts
:
return
"%s %s"
%
(
self
.
shortcuts
[
args
[
0
]
]
,
" "
.
join
(
args
[
1
:
]
)
)
else
:
return
line

Allow commands to have a last parameter of cookie = somevalue


def drawdaisy(x, y, color='#fefefe'):
    """
    Draw a daisy at x, y
    """
    # save location, size etc
    _ctx.push()

    # save fill and stroke
    _fill =_ctx.fill()
    _stroke = _ctx.stroke()

    sc = (1.0 / _ctx.HEIGHT) * float(y * 0.5) * 4.0

    # draw stalk
    _ctx.strokewidth(sc * 2.0)
    _ctx.stroke('#3B240B')

    _ctx.line(x + (sin(x * 0.1) * 10.0), y + 80, x + sin(_ctx.FRAME * 0.1), y)

    # draw flower
    _ctx.translate(-20, 0)
    _ctx.scale(sc)

    # draw petals
    _ctx.fill(color)
    _ctx.nostroke()
    for angle in xrange(0, 360, 45):
        _ctx.rotate(degrees=45)
        _ctx.rect(x, y, 40, 8, 1)

    # draw centre
    _ctx.fill('#F7FE2E')
    _ctx.ellipse(x + 15, y, 10, 10)

    # restore fill and stroke
    _ctx.fill(_fill)
    _ctx.stroke(_stroke)

    # restore location, size etc
    _ctx.pop()

def drawdaisy ( x , y , color = '#fefefe' ) : _ctx . push ( ) _fill = _ctx . fill ( ) _stroke = _ctx . stroke ( ) sc = ( 1.0 / _ctx . HEIGHT ) * float ( y * 0.5 ) * 4.0 _ctx . strokewidth ( sc * 2.0 ) _ctx . stroke ( '#3B240B' ) _ctx . line ( x + ( sin ( x * 0.1 ) * 10.0 ) , y + 80 , x + sin ( _ctx . FRAME * 0.1 ) , y ) _ctx . translate ( - 20 , 0 ) _ctx . scale ( sc ) _ctx . fill ( color ) _ctx . nostroke ( ) for angle in xrange ( 0 , 360 , 45 ) : _ctx . rotate ( degrees = 45 ) _ctx . rect ( x , y , 40 , 8 , 1 ) _ctx . fill ( '#F7FE2E' ) _ctx . ellipse ( x + 15 , y , 10 , 10 ) _ctx . fill ( _fill ) _ctx . stroke ( _stroke ) _ctx . pop ( )


def get_source(self, doc):
        """
        Grab contents of 'doc' and return it

        :param doc: The active document
        :return:
        """
        start_iter = doc.get_start_iter()
        end_iter = doc.get_end_iter()
        source = doc.get_text(start_iter, end_iter, False)
        return source

def get_source ( self , doc ) : start_iter = doc . get_start_iter ( ) end_iter = doc . get_end_iter ( ) source = doc . get_text ( start_iter , end_iter , False ) return source


def loadGrammar(self, grammar, searchpaths=None):
        """load context-free grammar"""
        self.grammar = self._load(grammar, searchpaths=searchpaths)
        self.refs = {}
        for ref in self.grammar.getElementsByTagName("ref"):
            self.refs[ref.attributes["id"].value] = ref

def loadGrammar ( self , grammar , searchpaths = None ) : self . grammar = self . _load ( grammar , searchpaths = searchpaths ) self . refs = { } for ref in self . grammar . getElementsByTagName ( "ref" ) : self . refs [ ref . attributes [ "id" ] . value ] = ref


def refresh(self):
        """reset output buffer, re-parse entire source file, and return output

        Since parsing involves a good deal of randomness, this is an
        easy way to get new output without having to reload a grammar file
        each time.
        """
        self.reset()
        self.parse(self.source)
        return self.output()

def refresh ( self ) : self . reset ( ) self . parse ( self . source ) return self . output ( )

reset output buffer re - parse entire source file and return output Since parsing involves a good deal of randomness this is an easy way to get new output without having to reload a grammar file each time .


def randomChildElement(self, node):
        """choose a random child element of a node

        This is a utility method used by do_xref and do_choice.
        """
        choices = [e for e in node.childNodes
                   if e.nodeType == e.ELEMENT_NODE]
        chosen = random.choice(choices)
        if _debug:
            sys.stderr.write('%s available choices: %s\n' % \
                (len(choices), [e.toxml() for e in choices]))
            sys.stderr.write('Chosen: %s\n' % chosen.toxml())
        return chosen

def
randomChildElement
(
self
,
node
)
:
choices
=
[
e
for
e
in
node
.
childNodes
if
e
.
nodeType
==
e
.
ELEMENT_NODE
]
chosen
=
random
.
choice
(
choices
)
if
_debug
:
sys
.
stderr
.
write
(
'%s available choices: %s\n'
%
(
len
(
choices
)
,
[
e
.
toxml
(
)
for
e
in
choices
]
)
)
sys
.
stderr
.
write
(
'Chosen: %s\n'
%
chosen
.
toxml
(
)
)
return
chosen

choose a random child element of a node This is a utility method used by do_xref and do_choice .


def replace_entities(ustring, placeholder=" "):

    """Replaces HTML special characters by readable characters.

    As taken from Leif K-Brooks algorithm on:
    http://groups-beta.google.com/group/comp.lang.python

    """

    def _repl_func(match):
        try:
            if match.group(1): # Numeric character reference
                return unichr( int(match.group(2)) )
            else:
                try: return cp1252[ unichr(int(match.group(3))) ].strip()
                except: return unichr( name2codepoint[match.group(3)] )
        except:
            return placeholder

    # Force to Unicode.
    if not isinstance(ustring, unicode):
        ustring = UnicodeDammit(ustring).unicode

    # Don't want some weird unicode character here
    # that truncate_spaces() doesn't know of:
    ustring = ustring.replace(" ", " ")

    # The ^> makes sure nothing inside a tag (i.e. href with query arguments) gets processed.
    _entity_re = re.compile(r'&(?:(#)(\d+)|([^;^> ]+));')
    return _entity_re.sub(_repl_func, ustring)

def
replace_entities
(
ustring
,
placeholder
=
" "
)
:
def
_repl_func
(
match
)
:
try
:
if
match
.
group
(
1
)
:
return
unichr
(
int
(
match
.
group
(
2
)
)
)
else
:
try
:
return
cp1252
[
unichr
(
int
(
match
.
group
(
3
)
)
)
]
.
strip
(
)
except
:
return
unichr
(
name2codepoint
[
match
.
group
(
3
)
]
)
except
:
return
placeholder
if
not
isinstance
(
ustring
,
unicode
)
:
ustring
=
UnicodeDammit
(
ustring
)
.
unicode
ustring
=
ustring
.
replace
(
" "
,
" "
)
_entity_re
=
re
.
compile
(
r'&(?:(#)(\d+)|([^;^> ]+));'
)
return
_entity_re
.
sub
(
_repl_func
,
ustring
)

Replaces HTML special characters by readable characters .


def not_found(url, wait=10):

    """ Returns True when the url generates a "404 Not Found" error.
    """

    try: connection = open(url, wait)
    except HTTP404NotFound:
        return True
    except:
        return False

    return False

def not_found ( url , wait = 10 ) : try : connection = open ( url , wait ) except HTTP404NotFound : return True except : return False return False

Returns True when the url generates a 404 Not Found error .


def is_type(url, types=[], wait=10):

    """ Determine the MIME-type of the document behind the url.

    MIME is more reliable than simply checking the document extension.
    Returns True when the MIME-type starts with anything in the list of types.

    """

    # Types can also be a single string for convenience.
    if isinstance(types, str):
        types = [types]

    try: connection = open(url, wait)
    except:
        return False

    type = connection.info()["Content-Type"]
    for t in types:
        if type.startswith(t): return True

    return False

def is_type ( url , types = [ ] , wait = 10 ) : if isinstance ( types , str ) : types = [ types ] try : connection = open ( url , wait ) except : return False type = connection . info ( ) [ "Content-Type" ] for t in types : if type . startswith ( t ) : return True return False

Determine the MIME - type of the document behind the url .


def requirements(debug=True, with_examples=True, with_pgi=None):
    """
    Build requirements based on flags

    :param with_pgi: Use 'pgi' instead of 'gi' - False on CPython, True elsewhere
    :param with_examples:
    :return:
    """
    reqs = list(BASE_REQUIREMENTS)
    if with_pgi is None:
        with_pgi = is_jython

    if debug:
        print("setup options: ")
        print("with_pgi:      ", "yes" if with_pgi else "no")
        print("with_examples: ", "yes" if with_examples else "no")
    if with_pgi:
        reqs.append("pgi")
        if debug:
            print("warning, as of April 2019 typography does not work with pgi")
    else:
        reqs.append(PYGOBJECT)

    if with_examples:
        reqs.extend(EXAMPLE_REQUIREMENTS)

    if debug:
        print("")
        print("")
        for req in reqs:
            print(req)
    return reqs


def image(self, path, x, y, width=None, height=None, alpha=1.0, data=None, draw=True, **kwargs):
        '''Draws a image form path, in x,y and resize it to width, height dimensions.
        '''
        return self.Image(path, x, y, width, height, alpha, data, **kwargs)

def image ( self , path , x , y , width = None , height = None , alpha = 1.0 , data = None , draw = True , ** kwargs ) : return self . Image ( path , x , y , width , height , alpha , data , ** kwargs )

Draws a image form path in x y and resize it to width height dimensions .


def rect(self, x, y, width, height, roundness=0.0, draw=True, **kwargs):
        '''
        Draw a rectangle from x, y of width, height.

        :param startx: top left x-coordinate
        :param starty: top left y-coordinate

        :param width: height  Size of rectangle.
        :roundness: Corner roundness defaults to 0.0 (a right-angle).
        :draw: If True draws immediately.
        :fill: Optionally pass a fill color.

        :return: path representing the rectangle.

        '''
        path = self.BezierPath(**kwargs)
        path.rect(x, y, width, height, roundness, self.rectmode)
        if draw:
            path.draw()
        return path

def rect ( self , x , y , width , height , roundness = 0.0 , draw = True , ** kwargs ) : path = self . BezierPath ( ** kwargs ) path . rect ( x , y , width , height , roundness , self . rectmode ) if draw : path . draw ( ) return path

Draw a rectangle from x y of width height .


def rectmode(self, mode=None):
        '''
        Set the current rectmode.

        :param mode: CORNER, CENTER, CORNERS
        :return: rectmode if mode is None or valid.
        '''
        if mode in (self.CORNER, self.CENTER, self.CORNERS):
            self.rectmode = mode
            return self.rectmode
        elif mode is None:
            return self.rectmode
        else:
            raise ShoebotError(_("rectmode: invalid input"))

def
rectmode
(
self
,
mode
=
)
:
if
mode
in
(
self
.
CORNER
,
self
.
CENTER
,
self
.
CORNERS
)
:
self
.
rectmode
=
mode
return
self
.
rectmode
elif
mode
is
:
return
self
.
rectmode
else
:
raise
ShoebotError
(
_
(
"rectmode: invalid input"
)
)


def ellipsemode(self, mode=None):
        '''
        Set the current ellipse drawing mode.

        :param mode: CORNER, CENTER, CORNERS
        :return: ellipsemode if mode is None or valid.
        '''
        if mode in (self.CORNER, self.CENTER, self.CORNERS):
            self.ellipsemode = mode
            return self.ellipsemode
        elif mode is None:
            return self.ellipsemode
        else:
            raise ShoebotError(_("ellipsemode: invalid input"))

def
ellipsemode
(
self
,
mode
=
)
:
if
mode
in
(
self
.
CORNER
,
self
.
CENTER
,
self
.
CORNERS
)
:
self
.
ellipsemode
=
mode
return
self
.
ellipsemode
elif
mode
is
:
return
self
.
ellipsemode
else
:
raise
ShoebotError
(
_
(
"ellipsemode: invalid input"
)
)


def arrow(self, x, y, width, type=NORMAL, draw=True, **kwargs):
        '''Draw an arrow.

        Arrows can be two types: NORMAL or FORTYFIVE.

        :param x: top left x-coordinate
        :param y: top left y-coordinate
        :param width: width of arrow
        :param type:  NORMAL or FORTYFIVE
        :draw:  If True draws arrow immediately

        :return: Path object representing the arrow.
        '''
        # Taken from Nodebox
        path = self.BezierPath(**kwargs)
        if type == self.NORMAL:
            head = width * .4
            tail = width * .2
            path.moveto(x, y)
            path.lineto(x - head, y + head)
            path.lineto(x - head, y + tail)
            path.lineto(x - width, y + tail)
            path.lineto(x - width, y - tail)
            path.lineto(x - head, y - tail)
            path.lineto(x - head, y - head)
            path.lineto(x, y)
        elif type == self.FORTYFIVE:
            head = .3
            tail = 1 + head
            path.moveto(x, y)
            path.lineto(x, y + width * (1 - head))
            path.lineto(x - width * head, y + width)
            path.lineto(x - width * head, y + width * tail * .4)
            path.lineto(x - width * tail * .6, y + width)
            path.lineto(x - width, y + width * tail * .6)
            path.lineto(x - width * tail * .4, y + width * head)
            path.lineto(x - width, y + width * head)
            path.lineto(x - width * (1 - head), y)
            path.lineto(x, y)
        else:
            raise NameError(_("arrow: available types for arrow() are NORMAL and FORTYFIVE\n"))
        if draw:
            path.draw()
        return path


def star(self, startx, starty, points=20, outer=100, inner=50, draw=True, **kwargs):
        '''Draws a star.
        '''
        # Taken from Nodebox.
        self.beginpath(**kwargs)
        self.moveto(startx, starty + outer)

        for i in range(1, int(2 * points)):
            angle = i * pi / points
            x = sin(angle)
            y = cos(angle)
            if i % 2:
                radius = inner
            else:
                radius = outer
            x = startx + radius * x
            y = starty + radius * y
            self.lineto(x, y)

        return self.endpath(draw)

def star ( self , startx , starty , points = 20 , outer = 100 , inner = 50 , draw = True , ** kwargs ) : self . beginpath ( ** kwargs ) self . moveto ( startx , starty + outer ) for i in range ( 1 , int ( 2 * points ) ) : angle = i * pi / points x = sin ( angle ) y = cos ( angle ) if i % 2 : radius = inner else : radius = outer x = startx + radius * x y = starty + radius * y self . lineto ( x , y ) return self . endpath ( draw )


def relmoveto(self, x, y):
        '''Move relatively to the last point.'''
        if self._path is None:
            raise ShoebotError(_("No current path. Use beginpath() first."))
        self._path.relmoveto(x, y)


def rellineto(self, x, y):
        '''Draw a line using relative coordinates.'''
        if self._path is None:
            raise ShoebotError(_("No current path. Use beginpath() first."))
        self._path.rellineto(x, y)

Draw a line using relative coordinates .


def relcurveto(self, h1x, h1y, h2x, h2y, x, y):
        '''Draws a curve relatively to the last point.
        '''
        if self._path is None:
            raise ShoebotError(_("No current path. Use beginpath() first."))
        self._path.relcurveto(h1x, h1y, h2x, h2y, x, y)

Draws a curve relatively to the last point .


def transform(self, mode=None):
        '''
        Set the current transform mode.

        :param mode: CENTER or CORNER'''
        if mode:
            self._canvas.mode = mode
        return self._canvas.mode

def transform ( self , mode = None ) : if mode : self . _canvas . mode = mode return self . _canvas . mode


def scale(self, x=1, y=None):
        '''
        Set a scale at which to draw objects.

        1.0 draws objects at their natural size

        :param x: Scale on the horizontal plane
        :param y: Scale on the vertical plane
        '''
        if not y:
            y = x
        if x == 0:
            # Cairo borks on zero values
            x = 1
        if y == 0:
            y = 1
        self._canvas.scale(x, y)

def scale ( self , x = 1 , y = None ) : if not y : y = x if x == 0 : x = 1 if y == 0 : y = 1 self . _canvas . scale ( x , y )


def nostroke(self):
        ''' Stop applying strokes to new paths.

        :return: stroke color before nostroke was called.
        '''
        c = self._canvas.strokecolor
        self._canvas.strokecolor = None
        return c

def nostroke ( self ) : c = self . _canvas . strokecolor self . _canvas . strokecolor = None return c


def strokewidth(self, w=None):
        '''Set the stroke width.

        :param w: Stroke width.
        :return: If no width was specified then current width is returned.
        '''
        if w is not None:
            self._canvas.strokewidth = w
        else:
            return self._canvas.strokewidth

def strokewidth ( self , w = None ) : if w is not None : self . _canvas . strokewidth = w else : return self . _canvas . strokewidth


def font(self, fontpath=None, fontsize=None):
        '''Set the font to be used with new text instances.

        :param fontpath: path to truetype or opentype font.
        :param fontsize: size of font

        :return: current current fontpath (if fontpath param not set)
        Accepts TrueType and OpenType files. Depends on FreeType being
        installed.'''
        if fontpath is not None:
            self._canvas.fontfile = fontpath
        else:
            return self._canvas.fontfile
        if fontsize is not None:
            self._canvas.fontsize = fontsize

def font ( self , fontpath = None , fontsize = None ) : if fontpath is not None : self . _canvas . fontfile = fontpath else : return self . _canvas . fontfile if fontsize is not None : self . _canvas . fontsize = fontsize

Set the font to be used with new text instances .


def fontsize(self, fontsize=None):
        '''
        Set or return size of current font.

        :param fontsize: Size of font.
        :return: Size of font (if fontsize was not specified)
        '''
        if fontsize is not None:
            self._canvas.fontsize = fontsize
        else:
            return self._canvas.fontsize

def fontsize ( self , fontsize = None ) : if fontsize is not None : self . _canvas . fontsize = fontsize else : return self . _canvas . fontsize


def text(self, txt, x, y, width=None, height=1000000, outline=False, draw=True, **kwargs):
        '''
        Draws a string of text according to current font settings.

        :param txt: Text to output
        :param x: x-coordinate of the top left corner
        :param y: y-coordinate of the top left corner
        :param width: text width
        :param height: text height
        :param outline: If True draws outline text (defaults to False)
        :param draw: Set to False to inhibit immediate drawing (defaults to True)
        :return: Path object representing the text.
        '''
        txt = self.Text(txt, x, y, width, height, outline=outline, ctx=None, **kwargs)
        if outline:
            path = txt.path
            if draw:
                path.draw()
            return path
        else:
            return txt

def text ( self , txt , x , y , width = None , height = 1000000 , outline = False , draw = True , ** kwargs ) : txt = self . Text ( txt , x , y , width , height , outline = outline , ctx = None , ** kwargs ) if outline : path = txt . path if draw : path . draw ( ) return path else : return txt

Draws a string of text according to current font settings .


def textheight(self, txt, width=None):
        '''Returns the height of a string of text according to the current
        font settings.

        :param txt: string to measure
        :param width: width of a line of text in a block
        '''
        w = width
        return self.textmetrics(txt, width=w)[1]

def textheight ( self , txt , width = None ) : w = width return self . textmetrics ( txt , width = w ) [ 1 ]

Returns the height of a string of text according to the current font settings .


def graph_background(s):

    """ Graph background color.
    """

    if s.background == None:
        s._ctx.background(None)
    else:
        s._ctx.background(s.background)

    if s.depth:
        try:
            clr = colors.color(s.background).darker(0.2)
            p = s._ctx.rect(0, 0, s._ctx.WIDTH, s._ctx.HEIGHT, draw=False)
            colors.gradientfill(p, clr, clr.lighter(0.35))
            colors.shadow(dx=0, dy=0, blur=2, alpha=0.935, clr=s.background)
        except:
            pass

def graph_background ( s ) : if s . background == None : s . _ctx . background ( None ) else : s . _ctx . background ( s . background ) if s . depth : try : clr = colors . color ( s . background ) . darker ( 0.2 ) p = s . _ctx . rect ( 0 , 0 , s . _ctx . WIDTH , s . _ctx . HEIGHT , draw = False ) colors . gradientfill ( p , clr , clr . lighter ( 0.35 ) ) colors . shadow ( dx = 0 , dy = 0 , blur = 2 , alpha = 0.935 , clr = s . background ) except : pass