def _convert_boolean_config_value(config, name, default=True):
        """Convert the named field to bool.

        The current value should be one of the strings "yes" or "no".  It will
        be replaced with its boolean counterpart.  If the field is not present
        in the config object, the default value is used.

        :param config: the config section where to set the option
        :type config: configobj.ConfigObj
        :param name: the name of the option to convert
        :type name: str
        :param default: the default value to use if the option was not
            previously set
        :type default: bool
        :returns: None

        """
        if name not in config:
            config[name] = default
        elif config[name] == "yes":
            config[name] = True
        elif config[name] == "no":
            config[name] = False
        else:
            raise ValueError("Error in config file\nInvalid value for %s "
                             "parameter\nPossible values: yes, no" % name)


def new_contact(cls, address_book, supported_private_objects, version,
            localize_dates):
        """Use this to create a new and empty contact."""
        return cls(address_book, None, supported_private_objects, version,
                localize_dates)

def new_contact ( cls , address_book , supported_private_objects , version , localize_dates ) : return cls ( address_book , None , supported_private_objects , version , localize_dates )

Use this to create a new and empty contact .


def from_file(cls, address_book, filename, supported_private_objects,
            localize_dates):
        """
        Use this if you want to create a new contact from an existing .vcf
        file.
        """
        return cls(address_book, filename, supported_private_objects, None,
                localize_dates)

def from_file ( cls , address_book , filename , supported_private_objects , localize_dates ) : return cls ( address_book , filename , supported_private_objects , None , localize_dates )

Use this if you want to create a new contact from an existing . vcf file .


def from_user_input(cls, address_book, user_input,
                        supported_private_objects, version, localize_dates):
        """Use this if you want to create a new contact from user input."""
        contact = cls(address_book, None, supported_private_objects, version,
                localize_dates)
        contact._process_user_input(user_input)
        return contact

def from_user_input ( cls , address_book , user_input , supported_private_objects , version , localize_dates ) : contact = cls ( address_book , None , supported_private_objects , version , localize_dates ) contact . _process_user_input ( user_input ) return contact

Use this if you want to create a new contact from user input .


def from_existing_contact_with_new_user_input(cls, contact, user_input,
            localize_dates):
        """
        Use this if you want to clone an existing contact and  replace its data
        with new user input in one step.
        """
        contact = cls(contact.address_book, contact.filename,
                      contact.supported_private_objects, None, localize_dates)
        contact._process_user_input(user_input)
        return contact

Use this if you want to clone an existing contact and replace its data with new user input in one step .


def _get_names_part(self, part):
        """Get some part of the "N" entry in the vCard as a list

        :param part: the name to get e.g. "prefix" or "given"
        :type part: str
        :returns: a list of entries for this name part
        :rtype: list(str)

        """
        try:
            the_list = getattr(self.vcard.n.value, part)
        except AttributeError:
            return []
        else:
            # check if list only contains empty strings
            if not ''.join(the_list):
                return []
        return the_list if isinstance(the_list, list) else [the_list]

def _get_names_part ( self , part ) : try : the_list = getattr ( self . vcard . n . value , part ) except AttributeError : return [ ] else : if not '' . join ( the_list ) : return [ ] return the_list if isinstance ( the_list , list ) else [ the_list ]

Get some part of the N entry in the vCard as a list


def _add_category(self, categories):
        """ categories variable must be a list """
        categories_obj = self.vcard.add('categories')
        categories_obj.value = helpers.convert_to_vcard(
            "category", categories, ObjectType.list_with_strings)

def _add_category ( self , categories ) : categories_obj = self . vcard . add ( 'categories' ) categories_obj . value = helpers . convert_to_vcard ( "category" , categories , ObjectType . list_with_strings )


def _parse_type_value(types, value, supported_types):
        """Parse type value of phone numbers, email and post addresses.

        :param types: list of type values
        :type types: list(str)
        :param value: the corresponding label, required for more verbose
            exceptions
        :type value: str
        :param supported_types: all allowed standard types
        :type supported_types: list(str)
        :returns: tuple of standard and custom types and pref integer
        :rtype: tuple(list(str), list(str), int)

        """
        custom_types = []
        standard_types = []
        pref = 0
        for type in types:
            type = type.strip()
            if type:
                if type.lower() in supported_types:
                    standard_types.append(type)
                elif type.lower() == "pref":
                    pref += 1
                elif re.match(r"^pref=\d{1,2}$", type.lower()):
                    pref += int(type.split("=")[1])
                else:
                    if type.lower().startswith("x-"):
                        custom_types.append(type[2:])
                        standard_types.append(type)
                    else:
                        custom_types.append(type)
                        standard_types.append("X-{}".format(type))
        return (standard_types, custom_types, pref)

def _parse_type_value ( types , value , supported_types ) : custom_types = [ ] standard_types = [ ] pref = 0 for type in types : type = type . strip ( ) if type : if type . lower ( ) in supported_types : standard_types . append ( type ) elif type . lower ( ) == "pref" : pref += 1 elif re . match ( r"^pref=\d{1,2}$" , type . lower ( ) ) : pref += int ( type . split ( "=" ) [ 1 ] ) else : if type . lower ( ) . startswith ( "x-" ) : custom_types . append ( type [ 2 : ] ) standard_types . append ( type ) else : custom_types . append ( type ) standard_types . append ( "X-{}" . format ( type ) ) return ( standard_types , custom_types , pref )

Parse type value of phone numbers email and post addresses .


def list_to_string(input, delimiter):
    """converts list to string recursively so that nested lists are supported

    :param input: a list of strings and lists of strings (and so on recursive)
    :type input: list
    :param delimiter: the deimiter to use when joining the items
    :type delimiter: str
    :returns: the recursively joined list
    :rtype: str
    """
    if isinstance(input, list):
        return delimiter.join(
            list_to_string(item, delimiter) for item in input)
    return input

def list_to_string ( input , delimiter ) : if isinstance ( input , list ) : return delimiter . join ( list_to_string ( item , delimiter ) for item in input ) return input


def string_to_date(input):
    """Convert string to date object.

    :param input: the date string to parse
    :type input: str
    :returns: the parsed datetime object
    :rtype: datetime.datetime
    """
    # try date formats --mmdd, --mm-dd, yyyymmdd, yyyy-mm-dd and datetime
    # formats yyyymmddThhmmss, yyyy-mm-ddThh:mm:ss, yyyymmddThhmmssZ,
    # yyyy-mm-ddThh:mm:ssZ.
    for format_string in ("--%m%d", "--%m-%d", "%Y%m%d", "%Y-%m-%d",
                          "%Y%m%dT%H%M%S", "%Y-%m-%dT%H:%M:%S",
                          "%Y%m%dT%H%M%SZ", "%Y-%m-%dT%H:%M:%SZ"):
        try:
            return datetime.strptime(input, format_string)
        except ValueError:
            pass
    # try datetime formats yyyymmddThhmmsstz and yyyy-mm-ddThh:mm:sstz where tz
    # may look like -06:00.
    for format_string in ("%Y%m%dT%H%M%S%z", "%Y-%m-%dT%H:%M:%S%z"):
        try:
            return datetime.strptime(''.join(input.rsplit(":", 1)),
                                     format_string)
        except ValueError:
            pass
    raise ValueError

def string_to_date ( input ) : for format_string in ( "--%m%d" , "--%m-%d" , "%Y%m%d" , "%Y-%m-%d" , "%Y%m%dT%H%M%S" , "%Y-%m-%dT%H:%M:%S" , "%Y%m%dT%H%M%SZ" , "%Y-%m-%dT%H:%M:%SZ" ) : try : return datetime . strptime ( input , format_string ) except ValueError : pass for format_string in ( "%Y%m%dT%H%M%S%z" , "%Y-%m-%dT%H:%M:%S%z" ) : try : return datetime . strptime ( '' . join ( input . rsplit ( ":" , 1 ) ) , format_string ) except ValueError : pass raise ValueError


def _compare_uids(uid1, uid2):
        """Calculate the minimum length of initial substrings of uid1 and uid2
        for them to be different.

        :param uid1: first uid to compare
        :type uid1: str
        :param uid2: second uid to compare
        :type uid2: str
        :returns: the length of the shortes unequal initial substrings
        :rtype: int
        """
        sum = 0
        for char1, char2 in zip(uid1, uid2):
            if char1 == char2:
                sum += 1
            else:
                break
        return sum

def _compare_uids ( uid1 , uid2 ) : sum = 0 for char1 , char2 in zip ( uid1 , uid2 ) : if char1 == char2 : sum += 1 else : break return sum

Calculate the minimum length of initial substrings of uid1 and uid2 for them to be different .


def _search_all(self, query):
        """Search in all fields for contacts matching query.

        :param query: the query to search for
        :type query: str
        :yields: all found contacts
        :rtype: generator(carddav_object.CarddavObject)

        """
        regexp = re.compile(query, re.IGNORECASE | re.DOTALL)
        for contact in self.contacts.values():
            # search in all contact fields
            contact_details = contact.print_vcard()
            if regexp.search(contact_details) is not None:
                yield contact
            else:
                # find phone numbers with special chars like /
                clean_contact_details = re.sub("[^a-zA-Z0-9\n]", "",
                                               contact_details)
                if regexp.search(clean_contact_details) is not None \
                        and len(re.sub("\D", "", query)) >= 3:
                    yield contact

def _search_all ( self , query ) : regexp = re . compile ( query , re . IGNORECASE | re . DOTALL ) for contact in self . contacts . values ( ) : contact_details = contact . print_vcard ( ) if regexp . search ( contact_details ) is not None : yield contact else : clean_contact_details = re . sub ( "[^a-zA-Z0-9\n]" , "" , contact_details ) if regexp . search ( clean_contact_details ) is not None and len ( re . sub ( "\D" , "" , query ) ) >= 3 : yield contact

Search in all fields for contacts matching query .


def _search_names(self, query):
        """Search in the name filed for contacts matching query.

        :param query: the query to search for
        :type query: str
        :yields: all found contacts
        :rtype: generator(carddav_object.CarddavObject)

        """
        regexp = re.compile(query, re.IGNORECASE | re.DOTALL)
        for contact in self.contacts.values():
            # only search in contact name
            if regexp.search(contact.get_full_name()) is not None:
                yield contact

def _search_names ( self , query ) : regexp = re . compile ( query , re . IGNORECASE | re . DOTALL ) for contact in self . contacts . values ( ) : if regexp . search ( contact . get_full_name ( ) ) is not None : yield contact

Search in the name filed for contacts matching query .


def _search_uid(self, query):
        """Search for contacts with a matching uid.

        :param query: the query to search for
        :type query: str
        :yields: all found contacts
        :rtype: generator(carddav_object.CarddavObject)

        """
        try:
            # First we treat the argument as a full UID and try to match it
            # exactly.
            yield self.contacts[query]
        except KeyError:
            # If that failed we look for all contacts whos UID start with the
            # given query.
            for uid in self.contacts:
                if uid.startswith(query):
                    yield self.contacts[uid]

def _search_uid ( self , query ) : try : yield self . contacts [ query ] except KeyError : for uid in self . contacts : if uid . startswith ( query ) : yield self . contacts [ uid ]

Search for contacts with a matching uid .


def search(self, query, method="all"):
        """Search this address book for contacts matching the query.

        The method can be one of "all", "name" and "uid".  The backend for this
        address book migth be load()ed if needed.

        :param query: the query to search for
        :type query: str
        :param method: the type of fileds to use when seaching
        :type method: str
        :returns: all found contacts
        :rtype: list(carddav_object.CarddavObject)

        """
        logging.debug('address book %s, searching with %s', self.name, query)
        if not self._loaded:
            self.load(query)
        if method == "all":
            search_function = self._search_all
        elif method == "name":
            search_function = self._search_names
        elif method == "uid":
            search_function = self._search_uid
        else:
            raise ValueError('Only the search methods "all", "name" and "uid" '
                             'are supported.')
        return list(search_function(query))

Search this address book for contacts matching the query .


def get_short_uid_dict(self, query=None):
        """Create a dictionary of shortend UIDs for all contacts.

        All arguments are only used if the address book is not yet initialized
        and will just be handed to self.load().

        :param query: see self.load()
        :type query: str
        :returns: the contacts mapped by the shortes unique prefix of their UID
        :rtype: dict(str: CarddavObject)
        """
        if self._short_uids is None:
            if not self._loaded:
                self.load(query)
            if not self.contacts:
                self._short_uids = {}
            elif len(self.contacts) == 1:
                self._short_uids = {uid[0:1]: contact
                                    for uid, contact in self.contacts.items()}
            else:
                self._short_uids = {}
                sorted_uids = sorted(self.contacts)
                # Prepare for the loop; the first and last items are handled
                # seperatly.
                item0, item1 = sorted_uids[:2]
                same1 = self._compare_uids(item0, item1)
                self._short_uids[item0[:same1 + 1]] = self.contacts[item0]
                for item_new in sorted_uids[2:]:
                    # shift the items and the common prefix lenght one further
                    item0, item1 = item1, item_new
                    same0, same1 = same1, self._compare_uids(item0, item1)
                    # compute the final prefix length for item1
                    same = max(same0, same1)
                    self._short_uids[item0[:same + 1]] = self.contacts[item0]
                # Save the last item.
                self._short_uids[item1[:same1 + 1]] = self.contacts[item1]
        return self._short_uids

def get_short_uid_dict ( self , query = None ) : if self . _short_uids is None : if not self . _loaded : self . load ( query ) if not self . contacts : self . _short_uids = { } elif len ( self . contacts ) == 1 : self . _short_uids = { uid [ 0 : 1 ] : contact for uid , contact in self . contacts . items ( ) } else : self . _short_uids = { } sorted_uids = sorted ( self . contacts ) item0 , item1 = sorted_uids [ : 2 ] same1 = self . _compare_uids ( item0 , item1 ) self . _short_uids [ item0 [ : same1 + 1 ] ] = self . contacts [ item0 ] for item_new in sorted_uids [ 2 : ] : item0 , item1 = item1 , item_new same0 , same1 = same1 , self . _compare_uids ( item0 , item1 ) same = max ( same0 , same1 ) self . _short_uids [ item0 [ : same + 1 ] ] = self . contacts [ item0 ] self . _short_uids [ item1 [ : same1 + 1 ] ] = self . contacts [ item1 ] return self . _short_uids

Create a dictionary of shortend UIDs for all contacts .


def get_short_uid(self, uid):
        """Get the shortend UID for the given UID.

        :param uid: the full UID to shorten
        :type uid: str
        :returns: the shortend uid or the empty string
        :rtype: str
        """
        if uid:
            short_uids = self.get_short_uid_dict()
            for length_of_uid in range(len(uid), 0, -1):
                if short_uids.get(uid[:length_of_uid]) is not None:
                    return uid[:length_of_uid]
        return ""

def get_short_uid ( self , uid ) : if uid : short_uids = self . get_short_uid_dict ( ) for length_of_uid in range ( len ( uid ) , 0 , - 1 ) : if short_uids . get ( uid [ : length_of_uid ] ) is not None : return uid [ : length_of_uid ] return ""

Get the shortend UID for the given UID .


def _find_vcard_files(self, search=None, search_in_source_files=False):
        """Find all vcard files inside this address book.

        If a search string is given only files which contents match that will
        be returned.

        :param search: a regular expression to limit the results
        :type search: str
        :param search_in_source_files: apply search regexp directly on the .vcf files to speed up parsing (less accurate)
        :type search_in_source_files: bool
        :returns: the paths of the vcard files
        :rtype: generator

        """
        files = glob.glob(os.path.join(self.path, "*.vcf"))
        if search and search_in_source_files:
            for filename in files:
                with open(filename, "r") as filehandle:
                    if re.search(search, filehandle.read(),
                                 re.IGNORECASE | re.DOTALL):
                        yield filename
        else:
            yield from files

def _find_vcard_files ( self , search = None , search_in_source_files = False ) : files = glob . glob ( os . path . join ( self . path , "*.vcf" ) ) if search and search_in_source_files : for filename in files : with open ( filename , "r" ) as filehandle : if re . search ( search , filehandle . read ( ) , re . IGNORECASE | re . DOTALL ) : yield filename else : yield from files

Find all vcard files inside this address book .


def load(self, query=None, search_in_source_files=False):
        """Load all vcard files in this address book from disk.

        If a search string is given only files which contents match that will
        be loaded.

        :param query: a regular expression to limit the results
        :type query: str
        :param search_in_source_files: apply search regexp directly on the .vcf files to speed up parsing (less accurate)
        :type search_in_source_files: bool
        :returns: the number of successfully loaded cards and the number of
            errors
        :rtype: int, int
        :throws: AddressBookParseError
        """
        if self._loaded:
            return
        logging.debug('Loading Vdir %s with query %s', self.name, query)
        errors = 0
        for filename in self._find_vcard_files(
                search=query, search_in_source_files=search_in_source_files):
            try:
                card = CarddavObject.from_file(self, filename,
                                               self._private_objects,
                                               self._localize_dates)
            except (IOError, vobject.base.ParseError) as err:
                verb = "open" if isinstance(err, IOError) else "parse"
                logging.debug("Error: Could not %s file %s\n%s", verb,
                              filename, err)
                if self._skip:
                    errors += 1
                else:
                    # FIXME: This should throw an apropriate exception and the
                    # sys.exit should be called somewhere closer to the command
                    # line parsing.
                    logging.error(
                        "The vcard file %s of address book %s could not be "
                        "parsed\nUse --debug for more information or "
                        "--skip-unparsable to proceed", filename, self.name)
                    sys.exit(2)
            else:
                uid = card.get_uid()
                if not uid:
                    logging.warning("Card %s from address book %s has no UID "
                                    "and will not be availbale.", card,
                                    self.name)
                elif uid in self.contacts:
                    logging.warning(
                        "Card %s and %s from address book %s have the same "
                        "UID. The former will not be availbale.", card,
                        self.contacts[uid], self.name)
                else:
                    self.contacts[uid] = card
        self._loaded = True
        if errors:
            logging.warning(
                "%d of %d vCard files of address book %s could not be parsed.",
                errors, len(self.contacts) + errors, self)
        logging.debug('Loded %s contacts from address book %s.',
                      len(self.contacts), self.name)

Load all vcard files in this address book from disk .


def get_abook(self, name):
        """Get one of the backing abdress books by its name,

        :param name: the name of the address book to get
        :type name: str
        :returns: the matching address book or None
        :rtype: AddressBook or NoneType

        """
        for abook in self._abooks:
            if abook.name == name:
                return abook

def get_abook ( self , name ) : for abook in self . _abooks : if abook . name == name : return abook


def avail_archs(self):
        ''' Initialize the dictionary of architectures for assembling via keystone'''

        return {
            ARM32:   (KS_ARCH_ARM,     KS_MODE_ARM),
            ARM64:   (KS_ARCH_ARM64,   KS_MODE_LITTLE_ENDIAN),
            ARM_TB:  (KS_ARCH_ARM,     KS_MODE_THUMB),
            HEXAGON: (KS_ARCH_HEXAGON, KS_MODE_BIG_ENDIAN),
            MIPS32:  (KS_ARCH_MIPS,    KS_MODE_MIPS32),
            MIPS64:  (KS_ARCH_MIPS,    KS_MODE_MIPS64),
            PPC32:   (KS_ARCH_PPC,     KS_MODE_PPC32),
            PPC64:   (KS_ARCH_PPC,     KS_MODE_PPC64),
            SPARC32: (KS_ARCH_SPARC,   KS_MODE_SPARC32),
            SPARC64: (KS_ARCH_SPARC,   KS_MODE_SPARC64),
            SYSTEMZ: (KS_ARCH_SYSTEMZ, KS_MODE_BIG_ENDIAN),
            X86_16:  (KS_ARCH_X86,     KS_MODE_16),
            X86_32:  (KS_ARCH_X86,     KS_MODE_32),
            X86_64:  (KS_ARCH_X86,     KS_MODE_64),
        }


def avail_archs(self):
        ''' Initialize the dictionary of architectures for disassembling via capstone'''

        return {
            ARM32:   (CS_ARCH_ARM,   CS_MODE_ARM),
            ARM64:   (CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN),
            ARM_TB:  (CS_ARCH_ARM,   CS_MODE_THUMB),
            MIPS32:  (CS_ARCH_MIPS,  CS_MODE_MIPS32),
            MIPS64:  (CS_ARCH_MIPS,  CS_MODE_MIPS64),
            SPARC32: (CS_ARCH_SPARC, CS_MODE_BIG_ENDIAN),
            SPARC64: (CS_ARCH_SPARC, CS_MODE_V9),
            SYSTEMZ: (CS_ARCH_SYSZ,  CS_MODE_BIG_ENDIAN),
            X86_16:  (CS_ARCH_X86,   CS_MODE_16),
            X86_32:  (CS_ARCH_X86,   CS_MODE_32),
            X86_64:  (CS_ARCH_X86,   CS_MODE_64),
        }


def getargspec_permissive(func):
    """
    An `inspect.getargspec` with a relaxed sanity check to support Cython.

    Motivation:

        A Cython-compiled function is *not* an instance of Python's
        types.FunctionType.  That is the sanity check the standard Py2
        library uses in `inspect.getargspec()`.  So, an exception is raised
        when calling `argh.dispatch_command(cythonCompiledFunc)`.  However,
        the CyFunctions do have perfectly usable `.func_code` and
        `.func_defaults` which is all `inspect.getargspec` needs.

        This function just copies `inspect.getargspec()` from the standard
        library but relaxes the test to a more duck-typing one of having
        both `.func_code` and `.func_defaults` attributes.
    """
    if inspect.ismethod(func):
        func = func.im_func

    # Py2 Stdlib uses isfunction(func) which is too strict for Cython-compiled
    # functions though such have perfectly usable func_code, func_defaults.
    if not (hasattr(func, "func_code") and hasattr(func, "func_defaults")):
        raise TypeError('{!r} missing func_code or func_defaults'.format(func))

    args, varargs, varkw = inspect.getargs(func.func_code)
    return inspect.ArgSpec(args, varargs, varkw, func.func_defaults)

An inspect . getargspec with a relaxed sanity check to support Cython .


def dispatch(parser, argv=None, add_help_command=True,
             completion=True, pre_call=None,
             output_file=sys.stdout, errors_file=sys.stderr,
             raw_output=False, namespace=None,
             skip_unknown_args=False):
    """
    Parses given list of arguments using given parser, calls the relevant
    function and prints the result.

    The target function should expect one positional argument: the
    :class:`argparse.Namespace` object. However, if the function is decorated with
    :func:`~argh.decorators.plain_signature`, the positional and named
    arguments from the namespace object are passed to the function instead
    of the object itself.

    :param parser:

        the ArgumentParser instance.

    :param argv:

        a list of strings representing the arguments. If `None`, ``sys.argv``
        is used instead. Default is `None`.

    :param add_help_command:

        if `True`, converts first positional argument "help" to a keyword
        argument so that ``help foo`` becomes ``foo --help`` and displays usage
        information for "foo". Default is `True`.

    :param output_file:

        A file-like object for output. If `None`, the resulting lines are
        collected and returned as a string. Default is ``sys.stdout``.

    :param errors_file:

        Same as `output_file` but for ``sys.stderr``.

    :param raw_output:

        If `True`, results are written to the output file raw, without adding
        whitespaces or newlines between yielded strings. Default is `False`.

    :param completion:

        If `True`, shell tab completion is enabled. Default is `True`. (You
        will also need to install it.)  See :mod:`argh.completion`.

    :param skip_unknown_args:

        If `True`, unknown arguments do not cause an error
        (`ArgumentParser.parse_known_args` is used).

    :param namespace:

        An `argparse.Namespace`-like object.  By default an
        :class:`ArghNamespace` object is used.  Please note that support for
        combined default and nested functions may be broken if a different
        type of object is forced.

    By default the exceptions are not wrapped and will propagate. The only
    exception that is always wrapped is :class:`~argh.exceptions.CommandError`
    which is interpreted as an expected event so the traceback is hidden.
    You can also mark arbitrary exceptions as "wrappable" by using the
    :func:`~argh.decorators.wrap_errors` decorator.
    """
    if completion:
        autocomplete(parser)

    if argv is None:
        argv = sys.argv[1:]

    if add_help_command:
        if argv and argv[0] == 'help':
            argv.pop(0)
            argv.append('--help')

    if skip_unknown_args:
        parse_args = parser.parse_known_args
    else:
        parse_args = parser.parse_args

    if not namespace:
        namespace = ArghNamespace()

    # this will raise SystemExit if parsing fails
    namespace_obj = parse_args(argv, namespace=namespace)

    function = _get_function_from_namespace_obj(namespace_obj)

    if function:
        lines = _execute_command(function, namespace_obj, errors_file,
                                 pre_call=pre_call)
    else:
        # no commands declared, can't dispatch; display help message
        lines = [parser.format_usage()]

    if output_file is None:
        # user wants a string; we create an internal temporary file-like object
        # and will return its contents as a string
        if sys.version_info < (3,0):
            f = compat.BytesIO()
        else:
            f = compat.StringIO()
    else:
        # normally this is stdout; can be any file
        f = output_file

    for line in lines:
        # print the line as soon as it is generated to ensure that it is
        # displayed to the user before anything else happens, e.g.
        # raw_input() is called

        io.dump(line, f)
        if not raw_output:
            # in most cases user wants one message per line
            io.dump('\n', f)

    if output_file is None:
        # user wanted a string; return contents of our temporary file-like obj
        f.seek(0)
        return f.read()

def dispatch ( parser , argv = None , add_help_command = True , completion = True , pre_call = None , output_file = sys . stdout , errors_file = sys . stderr , raw_output = False , namespace = None , skip_unknown_args = False ) : if completion : autocomplete ( parser ) if argv is None : argv = sys . argv [ 1 : ] if add_help_command : if argv and argv [ 0 ] == 'help' : argv . pop ( 0 ) argv . append ( '--help' ) if skip_unknown_args : parse_args = parser . parse_known_args else : parse_args = parser . parse_args if not namespace : namespace = ArghNamespace ( ) namespace_obj = parse_args ( argv , namespace = namespace ) function = _get_function_from_namespace_obj ( namespace_obj ) if function : lines = _execute_command ( function , namespace_obj , errors_file , pre_call = pre_call ) else : lines = [ parser . format_usage ( ) ] if output_file is None : if sys . version_info < ( 3 , 0 ) : f = compat . BytesIO ( ) else : f = compat . StringIO ( ) else : f = output_file for line in lines : io . dump ( line , f ) if not raw_output : io . dump ( '\n' , f ) if output_file is None : f . seek ( 0 ) return f . read ( )

Parses given list of arguments using given parser calls the relevant function and prints the result .


def safe_input(prompt):
    """
    Prompts user for input. Correctly handles prompt message encoding.
    """

    if sys.version_info < (3,0):
        if isinstance(prompt, compat.text_type):
            # Python 2.x: unicode →  bytes
            encoding = locale.getpreferredencoding() or 'utf-8'
            prompt = prompt.encode(encoding)
    else:
        if not isinstance(prompt, compat.text_type):
            # Python 3.x: bytes →  unicode
            prompt = prompt.decode()

    return _input(prompt)

def safe_input ( prompt ) : if sys . version_info < ( 3 , 0 ) : if isinstance ( prompt , compat . text_type ) : encoding = locale . getpreferredencoding ( ) or 'utf-8' prompt = prompt . encode ( encoding ) else : if not isinstance ( prompt , compat . text_type ) : prompt = prompt . decode ( ) return _input ( prompt )

Prompts user for input . Correctly handles prompt message encoding .


def encode_output(value, output_file):
    """
    Encodes given value so it can be written to given file object.

    Value may be Unicode, binary string or any other data type.

    The exact behaviour depends on the Python version:

    Python 3.x

        `sys.stdout` is a `_io.TextIOWrapper` instance that accepts `str`
        (unicode) and breaks on `bytes`.

        It is OK to simply assume that everything is Unicode unless special
        handling is introduced in the client code.

        Thus, no additional processing is performed.

    Python 2.x

        `sys.stdout` is a file-like object that accepts `str` (bytes)
        and breaks when `unicode` is passed to `sys.stdout.write()`.

        We can expect both Unicode and bytes. They need to be encoded so as
        to match the file object encoding.

        The output is binary if the object doesn't explicitly require Unicode.

    """
    if sys.version_info > (3,0):
        # Python 3:  whatever → unicode
        return compat.text_type(value)
    else:
        # Python 2:  handle special cases
        stream_encoding = getattr(output_file, 'encoding', None)
        if stream_encoding:
            if stream_encoding.upper() == 'UTF-8':
                return compat.text_type(value)
            else:
                return value.encode(stream_encoding, 'ignore')
        else:
            # no explicit encoding requirements; force binary
            if isinstance(value, compat.text_type):
                # unicode → binary
                return value.encode('utf-8')
            else:
                return str(value)

def encode_output ( value , output_file ) : if sys . version_info > ( 3 , 0 ) : return compat . text_type ( value ) else : stream_encoding = getattr ( output_file , 'encoding' , None ) if stream_encoding : if stream_encoding . upper ( ) == 'UTF-8' : return compat . text_type ( value ) else : return value . encode ( stream_encoding , 'ignore' ) else : if isinstance ( value , compat . text_type ) : return value . encode ( 'utf-8' ) else : return str ( value )

Encodes given value so it can be written to given file object .


def _guess(kwargs):
    """
    Adds types, actions, etc. to given argument specification.
    For example, ``default=3`` implies ``type=int``.

    :param arg: a :class:`argh.utils.Arg` instance
    """
    guessed = {}

    # Parser actions that accept argument 'type'
    TYPE_AWARE_ACTIONS = 'store', 'append'

    # guess type/action from default value
    value = kwargs.get('default')
    if value is not None:
        if isinstance(value, bool):
            if kwargs.get('action') is None:
                # infer action from default value
                guessed['action'] = 'store_false' if value else 'store_true'
        elif kwargs.get('type') is None:
            # infer type from default value
            # (make sure that action handler supports this keyword)
            if kwargs.get('action', 'store') in TYPE_AWARE_ACTIONS:
                guessed['type'] = type(value)

    # guess type from choices (first item)
    if kwargs.get('choices') and 'type' not in list(guessed) + list(kwargs):
        guessed['type'] = type(kwargs['choices'][0])

    return dict(kwargs, **guessed)

def _guess ( kwargs ) : guessed = { } TYPE_AWARE_ACTIONS = 'store' , 'append' value = kwargs . get ( 'default' ) if value is not None : if isinstance ( value , bool ) : if kwargs . get ( 'action' ) is None : guessed [ 'action' ] = 'store_false' if value else 'store_true' elif kwargs . get ( 'type' ) is None : if kwargs . get ( 'action' , 'store' ) in TYPE_AWARE_ACTIONS : guessed [ 'type' ] = type ( value ) if kwargs . get ( 'choices' ) and 'type' not in list ( guessed ) + list ( kwargs ) : guessed [ 'type' ] = type ( kwargs [ 'choices' ] [ 0 ] ) return dict ( kwargs , ** guessed )

Adds types actions etc . to given argument specification . For example default = 3 implies type = int .


def add_commands(parser, functions, namespace=None, namespace_kwargs=None,
                 func_kwargs=None,
                 # deprecated args:
                 title=None, description=None, help=None):
    """
    Adds given functions as commands to given parser.

    :param parser:

        an :class:`argparse.ArgumentParser` instance.

    :param functions:

        a list of functions. A subparser is created for each of them.
        If the function is decorated with :func:`~argh.decorators.arg`, the
        arguments are passed to :class:`argparse.ArgumentParser.add_argument`.
        See also :func:`~argh.dispatching.dispatch` for requirements
        concerning function signatures. The command name is inferred from the
        function name. Note that the underscores in the name are replaced with
        hyphens, i.e. function name "foo_bar" becomes command name "foo-bar".

    :param namespace:

        an optional string representing the group of commands. For example, if
        a command named "hello" is added without the namespace, it will be
        available as "prog.py hello"; if the namespace if specified as "greet",
        then the command will be accessible as "prog.py greet hello". The
        namespace itself is not callable, so "prog.py greet" will fail and only
        display a help message.

    :param func_kwargs:

        a `dict` of keyword arguments to be passed to each nested ArgumentParser
        instance created per command (i.e. per function).  Members of this
        dictionary have the highest priority, so a function's docstring is
        overridden by a `help` in `func_kwargs` (if present).

    :param namespace_kwargs:

        a `dict` of keyword arguments to be passed to the nested ArgumentParser
        instance under given `namespace`.

    Deprecated params that should be moved into `namespace_kwargs`:

    :param title:

        passed to :meth:`argparse.ArgumentParser.add_subparsers` as `title`.

        .. deprecated:: 0.26.0

           Please use `namespace_kwargs` instead.

    :param description:

        passed to :meth:`argparse.ArgumentParser.add_subparsers` as
        `description`.

        .. deprecated:: 0.26.0

           Please use `namespace_kwargs` instead.

    :param help:

        passed to :meth:`argparse.ArgumentParser.add_subparsers` as `help`.

        .. deprecated:: 0.26.0

           Please use `namespace_kwargs` instead.

    .. note::

        This function modifies the parser object. Generally side effects are
        bad practice but we don't seem to have any choice as ArgumentParser is
        pretty opaque.
        You may prefer :class:`~argh.helpers.ArghParser.add_commands` for a bit
        more predictable API.

    .. note::

       An attempt to add commands to a parser which already has a default
       function (e.g. added with :func:`~argh.assembling.set_default_command`)
       results in `AssemblingError`.

    """
    # FIXME "namespace" is a correct name but it clashes with the "namespace"
    # that represents arguments (argparse.Namespace and our ArghNamespace).
    # We should rename the argument here.

    if DEST_FUNCTION in parser._defaults:
        _require_support_for_default_command_with_subparsers()

    namespace_kwargs = namespace_kwargs or {}

    # FIXME remove this by 1.0
    #
    if title:
        warnings.warn('argument `title` is deprecated in add_commands(),'
                      ' use `parser_kwargs` instead', DeprecationWarning)
        namespace_kwargs['description'] = title
    if help:
        warnings.warn('argument `help` is deprecated in add_commands(),'
                      ' use `parser_kwargs` instead', DeprecationWarning)
        namespace_kwargs['help'] = help
    if description:
        warnings.warn('argument `description` is deprecated in add_commands(),'
                      ' use `parser_kwargs` instead', DeprecationWarning)
        namespace_kwargs['description'] = description
    #
    # /

    subparsers_action = get_subparsers(parser, create=True)

    if namespace:
        # Make a nested parser and init a deeper _SubParsersAction under it.

        # Create a named group of commands.  It will be listed along with
        # root-level commands in ``app.py --help``; in that context its `title`
        # can be used as a short description on the right side of its name.
        # Normally `title` is shown above the list of commands
        # in ``app.py my-namespace --help``.
        subsubparser_kw = {
            'help': namespace_kwargs.get('title'),
        }
        subsubparser = subparsers_action.add_parser(namespace, **subsubparser_kw)
        subparsers_action = subsubparser.add_subparsers(**namespace_kwargs)
    else:
        assert not namespace_kwargs, ('`parser_kwargs` only makes sense '
                                      'with `namespace`.')

    for func in functions:
        cmd_name, func_parser_kwargs = _extract_command_meta_from_func(func)

        # override any computed kwargs by manually supplied ones
        if func_kwargs:
            func_parser_kwargs.update(func_kwargs)

        # create and set up the parser for this command
        command_parser = subparsers_action.add_parser(cmd_name, **func_parser_kwargs)
        set_default_command(command_parser, func)

Adds given functions as commands to given parser .


def named(new_name):
    """
    Sets given string as command name instead of the function name.
    The string is used verbatim without further processing.

    Usage::

        @named('load')
        def do_load_some_stuff_and_keep_the_original_function_name(args):
            ...

    The resulting command will be available only as ``load``.  To add aliases
    without renaming the command, check :func:`aliases`.

    .. versionadded:: 0.19
    """
    def wrapper(func):
        setattr(func, ATTR_NAME, new_name)
        return func
    return wrapper

def named ( new_name ) : def wrapper ( func ) : setattr ( func , ATTR_NAME , new_name ) return func return wrapper

Sets given string as command name instead of the function name . The string is used verbatim without further processing .


def arg(*args, **kwargs):
    """
    Declares an argument for given function. Does not register the function
    anywhere, nor does it modify the function in any way.

    The signature of the decorator matches that of
    :meth:`argparse.ArgumentParser.add_argument`, only some keywords are not
    required if they can be easily guessed (e.g. you don't have to specify type
    or action when an `int` or `bool` default value is supplied).

    Typical use cases:

    - In combination with :func:`expects_obj` (which is not recommended);
    - in combination with ordinary function signatures to add details that
      cannot be expressed with that syntax (e.g. help message).

    Usage::

        from argh import arg

        @arg('path', help='path to the file to load')
        @arg('--format', choices=['yaml','json'])
        @arg('-v', '--verbosity', choices=range(0,3), default=2)
        def load(path, something=None, format='json', dry_run=False, verbosity=1):
            loaders = {'json': json.load, 'yaml': yaml.load}
            loader = loaders[args.format]
            data = loader(args.path)
            if not args.dry_run:
                if verbosity < 1:
                    print('saving to the database')
                put_to_database(data)

    In this example:

    - `path` declaration is extended with `help`;
    - `format` declaration is extended with `choices`;
    - `dry_run` declaration is not duplicated;
    - `verbosity` is extended with `choices` and the default value is
      overridden.  (If both function signature and `@arg` define a default
      value for an argument, `@arg` wins.)

    .. note::

        It is recommended to avoid using this decorator unless there's no way
        to tune the argument's behaviour or presentation using ordinary
        function signatures.  Readability counts, don't repeat yourself.

    """
    def wrapper(func):
        declared_args = getattr(func, ATTR_ARGS, [])
        # The innermost decorator is called first but appears last in the code.
        # We need to preserve the expected order of positional arguments, so
        # the outermost decorator inserts its value before the innermost's:
        declared_args.insert(0, dict(option_strings=args, **kwargs))
        setattr(func, ATTR_ARGS, declared_args)
        return func
    return wrapper

def arg ( * args , ** kwargs ) : def wrapper ( func ) : declared_args = getattr ( func , ATTR_ARGS , [ ] ) declared_args . insert ( 0 , dict ( option_strings = args , ** kwargs ) ) setattr ( func , ATTR_ARGS , declared_args ) return func return wrapper

Declares an argument for given function . Does not register the function anywhere nor does it modify the function in any way .


def confirm(action, default=None, skip=False):
    """
    A shortcut for typical confirmation prompt.

    :param action:

        a string describing the action, e.g. "Apply changes". A question mark
        will be appended.

    :param default:

        `bool` or `None`. Determines what happens when user hits :kbd:`Enter`
        without typing in a choice. If `True`, default choice is "yes". If
        `False`, it is "no". If `None` the prompt keeps reappearing until user
        types in a choice (not necessarily acceptable) or until the number of
        iteration reaches the limit. Default is `None`.

    :param skip:

        `bool`; if `True`, no interactive prompt is used and default choice is
        returned (useful for batch mode). Default is `False`.

    Usage::

        def delete(key, silent=False):
            item = db.get(Item, args.key)
            if confirm('Delete '+item.title, default=True, skip=silent):
                item.delete()
                print('Item deleted.')
            else:
                print('Operation cancelled.')

    Returns `None` on `KeyboardInterrupt` event.
    """
    MAX_ITERATIONS = 3
    if skip:
        return default
    else:
        defaults = {
            None: ('y','n'),
            True: ('Y','n'),
            False: ('y','N'),
        }
        y, n = defaults[default]
        prompt = text_type('{action}? ({y}/{n})').format(**locals())
        choice = None
        try:
            if default is None:
                cnt = 1
                while not choice and cnt < MAX_ITERATIONS:
                    choice = safe_input(prompt)
                    cnt += 1
            else:
                choice = safe_input(prompt)
        except KeyboardInterrupt:
            return None
    if choice in ('yes', 'y', 'Y'):
        return True
    if choice in ('no', 'n', 'N'):
        return False
    if default is not None:
        return default
    return None

def
confirm
(
action
,
default
=
,
skip
=
False
)
:
MAX_ITERATIONS
=
3
if
skip
:
return
default
else
:
defaults
=
{
:
(
'y'
,
'n'
)
,
True
:
(
'Y'
,
'n'
)
,
False
:
(
'y'
,
'N'
)
,
}
y
,
n
=
defaults
[
default
]
prompt
=
text_type
(
'{action}? ({y}/{n})'
)
.
format
(
**
locals
(
)
)
choice
=
try
:
if
default
is
:
cnt
=
1
while
not
choice
and
cnt
<
MAX_ITERATIONS
:
choice
=
safe_input
(
prompt
)
cnt
+=
1
else
:
choice
=
safe_input
(
prompt
)
except
KeyboardInterrupt
:
return
if
choice
in
(
'yes'
,
'y'
,
'Y'
)
:
return
True
if
choice
in
(
'no'
,
'n'
,
'N'
)
:
return
False
if
default
is
not
:
return
default
return

A shortcut for typical confirmation prompt .


def replace(self, **kwargs):
        '''
        Copy the Query object, optionally replacing the filters, order_by, or
        limit information on the copy. This is mostly an internal detail that
        you can ignore.
        '''
        data = {
            'model': self._model,
            'filters': self._filters,
            'order_by': self._order_by,
            'limit': self._limit,
            'select': self._select,
        }
        data.update(**kwargs)
        return Query(**data)

def replace ( self , ** kwargs ) : data = { 'model' : self . _model , 'filters' : self . _filters , 'order_by' : self . _order_by , 'limit' : self . _limit , 'select' : self . _select , } data . update ( ** kwargs ) return Query ( ** data )

Copy the Query object optionally replacing the filters order_by or limit information on the copy . This is mostly an internal detail that you can ignore .


def like(self, **kwargs):
        '''
        When provided with keyword arguments of the form ``col=pattern``, this
        will limit the entities returned to those that include the provided
        pattern. Note that 'like' queries require that the ``prefix=True``
        option must have been provided as part of the column definition.

        Patterns allow for 4 wildcard characters, whose semantics are as
        follows:

            * *?* - will match 0 or 1 of any character
            * *\** - will match 0 or more of any character
            * *+* - will match 1 or more of any character
            * *!* - will match exactly 1 of any character

        As an example, imagine that you have enabled the required prefix
        matching on your ``User.email`` column. And lets say that you want to
        find everyone with an email address that contains the name 'frank'
        before the ``@`` sign. You can use either of the following patterns
        to discover those users.

            * *\*frank\*@*
            * *\*frank\*@*

        .. note:: Like queries implicitly start at the beginning of strings
          checked, so if you want to match a pattern that doesn't start at
          the beginning of a string, you should prefix it with one of the
          wildcard characters (like ``*`` as we did with the 'frank' pattern).
        '''
        new = []
        for k, v in kwargs.items():
            v = self._check(k, v, 'like')
            new.append(Pattern(k, v))
        return self.replace(filters=self._filters+tuple(new))

def like ( self , ** kwargs ) : new = [ ] for k , v in kwargs . items ( ) : v = self . _check ( k , v , 'like' ) new . append ( Pattern ( k , v ) ) return self . replace ( filters = self . _filters + tuple ( new ) )

When provided with keyword arguments of the form col = pattern this will limit the entities returned to those that include the provided pattern . Note that like queries require that the prefix = True option must have been provided as part of the column definition .


def cached_result(self, timeout):
        '''
        This will execute the query, returning the key where a ZSET of your
        results will be stored for pagination, further operations, etc.

        The timeout must be a positive integer number of seconds for which to
        set the expiration time on the key (this is to ensure that any cached
        query results are eventually deleted, unless you make the explicit
        step to use the PERSIST command).

        .. note:: Limit clauses are ignored and not passed.

        Usage::

            ukey = User.query.endswith(email='@gmail.com').cached_result(30)
            for i in xrange(0, conn.zcard(ukey), 100):
                # refresh the expiration
                conn.expire(ukey, 30)
                users = User.get(conn.zrange(ukey, i, i+99))
                ...
        '''
        if not (self._filters or self._order_by):
            raise QueryError("You are missing filter or order criteria")
        timeout = int(timeout)
        if timeout < 1:
            raise QueryError("You must specify a timeout >= 1, you gave %r"%timeout)
        return self._model._gindex.search(
            _connect(self._model), self._filters, self._order_by, timeout=timeout)

This will execute the query returning the key where a ZSET of your results will be stored for pagination further operations etc .


def first(self):
        '''
        Returns only the first result from the query, if any.
        '''
        lim = [0, 1]
        if self._limit:
            lim[0] = self._limit[0]
        if not self._filters and not self._order_by:
            for ent in self:
                return ent
            return None
        ids = self.limit(*lim)._search()
        if ids:
            return self._model.get(ids[0])
        return None

def first ( self ) : lim = [ 0 , 1 ] if self . _limit : lim [ 0 ] = self . _limit [ 0 ] if not self . _filters and not self . _order_by : for ent in self : return ent return None ids = self . limit ( * lim ) . _search ( ) if ids : return self . _model . get ( ids [ 0 ] ) return None

Returns only the first result from the query if any .


def delete(self, blocksize=100):
        '''
        Will delete the entities that match at the time the query is executed.

        Used like::

            MyModel.query.filter(email=...).delete()
            MyModel.query.endswith(email='@host.com').delete()

        .. warning:: can't be used on models on either side of a ``OneToMany``,
            ``ManyToOne``, or ``OneToOne`` relationship.
        '''

        from .columns import MODELS_REFERENCED
        if not self._model._no_fk or self._model._namespace in MODELS_REFERENCED:
            raise QueryError("Can't delete entities of models with foreign key relationships")

        de = []
        i = 0
        for result in self.iter_result(pagesize=blocksize):
            de.append(result)
            i += 1
            if i >= blocksize:
                session.delete(de) # one round-trip to delete "chunk" items
                del de[:]
                i = 0

        if de:
            session.delete(de)

Will delete the entities that match at the time the query is executed .


def _on_delete(ent):
    '''
    This function handles all on_delete semantics defined on OneToMany columns.

    This function only exists because 'cascade' is *very* hard to get right.
    '''
    seen_d = set([ent._pk])
    to_delete = [ent]
    seen_s = set()
    to_save = []

    def _set_default(ent, attr, de=NULL):
        pk = ent._pk
        if pk in seen_d:
            # going to be deleted, don't need to modify
            return

        col = ent.__class__._columns[attr]
        de = de if de is not NULL else col._default
        if de in (None, NULL):
            setattr(ent, attr, None)
        elif callable(col._default):
            setattr(ent, attr, col._default())
        else:
            setattr(ent, attr, col._default)

        if pk not in seen_s:
            seen_s.add(pk)
            to_save.append(ent)

    for self in to_delete:
        for tbl, attr, action in MODELS_REFERENCED.get(self._namespace, ()):
            if action == 'no action':
                continue

            refs = MODELS[tbl].get_by(**{attr: self.id})
            if not refs:
                continue

            if action == 'restrict':
                # raise the exception here for a better traceback
                raise _restrict(self, attr, refs)
            elif action == 'set null':
                for ref in refs:
                    _set_default(ref, attr, None)
                continue
            elif action == 'set default':
                for ref in refs:
                    _set_default(ref, attr)
                continue

            # otherwise col._on_delete == 'cascade'
            for ent in (refs if isinstance(refs, list) else [refs]):
                if ent._pk not in seen_d:
                    seen_d.add(ent._pk)
                    to_delete.append(ent)

    # If we got here, then to_delete includes all items to delete. Let's delete
    # them!
    for self in to_delete:
        self.delete(skip_on_delete_i_really_mean_it=SKIP_ON_DELETE)
    for self in to_save:
        # Careful not to resurrect deleted entities
        if self._pk not in seen_d:
            self.save()

def
_on_delete
(
ent
)
:
seen_d
=
set
(
[
ent
.
_pk
]
)
to_delete
=
[
ent
]
seen_s
=
set
(
)
to_save
=
[
]
def
_set_default
(
ent
,
attr
,
de
=
)
:
pk
=
ent
.
_pk
if
pk
in
seen_d
:
return
col
=
ent
.
__class__
.
_columns
[
attr
]
de
=
de
if
de
is
not
else
col
.
_default
if
de
in
(
,
)
:
setattr
(
ent
,
attr
,
)
elif
callable
(
col
.
_default
)
:
setattr
(
ent
,
attr
,
col
.
_default
(
)
)
else
:
setattr
(
ent
,
attr
,
col
.
_default
)
if
pk
not
in
seen_s
:
seen_s
.
add
(
pk
)
to_save
.
append
(
ent
)
for
self
in
to_delete
:
for
tbl
,
attr
,
action
in
MODELS_REFERENCED
.
get
(
self
.
_namespace
,
(
)
)
:
if
action
==
'no action'
:
continue
refs
=
MODELS
[
tbl
]
.
get_by
(
**
{
attr
:
self
.
id
}
)
if
not
refs
:
continue
if
action
==
'restrict'
:
raise
_restrict
(
self
,
attr
,
refs
)
elif
action
==
'set null'
:
for
ref
in
refs
:
_set_default
(
ref
,
attr
,
)
continue
elif
action
==
'set default'
:
for
ref
in
refs
:
_set_default
(
ref
,
attr
)
continue
for
ent
in
(
refs
if
isinstance
(
refs
,
list
)
else
[
refs
]
)
:
if
ent
.
_pk
not
in
seen_d
:
seen_d
.
add
(
ent
.
_pk
)
to_delete
.
append
(
ent
)
for
self
in
to_delete
:
self
.
delete
(
skip_on_delete_i_really_mean_it
=
SKIP_ON_DELETE
)
for
self
in
to_save
:
if
self
.
_pk
not
in
seen_d
:
self
.
save
(
)

This function handles all on_delete semantics defined on OneToMany columns .


def redis_prefix_lua(conn, dest, index, prefix, is_first, pattern=None):
    '''
    Performs the actual prefix, suffix, and pattern match operations.
    '''
    tkey = '%s:%s'%(index.partition(':')[0], uuid.uuid4())
    start, end = _start_end(prefix)
    return _redis_prefix_lua(conn,
        [dest, tkey, index],
        [start, end, pattern or prefix, int(pattern is not None), int(bool(is_first))]
    )

def redis_prefix_lua ( conn , dest , index , prefix , is_first , pattern = None ) : tkey = '%s:%s' % ( index . partition ( ':' ) [ 0 ] , uuid . uuid4 ( ) ) start , end = _start_end ( prefix ) return _redis_prefix_lua ( conn , [ dest , tkey , index ] , [ start , end , pattern or prefix , int ( pattern is not None ) , int ( bool ( is_first ) ) ] )

Performs the actual prefix suffix and pattern match operations .


def estimate_work_lua(conn, index, prefix):
    '''
    Estimates the total work necessary to calculate the prefix match over the
    given index with the provided prefix.
    '''
    if index.endswith(':idx'):
        args = [] if not prefix else list(prefix)
        if args:
            args[0] = '-inf' if args[0] is None else repr(float(args[0]))
            args[1] = 'inf' if args[1] is None else repr(float(args[1]))
        return _estimate_work_lua(conn, [index], args, force_eval=True)
    elif index.endswith(':geo'):
        return _estimate_work_lua(conn, [index], filter(None, [prefix]), force_eval=True)

    start, end = _start_end(prefix)
    return _estimate_work_lua(conn, [index], [start, '(' + end], force_eval=True)

def estimate_work_lua ( conn , index , prefix ) : if index . endswith ( ':idx' ) : args = [ ] if not prefix else list ( prefix ) if args : args [ 0 ] = '-inf' if args [ 0 ] is None else repr ( float ( args [ 0 ] ) ) args [ 1 ] = 'inf' if args [ 1 ] is None else repr ( float ( args [ 1 ] ) ) return _estimate_work_lua ( conn , [ index ] , args , force_eval = True ) elif index . endswith ( ':geo' ) : return _estimate_work_lua ( conn , [ index ] , filter ( None , [ prefix ] ) , force_eval = True ) start , end = _start_end ( prefix ) return _estimate_work_lua ( conn , [ index ] , [ start , '(' + end ] , force_eval = True )

Estimates the total work necessary to calculate the prefix match over the given index with the provided prefix .


def search(self, conn, filters, order_by, offset=None, count=None, timeout=None):
        '''
        Search for model ids that match the provided filters.

        Arguments:

            * *filters* - A list of filters that apply to the search of one of
              the following two forms:

                1. ``'column:string'`` - a plain string will match a word in a
                   text search on the column

                .. note:: Read the documentation about the ``Query`` object
                  for what is actually passed during text search

                2. ``('column', min, max)`` - a numeric column range search,
                   between min and max (inclusive by default)

                .. note:: Read the documentation about the ``Query`` object
                  for information about open-ended ranges

                3. ``['column:string1', 'column:string2']`` - will match any
                   of the provided words in a text search on the column

                4. ``Prefix('column', 'prefix')`` - will match prefixes of
                   words in a text search on the column

                5. ``Suffix('column', 'suffix')`` - will match suffixes of
                   words in a text search on the column

                6. ``Pattern('column', 'pattern')`` - will match patterns over
                   words in a text search on the column

            * *order_by* - A string that names the numeric column by which to
              sort the results by. Prefixing with '-' will return results in
              descending order

            .. note:: While you can technically pass a non-numeric index as an
              *order_by* clause, the results will basically be to order the
              results by string comparison of the ids (10 will come before 2).

            .. note:: If you omit the ``order_by`` argument, results will be
              ordered by the last filter. If the last filter was a text
              filter, see the previous note. If the last filter was numeric,
              then results will be ordered by that result.

            * *offset* - A numeric starting offset for results
            * *count* - The maximum number of results to return from the query
        '''
        # prepare the filters
        pipe, intersect, temp_id = self._prepare(conn, filters)

        # handle ordering
        if order_by:
            reverse = order_by and order_by.startswith('-')
            order_clause = '%s:%s:idx'%(self.namespace, order_by.lstrip('-'))
            intersect(temp_id, {temp_id:0, order_clause: -1 if reverse else 1})

        # handle returning the temporary result key
        if timeout is not None:
            pipe.expire(temp_id, timeout)
            pipe.execute()
            return temp_id

        offset = offset if offset is not None else 0
        end = (offset + count - 1) if count and count > 0 else -1
        pipe.zrange(temp_id, offset, end)
        pipe.delete(temp_id)
        return pipe.execute()[-2]

def search ( self , conn , filters , order_by , offset = None , count = None , timeout = None ) : pipe , intersect , temp_id = self . _prepare ( conn , filters ) if order_by : reverse = order_by and order_by . startswith ( '-' ) order_clause = '%s:%s:idx' % ( self . namespace , order_by . lstrip ( '-' ) ) intersect ( temp_id , { temp_id : 0 , order_clause : - 1 if reverse else 1 } ) if timeout is not None : pipe . expire ( temp_id , timeout ) pipe . execute ( ) return temp_id offset = offset if offset is not None else 0 end = ( offset + count - 1 ) if count and count > 0 else - 1 pipe . zrange ( temp_id , offset , end ) pipe . delete ( temp_id ) return pipe . execute ( ) [ - 2 ]

Search for model ids that match the provided filters .


def count(self, conn, filters):
        '''
        Returns the count of the items that match the provided filters.

        For the meaning of what the ``filters`` argument means, see the
        ``.search()`` method docs.
        '''
        pipe, intersect, temp_id = self._prepare(conn, filters)
        pipe.zcard(temp_id)
        pipe.delete(temp_id)
        return pipe.execute()[-2]

def count ( self , conn , filters ) : pipe , intersect , temp_id = self . _prepare ( conn , filters ) pipe . zcard ( temp_id ) pipe . delete ( temp_id ) return pipe . execute ( ) [ - 2 ]

Returns the count of the items that match the provided filters .


def _connect(obj):
    '''
    Tries to get the _conn attribute from a model. Barring that, gets the
    global default connection using other methods.
    '''
    from .columns import MODELS
    if isinstance(obj, MODELS['Model']):
        obj = obj.__class__
    if hasattr(obj, '_conn'):
        return obj._conn
    if hasattr(obj, 'CONN'):
        return obj.CONN
    return get_connection()

def _connect ( obj ) : from . columns import MODELS if isinstance ( obj , MODELS [ 'Model' ] ) : obj = obj . __class__ if hasattr ( obj , '_conn' ) : return obj . _conn if hasattr ( obj , 'CONN' ) : return obj . CONN return get_connection ( )

Tries to get the _conn attribute from a model . Barring that gets the global default connection using other methods .


def FULL_TEXT(val):
    '''
    This is a basic full-text index keygen function. Words are lowercased, split
    by whitespace, and stripped of punctuation from both ends before an inverted
    index is created for term searching.
    '''
    if isinstance(val, float):
        val = repr(val)
    elif val in (None, ''):
        return None
    elif not isinstance(val, six.string_types):
        if six.PY3 and isinstance(val, bytes):
            val = val.decode('latin-1')
        else:
            val = str(val)
    r = sorted(set([x for x in [s.lower().strip(string.punctuation) for s in val.split()] if x]))
    if not isinstance(val, str):  # unicode on py2k
        return [s.encode('utf-8') for s in r]
    return r

def FULL_TEXT ( val ) : if isinstance ( val , float ) : val = repr ( val ) elif val in ( None , '' ) : return None elif not isinstance ( val , six . string_types ) : if six . PY3 and isinstance ( val , bytes ) : val = val . decode ( 'latin-1' ) else : val = str ( val ) r = sorted ( set ( [ x for x in [ s . lower ( ) . strip ( string . punctuation ) for s in val . split ( ) ] if x ] ) ) if not isinstance ( val , str ) : return [ s . encode ( 'utf-8' ) for s in r ] return r

This is a basic full - text index keygen function . Words are lowercased split by whitespace and stripped of punctuation from both ends before an inverted index is created for term searching .


def refresh_indices(model, block_size=100):
    '''
    This utility function will iterate over all entities of a provided model,
    refreshing their indices. This is primarily useful after adding an index
    on a column.

    Arguments:

        * *model* - the model whose entities you want to reindex
        * *block_size* - the maximum number of entities you want to fetch from
          Redis at a time, defaulting to 100

    This function will yield its progression through re-indexing all of your
    entities.

    Example use::

        for progress, total in refresh_indices(MyModel, block_size=200):
            print "%s of %s"%(progress, total)

    .. note:: This uses the session object to handle index refresh via calls to
      ``.commit()``. If you have any outstanding entities known in the
      session, they will be committed.
    '''
    conn = _connect(model)
    max_id = int(conn.get('%s:%s:'%(model._namespace, model._pkey)) or '0')
    block_size = max(block_size, 10)
    for i in range(1, max_id+1, block_size):
        # fetches entities, keeping a record in the session
        models = model.get(list(range(i, i+block_size)))
        models # for pyflakes
        # re-save un-modified data, resulting in index-only updates
        session.commit(all=True)
        yield min(i+block_size, max_id), max_id

def refresh_indices ( model , block_size = 100 ) : conn = _connect ( model ) max_id = int ( conn . get ( '%s:%s:' % ( model . _namespace , model . _pkey ) ) or '0' ) block_size = max ( block_size , 10 ) for i in range ( 1 , max_id + 1 , block_size ) : models = model . get ( list ( range ( i , i + block_size ) ) ) models session . commit ( all = True ) yield min ( i + block_size , max_id ) , max_id

This utility function will iterate over all entities of a provided model refreshing their indices . This is primarily useful after adding an index on a column .


def clean_old_index(model, block_size=100, **kwargs):
    '''
    This utility function will clean out old index data that was accidentally
    left during item deletion in rom versions <= 0.27.0 . You should run this
    after you have upgraded all of your clients to version 0.28.0 or later.

    Arguments:

        * *model* - the model whose entities you want to reindex
        * *block_size* - the maximum number of items to check at a time
          defaulting to 100

    This function will yield its progression through re-checking all of the
    data that could be left over.

    Example use::

        for progress, total in clean_old_index(MyModel, block_size=200):
            print "%s of %s"%(progress, total)
    '''

    conn = _connect(model)
    version = list(map(int, conn.info()['redis_version'].split('.')[:2]))
    has_hscan = version >= [2, 8]
    pipe = conn.pipeline(True)
    prefix = '%s:'%model._namespace
    index = prefix + ':'
    block_size = max(block_size, 10)

    force_hscan = kwargs.get('force_hscan', False)
    if (has_hscan or force_hscan) and force_hscan is not None:
        max_id = conn.hlen(index)
        cursor = None
        scanned = 0
        while cursor != b'0':
            cursor, remove = _scan_index_lua(conn, [index, prefix], [cursor or '0', block_size, 0, 0])
            if remove:
                _clean_index_lua(conn, [model._namespace], remove)

            scanned += block_size
            if scanned > max_id:
                max_id = scanned + 1
            yield scanned, max_id

        # need to scan over unique indexes :/
        for uniq in chain(model._unique, model._cunique):
            name = uniq if isinstance(uniq, six.string_types) else ':'.join(uniq)
            idx = prefix + name + ':uidx'

            cursor = None
            while cursor != b'0':
                cursor, remove = _scan_index_lua(conn, [idx, prefix], [cursor or '0', block_size, 1, 0])
                if remove:
                    conn.hdel(idx, *remove)

                scanned += block_size
                if scanned > max_id:
                    max_id = scanned + 1
                yield scanned, max_id
    else:
        if model._unique or model._cunique:
            if has_hscan:
                warnings.warn("You have disabled the use of HSCAN to clean up indexes, this will prevent unique index cleanup", stacklevel=2)
            else:
                warnings.warn("Unique indexes cannot be cleaned up in Redis versions prior to 2.8", stacklevel=2)

        max_id = int(conn.get('%s%s:'%(prefix, model._pkey)) or '0')
        for i in range(1, max_id+1, block_size):
            ids = list(range(i, min(i+block_size, max_id+1)))
            for id in ids:
                pipe.exists(prefix + str(id))
                pipe.hexists(index, id)

            result = iter(pipe.execute())
            remove = [id for id, ent, ind in zip(ids, result, result) if ind and not ent]
            if remove:
                _clean_index_lua(conn, [model._namespace], remove)

            yield min(i+block_size, max_id-1), max_id

    yield max_id, max_id

This utility function will clean out old index data that was accidentally left during item deletion in rom versions < = 0 . 27 . 0 . You should run this after you have upgraded all of your clients to version 0 . 28 . 0 or later .


def add(self, obj):
        '''
        Adds an entity to the session.
        '''
        if self.null_session:
            return
        self._init()
        pk = obj._pk
        if not pk.endswith(':None'):
            self.known[pk] = obj
            self.wknown[pk] = obj

def add ( self , obj ) : if self . null_session : return self . _init ( ) pk = obj . _pk if not pk . endswith ( ':None' ) : self . known [ pk ] = obj self . wknown [ pk ] = obj


def get(self, pk):
        '''
        Fetches an entity from the session based on primary key.
        '''
        self._init()
        return self.known.get(pk) or self.wknown.get(pk)

def get ( self , pk ) : self . _init ( ) return self . known . get ( pk ) or self . wknown . get ( pk )

Fetches an entity from the session based on primary key .


def redis_writer_lua(conn, pkey, namespace, id, unique, udelete, delete,
                     data, keys, scored, prefix, suffix, geo, old_data, is_delete):
    '''
    ... Actually write data to Redis. This is an internal detail. Please don't
    call me directly.
    '''
    ldata = []
    for pair in data.items():
        ldata.extend(pair)

    for item in prefix:
        item.append(_prefix_score(item[-1]))
    for item in suffix:
        item.append(_prefix_score(item[-1]))

    data = [json.dumps(x, default=_fix_bytes) for x in
            (unique, udelete, delete, ldata, keys, scored, prefix, suffix, geo, is_delete, old_data)]
    result = _redis_writer_lua(conn, [], [namespace, id] + data)

    if isinstance(conn, _Pipeline):
        # we're in a pipelined write situation, don't parse the pipeline :P
        return

    if six.PY3:
        result = result.decode()

    result = json.loads(result)
    if 'unique' in result:
        result = result['unique']
        raise UniqueKeyViolation(
            "Value %r for %s:%s:uidx not distinct (failed for pk=%s)"%(
                unique[result], namespace, result, id),
            namespace, id)

    if 'race' in result:
        result = result['race']
        if pkey in result:
            raise EntityDeletedError(
                "Entity %s:%s deleted by another writer; use .save(force=True) to re-save"%(
                    namespace, id),
                namespace, id)

        raise DataRaceError(
            "%s:%s Column(s) %r updated by another writer, write aborted!"%(
                namespace, id, result),
            namespace, id)

... Actually write data to Redis . This is an internal detail . Please don t call me directly .


def save(self, full=False, force=False):
        '''
        Saves the current entity to Redis. Will only save changed data by
        default, but you can force a full save by passing ``full=True``.

        If the underlying entity was deleted and you want to re-save the entity,
        you can pass ``force=True`` to force a full re-save of the entity.
        '''
        # handle the pre-commit hooks
        was_new = self._new
        if was_new:
            self._before_insert()
        else:
            self._before_update()

        new = self.to_dict()
        ret, data = self._apply_changes(
            self._last, new, full or self._new or force, is_new=self._new or force)
        self._last = data
        self._new = False
        self._modified = False
        self._deleted = False
        # handle the post-commit hooks
        if was_new:
            self._after_insert()
        else:
            self._after_update()
        return ret

def save ( self , full = False , force = False ) : was_new = self . _new if was_new : self . _before_insert ( ) else : self . _before_update ( ) new = self . to_dict ( ) ret , data = self . _apply_changes ( self . _last , new , full or self . _new or force , is_new = self . _new or force ) self . _last = data self . _new = False self . _modified = False self . _deleted = False if was_new : self . _after_insert ( ) else : self . _after_update ( ) return ret

Saves the current entity to Redis . Will only save changed data by default but you can force a full save by passing full = True .


def delete(self, **kwargs):
        '''
        Deletes the entity immediately. Also performs any on_delete operations
        specified as part of column definitions.
        '''
        if kwargs.get('skip_on_delete_i_really_mean_it') is not SKIP_ON_DELETE:
            # handle the pre-commit hook
            self._before_delete()
            # handle any foreign key references + cascade options
            _on_delete(self)

        session.forget(self)
        self._apply_changes(self._last, {}, delete=True, _conn=kwargs.get('_conn'))
        self._modified = True
        self._deleted = True
        # handle the post-commit hooks
        if kwargs.get('skip_on_delete_i_really_mean_it') is not SKIP_ON_DELETE:
            self._after_delete()

def delete ( self , ** kwargs ) : if kwargs . get ( 'skip_on_delete_i_really_mean_it' ) is not SKIP_ON_DELETE : self . _before_delete ( ) _on_delete ( self ) session . forget ( self ) self . _apply_changes ( self . _last , { } , delete = True , _conn = kwargs . get ( '_conn' ) ) self . _modified = True self . _deleted = True if kwargs . get ( 'skip_on_delete_i_really_mean_it' ) is not SKIP_ON_DELETE : self . _after_delete ( )

Deletes the entity immediately . Also performs any on_delete operations specified as part of column definitions .


def get(cls, ids):
        '''
        Will fetch one or more entities of this type from the session or
        Redis.

        Used like::

            MyModel.get(5)
            MyModel.get([1, 6, 2, 4])

        Passing a list or a tuple will return multiple entities, in the same
        order that the ids were passed.
        '''
        conn = _connect(cls)
        # prepare the ids
        single = not isinstance(ids, (list, tuple, set, frozenset))
        if single:
            ids = [ids]
        pks = ['%s:%s'%(cls._namespace, id) for id in map(int, ids)]
        # get from the session, if possible
        out = list(map(session.get, pks))
        # if we couldn't get an instance from the session, load from Redis
        if None in out:
            pipe = conn.pipeline(True)
            idxs = []
            # Fetch missing data
            for i, data in enumerate(out):
                if data is None:
                    idxs.append(i)
                    pipe.hgetall(pks[i])
            # Update output list
            for i, data in zip(idxs, pipe.execute()):
                if data:
                    if six.PY3:
                        data = dict((k.decode(), v.decode()) for k, v in data.items())
                    out[i] = cls(_loading=True, **data)
            # Get rid of missing models
            out = [x for x in out if x]
        if single:
            return out[0] if out else None
        return out

def get ( cls , ids ) : conn = _connect ( cls ) single = not isinstance ( ids , ( list , tuple , set , frozenset ) ) if single : ids = [ ids ] pks = [ '%s:%s' % ( cls . _namespace , id ) for id in map ( int , ids ) ] out = list ( map ( session . get , pks ) ) if None in out : pipe = conn . pipeline ( True ) idxs = [ ] for i , data in enumerate ( out ) : if data is None : idxs . append ( i ) pipe . hgetall ( pks [ i ] ) for i , data in zip ( idxs , pipe . execute ( ) ) : if data : if six . PY3 : data = dict ( ( k . decode ( ) , v . decode ( ) ) for k , v in data . items ( ) ) out [ i ] = cls ( _loading = True , ** data ) out = [ x for x in out if x ] if single : return out [ 0 ] if out else None return out

Will fetch one or more entities of this type from the session or Redis .


def register(cls, type, reduce_func):
        """Attach a reducer function to a given type in the dispatch table."""
        if sys.version_info < (3,):
            # Python 2 pickler dispatching is not explicitly customizable.
            # Let us use a closure to workaround this limitation.
            def dispatcher(cls, obj):
                reduced = reduce_func(obj)
                cls.save_reduce(obj=obj, *reduced)
            cls.dispatch_table[type] = dispatcher
        else:
            cls.dispatch_table[type] = reduce_func

def register ( cls , type , reduce_func ) : if sys . version_info < ( 3 , ) : def dispatcher ( cls , obj ) : reduced = reduce_func ( obj ) cls . save_reduce ( obj = obj , * reduced ) cls . dispatch_table [ type ] = dispatcher else : cls . dispatch_table [ type ] = reduce_func

Attach a reducer function to a given type in the dispatch table .


def _sem_open(name, value=None):
    """ Construct or retrieve a semaphore with the given name

    If value is None, try to retrieve an existing named semaphore.
    Else create a new semaphore with the given value
    """
    if value is None:
        handle = pthread.sem_open(ctypes.c_char_p(name), 0)
    else:
        handle = pthread.sem_open(ctypes.c_char_p(name), SEM_OFLAG, SEM_PERM,
                                  ctypes.c_int(value))

    if handle == SEM_FAILURE:
        e = ctypes.get_errno()
        if e == errno.EEXIST:
            raise FileExistsError("a semaphore named %s already exists" % name)
        elif e == errno.ENOENT:
            raise FileNotFoundError('cannot find semaphore named %s' % name)
        elif e == errno.ENOSYS:
            raise NotImplementedError('No semaphore implementation on this '
                                      'system')
        else:
            raiseFromErrno()

    return handle

def
_sem_open
(
name
,
value
=
)
:
if
value
is
:
handle
=
pthread
.
sem_open
(
ctypes
.
c_char_p
(
name
)
,
0
)
else
:
handle
=
pthread
.
sem_open
(
ctypes
.
c_char_p
(
name
)
,
SEM_OFLAG
,
SEM_PERM
,
ctypes
.
c_int
(
value
)
)
if
handle
==
SEM_FAILURE
:
e
=
ctypes
.
get_errno
(
)
if
e
==
errno
.
EEXIST
:
raise
FileExistsError
(
"a semaphore named %s already exists"
%
name
)
elif
e
==
errno
.
ENOENT
:
raise
FileNotFoundError
(
'cannot find semaphore named %s'
%
name
)
elif
e
==
errno
.
ENOSYS
:
raise
NotImplementedError
(
'No semaphore implementation on this '
'system'
)
else
:
raiseFromErrno
(
)
return
handle


def cpu_count():
    """Return the number of CPUs the current process can use.

    The returned number of CPUs accounts for:
     * the number of CPUs in the system, as given by
       ``multiprocessing.cpu_count``;
     * the CPU affinity settings of the current process
       (available with Python 3.4+ on some Unix systems);
     * CFS scheduler CPU bandwidth limit (available on Linux only, typically
       set by docker and similar container orchestration systems);
     * the value of the LOKY_MAX_CPU_COUNT environment variable if defined.
    and is given as the minimum of these constraints.
    It is also always larger or equal to 1.
    """
    import math

    try:
        cpu_count_mp = mp.cpu_count()
    except NotImplementedError:
        cpu_count_mp = 1

    # Number of available CPUs given affinity settings
    cpu_count_affinity = cpu_count_mp
    if hasattr(os, 'sched_getaffinity'):
        try:
            cpu_count_affinity = len(os.sched_getaffinity(0))
        except NotImplementedError:
            pass

    # CFS scheduler CPU bandwidth limit
    # available in Linux since 2.6 kernel
    cpu_count_cfs = cpu_count_mp
    cfs_quota_fname = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
    cfs_period_fname = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
    if os.path.exists(cfs_quota_fname) and os.path.exists(cfs_period_fname):
        with open(cfs_quota_fname, 'r') as fh:
            cfs_quota_us = int(fh.read())
        with open(cfs_period_fname, 'r') as fh:
            cfs_period_us = int(fh.read())

        if cfs_quota_us > 0 and cfs_period_us > 0:
            # Make sure this quantity is an int as math.ceil returns a
            # float in python2.7. (See issue #165)
            cpu_count_cfs = int(math.ceil(cfs_quota_us / cfs_period_us))

    # User defined soft-limit passed as an loky specific environment variable.
    cpu_count_loky = int(os.environ.get('LOKY_MAX_CPU_COUNT', cpu_count_mp))
    aggregate_cpu_count = min(cpu_count_mp, cpu_count_affinity, cpu_count_cfs,
                              cpu_count_loky)
    return max(aggregate_cpu_count, 1)

def cpu_count ( ) : import math try : cpu_count_mp = mp . cpu_count ( ) except NotImplementedError : cpu_count_mp = 1 cpu_count_affinity = cpu_count_mp if hasattr ( os , 'sched_getaffinity' ) : try : cpu_count_affinity = len ( os . sched_getaffinity ( 0 ) ) except NotImplementedError : pass cpu_count_cfs = cpu_count_mp cfs_quota_fname = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" cfs_period_fname = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" if os . path . exists ( cfs_quota_fname ) and os . path . exists ( cfs_period_fname ) : with open ( cfs_quota_fname , 'r' ) as fh : cfs_quota_us = int ( fh . read ( ) ) with open ( cfs_period_fname , 'r' ) as fh : cfs_period_us = int ( fh . read ( ) ) if cfs_quota_us > 0 and cfs_period_us > 0 : cpu_count_cfs = int ( math . ceil ( cfs_quota_us / cfs_period_us ) ) cpu_count_loky = int ( os . environ . get ( 'LOKY_MAX_CPU_COUNT' , cpu_count_mp ) ) aggregate_cpu_count = min ( cpu_count_mp , cpu_count_affinity , cpu_count_cfs , cpu_count_loky ) return max ( aggregate_cpu_count , 1 )

Return the number of CPUs the current process can use .


def _sendback_result(result_queue, work_id, result=None, exception=None):
    """Safely send back the given result or exception"""
    try:
        result_queue.put(_ResultItem(work_id, result=result,
                                     exception=exception))
    except BaseException as e:
        exc = _ExceptionWithTraceback(e)
        result_queue.put(_ResultItem(work_id, exception=exc))

def _sendback_result ( result_queue , work_id , result = None , exception = None ) : try : result_queue . put ( _ResultItem ( work_id , result = result , exception = exception ) ) except BaseException as e : exc = _ExceptionWithTraceback ( e ) result_queue . put ( _ResultItem ( work_id , exception = exc ) )


def _process_worker(call_queue, result_queue, initializer, initargs,
                    processes_management_lock, timeout, worker_exit_lock,
                    current_depth):
    """Evaluates calls from call_queue and places the results in result_queue.

    This worker is run in a separate process.

    Args:
        call_queue: A ctx.Queue of _CallItems that will be read and
            evaluated by the worker.
        result_queue: A ctx.Queue of _ResultItems that will written
            to by the worker.
        initializer: A callable initializer, or None
        initargs: A tuple of args for the initializer
        process_management_lock: A ctx.Lock avoiding worker timeout while some
            workers are being spawned.
        timeout: maximum time to wait for a new item in the call_queue. If that
            time is expired, the worker will shutdown.
        worker_exit_lock: Lock to avoid flagging the executor as broken on
            workers timeout.
        current_depth: Nested parallelism level, to avoid infinite spawning.
    """
    if initializer is not None:
        try:
            initializer(*initargs)
        except BaseException:
            _base.LOGGER.critical('Exception in initializer:', exc_info=True)
            # The parent will notice that the process stopped and
            # mark the pool broken
            return

    # set the global _CURRENT_DEPTH mechanism to limit recursive call
    global _CURRENT_DEPTH
    _CURRENT_DEPTH = current_depth
    _process_reference_size = None
    _last_memory_leak_check = None
    pid = os.getpid()

    mp.util.debug('Worker started with timeout=%s' % timeout)
    while True:
        try:
            call_item = call_queue.get(block=True, timeout=timeout)
            if call_item is None:
                mp.util.info("Shutting down worker on sentinel")
        except queue.Empty:
            mp.util.info("Shutting down worker after timeout %0.3fs"
                         % timeout)
            if processes_management_lock.acquire(block=False):
                processes_management_lock.release()
                call_item = None
            else:
                mp.util.info("Could not acquire processes_management_lock")
                continue
        except BaseException as e:
            previous_tb = traceback.format_exc()
            try:
                result_queue.put(_RemoteTraceback(previous_tb))
            except BaseException:
                # If we cannot format correctly the exception, at least print
                # the traceback.
                print(previous_tb)
            sys.exit(1)
        if call_item is None:
            # Notify queue management thread about clean worker shutdown
            result_queue.put(pid)
            with worker_exit_lock:
                return
        try:
            r = call_item()
        except BaseException as e:
            exc = _ExceptionWithTraceback(e)
            result_queue.put(_ResultItem(call_item.work_id, exception=exc))
        else:
            _sendback_result(result_queue, call_item.work_id, result=r)
            del r

        # Free the resource as soon as possible, to avoid holding onto
        # open files or shared memory that is not needed anymore
        del call_item

        if _USE_PSUTIL:
            if _process_reference_size is None:
                # Make reference measurement after the first call
                _process_reference_size = _get_memory_usage(pid, force_gc=True)
                _last_memory_leak_check = time()
                continue
            if time() - _last_memory_leak_check > _MEMORY_LEAK_CHECK_DELAY:
                mem_usage = _get_memory_usage(pid)
                _last_memory_leak_check = time()
                if mem_usage - _process_reference_size < _MAX_MEMORY_LEAK_SIZE:
                    # Memory usage stays within bounds: everything is fine.
                    continue

                # Check again memory usage; this time take the measurement
                # after a forced garbage collection to break any reference
                # cycles.
                mem_usage = _get_memory_usage(pid, force_gc=True)
                _last_memory_leak_check = time()
                if mem_usage - _process_reference_size < _MAX_MEMORY_LEAK_SIZE:
                    # The GC managed to free the memory: everything is fine.
                    continue

                # The process is leaking memory: let the master process
                # know that we need to start a new worker.
                mp.util.info("Memory leak detected: shutting down worker")
                result_queue.put(pid)
                with worker_exit_lock:
                    return
        else:
            # if psutil is not installed, trigger gc.collect events
            # regularly to limit potential memory leaks due to reference cycles
            if ((_last_memory_leak_check is None) or
                    (time() - _last_memory_leak_check >
                     _MEMORY_LEAK_CHECK_DELAY)):
                gc.collect()
                _last_memory_leak_check = time()

Evaluates calls from call_queue and places the results in result_queue .


def _add_call_item_to_queue(pending_work_items,
                            running_work_items,
                            work_ids,
                            call_queue):
    """Fills call_queue with _WorkItems from pending_work_items.

    This function never blocks.

    Args:
        pending_work_items: A dict mapping work ids to _WorkItems e.g.
            {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
        work_ids: A queue.Queue of work ids e.g. Queue([5, 6, ...]). Work ids
            are consumed and the corresponding _WorkItems from
            pending_work_items are transformed into _CallItems and put in
            call_queue.
        call_queue: A ctx.Queue that will be filled with _CallItems
            derived from _WorkItems.
    """
    while True:
        if call_queue.full():
            return
        try:
            work_id = work_ids.get(block=False)
        except queue.Empty:
            return
        else:
            work_item = pending_work_items[work_id]

            if work_item.future.set_running_or_notify_cancel():
                running_work_items += [work_id]
                call_queue.put(_CallItem(work_id,
                                         work_item.fn,
                                         work_item.args,
                                         work_item.kwargs),
                               block=True)
            else:
                del pending_work_items[work_id]
                continue

def _add_call_item_to_queue ( pending_work_items , running_work_items , work_ids , call_queue ) : while True : if call_queue . full ( ) : return try : work_id = work_ids . get ( block = False ) except queue . Empty : return else : work_item = pending_work_items [ work_id ] if work_item . future . set_running_or_notify_cancel ( ) : running_work_items += [ work_id ] call_queue . put ( _CallItem ( work_id , work_item . fn , work_item . args , work_item . kwargs ) , block = True ) else : del pending_work_items [ work_id ] continue


def _ensure_executor_running(self):
        """ensures all workers and management thread are running
        """
        with self._processes_management_lock:
            if len(self._processes) != self._max_workers:
                self._adjust_process_count()
            self._start_queue_management_thread()

def _ensure_executor_running ( self ) : with self . _processes_management_lock : if len ( self . _processes ) != self . _max_workers : self . _adjust_process_count ( ) self . _start_queue_management_thread ( )


def wrap_non_picklable_objects(obj, keep_wrapper=True):
    """Wrapper for non-picklable object to use cloudpickle to serialize them.

    Note that this wrapper tends to slow down the serialization process as it
    is done with cloudpickle which is typically slower compared to pickle. The
    proper way to solve serialization issues is to avoid defining functions and
    objects in the main scripts and to implement __reduce__ functions for
    complex classes.
    """
    if not cloudpickle:
        raise ImportError("could not import cloudpickle. Please install "
                          "cloudpickle to allow extended serialization. "
                          "(`pip install cloudpickle`).")

    # If obj is a  class, create a CloudpickledClassWrapper which instantiates
    # the object internally and wrap it directly in a CloudpickledObjectWrapper
    if inspect.isclass(obj):
        class CloudpickledClassWrapper(CloudpickledObjectWrapper):
            def __init__(self, *args, **kwargs):
                self._obj = obj(*args, **kwargs)
                self._keep_wrapper = keep_wrapper

        CloudpickledClassWrapper.__name__ = obj.__name__
        return CloudpickledClassWrapper

    # If obj is an instance of a class, just wrap it in a regular
    # CloudpickledObjectWrapper
    return _wrap_non_picklable_objects(obj, keep_wrapper=keep_wrapper)

Wrapper for non - picklable object to use cloudpickle to serialize them .


def start(self, initializer=None, initargs=()):
        '''Spawn a server process for this manager object'''
        assert self._state.value == State.INITIAL

        if (initializer is not None
                and not hasattr(initializer, '__call__')):
            raise TypeError('initializer must be a callable')

        # pipe over which we will retrieve address of server
        reader, writer = mp.Pipe(duplex=False)

        # spawn process which runs a server
        self._process = Process(
            target=type(self)._run_server,
            args=(self._registry, self._address, bytes(self._authkey),
                  self._serializer, writer, initializer, initargs),
        )
        ident = ':'.join(str(i) for i in self._process._identity)
        self._process.name = type(self).__name__ + '-' + ident
        self._process.start()

        # get address of server
        writer.close()
        self._address = reader.recv()
        reader.close()

        # register a finalizer
        self._state.value = State.STARTED
        self.shutdown = mp.util.Finalize(
            self, type(self)._finalize_manager,
            args=(self._process, self._address, self._authkey,
                  self._state, self._Client),
            exitpriority=0
        )

def
start
(
self
,
initializer
=
,
initargs
=
(
)
)
:
assert
self
.
_state
.
value
==
State
.
INITIAL
if
(
initializer
is
not
and
not
hasattr
(
initializer
,
'__call__'
)
)
:
raise
TypeError
(
'initializer must be a callable'
)
reader
,
writer
=
mp
.
Pipe
(
duplex
=
False
)
self
.
_process
=
Process
(
target
=
type
(
self
)
.
_run_server
,
args
=
(
self
.
_registry
,
self
.
_address
,
bytes
(
self
.
_authkey
)
,
self
.
_serializer
,
writer
,
initializer
,
initargs
)
,
)
ident
=
':'
.
join
(
str
(
i
)
for
i
in
self
.
_process
.
_identity
)
self
.
_process
.
name
=
type
(
self
)
.
__name__
+
'-'
+
ident
self
.
_process
.
start
(
)
writer
.
close
(
)
self
.
_address
=
reader
.
recv
(
)
reader
.
close
(
)
self
.
_state
.
value
=
State
.
STARTED
self
.
shutdown
=
mp
.
util
.
Finalize
(
self
,
type
(
self
)
.
_finalize_manager
,
args
=
(
self
.
_process
,
self
.
_address
,
self
.
_authkey
,
self
.
_state
,
self
.
_Client
)
,
exitpriority
=
0
)


def DupFd(fd):
    '''Return a wrapper for an fd.'''
    popen_obj = get_spawning_popen()
    if popen_obj is not None:
        return popen_obj.DupFd(popen_obj.duplicate_for_child(fd))
    elif HAVE_SEND_HANDLE and sys.version_info[:2] > (3, 3):
        from multiprocessing import resource_sharer
        return resource_sharer.DupFd(fd)
    else:
        raise TypeError(
            'Cannot pickle connection object. This object can only be '
            'passed when spawning a new process'
        )