def delete_matching_files_from_storage(self, root_folder, regex):
        """
        Delete files in `root_folder` which match `regex` before file ext.

        Example values:
            * root_folder = 'foo/'
            * self.name = 'bar.jpg'
            * regex = re.compile('-baz')

            Result:
                * foo/bar-baz.jpg <- Deleted
                * foo/bar-biz.jpg <- Not deleted
        """
        if not self.name:   # pragma: no cover
            return
        try:
            directory_list, file_list = self.storage.listdir(root_folder)
        except OSError:   # pragma: no cover
            pass
        else:
            folder, filename = os.path.split(self.name)
            basename, ext = os.path.splitext(filename)
            for f in file_list:
                if not f.startswith(basename) or not f.endswith(ext):   # pragma: no cover
                    continue
                tag = f[len(basename):-len(ext)]
                assert f == basename + tag + ext
                if regex.match(tag) is not None:
                    file_location = os.path.join(root_folder, f)
                    self.storage.delete(file_location)
                    cache.delete(
                        self.storage.url(file_location)
                    )
                    print(
                        "Deleted {file} (created from: {original})".format(
                            file=os.path.join(root_folder, f),
                            original=self.name
                        )
                    )

Delete files in root_folder which match regex before file ext .


def preprocess(self, image, image_format):
        """
        Preprocess an image.

        An API hook for image pre-processing. Calls any image format specific
        pre-processors (if defined). I.E. If `image_format` is 'JPEG', this
        method will look for a method named `preprocess_JPEG`, if found
        `image` will be passed to it.

        Arguments:
            * `image`: a PIL Image instance
            * `image_format`: str, a valid PIL format (i.e. 'JPEG' or 'GIF')

        Subclasses should return a 2-tuple:
            * [0]: A PIL Image instance.
            * [1]: A dictionary of additional keyword arguments to be used
                   when the instance is saved. If no additional keyword
                   arguments, return an empty dict ({}).
        """
        save_kwargs = {'format': image_format}

        # Ensuring image is properly rotated
        if hasattr(image, '_getexif'):
            exif_datadict = image._getexif()  # returns None if no EXIF data
            if exif_datadict is not None:
                exif = dict(exif_datadict.items())
                orientation = exif.get(EXIF_ORIENTATION_KEY, None)
                if orientation == 3:
                    image = image.transpose(Image.ROTATE_180)
                elif orientation == 6:
                    image = image.transpose(Image.ROTATE_270)
                elif orientation == 8:
                    image = image.transpose(Image.ROTATE_90)

        # Ensure any embedded ICC profile is preserved
        save_kwargs['icc_profile'] = image.info.get('icc_profile')

        if hasattr(self, 'preprocess_%s' % image_format):
            image, addl_save_kwargs = getattr(
                self,
                'preprocess_%s' % image_format
            )(image=image)
            save_kwargs.update(addl_save_kwargs)

        return image, save_kwargs

def preprocess ( self , image , image_format ) : save_kwargs = { 'format' : image_format } if hasattr ( image , '_getexif' ) : exif_datadict = image . _getexif ( ) if exif_datadict is not None : exif = dict ( exif_datadict . items ( ) ) orientation = exif . get ( EXIF_ORIENTATION_KEY , None ) if orientation == 3 : image = image . transpose ( Image . ROTATE_180 ) elif orientation == 6 : image = image . transpose ( Image . ROTATE_270 ) elif orientation == 8 : image = image . transpose ( Image . ROTATE_90 ) save_kwargs [ 'icc_profile' ] = image . info . get ( 'icc_profile' ) if hasattr ( self , 'preprocess_%s' % image_format ) : image , addl_save_kwargs = getattr ( self , 'preprocess_%s' % image_format ) ( image = image ) save_kwargs . update ( addl_save_kwargs ) return image , save_kwargs


def preprocess_GIF(self, image, **kwargs):
        """
        Receive a PIL Image instance of a GIF and return 2-tuple.

        Args:
            * [0]: Original Image instance (passed to `image`)
            * [1]: Dict with a transparency key (to GIF transparency layer)
        """
        if 'transparency' in image.info:
            save_kwargs = {'transparency': image.info['transparency']}
        else:
            save_kwargs = {}
        return (image, save_kwargs)

def preprocess_GIF ( self , image , ** kwargs ) : if 'transparency' in image . info : save_kwargs = { 'transparency' : image . info [ 'transparency' ] } else : save_kwargs = { } return ( image , save_kwargs )

Receive a PIL Image instance of a GIF and return 2 - tuple .


def preprocess_JPEG(self, image, **kwargs):
        """
        Receive a PIL Image instance of a JPEG and returns 2-tuple.

        Args:
            * [0]: Image instance, converted to RGB
            * [1]: Dict with a quality key (mapped to the value of `QUAL` as
                   defined by the `VERSATILEIMAGEFIELD_JPEG_RESIZE_QUALITY`
                   setting)
        """
        save_kwargs = {
            'progressive': VERSATILEIMAGEFIELD_PROGRESSIVE_JPEG,
            'quality': QUAL
        }
        if image.mode != 'RGB':
            image = image.convert('RGB')
        return (image, save_kwargs)

def preprocess_JPEG ( self , image , ** kwargs ) : save_kwargs = { 'progressive' : VERSATILEIMAGEFIELD_PROGRESSIVE_JPEG , 'quality' : QUAL } if image . mode != 'RGB' : image = image . convert ( 'RGB' ) return ( image , save_kwargs )

Receive a PIL Image instance of a JPEG and returns 2 - tuple .


def retrieve_image(self, path_to_image):
        """Return a PIL Image instance stored at `path_to_image`."""
        image = self.storage.open(path_to_image, 'rb')
        file_ext = path_to_image.rsplit('.')[-1]
        image_format, mime_type = get_image_metadata_from_file_ext(file_ext)

        return (
            Image.open(image),
            file_ext,
            image_format,
            mime_type
        )

def retrieve_image ( self , path_to_image ) : image = self . storage . open ( path_to_image , 'rb' ) file_ext = path_to_image . rsplit ( '.' ) [ - 1 ] image_format , mime_type = get_image_metadata_from_file_ext ( file_ext ) return ( Image . open ( image ) , file_ext , image_format , mime_type )

Return a PIL Image instance stored at path_to_image .


def save_image(self, imagefile, save_path, file_ext, mime_type):
        """
        Save an image to self.storage at `save_path`.

        Arguments:
            `imagefile`: Raw image data, typically a BytesIO instance.
            `save_path`: The path within self.storage where the image should
                         be saved.
            `file_ext`: The file extension of the image-to-be-saved.
            `mime_type`: A valid image mime type (as found in
                         versatileimagefield.utils)
        """
        file_to_save = InMemoryUploadedFile(
            imagefile,
            None,
            'foo.%s' % file_ext,
            mime_type,
            imagefile.tell(),
            None
        )
        file_to_save.seek(0)
        self.storage.save(save_path, file_to_save)

def save_image ( self , imagefile , save_path , file_ext , mime_type ) : file_to_save = InMemoryUploadedFile ( imagefile , None , 'foo.%s' % file_ext , mime_type , imagefile . tell ( ) , None ) file_to_save . seek ( 0 ) self . storage . save ( save_path , file_to_save )

Save an image to self . storage at save_path .


def ppoi_as_str(self):
        """Return PPOI value as a string."""
        return "%s__%s" % (
            str(self.ppoi[0]).replace('.', '-'),
            str(self.ppoi[1]).replace('.', '-')
        )

def ppoi_as_str ( self ) : return "%s__%s" % ( str ( self . ppoi [ 0 ] ) . replace ( '.' , '-' ) , str ( self . ppoi [ 1 ] ) . replace ( '.' , '-' ) )


def create_resized_image(self, path_to_image, save_path_on_storage,
                             width, height):
        """
        Create a resized image.

        `path_to_image`: The path to the image with the media directory to
                         resize. If `None`, the
                         VERSATILEIMAGEFIELD_PLACEHOLDER_IMAGE will be used.
        `save_path_on_storage`: Where on self.storage to save the resized image
        `width`: Width of resized image (int)
        `height`: Desired height of resized image (int)
        `filename_key`: A string that will be used in the sized image filename
                        to signify what operation was done to it.
                        Examples: 'crop' or 'scale'
        """
        image, file_ext, image_format, mime_type = self.retrieve_image(
            path_to_image
        )

        image, save_kwargs = self.preprocess(image, image_format)

        imagefile = self.process_image(
            image=image,
            image_format=image_format,
            save_kwargs=save_kwargs,
            width=width,
            height=height
        )
        self.save_image(imagefile, save_path_on_storage, file_ext, mime_type)

def create_resized_image ( self , path_to_image , save_path_on_storage , width , height ) : image , file_ext , image_format , mime_type = self . retrieve_image ( path_to_image ) image , save_kwargs = self . preprocess ( image , image_format ) imagefile = self . process_image ( image = image , image_format = image_format , save_kwargs = save_kwargs , width = width , height = height ) self . save_image ( imagefile , save_path_on_storage , file_ext , mime_type )


def render(self, name, value, attrs=None, renderer=None):
        """
        Render the widget as an HTML string.

        Overridden here to support Django < 1.11.
        """
        if self.has_template_widget_rendering:
            return super(ClearableFileInputWithImagePreview, self).render(
                name, value, attrs=attrs, renderer=renderer
            )
        else:
            context = self.get_context(name, value, attrs)
            return render_to_string(self.template_name, context)

def render ( self , name , value , attrs = None , renderer = None ) : if self . has_template_widget_rendering : return super ( ClearableFileInputWithImagePreview , self ) . render ( name , value , attrs = attrs , renderer = renderer ) else : context = self . get_context ( name , value , attrs ) return render_to_string ( self . template_name , context )


def get_context(self, name, value, attrs):
        """Get the context to render this widget with."""
        if self.has_template_widget_rendering:
            context = super(ClearableFileInputWithImagePreview, self).get_context(name, value, attrs)
        else:
            # Build the context manually.
            context = {}
            context['widget'] = {
                'name': name,
                'is_hidden': self.is_hidden,
                'required': self.is_required,
                'value': self._format_value(value),
                'attrs': self.build_attrs(self.attrs, attrs),
                'template_name': self.template_name,
                'type': self.input_type,
            }

        # It seems Django 1.11's ClearableFileInput doesn't add everything to the 'widget' key, so we can't use it
        # in MultiWidget. Add it manually here.
        checkbox_name = self.clear_checkbox_name(name)
        checkbox_id = self.clear_checkbox_id(checkbox_name)

        context['widget'].update({
            'checkbox_name': checkbox_name,
            'checkbox_id': checkbox_id,
            'is_initial': self.is_initial(value),
            'input_text': self.input_text,
            'initial_text': self.initial_text,
            'clear_checkbox_label': self.clear_checkbox_label,
        })

        if value and hasattr(value, "url"):
            context['widget'].update({
                'hidden_field_id': self.get_hidden_field_id(name),
                'point_stage_id': self.get_point_stage_id(name),
                'ppoi_id': self.get_ppoi_id(name),
                'sized_url': self.get_sized_url(value),
                'image_preview_id': self.image_preview_id(name),
            })

        return context

def get_context ( self , name , value , attrs ) : if self . has_template_widget_rendering : context = super ( ClearableFileInputWithImagePreview , self ) . get_context ( name , value , attrs ) else : context = { } context [ 'widget' ] = { 'name' : name , 'is_hidden' : self . is_hidden , 'required' : self . is_required , 'value' : self . _format_value ( value ) , 'attrs' : self . build_attrs ( self . attrs , attrs ) , 'template_name' : self . template_name , 'type' : self . input_type , } checkbox_name = self . clear_checkbox_name ( name ) checkbox_id = self . clear_checkbox_id ( checkbox_name ) context [ 'widget' ] . update ( { 'checkbox_name' : checkbox_name , 'checkbox_id' : checkbox_id , 'is_initial' : self . is_initial ( value ) , 'input_text' : self . input_text , 'initial_text' : self . initial_text , 'clear_checkbox_label' : self . clear_checkbox_label , } ) if value and hasattr ( value , "url" ) : context [ 'widget' ] . update ( { 'hidden_field_id' : self . get_hidden_field_id ( name ) , 'point_stage_id' : self . get_point_stage_id ( name ) , 'ppoi_id' : self . get_ppoi_id ( name ) , 'sized_url' : self . get_sized_url ( value ) , 'image_preview_id' : self . image_preview_id ( name ) , } ) return context

Get the context to render this widget with .


def build_attrs(self, base_attrs, extra_attrs=None):
        """Build an attribute dictionary."""
        attrs = base_attrs.copy()
        if extra_attrs is not None:
            attrs.update(extra_attrs)
        return attrs

def build_attrs ( self , base_attrs , extra_attrs = None ) : attrs = base_attrs . copy ( ) if extra_attrs is not None : attrs . update ( extra_attrs ) return attrs


def get_resized_path(path_to_image, width, height,
                     filename_key, storage):
    """
    Return a `path_to_image` location on `storage` as dictated by `width`, `height`
    and `filename_key`
    """
    containing_folder, filename = os.path.split(path_to_image)

    resized_filename = get_resized_filename(
        filename,
        width,
        height,
        filename_key
    )

    joined_path = os.path.join(*[
        VERSATILEIMAGEFIELD_SIZED_DIRNAME,
        containing_folder,
        resized_filename
    ]).replace(' ', '')  # Removing spaces so this path is memcached friendly

    return joined_path

def
get_resized_path
(
path_to_image
,
width
,
height
,
filename_key
,
storage
)
:
containing_folder
,
filename
=
os
.
path
.
split
(
path_to_image
)
resized_filename
=
get_resized_filename
(
filename
,
width
,
height
,
filename_key
)
joined_path
=
os
.
path
.
join
(
*
[
VERSATILEIMAGEFIELD_SIZED_DIRNAME
,
containing_folder
,
resized_filename
]
)
.
replace
(
' '
,
''
)
return
joined_path

Return a path_to_image location on storage as dictated by width height and filename_key


def get_filtered_path(path_to_image, filename_key, storage):
    """
    Return the 'filtered path'
    """
    containing_folder, filename = os.path.split(path_to_image)

    filtered_filename = get_filtered_filename(filename, filename_key)
    path_to_return = os.path.join(*[
        containing_folder,
        VERSATILEIMAGEFIELD_FILTERED_DIRNAME,
        filtered_filename
    ])
    # Removing spaces so this path is memcached key friendly
    path_to_return = path_to_return.replace(' ', '')
    return path_to_return

def
get_filtered_path
(
path_to_image
,
filename_key
,
storage
)
:
containing_folder
,
filename
=
os
.
path
.
split
(
path_to_image
)
filtered_filename
=
get_filtered_filename
(
filename
,
filename_key
)
path_to_return
=
os
.
path
.
join
(
*
[
containing_folder
,
VERSATILEIMAGEFIELD_FILTERED_DIRNAME
,
filtered_filename
]
)
path_to_return
=
path_to_return
.
replace
(
' '
,
''
)
return
path_to_return


def validate_versatileimagefield_sizekey_list(sizes):
    """
    Validate a list of size keys.

    `sizes`: An iterable of 2-tuples, both strings. Example:
    [
        ('large', 'url'),
        ('medium', 'crop__400x400'),
        ('small', 'thumbnail__100x100')
    ]
    """
    try:
        for key, size_key in sizes:
            size_key_split = size_key.split('__')
            if size_key_split[-1] != 'url' and (
                'x' not in size_key_split[-1]
            ):
                raise InvalidSizeKey(
                    "{0} is an invalid size. All sizes must be either "
                    "'url' or made up of at least two segments separated "
                    "by double underscores. Examples: 'crop__400x400', "
                    "filters__invert__url".format(size_key)
                )
    except ValueError:
        raise InvalidSizeKeySet(
            '{} is an invalid size key set. Size key sets must be an '
            'iterable of 2-tuples'.format(str(sizes))
        )
    return list(set(sizes))


def get_url_from_image_key(image_instance, image_key):
    """Build a URL from `image_key`."""
    img_key_split = image_key.split('__')
    if 'x' in img_key_split[-1]:
        size_key = img_key_split.pop(-1)
    else:
        size_key = None
    img_url = reduce(getattr, img_key_split, image_instance)
    if size_key:
        img_url = img_url[size_key].url
    return img_url


def get_rendition_key_set(key):
    """
    Retrieve a validated and prepped Rendition Key Set from
    settings.VERSATILEIMAGEFIELD_RENDITION_KEY_SETS
    """
    try:
        rendition_key_set = IMAGE_SETS[key]
    except KeyError:
        raise ImproperlyConfigured(
            "No Rendition Key Set exists at "
            "settings.VERSATILEIMAGEFIELD_RENDITION_KEY_SETS['{}']".format(key)
        )
    else:
        return validate_versatileimagefield_sizekey_list(rendition_key_set)

Retrieve a validated and prepped Rendition Key Set from settings . VERSATILEIMAGEFIELD_RENDITION_KEY_SETS


def format_instruction(insn):
    """
    Takes a raw `Instruction` and translates it into a human readable text
    representation. As of writing, the text representation for WASM is not yet
    standardized, so we just emit some generic format.
    """
    text = insn.op.mnemonic

    if not insn.imm:
        return text

    return text + ' ' + ', '.join([
        getattr(insn.op.imm_struct, x.name).to_string(
            getattr(insn.imm, x.name)
        )
        for x in insn.op.imm_struct._meta.fields
    ])

def
format_instruction
(
insn
)
:
text
=
insn
.
op
.
mnemonic
if
not
insn
.
imm
:
return
text
return
text
+
' '
+
', '
.
join
(
[
getattr
(
insn
.
op
.
imm_struct
,
x
.
name
)
.
to_string
(
getattr
(
insn
.
imm
,
x
.
name
)
)
for
x
in
insn
.
op
.
imm_struct
.
_meta
.
fields
]
)

Takes a raw Instruction and translates it into a human readable text representation . As of writing the text representation for WASM is not yet standardized so we just emit some generic format .


def format_function(
    func_body,
    func_type=None,
    indent=2,
    format_locals=True,
):
    """
    Takes a `FunctionBody` and optionally a `FunctionType`, yielding the string
    representation of the function line by line. The function type is required
    for formatting function parameter and return value information.
    """
    if func_type is None:
        yield 'func'
    else:
        param_section = ' (param {})'.format(' '.join(
            map(format_lang_type, func_type.param_types)
        )) if func_type.param_types else ''
        result_section = ' (result {})'.format(
            format_lang_type(func_type.return_type)
        ) if func_type.return_type else ''
        yield 'func' + param_section + result_section

    if format_locals and func_body.locals:
        yield '(locals {})'.format(' '.join(itertools.chain.from_iterable(
            itertools.repeat(format_lang_type(x.type), x.count)
            for x in func_body.locals
        )))

    level = 1
    for cur_insn in decode_bytecode(func_body.code):
        if cur_insn.op.flags & INSN_LEAVE_BLOCK:
            level -= 1
        yield ' ' * (level * indent) + format_instruction(cur_insn)
        if cur_insn.op.flags & INSN_ENTER_BLOCK:
            level += 1

def
format_function
(
func_body
,
func_type
=
,
indent
=
2
,
format_locals
=
True
,
)
:
if
func_type
is
:
yield
'func'
else
:
param_section
=
' (param {})'
.
format
(
' '
.
join
(
map
(
format_lang_type
,
func_type
.
param_types
)
)
)
if
func_type
.
param_types
else
''
result_section
=
' (result {})'
.
format
(
format_lang_type
(
func_type
.
return_type
)
)
if
func_type
.
return_type
else
''
yield
'func'
+
param_section
+
result_section
if
format_locals
and
func_body
.
locals
:
yield
'(locals {})'
.
format
(
' '
.
join
(
itertools
.
chain
.
from_iterable
(
itertools
.
repeat
(
format_lang_type
(
x
.
type
)
,
x
.
count
)
for
x
in
func_body
.
locals
)
)
)
level
=
1
for
cur_insn
in
decode_bytecode
(
func_body
.
code
)
:
if
cur_insn
.
op
.
flags
&
INSN_LEAVE_BLOCK
:
level
-=
1
yield
' '
*
(
level
*
indent
)
+
format_instruction
(
cur_insn
)
if
cur_insn
.
op
.
flags
&
INSN_ENTER_BLOCK
:
level
+=
1

Takes a FunctionBody and optionally a FunctionType yielding the string representation of the function line by line . The function type is required for formatting function parameter and return value information .


def decode_bytecode(bytecode):
    """Decodes raw bytecode, yielding `Instruction`s."""
    bytecode_wnd = memoryview(bytecode)
    while bytecode_wnd:
        opcode_id = byte2int(bytecode_wnd[0])
        opcode = OPCODE_MAP[opcode_id]

        if opcode.imm_struct is not None:
            offs, imm, _ = opcode.imm_struct.from_raw(None, bytecode_wnd[1:])
        else:
            imm = None
            offs = 0

        insn_len = 1 + offs
        yield Instruction(opcode, imm, insn_len)
        bytecode_wnd = bytecode_wnd[insn_len:]

def decode_bytecode ( bytecode ) : bytecode_wnd = memoryview ( bytecode ) while bytecode_wnd : opcode_id = byte2int ( bytecode_wnd [ 0 ] ) opcode = OPCODE_MAP [ opcode_id ] if opcode . imm_struct is not None : offs , imm , _ = opcode . imm_struct . from_raw ( None , bytecode_wnd [ 1 : ] ) else : imm = None offs = 0 insn_len = 1 + offs yield Instruction ( opcode , imm , insn_len ) bytecode_wnd = bytecode_wnd [ insn_len : ]

Decodes raw bytecode yielding Instruction s .


def decode_module(module, decode_name_subsections=False):
    """Decodes raw WASM modules, yielding `ModuleFragment`s."""
    module_wnd = memoryview(module)

    # Read & yield module header.
    hdr = ModuleHeader()
    hdr_len, hdr_data, _ = hdr.from_raw(None, module_wnd)
    yield ModuleFragment(hdr, hdr_data)
    module_wnd = module_wnd[hdr_len:]

    # Read & yield sections.
    while module_wnd:
        sec = Section()
        sec_len, sec_data, _ = sec.from_raw(None, module_wnd)

        # If requested, decode name subsections when encountered.
        if (
            decode_name_subsections and
            sec_data.id == SEC_UNK and
            sec_data.name == SEC_NAME
        ):
            sec_wnd = sec_data.payload
            while sec_wnd:
                subsec = NameSubSection()
                subsec_len, subsec_data, _ = subsec.from_raw(None, sec_wnd)
                yield ModuleFragment(subsec, subsec_data)
                sec_wnd = sec_wnd[subsec_len:]
        else:
            yield ModuleFragment(sec, sec_data)

        module_wnd = module_wnd[sec_len:]

def decode_module ( module , decode_name_subsections = False ) : module_wnd = memoryview ( module ) hdr = ModuleHeader ( ) hdr_len , hdr_data , _ = hdr . from_raw ( None , module_wnd ) yield ModuleFragment ( hdr , hdr_data ) module_wnd = module_wnd [ hdr_len : ] while module_wnd : sec = Section ( ) sec_len , sec_data , _ = sec . from_raw ( None , module_wnd ) if ( decode_name_subsections and sec_data . id == SEC_UNK and sec_data . name == SEC_NAME ) : sec_wnd = sec_data . payload while sec_wnd : subsec = NameSubSection ( ) subsec_len , subsec_data , _ = subsec . from_raw ( None , sec_wnd ) yield ModuleFragment ( subsec , subsec_data ) sec_wnd = sec_wnd [ subsec_len : ] else : yield ModuleFragment ( sec , sec_data ) module_wnd = module_wnd [ sec_len : ]

Decodes raw WASM modules yielding ModuleFragment s .


def deprecated_func(func):
    """Deprecates a function, printing a warning on the first usage."""

    # We use a mutable container here to work around Py2's lack of
    # the `nonlocal` keyword.
    first_usage = [True]

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        if first_usage[0]:
            warnings.warn(
                "Call to deprecated function {}.".format(func.__name__),
                DeprecationWarning,
            )
            first_usage[0] = False
        return func(*args, **kwargs)

    return wrapper

def
deprecated_func
(
func
)
:
first_usage
=
[
True
]
@
functools
.
wraps
(
func
)
def
wrapper
(
*
args
,
**
kwargs
)
:
if
first_usage
[
0
]
:
warnings
.
warn
(
"Call to deprecated function {}."
.
format
(
func
.
__name__
)
,
DeprecationWarning
,
)
first_usage
[
0
]
=
False
return
func
(
*
args
,
**
kwargs
)
return
wrapper

Deprecates a function printing a warning on the first usage .


def connect(self):
        """connect to the server"""
        if self.loop is None:  # pragma: no cover
            self.loop = asyncio.get_event_loop()
        t = asyncio.Task(
            self.loop.create_connection(
                self.config['protocol_factory'],
                self.config['host'], self.config['port'],
                ssl=self.config['ssl']),
            loop=self.loop)
        t.add_done_callback(self.connection_made)
        return t

def connect ( self ) : if self . loop is None : self . loop = asyncio . get_event_loop ( ) t = asyncio . Task ( self . loop . create_connection ( self . config [ 'protocol_factory' ] , self . config [ 'host' ] , self . config [ 'port' ] , ssl = self . config [ 'ssl' ] ) , loop = self . loop ) t . add_done_callback ( self . connection_made ) return t


def close(self):
        """Close the connection"""
        if self.pinger:
            self.pinger.cancel()
            self.pinger = None
        if getattr(self, 'protocol', None):
            self.protocol.close()

def close ( self ) : if self . pinger : self . pinger . cancel ( ) self . pinger = None if getattr ( self , 'protocol' , None ) : self . protocol . close ( )


def _read_result(self):
        """Parse read a response from the AGI and parse it.

        :return dict: The AGI response parsed into a dict.
        """
        response = yield from self.reader.readline()
        return parse_agi_result(response.decode(self.encoding)[:-1])

def _read_result ( self ) : response = yield from self . reader . readline ( ) return parse_agi_result ( response . decode ( self . encoding ) [ : - 1 ] )

Parse read a response from the AGI and parse it .


def handler(self, reader, writer):
        """AsyncIO coroutine handler to launch socket listening.

        :Example:

        ::

            @asyncio.coroutine
            def start(request):
                print('Receive a FastAGI request')
                print(['AGI variables:', request.headers])

            fa_app = Application()
            fa_app.add_route('calls/start', start)
            coro = asyncio.start_server(fa_app.handler, '0.0.0.0', 4574)
            server = loop.run_until_complete(coro)

        See https://docs.python.org/3/library/asyncio-stream.html
        """
        buffer = b''
        while b'\n\n' not in buffer:
            buffer += yield from reader.read(self.buf_size)
        lines = buffer[:-2].decode(self.default_encoding).split('\n')
        headers = OrderedDict([
            line.split(': ', 1) for line in lines if ': ' in line
        ])

        agi_network_script = headers.get('agi_network_script')
        log.info('Received FastAGI request from %r for "%s" route',
                 writer.get_extra_info('peername'), agi_network_script)
        log.debug("Asterisk Headers: %r", headers)

        if agi_network_script is not None:
            route = self._route.get(agi_network_script)
            if route is not None:
                request = Request(app=self,
                                  headers=headers,
                                  reader=reader, writer=writer,
                                  encoding=self.default_encoding)
                try:
                    yield from route(request)
                except BaseException:
                    log.exception(
                        'An exception has been raised for the request "%s"',
                        agi_network_script
                    )
            else:
                log.error('No route for the request "%s"', agi_network_script)
        else:
            log.error('No agi_network_script header for the request')
        log.debug("Closing client socket")
        writer.close()

AsyncIO coroutine handler to launch socket listening .


def parse_agi_result(line):
    """Parse AGI results using Regular expression.

    AGI Result examples::

        100 result=0 Trying...

        200 result=0

        200 result=-1

        200 result=132456

        200 result= (timeout)

        510 Invalid or unknown command

        520-Invalid command syntax. Proper usage follows:
        int() argument must be a string, a bytes-like object or a number, not
        'NoneType'

        HANGUP

    """
    # print("--------------\n", line)
    if line == 'HANGUP':
        return {'error': 'AGIResultHangup',
                'msg': 'User hungup during execution'}

    kwargs = dict(code=0, response="", line=line)
    m = re_code.search(line)
    try:
        kwargs.update(m.groupdict())
    except AttributeError:
        # None has no attribute groupdict
        pass
    return agi_code_check(**kwargs)

def
parse_agi_result
(
line
)
:
if
line
==
'HANGUP'
:
return
{
'error'
:
'AGIResultHangup'
,
'msg'
:
'User hungup during execution'
}
kwargs
=
dict
(
code
=
0
,
response
=
""
,
line
=
line
)
m
=
re_code
.
search
(
line
)
try
:
kwargs
.
update
(
m
.
groupdict
(
)
)
except
AttributeError
:
pass
return
agi_code_check
(
**
kwargs
)

Parse AGI results using Regular expression .


def agi_code_check(code=None, response=None, line=None):
    """
    Check the AGI code and return a dict to help on error handling.
    """
    code = int(code)
    response = response or ""
    result = {'status_code': code, 'result': ('', ''), 'msg': ''}
    if code == 100:
        result['msg'] = line
    elif code == 200:
        for key, value, data in re_kv.findall(response):
            result[key] = (value, data)
            # If user hangs up... we get 'hangup' in the data
            if data == 'hangup':
                return {
                    'error': 'AGIResultHangup',
                    'msg': 'User hungup during execution'}
            elif key == 'result' and value == '-1':
                return {
                    'error': 'AGIAppError',
                    'msg': 'Error executing application, or hangup'}
    elif code == 510:
        result['error'] = 'AGIInvalidCommand'
    elif code == 520:
        # AGI Usage error
        result['error'] = 'AGIUsageError'
        result['msg'] = line
    else:
        # Unhandled code or undefined response
        result['error'] = 'AGIUnknownError'
        result['msg'] = line
    return result

Check the AGI code and return a dict to help on error handling .


def reset(cls, uid=None):
        """Mostly used for unit testing. Allow to use a static uuid and reset
        all counter"""
        for instance in cls.instances:
            if uid:
                instance.uid = uid
            instance.generator = instance.get_generator()

def reset ( cls , uid = None ) : for instance in cls . instances : if uid : instance . uid = uid instance . generator = instance . get_generator ( )

Mostly used for unit testing . Allow to use a static uuid and reset all counter


def get_instances(self):
        """Mostly used for debugging"""
        return ["<%s prefix:%s (uid:%s)>" % (self.__class__.__name__,
                                             i.prefix, self.uid)
                for i in self.instances]

def
get_instances
(
self
)
:
return
[
"<%s prefix:%s (uid:%s)>"
%
(
self
.
__class__
.
__name__
,
i
.
prefix
,
self
.
uid
)
for
i
in
self
.
instances
]


def get_data(path):
    """
    Returns data from a package directory.
    'path' should be an absolute path.
    """
    # Run the imported setup to get the metadata.
    with FakeContext(path):
        with SetupMonkey() as sm:
            try:
                distro = run_setup('setup.py', stop_after='config')

                metadata = {'_setuptools': sm.used_setuptools}

                for k, v in distro.metadata.__dict__.items():
                    if k[0] == '_' or not v:
                        continue
                    if all(not x for x in v):
                        continue
                    metadata[k] = v

                if sm.used_setuptools:
                    for extras in ['cmdclass', 'zip_safe', 'test_suite']:
                        v = getattr(distro, extras, None)
                        if v is not None and v not in ([], {}):
                            metadata[extras] = v

            except ImportError as e:
                # Either there is no setup py, or it's broken.
                logging.exception(e)
                metadata = {}

    return metadata

def get_data ( path ) : with FakeContext ( path ) : with SetupMonkey ( ) as sm : try : distro = run_setup ( 'setup.py' , stop_after = 'config' ) metadata = { '_setuptools' : sm . used_setuptools } for k , v in distro . metadata . __dict__ . items ( ) : if k [ 0 ] == '_' or not v : continue if all ( not x for x in v ) : continue metadata [ k ] = v if sm . used_setuptools : for extras in [ 'cmdclass' , 'zip_safe' , 'test_suite' ] : v = getattr ( distro , extras , None ) if v is not None and v not in ( [ ] , { } ) : metadata [ extras ] = v except ImportError as e : logging . exception ( e ) metadata = { } return metadata

Returns data from a package directory . path should be an absolute path .


def get_primary_keys(model):
    """Get primary key properties for a SQLAlchemy model.

    :param model: SQLAlchemy model class
    """
    mapper = model.__mapper__
    return [mapper.get_property_by_column(column) for column in mapper.primary_key]

def get_primary_keys ( model ) : mapper = model . __mapper__ return [ mapper . get_property_by_column ( column ) for column in mapper . primary_key ]

Get primary key properties for a SQLAlchemy model .


def _deserialize(self, value, *args, **kwargs):
        """Deserialize a serialized value to a model instance.

        If the parent schema is transient, create a new (transient) instance.
        Otherwise, attempt to find an existing instance in the database.
        :param value: The value to deserialize.
        """
        if not isinstance(value, dict):
            if len(self.related_keys) != 1:
                self.fail(
                    "invalid",
                    value=value,
                    keys=[prop.key for prop in self.related_keys],
                )
            value = {self.related_keys[0].key: value}
        if self.transient:
            return self.related_model(**value)
        try:
            result = self._get_existing_instance(
                self.session.query(self.related_model), value
            )
        except NoResultFound:
            # The related-object DNE in the DB, but we still want to deserialize it
            # ...perhaps we want to add it to the DB later
            return self.related_model(**value)
        return result

def _deserialize ( self , value , * args , ** kwargs ) : if not isinstance ( value , dict ) : if len ( self . related_keys ) != 1 : self . fail ( "invalid" , value = value , keys = [ prop . key for prop in self . related_keys ] , ) value = { self . related_keys [ 0 ] . key : value } if self . transient : return self . related_model ( ** value ) try : result = self . _get_existing_instance ( self . session . query ( self . related_model ) , value ) except NoResultFound : return self . related_model ( ** value ) return result

Deserialize a serialized value to a model instance .


def _get_existing_instance(self, query, value):
        """Retrieve the related object from an existing instance in the DB.

        :param query: A SQLAlchemy `Query <sqlalchemy.orm.query.Query>` object.
        :param value: The serialized value to mapto an existing instance.
        :raises NoResultFound: if there is no matching record.
        """
        if self.columns:
            result = query.filter_by(
                **{prop.key: value.get(prop.key) for prop in self.related_keys}
            ).one()
        else:
            # Use a faster path if the related key is the primary key.
            result = query.get([value.get(prop.key) for prop in self.related_keys])
            if result is None:
                raise NoResultFound
        return result

def _get_existing_instance ( self , query , value ) : if self . columns : result = query . filter_by ( ** { prop . key : value . get ( prop . key ) for prop in self . related_keys } ) . one ( ) else : result = query . get ( [ value . get ( prop . key ) for prop in self . related_keys ] ) if result is None : raise NoResultFound return result

Retrieve the related object from an existing instance in the DB .


def get_declared_fields(mcs, klass, cls_fields, inherited_fields, dict_cls):
        """Updates declared fields with fields converted from the SQLAlchemy model
        passed as the `model` class Meta option.
        """
        opts = klass.opts
        Converter = opts.model_converter
        converter = Converter(schema_cls=klass)
        declared_fields = super(SchemaMeta, mcs).get_declared_fields(
            klass, cls_fields, inherited_fields, dict_cls
        )
        fields = mcs.get_fields(converter, opts, declared_fields, dict_cls)
        fields.update(declared_fields)
        return fields

def get_declared_fields ( mcs , klass , cls_fields , inherited_fields , dict_cls ) : opts = klass . opts Converter = opts . model_converter converter = Converter ( schema_cls = klass ) declared_fields = super ( SchemaMeta , mcs ) . get_declared_fields ( klass , cls_fields , inherited_fields , dict_cls ) fields = mcs . get_fields ( converter , opts , declared_fields , dict_cls ) fields . update ( declared_fields ) return fields

Updates declared fields with fields converted from the SQLAlchemy model passed as the model class Meta option .


def load(self, data, session=None, instance=None, transient=False, *args, **kwargs):
        """Deserialize data to internal representation.

        :param session: Optional SQLAlchemy session.
        :param instance: Optional existing instance to modify.
        :param transient: Optional switch to allow transient instantiation.
        """
        self._session = session or self._session
        self._transient = transient or self._transient
        if not (self.transient or self.session):
            raise ValueError("Deserialization requires a session")
        self.instance = instance or self.instance
        try:
            return super(ModelSchema, self).load(data, *args, **kwargs)
        finally:
            self.instance = None

def
load
(
self
,
data
,
session
=
,
instance
=
,
transient
=
False
,
*
args
,
**
kwargs
)
:
self
.
_session
=
session
or
self
.
_session
self
.
_transient
=
transient
or
self
.
_transient
if
not
(
self
.
transient
or
self
.
session
)
:
raise
ValueError
(
"Deserialization requires a session"
)
self
.
instance
=
instance
or
self
.
instance
try
:
return
super
(
ModelSchema
,
self
)
.
load
(
data
,
*
args
,
**
kwargs
)
finally
:
self
.
instance
=

Deserialize data to internal representation .


def _split_model_kwargs_association(self, data):
        """Split serialized attrs to ensure association proxies are passed separately.

        This is necessary for Python < 3.6.0, as the order in which kwargs are passed
        is non-deterministic, and associations must be parsed by sqlalchemy after their
        intermediate relationship, unless their `creator` has been set.

        Ignore invalid keys at this point - behaviour for unknowns should be
        handled elsewhere.

        :param data: serialized dictionary of attrs to split on association_proxy.
        """
        association_attrs = {
            key: value
            for key, value in iteritems(data)
            # association proxy
            if hasattr(getattr(self.opts.model, key, None), "remote_attr")
        }
        kwargs = {
            key: value
            for key, value in iteritems(data)
            if (hasattr(self.opts.model, key) and key not in association_attrs)
        }
        return kwargs, association_attrs

def _split_model_kwargs_association ( self , data ) : association_attrs = { key : value for key , value in iteritems ( data ) if hasattr ( getattr ( self . opts . model , key , None ) , "remote_attr" ) } kwargs = { key : value for key , value in iteritems ( data ) if ( hasattr ( self . opts . model , key ) and key not in association_attrs ) } return kwargs , association_attrs

Split serialized attrs to ensure association proxies are passed separately .


def gc():
    """Deletes old stellar tables that are not used anymore"""
    def after_delete(database):
        click.echo("Deleted table %s" % database)

    app = get_app()
    upgrade_from_old_version(app)
    app.delete_orphan_snapshots(after_delete)

def
gc
(
)
:
def
after_delete
(
database
)
:
click
.
echo
(
"Deleted table %s"
%
database
)
app
=
get_app
(
)
upgrade_from_old_version
(
app
)
app
.
delete_orphan_snapshots
(
after_delete
)


def snapshot(name):
    """Takes a snapshot of the database"""
    app = get_app()
    upgrade_from_old_version(app)
    name = name or app.default_snapshot_name

    if app.get_snapshot(name):
        click.echo("Snapshot with name %s already exists" % name)
        sys.exit(1)
    else:
        def before_copy(table_name):
            click.echo("Snapshotting database %s" % table_name)
        app.create_snapshot(name, before_copy=before_copy)

def
snapshot
(
name
)
:
app
=
get_app
(
)
upgrade_from_old_version
(
app
)
name
=
name
or
app
.
default_snapshot_name
if
app
.
get_snapshot
(
name
)
:
click
.
echo
(
"Snapshot with name %s already exists"
%
name
)
sys
.
exit
(
1
)
else
:
def
before_copy
(
table_name
)
:
click
.
echo
(
"Snapshotting database %s"
%
table_name
)
app
.
create_snapshot
(
name
,
before_copy
=
before_copy
)


def list():
    """Returns a list of snapshots"""
    snapshots = get_app().get_snapshots()

    click.echo('\n'.join(
        '%s: %s' % (
            s.snapshot_name,
            humanize.naturaltime(datetime.utcnow() - s.created_at)
        )
        for s in snapshots
    ))

def
list
(
)
:
snapshots
=
get_app
(
)
.
get_snapshots
(
)
click
.
echo
(
'\n'
.
join
(
'%s: %s'
%
(
s
.
snapshot_name
,
humanize
.
naturaltime
(
datetime
.
utcnow
(
)
s
.
created_at
)
)
for
s
in
snapshots
)
)


def restore(name):
    """Restores the database from a snapshot"""
    app = get_app()

    if not name:
        snapshot = app.get_latest_snapshot()
        if not snapshot:
            click.echo(
                "Couldn't find any snapshots for project %s" %
                load_config()['project_name']
            )
            sys.exit(1)
    else:
        snapshot = app.get_snapshot(name)
        if not snapshot:
            click.echo(
                "Couldn't find snapshot with name %s.\n"
                "You can list snapshots with 'stellar list'" % name
            )
            sys.exit(1)

    # Check if slaves are ready
    if not snapshot.slaves_ready:
        if app.is_copy_process_running(snapshot):
            sys.stdout.write(
                'Waiting for background process(%s) to finish' %
                snapshot.worker_pid
            )
            sys.stdout.flush()
            while not snapshot.slaves_ready:
                sys.stdout.write('.')
                sys.stdout.flush()
                sleep(1)
                app.db.session.refresh(snapshot)
            click.echo('')
        else:
            click.echo('Background process missing, doing slow restore.')
            app.inline_slave_copy(snapshot)

    app.restore(snapshot)
    click.echo('Restore complete.')


def remove(name):
    """Removes a snapshot"""
    app = get_app()

    snapshot = app.get_snapshot(name)
    if not snapshot:
        click.echo("Couldn't find snapshot %s" % name)
        sys.exit(1)

    click.echo("Deleting snapshot %s" % name)
    app.remove_snapshot(snapshot)
    click.echo("Deleted")

def
remove
(
name
)
:
app
=
get_app
(
)
snapshot
=
app
.
get_snapshot
(
name
)
if
not
snapshot
:
click
.
echo
(
"Couldn't find snapshot %s"
%
name
)
sys
.
exit
(
1
)
click
.
echo
(
"Deleting snapshot %s"
%
name
)
app
.
remove_snapshot
(
snapshot
)
click
.
echo
(
"Deleted"
)


def rename(old_name, new_name):
    """Renames a snapshot"""
    app = get_app()

    snapshot = app.get_snapshot(old_name)
    if not snapshot:
        click.echo("Couldn't find snapshot %s" % old_name)
        sys.exit(1)

    new_snapshot = app.get_snapshot(new_name)
    if new_snapshot:
        click.echo("Snapshot with name %s already exists" % new_name)
        sys.exit(1)

    app.rename_snapshot(snapshot, new_name)
    click.echo("Renamed snapshot %s to %s" % (old_name, new_name))

def
rename
(
old_name
,
new_name
)
:
app
=
get_app
(
)
snapshot
=
app
.
get_snapshot
(
old_name
)
if
not
snapshot
:
click
.
echo
(
"Couldn't find snapshot %s"
%
old_name
)
sys
.
exit
(
1
)
new_snapshot
=
app
.
get_snapshot
(
new_name
)
if
new_snapshot
:
click
.
echo
(
"Snapshot with name %s already exists"
%
new_name
)
sys
.
exit
(
1
)
app
.
rename_snapshot
(
snapshot
,
new_name
)
click
.
echo
(
"Renamed snapshot %s to %s"
%
(
old_name
,
new_name
)
)


def replace(name):
    """Replaces a snapshot"""
    app = get_app()

    snapshot = app.get_snapshot(name)
    if not snapshot:
        click.echo("Couldn't find snapshot %s" % name)
        sys.exit(1)

    app.remove_snapshot(snapshot)
    app.create_snapshot(name)
    click.echo("Replaced snapshot %s" % name)

def
replace
(
name
)
:
app
=
get_app
(
)
snapshot
=
app
.
get_snapshot
(
name
)
if
not
snapshot
:
click
.
echo
(
"Couldn't find snapshot %s"
%
name
)
sys
.
exit
(
1
)
app
.
remove_snapshot
(
snapshot
)
app
.
create_snapshot
(
name
)
click
.
echo
(
"Replaced snapshot %s"
%
name
)


def on_epoch_end(self) -> None:
        'Updates indexes after each epoch for shuffling'
        self.indexes = np.arange(self.nrows)
        if self.shuffle:
            np.random.shuffle(self.indexes)


def textacy_cleaner(text: str) -> str:
    """
    Defines the default function for cleaning text.

    This function operates over a list.
    """
    return preprocess_text(text,
                           fix_unicode=True,
                           lowercase=True,
                           transliterate=True,
                           no_urls=True,
                           no_emails=True,
                           no_phone_numbers=True,
                           no_numbers=True,
                           no_currency_symbols=True,
                           no_punct=True,
                           no_contractions=False,
                           no_accents=True)

def textacy_cleaner ( text : str ) -> str : return preprocess_text ( text , fix_unicode = True , lowercase = True , transliterate = True , no_urls = True , no_emails = True , no_phone_numbers = True , no_numbers = True , no_currency_symbols = True , no_punct = True , no_contractions = False , no_accents = True )

Defines the default function for cleaning text .


def apply_parallel(func: Callable,
                   data: List[Any],
                   cpu_cores: int = None) -> List[Any]:
    """
    Apply function to list of elements.

    Automatically determines the chunk size.
    """
    if not cpu_cores:
        cpu_cores = cpu_count()

    try:
        chunk_size = ceil(len(data) / cpu_cores)
        pool = Pool(cpu_cores)
        transformed_data = pool.map(func, chunked(data, chunk_size), chunksize=1)
    finally:
        pool.close()
        pool.join()
        return transformed_data

def apply_parallel ( func : Callable , data : List [ Any ] , cpu_cores : int = None ) -> List [ Any ] : if not cpu_cores : cpu_cores = cpu_count ( ) try : chunk_size = ceil ( len ( data ) / cpu_cores ) pool = Pool ( cpu_cores ) transformed_data = pool . map ( func , chunked ( data , chunk_size ) , chunksize = 1 ) finally : pool . close ( ) pool . join ( ) return transformed_data


def process_text_constructor(cleaner: Callable,
                             tokenizer: Callable,
                             append_indicators: bool,
                             start_tok: str,
                             end_tok: str):
    """Generate a function that will clean and tokenize text."""
    def process_text(text):
        if append_indicators:
            return [[start_tok] + tokenizer(cleaner(doc)) + [end_tok] for doc in text]
        return [tokenizer(cleaner(doc)) for doc in text]

    return process_text

def process_text_constructor ( cleaner : Callable , tokenizer : Callable , append_indicators : bool , start_tok : str , end_tok : str ) : def process_text ( text ) : if append_indicators : return [ [ start_tok ] + tokenizer ( cleaner ( doc ) ) + [ end_tok ] for doc in text ] return [ tokenizer ( cleaner ( doc ) ) for doc in text ] return process_text

Generate a function that will clean and tokenize text .


def process_text(self, text: List[str]) -> List[List[str]]:
        """Combine the cleaner and tokenizer."""
        process_text = process_text_constructor(cleaner=self.cleaner,
                                                tokenizer=self.tokenizer,
                                                append_indicators=self.append_indicators,
                                                start_tok=self.start_tok,
                                                end_tok=self.end_tok)
        return process_text(text)

def process_text ( self , text : List [ str ] ) -> List [ List [ str ] ] : process_text = process_text_constructor ( cleaner = self . cleaner , tokenizer = self . tokenizer , append_indicators = self . append_indicators , start_tok = self . start_tok , end_tok = self . end_tok ) return process_text ( text )


def parallel_process_text(self, data: List[str]) -> List[List[str]]:
        """Apply cleaner -> tokenizer."""
        process_text = process_text_constructor(cleaner=self.cleaner,
                                                tokenizer=self.tokenizer,
                                                append_indicators=self.append_indicators,
                                                start_tok=self.start_tok,
                                                end_tok=self.end_tok)
        n_cores = self.num_cores
        return flattenlist(apply_parallel(process_text, data, n_cores))

def parallel_process_text ( self , data : List [ str ] ) -> List [ List [ str ] ] : process_text = process_text_constructor ( cleaner = self . cleaner , tokenizer = self . tokenizer , append_indicators = self . append_indicators , start_tok = self . start_tok , end_tok = self . end_tok ) n_cores = self . num_cores return flattenlist ( apply_parallel ( process_text , data , n_cores ) )


def generate_doc_length_stats(self):
        """Analyze document length statistics for padding strategy"""
        heuristic = self.heuristic_pct
        histdf = (pd.DataFrame([(a, b) for a, b in self.document_length_histogram.items()],
                               columns=['bin', 'doc_count'])
                  .sort_values(by='bin'))
        histdf['cumsum_pct'] = histdf.doc_count.cumsum() / histdf.doc_count.sum()

        self.document_length_stats = histdf
        self.doc_length_huerestic = histdf.query(f'cumsum_pct >= {heuristic}').bin.head(1).values[0]
        logging.warning(' '.join(["Setting maximum document length to",
                                  f'{self.doc_length_huerestic} based upon',
                                  f'heuristic of {heuristic} percentile.\n',
                                  'See full histogram by insepecting the',
                                  "`document_length_stats` attribute."]))
        self.padding_maxlen = self.doc_length_huerestic


def token_count_pandas(self):
        """ See token counts as pandas dataframe"""
        freq_df = pd.DataFrame.from_dict(self.indexer.word_counts, orient='index')
        freq_df.columns = ['count']
        return freq_df.sort_values('count', ascending=False)

def token_count_pandas ( self ) : freq_df = pd . DataFrame . from_dict ( self . indexer . word_counts , orient = 'index' ) freq_df . columns = [ 'count' ] return freq_df . sort_values ( 'count' , ascending = False )


def map_param_type(param_type):
    """
    Perform param type mapping
    This requires a bit of logic since this isn't standardized.
    If a type doesn't map, assume str
    """
    main_type, sub_type = TYPE_INFO_RE.match(param_type).groups()

    if main_type in ('list', 'array'):
        # Handle no sub-type: "required list"
        if sub_type is not None:
            sub_type = sub_type.strip()

        if not sub_type:
            sub_type = 'str'

        # Handle list of pairs: "optional list<pair<callsign, path>>"
        sub_match = TYPE_INFO_RE.match(sub_type)
        if sub_match:
            sub_type = sub_match.group(1).lower()

        return [PARAM_TYPE_MAP.setdefault(sub_type, string_types)]

    return PARAM_TYPE_MAP.setdefault(main_type, string_types)

def map_param_type ( param_type ) : main_type , sub_type = TYPE_INFO_RE . match ( param_type ) . groups ( ) if main_type in ( 'list' , 'array' ) : if sub_type is not None : sub_type = sub_type . strip ( ) if not sub_type : sub_type = 'str' sub_match = TYPE_INFO_RE . match ( sub_type ) if sub_match : sub_type = sub_match . group ( 1 ) . lower ( ) return [ PARAM_TYPE_MAP . setdefault ( sub_type , string_types ) ] return PARAM_TYPE_MAP . setdefault ( main_type , string_types )

Perform param type mapping This requires a bit of logic since this isn t standardized . If a type doesn t map assume str


def parse_interfaces(interfaces):
    """
    Parse the conduit.query json dict response
    This performs the logic of parsing the non-standard params dict
        and then returning a dict Resource can understand
    """
    parsed_interfaces = collections.defaultdict(dict)

    for m, d in iteritems(interfaces):
        app, func = m.split('.', 1)

        method = parsed_interfaces[app][func] = {}

        # Make default assumptions since these aren't provided by Phab
        method['formats'] = ['json', 'human']
        method['method'] = 'POST'

        method['optional'] = {}
        method['required'] = {}

        for name, type_info in iteritems(dict(d['params'])):
            # Set the defaults
            optionality = 'required'
            param_type = 'string'

            # Usually in the format: <optionality> <param_type>
            type_info = TYPE_INFO_COMMENT_RE.sub('', type_info)
            info_pieces = TYPE_INFO_SPLITTER_RE.findall(type_info)
            for info_piece in info_pieces:
                if info_piece in ('optional', 'required'):
                    optionality = info_piece
                elif info_piece == 'ignored':
                    optionality = 'optional'
                    param_type = 'string'
                elif info_piece == 'nonempty':
                    optionality = 'required'
                elif info_piece == 'deprecated':
                    optionality = 'optional'
                else:
                    param_type = info_piece

            method[optionality][name] = map_param_type(param_type)

    return dict(parsed_interfaces)

def parse_interfaces ( interfaces ) : parsed_interfaces = collections . defaultdict ( dict ) for m , d in iteritems ( interfaces ) : app , func = m . split ( '.' , 1 ) method = parsed_interfaces [ app ] [ func ] = { } method [ 'formats' ] = [ 'json' , 'human' ] method [ 'method' ] = 'POST' method [ 'optional' ] = { } method [ 'required' ] = { } for name , type_info in iteritems ( dict ( d [ 'params' ] ) ) : optionality = 'required' param_type = 'string' type_info = TYPE_INFO_COMMENT_RE . sub ( '' , type_info ) info_pieces = TYPE_INFO_SPLITTER_RE . findall ( type_info ) for info_piece in info_pieces : if info_piece in ( 'optional' , 'required' ) : optionality = info_piece elif info_piece == 'ignored' : optionality = 'optional' param_type = 'string' elif info_piece == 'nonempty' : optionality = 'required' elif info_piece == 'deprecated' : optionality = 'optional' else : param_type = info_piece method [ optionality ] [ name ] = map_param_type ( param_type ) return dict ( parsed_interfaces )

Parse the conduit . query json dict response This performs the logic of parsing the non - standard params dict and then returning a dict Resource can understand


def inverse(self):
        """The inverse of this bidict.

        *See also* :attr:`inv`
        """
        # Resolve and return a strong reference to the inverse bidict.
        # One may be stored in self._inv already.
        if self._inv is not None:
            return self._inv
        # Otherwise a weakref is stored in self._invweak. Try to get a strong ref from it.
        inv = self._invweak()
        if inv is not None:
            return inv
        # Refcount of referent must have dropped to zero, as in `bidict().inv.inv`. Init a new one.
        self._init_inv()  # Now this bidict will retain a strong ref to its inverse.
        return self._inv

def inverse ( self ) : if self . _inv is not None : return self . _inv inv = self . _invweak ( ) if inv is not None : return inv self . _init_inv ( ) return self . _inv


def _update_with_rollback(self, on_dup, *args, **kw):
        """Update, rolling back on failure."""
        writelog = []
        appendlog = writelog.append
        dedup_item = self._dedup_item
        write_item = self._write_item
        for (key, val) in _iteritems_args_kw(*args, **kw):
            try:
                dedup_result = dedup_item(key, val, on_dup)
            except DuplicationError:
                undo_write = self._undo_write
                for dedup_result, write_result in reversed(writelog):
                    undo_write(dedup_result, write_result)
                raise
            if dedup_result is not _NOOP:
                write_result = write_item(key, val, dedup_result)
                appendlog((dedup_result, write_result))

def _update_with_rollback ( self , on_dup , * args , ** kw ) : writelog = [ ] appendlog = writelog . append dedup_item = self . _dedup_item write_item = self . _write_item for ( key , val ) in _iteritems_args_kw ( * args , ** kw ) : try : dedup_result = dedup_item ( key , val , on_dup ) except DuplicationError : undo_write = self . _undo_write for dedup_result , write_result in reversed ( writelog ) : undo_write ( dedup_result , write_result ) raise if dedup_result is not _NOOP : write_result = write_item ( key , val , dedup_result ) appendlog ( ( dedup_result , write_result ) )


def copy(self):
        """A shallow copy."""
        # Could just ``return self.__class__(self)`` here instead, but the below is faster. It uses
        # __new__ to create a copy instance while bypassing its __init__, which would result
        # in copying this bidict's items into the copy instance one at a time. Instead, make whole
        # copies of each of the backing mappings, and make them the backing mappings of the copy,
        # avoiding copying items one at a time.
        copy = self.__class__.__new__(self.__class__)
        copy._fwdm = self._fwdm.copy()  # pylint: disable=protected-access
        copy._invm = self._invm.copy()  # pylint: disable=protected-access
        copy._init_inv()  # pylint: disable=protected-access
        return copy

def copy ( self ) : copy = self . __class__ . __new__ ( self . __class__ ) copy . _fwdm = self . _fwdm . copy ( ) copy . _invm = self . _invm . copy ( ) copy . _init_inv ( ) return copy


def copy(self):
        """A shallow copy of this ordered bidict."""
        # Fast copy implementation bypassing __init__. See comments in :meth:`BidictBase.copy`.
        copy = self.__class__.__new__(self.__class__)
        sntl = _Sentinel()
        fwdm = self._fwdm.copy()
        invm = self._invm.copy()
        cur = sntl
        nxt = sntl.nxt
        for (key, val) in iteritems(self):
            nxt = _Node(cur, sntl)
            cur.nxt = fwdm[key] = invm[val] = nxt
            cur = nxt
        sntl.prv = nxt
        copy._sntl = sntl  # pylint: disable=protected-access
        copy._fwdm = fwdm  # pylint: disable=protected-access
        copy._invm = invm  # pylint: disable=protected-access
        copy._init_inv()  # pylint: disable=protected-access
        return copy

def copy ( self ) : copy = self . __class__ . __new__ ( self . __class__ ) sntl = _Sentinel ( ) fwdm = self . _fwdm . copy ( ) invm = self . _invm . copy ( ) cur = sntl nxt = sntl . nxt for ( key , val ) in iteritems ( self ) : nxt = _Node ( cur , sntl ) cur . nxt = fwdm [ key ] = invm [ val ] = nxt cur = nxt sntl . prv = nxt copy . _sntl = sntl copy . _fwdm = fwdm copy . _invm = invm copy . _init_inv ( ) return copy


def equals_order_sensitive(self, other):
        """Order-sensitive equality check.

        *See also* :ref:`eq-order-insensitive`
        """
        # Same short-circuit as BidictBase.__eq__. Factoring out not worth function call overhead.
        if not isinstance(other, Mapping) or len(self) != len(other):
            return False
        return all(i == j for (i, j) in izip(iteritems(self), iteritems(other)))

def equals_order_sensitive ( self , other ) : if not isinstance ( other , Mapping ) or len ( self ) != len ( other ) : return False return all ( i == j for ( i , j ) in izip ( iteritems ( self ) , iteritems ( other ) ) )


def inverted(arg):
    """Yield the inverse items of the provided object.

    If *arg* has a :func:`callable` ``__inverted__`` attribute,
    return the result of calling it.

    Otherwise, return an iterator over the items in `arg`,
    inverting each item on the fly.

    *See also* :attr:`bidict.BidirectionalMapping.__inverted__`
    """
    inv = getattr(arg, '__inverted__', None)
    if callable(inv):
        return inv()
    return ((val, key) for (key, val) in _iteritems_mapping_or_iterable(arg))

def inverted ( arg ) : inv = getattr ( arg , '__inverted__' , None ) if callable ( inv ) : return inv ( ) return ( ( val , key ) for ( key , val ) in _iteritems_mapping_or_iterable ( arg ) )

Yield the inverse items of the provided object .


def clear(self):
        """Remove all items."""
        self._fwdm.clear()
        self._invm.clear()
        self._sntl.nxt = self._sntl.prv = self._sntl

def clear ( self ) : self . _fwdm . clear ( ) self . _invm . clear ( ) self . _sntl . nxt = self . _sntl . prv = self . _sntl


def move_to_end(self, key, last=True):
        """Move an existing key to the beginning or end of this ordered bidict.

        The item is moved to the end if *last* is True, else to the beginning.

        :raises KeyError: if the key does not exist
        """
        node = self._fwdm[key]
        node.prv.nxt = node.nxt
        node.nxt.prv = node.prv
        sntl = self._sntl
        if last:
            last = sntl.prv
            node.prv = last
            node.nxt = sntl
            sntl.prv = last.nxt = node
        else:
            first = sntl.nxt
            node.prv = sntl
            node.nxt = first
            sntl.nxt = first.prv = node

def move_to_end ( self , key , last = True ) : node = self . _fwdm [ key ] node . prv . nxt = node . nxt node . nxt . prv = node . prv sntl = self . _sntl if last : last = sntl . prv node . prv = last node . nxt = sntl sntl . prv = last . nxt = node else : first = sntl . nxt node . prv = sntl node . nxt = first sntl . nxt = first . prv = node

Move an existing key to the beginning or end of this ordered bidict .


def write_temp_file(text=""):
    """Create a new temporary file and write some initial text to it.

    :param text: the text to write to the temp file
    :type text: str
    :returns: the file name of the newly created temp file
    :rtype: str

    """
    with NamedTemporaryFile(mode='w+t', suffix='.yml', delete=False) \
         as tempfile:
        tempfile.write(text)
        return tempfile.name

def write_temp_file ( text = "" ) : with NamedTemporaryFile ( mode = 'w+t' , suffix = '.yml' , delete = False ) as tempfile : tempfile . write ( text ) return tempfile . name

Create a new temporary file and write some initial text to it .


def get_contacts(address_books, query, method="all", reverse=False,
                 group=False, sort="first_name"):
    """Get a list of contacts from one or more address books.

    :param address_books: the address books to search
    :type address_books: list(address_book.AddressBook)
    :param query: a search query to select contacts
    :type quer: str
    :param method: the search method, one of "all", "name" or "uid"
    :type method: str
    :param reverse: reverse the order of the returned contacts
    :type reverse: bool
    :param group: group results by address book
    :type group: bool
    :param sort: the field to use for sorting, one of "first_name", "last_name"
    :type sort: str
    :returns: contacts from the address_books that match the query
    :rtype: list(CarddavObject)

    """
    # Search for the contacts in all address books.
    contacts = []
    for address_book in address_books:
        contacts.extend(address_book.search(query, method=method))
    # Sort the contacts.
    if group:
        if sort == "first_name":
            return sorted(contacts, reverse=reverse, key=lambda x: (
                unidecode(x.address_book.name).lower(),
                unidecode(x.get_first_name_last_name()).lower()))
        elif sort == "last_name":
            return sorted(contacts, reverse=reverse, key=lambda x: (
                unidecode(x.address_book.name).lower(),
                unidecode(x.get_last_name_first_name()).lower()))
        else:
            raise ValueError('sort must be "first_name" or "last_name" not '
                             '{}.'.format(sort))
    else:
        if sort == "first_name":
            return sorted(contacts, reverse=reverse, key=lambda x:
                          unidecode(x.get_first_name_last_name()).lower())
        elif sort == "last_name":
            return sorted(contacts, reverse=reverse, key=lambda x:
                          unidecode(x.get_last_name_first_name()).lower())
        else:
            raise ValueError('sort must be "first_name" or "last_name" not '
                             '{}.'.format(sort))

Get a list of contacts from one or more address books .


def merge_args_into_config(args, config):
    """Merge the parsed arguments from argparse into the config object.

    :param args: the parsed command line arguments
    :type args: argparse.Namespace
    :param config: the parsed config file
    :type config: config.Config
    :returns: the merged config object
    :rtype: config.Config

    """
    # display by name: first or last name
    if "display" in args and args.display:
        config.set_display_by_name(args.display)
    # group by address book
    if "group_by_addressbook" in args and args.group_by_addressbook:
        config.set_group_by_addressbook(True)
    # reverse contact list
    if "reverse" in args and args.reverse:
        config.set_reverse(True)
    # sort criteria: first or last name
    if "sort" in args and args.sort:
        config.sort = args.sort
    # preferred vcard version
    if "vcard_version" in args and args.vcard_version:
        config.set_preferred_vcard_version(args.vcard_version)
    # search in source files
    if "search_in_source_files" in args and args.search_in_source_files:
        config.set_search_in_source_files(True)
    # skip unparsable vcards
    if "skip_unparsable" in args and args.skip_unparsable:
        config.set_skip_unparsable(True)
    # If the user could but did not specify address books on the command line
    # it means they want to use all address books in that place.
    if "addressbook" in args and not args.addressbook:
        args.addressbook = [abook.name for abook in config.abooks]
    if "target_addressbook" in args and not args.target_addressbook:
        args.target_addressbook = [abook.name for abook in config.abooks]

def merge_args_into_config ( args , config ) : if "display" in args and args . display : config . set_display_by_name ( args . display ) if "group_by_addressbook" in args and args . group_by_addressbook : config . set_group_by_addressbook ( True ) if "reverse" in args and args . reverse : config . set_reverse ( True ) if "sort" in args and args . sort : config . sort = args . sort if "vcard_version" in args and args . vcard_version : config . set_preferred_vcard_version ( args . vcard_version ) if "search_in_source_files" in args and args . search_in_source_files : config . set_search_in_source_files ( True ) if "skip_unparsable" in args and args . skip_unparsable : config . set_skip_unparsable ( True ) if "addressbook" in args and not args . addressbook : args . addressbook = [ abook . name for abook in config . abooks ] if "target_addressbook" in args and not args . target_addressbook : args . target_addressbook = [ abook . name for abook in config . abooks ]

Merge the parsed arguments from argparse into the config object .


def load_address_books(names, config, search_queries):
    """Load all address books with the given names from the config.

    :param names: the address books to load
    :type names: list(str)
    :param config: the config instance to use when looking up address books
    :type config: config.Config
    :param search_queries: a mapping of address book names to search queries
    :type search_queries: dict
    :yields: the loaded address books
    :ytype: addressbook.AddressBook

    """
    all_names = {str(book) for book in config.abooks}
    if not names:
        names = all_names
    elif not all_names.issuperset(names):
        sys.exit('Error: The entered address books "{}" do not exist.\n'
                 'Possible values are: {}'.format(
                     '", "'.join(set(names) - all_names),
                     ', '.join(all_names)))
    # load address books which are defined in the configuration file
    for name in names:
        address_book = config.abook.get_abook(name)
        address_book.load(search_queries[address_book.name],
                search_in_source_files=config.search_in_source_files())
        yield address_book

Load all address books with the given names from the config .


def prepare_search_queries(args):
    """Prepare the search query string from the given command line args.

    Each address book can get a search query string to filter vcards befor
    loading them.  Depending on the question if the address book is used for
    source or target searches different regexes have to be combined into one
    search string.

    :param args: the parsed command line
    :type args: argparse.Namespace
    :returns: a dict mapping abook names to their loading queries, if the query
        is None it means that all cards should be loaded
    :rtype: dict(str:str or None)

    """
    # get all possible search queries for address book parsing
    source_queries = []
    target_queries = []
    if "source_search_terms" in args and args.source_search_terms:
        escaped_term = ".*".join(re.escape(x)
                                 for x in args.source_search_terms)
        source_queries.append(escaped_term)
        args.source_search_terms = escaped_term
    if "search_terms" in args and args.search_terms:
        escaped_term = ".*".join(re.escape(x) for x in args.search_terms)
        source_queries.append(escaped_term)
        args.search_terms = escaped_term
    if "target_contact" in args and args.target_contact:
        escaped_term = re.escape(args.target_contact)
        target_queries.append(escaped_term)
        args.target_contact = escaped_term
    if "uid" in args and args.uid:
        source_queries.append(args.uid)
    if "target_uid" in args and args.target_uid:
        target_queries.append(args.target_uid)
    # create and return regexp, None means that no query is given and hence all
    # contacts should be searched.
    source_queries = "^.*(%s).*$" % ')|('.join(source_queries) \
        if source_queries else None
    target_queries = "^.*(%s).*$" % ')|('.join(target_queries) \
        if target_queries else None
    logging.debug('Created source query regex: %s', source_queries)
    logging.debug('Created target query regex: %s', target_queries)
    # Get all possible search queries for address book parsing, always
    # depending on the fact if the address book is used to find source or
    # target contacts or both.
    queries = {abook.name: [] for abook in config.abook._abooks}
    for name in queries:
        if "addressbook" in args and name in args.addressbook:
            queries[name].append(source_queries)
        if "target_addressbook" in args and name in args.target_addressbook:
            queries[name].append(target_queries)
        # If None is included in the search queries of an address book it means
        # that either no source or target query was given and this address book
        # is affected by this.  All contacts should be loaded from that address
        # book.
        if None in queries[name]:
            queries[name] = None
        else:
            queries[name] = "({})".format(')|('.join(queries[name]))
    logging.debug('Created query regex: %s', queries)
    return queries

def
prepare_search_queries
(
args
)
:
source_queries
=
[
]
target_queries
=
[
]
if
"source_search_terms"
in
args
and
args
.
source_search_terms
:
escaped_term
=
".*"
.
join
(
re
.
escape
(
x
)
for
x
in
args
.
source_search_terms
)
source_queries
.
append
(
escaped_term
)
args
.
source_search_terms
=
escaped_term
if
"search_terms"
in
args
and
args
.
search_terms
:
escaped_term
=
".*"
.
join
(
re
.
escape
(
x
)
for
x
in
args
.
search_terms
)
source_queries
.
append
(
escaped_term
)
args
.
search_terms
=
escaped_term
if
"target_contact"
in
args
and
args
.
target_contact
:
escaped_term
=
re
.
escape
(
args
.
target_contact
)
target_queries
.
append
(
escaped_term
)
args
.
target_contact
=
escaped_term
if
"uid"
in
args
and
args
.
uid
:
source_queries
.
append
(
args
.
uid
)
if
"target_uid"
in
args
and
args
.
target_uid
:
target_queries
.
append
(
args
.
target_uid
)
source_queries
=
"^.*(%s).*$"
%
')|('
.
join
(
source_queries
)
if
source_queries
else
target_queries
=
"^.*(%s).*$"
%
')|('
.
join
(
target_queries
)
if
target_queries
else
logging
.
debug
(
'Created source query regex: %s'
,
source_queries
)
logging
.
debug
(
'Created target query regex: %s'
,
target_queries
)
queries
=
{
abook
.
name
:
[
]
for
abook
in
config
.
abook
.
_abooks
}
for
name
in
queries
:
if
"addressbook"
in
args
and
name
in
args
.
addressbook
:
queries
[
name
]
.
append
(
source_queries
)
if
"target_addressbook"
in
args
and
name
in
args
.
target_addressbook
:
queries
[
name
]
.
append
(
target_queries
)
if
in
queries
[
name
]
:
queries
[
name
]
=
else
:
queries
[
name
]
=
"({})"
.
format
(
')|('
.
join
(
queries
[
name
]
)
)
logging
.
debug
(
'Created query regex: %s'
,
queries
)
return
queries

Prepare the search query string from the given command line args .


def new_subcommand(selected_address_books, input_from_stdin_or_file,
                   open_editor):
    """Create a new contact.

    :param selected_address_books: a list of addressbooks that were selected on
        the command line
    :type selected_address_books: list of address_book.AddressBook
    :param input_from_stdin_or_file: the data for the new contact as a yaml
        formatted string
    :type input_from_stdin_or_file: str
    :param open_editor: whether to open the new contact in the edior after
        creation
    :type open_editor: bool
    :returns: None
    :rtype: None

    """
    # ask for address book, in which to create the new contact
    selected_address_book = choose_address_book_from_list(
        "Select address book for new contact", selected_address_books)
    if selected_address_book is None:
        print("Error: address book list is empty")
        sys.exit(1)
    # if there is some data in stdin
    if input_from_stdin_or_file:
        # create new contact from stdin
        try:
            new_contact = CarddavObject.from_user_input(
                selected_address_book, input_from_stdin_or_file,
                config.get_supported_private_objects(),
                config.get_preferred_vcard_version(),
                config.localize_dates())
        except ValueError as err:
            print(err)
            sys.exit(1)
        else:
            new_contact.write_to_file()
        if open_editor:
            modify_existing_contact(new_contact)
        else:
            print("Creation successful\n\n%s" % new_contact.print_vcard())
    else:
        create_new_contact(selected_address_book)

def
new_subcommand
(
selected_address_books
,
input_from_stdin_or_file
,
open_editor
)
:
selected_address_book
=
choose_address_book_from_list
(
"Select address book for new contact"
,
selected_address_books
)
if
selected_address_book
is
:
print
(
"Error: address book list is empty"
)
sys
.
exit
(
1
)
if
input_from_stdin_or_file
:
try
:
new_contact
=
CarddavObject
.
from_user_input
(
selected_address_book
,
input_from_stdin_or_file
,
config
.
get_supported_private_objects
(
)
,
config
.
get_preferred_vcard_version
(
)
,
config
.
localize_dates
(
)
)
except
ValueError
as
err
:
print
(
err
)
sys
.
exit
(
1
)
else
:
new_contact
.
write_to_file
(
)
if
open_editor
:
modify_existing_contact
(
new_contact
)
else
:
print
(
"Creation successful\n\n%s"
%
new_contact
.
print_vcard
(
)
)
else
:
create_new_contact
(
selected_address_book
)


def birthdays_subcommand(vcard_list, parsable):
    """Print birthday contact table.

    :param vcard_list: the vcards to search for matching entries which should
        be printed
    :type vcard_list: list of carddav_object.CarddavObject
    :param parsable: machine readable output: columns devided by tabulator (\t)
    :type parsable: bool
    :returns: None
    :rtype: None

    """
    # filter out contacts without a birthday date
    vcard_list = [
        vcard for vcard in vcard_list if vcard.get_birthday() is not None]
    # sort by date (month and day)
    # The sort function should work for strings and datetime objects.  All
    # strings will besorted before any datetime objects.
    vcard_list.sort(
        key=lambda x: (x.get_birthday().month, x.get_birthday().day)
        if isinstance(x.get_birthday(), datetime.datetime)
        else (0, 0, x.get_birthday()))
    # add to string list
    birthday_list = []
    for vcard in vcard_list:
        date = vcard.get_birthday()
        if parsable:
            if config.display_by_name() == "first_name":
                birthday_list.append("%04d.%02d.%02d\t%s"
                                     % (date.year, date.month, date.day,
                                        vcard.get_first_name_last_name()))
            else:
                birthday_list.append("%04d.%02d.%02d\t%s"
                                     % (date.year, date.month, date.day,
                                        vcard.get_last_name_first_name()))
        else:
            if config.display_by_name() == "first_name":
                birthday_list.append("%s\t%s"
                                     % (vcard.get_first_name_last_name(),
                                        vcard.get_formatted_birthday()))
            else:
                birthday_list.append("%s\t%s"
                                     % (vcard.get_last_name_first_name(),
                                        vcard.get_formatted_birthday()))
    if birthday_list:
        if parsable:
            print('\n'.join(birthday_list))
        else:
            list_birthdays(birthday_list)
    else:
        if not parsable:
            print("Found no birthdays")
        sys.exit(1)

def
birthdays_subcommand
(
vcard_list
,
parsable
)
:
vcard_list
=
[
vcard
for
vcard
in
vcard_list
if
vcard
.
get_birthday
(
)
is
not
]
vcard_list
.
sort
(
key
=
lambda
x
:
(
x
.
get_birthday
(
)
.
month
,
x
.
get_birthday
(
)
.
day
)
if
isinstance
(
x
.
get_birthday
(
)
,
datetime
.
datetime
)
else
(
0
,
0
,
x
.
get_birthday
(
)
)
)
birthday_list
=
[
]
for
vcard
in
vcard_list
:
date
=
vcard
.
get_birthday
(
)
if
parsable
:
if
config
.
display_by_name
(
)
==
"first_name"
:
birthday_list
.
append
(
"%04d.%02d.%02d\t%s"
%
(
date
.
year
,
date
.
month
,
date
.
day
,
vcard
.
get_first_name_last_name
(
)
)
)
else
:
birthday_list
.
append
(
"%04d.%02d.%02d\t%s"
%
(
date
.
year
,
date
.
month
,
date
.
day
,
vcard
.
get_last_name_first_name
(
)
)
)
else
:
if
config
.
display_by_name
(
)
==
"first_name"
:
birthday_list
.
append
(
"%s\t%s"
%
(
vcard
.
get_first_name_last_name
(
)
,
vcard
.
get_formatted_birthday
(
)
)
)
else
:
birthday_list
.
append
(
"%s\t%s"
%
(
vcard
.
get_last_name_first_name
(
)
,
vcard
.
get_formatted_birthday
(
)
)
)
if
birthday_list
:
if
parsable
:
print
(
'\n'
.
join
(
birthday_list
)
)
else
:
list_birthdays
(
birthday_list
)
else
:
if
not
parsable
:
print
(
"Found no birthdays"
)
sys
.
exit
(
1
)


def phone_subcommand(search_terms, vcard_list, parsable):
    """Print a phone application friendly contact table.

    :param search_terms: used as search term to filter the contacts before
        printing
    :type search_terms: str
    :param vcard_list: the vcards to search for matching entries which should
        be printed
    :type vcard_list: list of carddav_object.CarddavObject
    :param parsable: machine readable output: columns devided by tabulator (\t)
    :type parsable: bool
    :returns: None
    :rtype: None

    """
    all_phone_numbers_list = []
    matching_phone_number_list = []
    for vcard in vcard_list:
        for type, number_list in sorted(vcard.get_phone_numbers().items(),
                                        key=lambda k: k[0].lower()):
            for number in sorted(number_list):
                if config.display_by_name() == "first_name":
                    name = vcard.get_first_name_last_name()
                else:
                    name = vcard.get_last_name_first_name()
                # create output lines
                line_formatted = "\t".join([name, type, number])
                line_parsable = "\t".join([number, name, type])
                if parsable:
                    # parsable option: start with phone number
                    phone_number_line = line_parsable
                else:
                    # else: start with name
                    phone_number_line = line_formatted
                if re.search(search_terms,
                             "%s\n%s" % (line_formatted, line_parsable),
                             re.IGNORECASE | re.DOTALL):
                    matching_phone_number_list.append(phone_number_line)
                elif len(re.sub("\D", "", search_terms)) >= 3:
                    # The user likely searches for a phone number cause the
                    # search string contains at least three digits.  So we
                    # remove all non-digit chars from the phone number field
                    # and match against that.
                    if re.search(re.sub("\D", "", search_terms),
                                 re.sub("\D", "", number), re.IGNORECASE):
                        matching_phone_number_list.append(phone_number_line)
                # collect all phone numbers in a different list as fallback
                all_phone_numbers_list.append(phone_number_line)
    if matching_phone_number_list:
        if parsable:
            print('\n'.join(matching_phone_number_list))
        else:
            list_phone_numbers(matching_phone_number_list)
    elif all_phone_numbers_list:
        if parsable:
            print('\n'.join(all_phone_numbers_list))
        else:
            list_phone_numbers(all_phone_numbers_list)
    else:
        if not parsable:
            print("Found no phone numbers")
        sys.exit(1)

def
phone_subcommand
(
search_terms
,
vcard_list
,
parsable
)
:
all_phone_numbers_list
=
[
]
matching_phone_number_list
=
[
]
for
vcard
in
vcard_list
:
for
type
,
number_list
in
sorted
(
vcard
.
get_phone_numbers
(
)
.
items
(
)
,
key
=
lambda
k
:
k
[
0
]
.
lower
(
)
)
:
for
number
in
sorted
(
number_list
)
:
if
config
.
display_by_name
(
)
==
"first_name"
:
name
=
vcard
.
get_first_name_last_name
(
)
else
:
name
=
vcard
.
get_last_name_first_name
(
)
line_formatted
=
"\t"
.
join
(
[
name
,
type
,
number
]
)
line_parsable
=
"\t"
.
join
(
[
number
,
name
,
type
]
)
if
parsable
:
phone_number_line
=
line_parsable
else
:
phone_number_line
=
line_formatted
if
re
.
search
(
search_terms
,
"%s\n%s"
%
(
line_formatted
,
line_parsable
)
,
re
.
IGNORECASE
|
re
.
DOTALL
)
:
matching_phone_number_list
.
append
(
phone_number_line
)
elif
len
(
re
.
sub
(
"\D"
,
""
,
search_terms
)
)
>=
3
:
if
re
.
search
(
re
.
sub
(
"\D"
,
""
,
search_terms
)
,
re
.
sub
(
"\D"
,
""
,
number
)
,
re
.
IGNORECASE
)
:
matching_phone_number_list
.
append
(
phone_number_line
)
all_phone_numbers_list
.
append
(
phone_number_line
)
if
matching_phone_number_list
:
if
parsable
:
print
(
'\n'
.
join
(
matching_phone_number_list
)
)
else
:
list_phone_numbers
(
matching_phone_number_list
)
elif
all_phone_numbers_list
:
if
parsable
:
print
(
'\n'
.
join
(
all_phone_numbers_list
)
)
else
:
list_phone_numbers
(
all_phone_numbers_list
)
else
:
if
not
parsable
:
print
(
"Found no phone numbers"
)
sys
.
exit
(
1
)

Print a phone application friendly contact table .


def list_subcommand(vcard_list, parsable):
    """Print a user friendly contacts table.

    :param vcard_list: the vcards to print
    :type vcard_list: list of carddav_object.CarddavObject
    :param parsable: machine readable output: columns devided by tabulator (\t)
    :type parsable: bool
    :returns: None
    :rtype: None

    """
    if not vcard_list:
        if not parsable:
            print("Found no contacts")
        sys.exit(1)
    elif parsable:
        contact_line_list = []
        for vcard in vcard_list:
            if config.display_by_name() == "first_name":
                name = vcard.get_first_name_last_name()
            else:
                name = vcard.get_last_name_first_name()
            contact_line_list.append('\t'.join([vcard.get_uid(), name,
                                                vcard.address_book.name]))
        print('\n'.join(contact_line_list))
    else:
        list_contacts(vcard_list)

def
list_subcommand
(
vcard_list
,
parsable
)
:
if
not
vcard_list
:
if
not
parsable
:
print
(
"Found no contacts"
)
sys
.
exit
(
1
)
elif
parsable
:
contact_line_list
=
[
]
for
vcard
in
vcard_list
:
if
config
.
display_by_name
(
)
==
"first_name"
:
name
=
vcard
.
get_first_name_last_name
(
)
else
:
name
=
vcard
.
get_last_name_first_name
(
)
contact_line_list
.
append
(
'\t'
.
join
(
[
vcard
.
get_uid
(
)
,
name
,
vcard
.
address_book
.
name
]
)
)
print
(
'\n'
.
join
(
contact_line_list
)
)
else
:
list_contacts
(
vcard_list
)


def modify_subcommand(selected_vcard, input_from_stdin_or_file, open_editor):
    """Modify a contact in an external editor.

    :param selected_vcard: the contact to modify
    :type selected_vcard: carddav_object.CarddavObject
    :param input_from_stdin_or_file: new data from stdin (or a file) that
        should be incorperated into the contact, this should be a yaml
        formatted string
    :type input_from_stdin_or_file: str
    :param open_editor: whether to open the new contact in the edior after
        creation
    :type open_editor: bool
    :returns: None
    :rtype: None

    """
    # show warning, if vcard version of selected contact is not 3.0 or 4.0
    if selected_vcard.get_version() not in config.supported_vcard_versions:
        print("Warning:\nThe selected contact is based on vcard version %s "
              "but khard only supports the creation and modification of vcards"
              " with version 3.0 and 4.0.\nIf you proceed, the contact will be"
              " converted to vcard version %s but beware: This could corrupt "
              "the contact file or cause data loss."
              % (selected_vcard.get_version(),
                 config.get_preferred_vcard_version()))
        while True:
            input_string = input("Do you want to proceed anyway (y/n)? ")
            if input_string.lower() in ["", "n", "q"]:
                print("Canceled")
                sys.exit(0)
            if input_string.lower() == "y":
                break
    # if there is some data in stdin
    if input_from_stdin_or_file:
        # create new contact from stdin
        try:
            new_contact = \
                CarddavObject.from_existing_contact_with_new_user_input(
                    selected_vcard, input_from_stdin_or_file,
                    config.localize_dates())
        except ValueError as err:
            print(err)
            sys.exit(1)
        if selected_vcard == new_contact:
            print("Nothing changed\n\n%s" % new_contact.print_vcard())
        else:
            print("Modification\n\n%s\n" % new_contact.print_vcard())
            while True:
                input_string = input("Do you want to proceed (y/n)? ")
                if input_string.lower() in ["", "n", "q"]:
                    print("Canceled")
                    break
                if input_string.lower() == "y":
                    new_contact.write_to_file(overwrite=True)
                    if open_editor:
                        modify_existing_contact(new_contact)
                    else:
                        print("Done")
                    break
    else:
        modify_existing_contact(selected_vcard)

Modify a contact in an external editor .


def remove_subcommand(selected_vcard, force):
    """Remove a contact from the addressbook.

    :param selected_vcard: the contact to delete
    :type selected_vcard: carddav_object.CarddavObject
    :param force: delete without confirmation
    :type force: bool
    :returns: None
    :rtype: None

    """
    if not force:
        while True:
            input_string = input(
                "Deleting contact %s from address book %s. Are you sure? "
                "(y/n): " % (selected_vcard, selected_vcard.address_book))
            if input_string.lower() in ["", "n", "q"]:
                print("Canceled")
                sys.exit(0)
            if input_string.lower() == "y":
                break
    selected_vcard.delete_vcard_file()
    print("Contact %s deleted successfully" % selected_vcard.get_full_name())


def source_subcommand(selected_vcard, editor):
    """Open the vcard file for a contact in an external editor.

    :param selected_vcard: the contact to edit
    :type selected_vcard: carddav_object.CarddavObject
    :param editor: the eitor command to use
    :type editor: str
    :returns: None
    :rtype: None

    """
    child = subprocess.Popen([editor, selected_vcard.filename])
    child.communicate()

def source_subcommand ( selected_vcard , editor ) : child = subprocess . Popen ( [ editor , selected_vcard . filename ] ) child . communicate ( )

Open the vcard file for a contact in an external editor .


def merge_subcommand(vcard_list, selected_address_books, search_terms,
                     target_uid):
    """Merge two contacts into one.

    :param vcard_list: the vcards from which to choose contacts for mergeing
    :type vcard_list: list of carddav_object.CarddavObject
    :param selected_address_books: the addressbooks to use to find the target
        contact
    :type selected_address_books: list(addressbook.AddressBook)
    :param search_terms: the search terms to find the target contact
    :type search_terms: str
    :param target_uid: the uid of the target contact or empty
    :type target_uid: str
    :returns: None
    :rtype: None

    """
    # Check arguments.
    if target_uid != "" and search_terms != "":
        print("You can not specify a target uid and target search terms for a "
              "merge.")
        sys.exit(1)
    # Find possible target contacts.
    if target_uid != "":
        target_vcards = get_contacts(selected_address_books, target_uid,
                                     method="uid")
        # We require that the uid given can uniquely identify a contact.
        if len(target_vcards) != 1:
            if not target_vcards:
                print("Found no contact for target uid %s" % target_uid)
            else:
                print("Found multiple contacts for target uid %s" % target_uid)
                for vcard in target_vcards:
                    print("    %s: %s" % (vcard, vcard.get_uid()))
            sys.exit(1)
    else:
        target_vcards = get_contact_list_by_user_selection(
            selected_address_books, search_terms, False)
    # get the source vcard, from which to merge
    source_vcard = choose_vcard_from_list("Select contact from which to merge",
                                          vcard_list)
    if source_vcard is None:
        print("Found no source contact for merging")
        sys.exit(1)
    else:
        print("Merge from %s from address book %s\n\n"
              % (source_vcard, source_vcard.address_book))
    # get the target vcard, into which to merge
    target_vcard = choose_vcard_from_list("Select contact into which to merge",
                                          target_vcards)
    if target_vcard is None:
        print("Found no target contact for merging")
        sys.exit(1)
    else:
        print("Merge into %s from address book %s\n\n"
              % (target_vcard, target_vcard.address_book))
    # merging
    if source_vcard == target_vcard:
        print("The selected contacts are already identical")
    else:
        merge_existing_contacts(source_vcard, target_vcard, True)


def copy_or_move_subcommand(action, vcard_list, target_address_book_list):
    """Copy or move a contact to a different address book.

    :action: the string "copy" or "move" to indicate what to do
    :type action: str
    :param vcard_list: the contact list from which to select one for the action
    :type vcard_list: list of carddav_object.CarddavObject
    :param target_address_book_list: the list of target address books
    :type target_address_book_list: list(addressbook.AddressBook)
    :returns: None
    :rtype: None

    """
    # get the source vcard, which to copy or move
    source_vcard = choose_vcard_from_list(
        "Select contact to %s" % action.title(), vcard_list)
    if source_vcard is None:
        print("Found no contact")
        sys.exit(1)
    else:
        print("%s contact %s from address book %s"
              % (action.title(), source_vcard, source_vcard.address_book))

    # get target address book
    if len(target_address_book_list) == 1 \
            and target_address_book_list[0] == source_vcard.address_book:
        print("The address book %s already contains the contact %s"
              % (target_address_book_list[0], source_vcard))
        sys.exit(1)
    else:
        available_address_books = [abook for abook in target_address_book_list
                                   if abook != source_vcard.address_book]
        selected_target_address_book = choose_address_book_from_list(
            "Select target address book", available_address_books)
        if selected_target_address_book is None:
            print("Error: address book list is empty")
            sys.exit(1)

    # check if a contact already exists in the target address book
    target_vcard = choose_vcard_from_list(
        "Select target contact which to overwrite",
        get_contact_list_by_user_selection([selected_target_address_book],
                                           source_vcard.get_full_name(), True))
    # If the target contact doesn't exist, move or copy the source contact into
    # the target address book without further questions.
    if target_vcard is None:
        copy_contact(source_vcard, selected_target_address_book,
                     action == "move")
    else:
        if source_vcard == target_vcard:
            # source and target contact are identical
            print("Target contact: %s" % target_vcard)
            if action == "move":
                copy_contact(source_vcard, selected_target_address_book, True)
            else:
                print("The selected contacts are already identical")
        else:
            # source and target contacts are different
            # either overwrite the target one or merge into target contact
            print("The address book %s already contains the contact %s\n\n"
                  "Source\n\n%s\n\nTarget\n\n%s\n\n"
                  "Possible actions:\n"
                  "  a: %s anyway\n"
                  "  m: Merge from source into target contact\n"
                  "  o: Overwrite target contact\n"
                  "  q: Quit" % (
                      target_vcard.address_book, source_vcard,
                      source_vcard.print_vcard(), target_vcard.print_vcard(),
                      "Move" if action == "move" else "Copy"))
            while True:
                input_string = input("Your choice: ")
                if input_string.lower() == "a":
                    copy_contact(source_vcard, selected_target_address_book,
                                 action == "move")
                    break
                if input_string.lower() == "o":
                    copy_contact(source_vcard, selected_target_address_book,
                                 action == "move")
                    target_vcard.delete_vcard_file()
                    break
                if input_string.lower() == "m":
                    merge_existing_contacts(source_vcard, target_vcard,
                                            action == "move")
                    break
                if input_string.lower() in ["", "q"]:
                    print("Canceled")
                    break

Copy or move a contact to a different address book .


def get_action(cls, alias):
        """Find the name of the action for the supplied alias.  If no action is
        asociated with the given alias, None is returned.

        :param alias: the alias to look up
        :type alias: str
        :rturns: the name of the corresponding action or None
        :rtype: str or NoneType

        """
        for action, alias_list in cls.action_map.items():
            if alias in alias_list:
                return action
        return None

def get_action ( cls , alias ) : for action , alias_list in cls . action_map . items ( ) : if alias in alias_list : return action return None

Find the name of the action for the supplied alias . If no action is asociated with the given alias None is returned .