def cp_single_file(self, pool, source, target, delete_source):
    '''Copy a single file or a directory by adding a task into queue'''
    if source[-1] == PATH_SEP:
      if self.opt.recursive:
        basepath = S3URL(source).path
        for f in (f for f in self.s3walk(source) if not f['is_dir']):
          pool.copy(f['name'], os.path.join(target, os.path.relpath(S3URL(f['name']).path, basepath)), delete_source=delete_source)
      else:
        message('omitting directory "%s".' % source)
    else:
      pool.copy(source, target, delete_source=delete_source)

def
cp_single_file
(
self
,
pool
,
source
,
target
,
delete_source
)
:
if
source
[
1
]
==
PATH_SEP
:
if
self
.
opt
.
recursive
:
basepath
=
S3URL
(
source
)
.
path
for
f
in
(
f
for
f
in
self
.
s3walk
(
source
)
if
not
f
[
'is_dir'
]
)
:
pool
.
copy
(
f
[
'name'
]
,
os
.
path
.
join
(
target
,
os
.
path
.
relpath
(
S3URL
(
f
[
'name'
]
)
.
path
,
basepath
)
)
,
delete_source
=
delete_source
)
else
:
message
(
'omitting directory "%s".'
%
source
)
else
:
pool
.
copy
(
source
,
target
,
delete_source
=
delete_source
)


def cp_files(self, source, target, delete_source=False):
    '''Copy files
       This function can handle multiple files if source S3 URL has wildcard
       characters. It also handles recursive mode by copying all files and
       keep the directory structure.
    '''
    pool = ThreadPool(ThreadUtil, self.opt)
    source = self.source_expand(source)

    if target[-1] == PATH_SEP:
      for src in source:
        self.cp_single_file(pool, src, os.path.join(target, self.get_basename(S3URL(src).path)), delete_source)
    else:
      if len(source) > 1:
        raise Failure('Target "%s" is not a directory (with a trailing slash).' % target)
        # Copy file if it exists otherwise do nothing
      elif len(source) == 1:
        self.cp_single_file(pool, source[0], target, delete_source)
      else:
        # Source expand may return empty list only if ignore-empty-source is set to true
        pass

    pool.join()

Copy files This function can handle multiple files if source S3 URL has wildcard characters . It also handles recursive mode by copying all files and keep the directory structure .


def del_files(self, source):
    '''Delete files on S3'''
    src_files = []
    for obj in self.s3walk(source):
      if not obj['is_dir']: # ignore directories
        src_files.append(obj['name'])

    pool = ThreadPool(ThreadUtil, self.opt)
    pool.batch_delete(src_files)
    pool.join()

def del_files ( self , source ) : src_files = [ ] for obj in self . s3walk ( source ) : if not obj [ 'is_dir' ] : src_files . append ( obj [ 'name' ] ) pool = ThreadPool ( ThreadUtil , self . opt ) pool . batch_delete ( src_files ) pool . join ( )


def relative_dir_walk(self, dir):
    '''Generic version of directory walk. Return file list without base path
       for comparison.
    '''
    result = []

    if S3URL.is_valid(dir):
      basepath = S3URL(dir).path
      for f in (f for f in self.s3walk(dir) if not f['is_dir']):
        result.append(os.path.relpath(S3URL(f['name']).path, basepath))
    else:
      for f in (f for f in self.local_walk(dir) if not os.path.isdir(f)):
        result.append(os.path.relpath(f, dir))

    return result

def relative_dir_walk ( self , dir ) : result = [ ] if S3URL . is_valid ( dir ) : basepath = S3URL ( dir ) . path for f in ( f for f in self . s3walk ( dir ) if not f [ 'is_dir' ] ) : result . append ( os . path . relpath ( S3URL ( f [ 'name' ] ) . path , basepath ) ) else : for f in ( f for f in self . local_walk ( dir ) if not os . path . isdir ( f ) ) : result . append ( os . path . relpath ( f , dir ) ) return result

Generic version of directory walk . Return file list without base path for comparison .


def dsync_files(self, source, target):
    '''Sync directory to directory.'''
    src_s3_url = S3URL.is_valid(source)
    dst_s3_url = S3URL.is_valid(target)

    source_list = self.relative_dir_walk(source)
    if len(source_list) == 0 or '.' in source_list:
      raise Failure('Sync command need to sync directory to directory.')

    sync_list = [(os.path.join(source, f), os.path.join(target, f)) for f in source_list]

    pool = ThreadPool(ThreadUtil, self.opt)
    if src_s3_url and not dst_s3_url:
      for src, dest in sync_list:
        pool.download(src, dest)
    elif not src_s3_url and dst_s3_url:
      for src, dest in sync_list:
        pool.upload(src, dest)
    elif src_s3_url and dst_s3_url:
      for src, dest in sync_list:
        pool.copy(src, dest)
    else:
      raise InvalidArgument('Cannot sync two local directories.')
    pool.join()

    if self.opt.delete_removed:
      target_list = self.relative_dir_walk(target)
      remove_list = [os.path.join(target, f) for f in (set(target_list) - set(source_list))]

      if S3URL.is_valid(target):
        pool = ThreadPool(ThreadUtil, self.opt)
        pool.batch_delete(remove_list)
        pool.join()
      else:
        for f in remove_list:
          try:
            os.unlink(f)
            message('Delete %s', f)
          except:
            pass


def file_hash(self, filename, block_size=2**20):
    '''Calculate MD5 hash code for a local file'''
    m = hashlib.md5()
    with open(filename, 'rb') as f:
      while True:
        data = f.read(block_size)
        if not data:
          break
        m.update(data)
    return m.hexdigest()

def file_hash ( self , filename , block_size = 2 ** 20 ) : m = hashlib . md5 ( ) with open ( filename , 'rb' ) as f : while True : data = f . read ( block_size ) if not data : break m . update ( data ) return m . hexdigest ( )


def get_md5(self):
    '''Get or calculate MD5 value of the local file.'''
    if self.md5 is None:
      self.md5 = self.file_hash(self.filename)
    return self.md5

def get_md5 ( self ) : if self . md5 is None : self . md5 = self . file_hash ( self . filename ) return self . md5

Get or calculate MD5 value of the local file .


def mkdirs(self, target):
    '''Ensure all directories are created for a given target file.'''
    path = os.path.dirname(target)
    if path and path != PATH_SEP and not os.path.isdir(path):
      # Multi-threading means there will be intervleaved execution
      # between the check and creation of the directory.
      try:
        os.makedirs(path)
      except OSError as ose:
        if ose.errno != errno.EEXIST:
          raise Failure('Unable to create directory (%s)' % (path,))

def
mkdirs
(
self
,
target
)
:
path
=
os
.
path
.
dirname
(
target
)
if
path
and
path
!=
PATH_SEP
and
not
os
.
path
.
isdir
(
path
)
:
try
:
os
.
makedirs
(
path
)
except
OSError
as
ose
:
if
ose
.
errno
!=
errno
.
EEXIST
:
raise
Failure
(
'Unable to create directory (%s)'
%
(
path
,
)
)

Ensure all directories are created for a given target file .


def sync_check(self, md5cache, remoteKey):
    '''Check MD5 for a local file and a remote file.
       Return True if they have the same md5 hash, otherwise False.
    '''
    if not remoteKey:
      return False
    if not os.path.exists(md5cache.filename):
      return False
    localmd5 = md5cache.get_md5()

    # check multiple md5 locations
    return ('ETag' in remoteKey and remoteKey['ETag'] == '"%s"' % localmd5) or \
           ('md5' in remoteKey and remoteKey['md5'] == localmd5) or \
           ('md5' in remoteKey['Metadata'] and remoteKey['Metadata']['md5'] == localmd5)

def sync_check ( self , md5cache , remoteKey ) : if not remoteKey : return False if not os . path . exists ( md5cache . filename ) : return False localmd5 = md5cache . get_md5 ( ) return ( 'ETag' in remoteKey and remoteKey [ 'ETag' ] == '"%s"' % localmd5 ) or ( 'md5' in remoteKey and remoteKey [ 'md5' ] == localmd5 ) or ( 'md5' in remoteKey [ 'Metadata' ] and remoteKey [ 'Metadata' ] [ 'md5' ] == localmd5 )

Check MD5 for a local file and a remote file . Return True if they have the same md5 hash otherwise False .


def partial_match(self, path, filter_path):
    '''Partially match a path and a filter_path with wildcards.
       This function will return True if this path partially match a filter path.
       This is used for walking through directories with multiple level wildcard.
    '''
    if not path or not filter_path:
      return True

    # trailing slash normalization
    if path[-1] == PATH_SEP:
      path = path[0:-1]
    if filter_path[-1] == PATH_SEP:
      filter_path += '*'

    pi = path.split(PATH_SEP)
    fi = filter_path.split(PATH_SEP)

    # Here, if we are in recursive mode, we allow the pi to be longer than fi.
    # Otherwise, length of pi should be equal or less than the lenght of fi.
    min_len = min(len(pi), len(fi))
    matched = fnmatch.fnmatch(PATH_SEP.join(pi[0:min_len]), PATH_SEP.join(fi[0:min_len]))
    return matched and (self.opt.recursive or len(pi) <= len(fi))

def partial_match ( self , path , filter_path ) : if not path or not filter_path : return True if path [ - 1 ] == PATH_SEP : path = path [ 0 : - 1 ] if filter_path [ - 1 ] == PATH_SEP : filter_path += '*' pi = path . split ( PATH_SEP ) fi = filter_path . split ( PATH_SEP ) min_len = min ( len ( pi ) , len ( fi ) ) matched = fnmatch . fnmatch ( PATH_SEP . join ( pi [ 0 : min_len ] ) , PATH_SEP . join ( fi [ 0 : min_len ] ) ) return matched and ( self . opt . recursive or len ( pi ) <= len ( fi ) )

Partially match a path and a filter_path with wildcards . This function will return True if this path partially match a filter path . This is used for walking through directories with multiple level wildcard .


def s3walk(self, s3url, s3dir, filter_path, result):
    '''Thread worker for s3walk.
       Recursively walk into all subdirectories if they still match the filter
       path partially.
    '''

    paginator = self.s3.get_paginator('list_objects')
    filter_path_level = filter_path.count(PATH_SEP)

    for page in paginator.paginate(Bucket=s3url.bucket, Prefix=s3dir, Delimiter=PATH_SEP, PaginationConfig={'PageSize': 1000}):
      # Get subdirectories first.
      for obj in page.get('CommonPrefixes') or []:
        obj_name = obj['Prefix']

        if not self.partial_match(obj_name, filter_path):
          continue

        if self.opt.recursive or (obj_name.count(PATH_SEP) != filter_path_level + 1):
          self.pool.s3walk(s3url, obj_name, filter_path, result)
        else:
          self.conditional(result, {
            'name': S3URL.combine(s3url.proto, s3url.bucket, obj_name),
            'is_dir': True,
            'size': 0,
            'last_modified': None
          })

      # Then get all items in this folder.
      for obj in page.get('Contents') or []:
        obj_name = obj['Key']
        if not self.partial_match(obj_name, filter_path):
          continue

        if self.opt.recursive or obj_name.count(PATH_SEP) == filter_path_level:
          self.conditional(result, {
            'name': S3URL.combine(s3url.proto, s3url.bucket, obj_name),
            'is_dir': False,
            'size': obj['Size'],
            'last_modified': obj['LastModified']
          })

def s3walk ( self , s3url , s3dir , filter_path , result ) : paginator = self . s3 . get_paginator ( 'list_objects' ) filter_path_level = filter_path . count ( PATH_SEP ) for page in paginator . paginate ( Bucket = s3url . bucket , Prefix = s3dir , Delimiter = PATH_SEP , PaginationConfig = { 'PageSize' : 1000 } ) : for obj in page . get ( 'CommonPrefixes' ) or [ ] : obj_name = obj [ 'Prefix' ] if not self . partial_match ( obj_name , filter_path ) : continue if self . opt . recursive or ( obj_name . count ( PATH_SEP ) != filter_path_level + 1 ) : self . pool . s3walk ( s3url , obj_name , filter_path , result ) else : self . conditional ( result , { 'name' : S3URL . combine ( s3url . proto , s3url . bucket , obj_name ) , 'is_dir' : True , 'size' : 0 , 'last_modified' : None } ) for obj in page . get ( 'Contents' ) or [ ] : obj_name = obj [ 'Key' ] if not self . partial_match ( obj_name , filter_path ) : continue if self . opt . recursive or obj_name . count ( PATH_SEP ) == filter_path_level : self . conditional ( result , { 'name' : S3URL . combine ( s3url . proto , s3url . bucket , obj_name ) , 'is_dir' : False , 'size' : obj [ 'Size' ] , 'last_modified' : obj [ 'LastModified' ] } )

Thread worker for s3walk . Recursively walk into all subdirectories if they still match the filter path partially .


def conditional(self, result, obj):
    '''Check all file item with given conditions.'''
    fileonly = (self.opt.last_modified_before is not None) or (self.opt.last_modified_after is not None)

    if obj['is_dir']:
      if not fileonly:
        result.append(obj)
      return

    if (self.opt.last_modified_before is not None) and obj['last_modified'] >= self.opt.last_modified_before:
      return

    if (self.opt.last_modified_after is not None) and obj['last_modified'] <= self.opt.last_modified_after:
      return

    result.append(obj)

def conditional ( self , result , obj ) : fileonly = ( self . opt . last_modified_before is not None ) or ( self . opt . last_modified_after is not None ) if obj [ 'is_dir' ] : if not fileonly : result . append ( obj ) return if ( self . opt . last_modified_before is not None ) and obj [ 'last_modified' ] >= self . opt . last_modified_before : return if ( self . opt . last_modified_after is not None ) and obj [ 'last_modified' ] <= self . opt . last_modified_after : return result . append ( obj )

Check all file item with given conditions .


def get_file_privilege(self, source):
    '''Get privileges of a local file'''
    try:
      return str(oct(os.stat(source).st_mode)[-3:])
    except Exception as e:
      raise Failure('Could not get stat for %s, error_message = %s', source, e)


def lookup(self, s3url):
    '''Get the s3 object with the S3 URL. Return None if not exist.'''
    try:
      return self.s3.head_object(Bucket=s3url.bucket, Key=s3url.path)
    except BotoClient.ClientError as e:
      if e.response['ResponseMetadata']['HTTPStatusCode'] == 404:
        return None
      else:
        raise e

def lookup ( self , s3url ) : try : return self . s3 . head_object ( Bucket = s3url . bucket , Key = s3url . path ) except BotoClient . ClientError as e : if e . response [ 'ResponseMetadata' ] [ 'HTTPStatusCode' ] == 404 : return None else : raise e

Get the s3 object with the S3 URL . Return None if not exist .


def read_file_chunk(self, source, pos, chunk):
    '''Read local file chunk'''
    if chunk==0:
        return StringIO()
    data = None
    with open(source, 'rb') as f:
      f.seek(pos)
      data = f.read(chunk)
    if not data:
      raise Failure('Unable to read data from source: %s' % source)
    return StringIO(data)

def
read_file_chunk
(
self
,
source
,
pos
,
chunk
)
:
if
chunk
==
0
:
return
StringIO
(
)
data
=
with
open
(
source
,
'rb'
)
as
f
:
f
.
seek
(
pos
)
data
=
f
.
read
(
chunk
)
if
not
data
:
raise
Failure
(
'Unable to read data from source: %s'
%
source
)
return
StringIO
(
data
)


def upload(self, source, target, mpi=None, pos=0, chunk=0, part=0):
    '''Thread worker for upload operation.'''
    s3url = S3URL(target)
    obj = self.lookup(s3url)

    # Initialization: Set up multithreaded uploads.
    if not mpi:
      fsize = os.path.getsize(source)
      md5cache = LocalMD5Cache(source)

      # optional checks
      if self.opt.dry_run:
        message('%s => %s', source, target)
        return
      elif self.opt.sync_check and self.sync_check(md5cache, obj):
        message('%s => %s (synced)', source, target)
        return
      elif not self.opt.force and obj:
        raise Failure('File already exists: %s' % target)

      if fsize < self.opt.max_singlepart_upload_size:
        data = self.read_file_chunk(source, 0, fsize)
        self.s3.put_object(Bucket=s3url.bucket,
                           Key=s3url.path,
                           Body=data,
                           Metadata={'md5': md5cache.get_md5(),
                                     'privilege': self.get_file_privilege(source)})
        message('%s => %s', source, target)
        return

      # Here we need to have our own md5 value because multipart upload calculates
      # different md5 values.
      response = self.s3.create_multipart_upload(Bucket=s3url.bucket,
                                                 Key=s3url.path,
                                                 Metadata={'md5': md5cache.get_md5(),
                                                           'privilege': self.get_file_privilege(source)})
      upload_id = response['UploadId']

      for args in self.get_file_splits(upload_id, source, target, fsize, self.opt.multipart_split_size):
        self.pool.upload(*args)
      return

    data = self.read_file_chunk(source, pos, chunk)
    response = self.s3.upload_part(Bucket=s3url.bucket, Key=s3url.path, UploadId=mpi.id, Body=data, PartNumber=part)

    # Finalize
    if mpi.complete({'ETag': response['ETag'], 'PartNumber': part}):
      try:
        self.s3.complete_multipart_upload(Bucket=s3url.bucket, Key=s3url.path, UploadId=mpi.id, MultipartUpload={'Parts': mpi.sorted_parts()})
        message('%s => %s', source, target)
      except Exception as e:
        message('Unable to complete upload: %s', str(e))
        self.s3.abort_multipart_upload(Bucket=s3url.bucket, Key=s3url.path, UploadId=mpi.id)
        raise RetryFailure('Upload failed: Unable to complete upload %s.' % source)


def _verify_file_size(self, obj, downloaded_file):
    '''Verify the file size of the downloaded file.'''
    file_size = os.path.getsize(downloaded_file)
    if int(obj['ContentLength']) != file_size:
      raise RetryFailure('Downloaded file size inconsistent: %s' % (repr(obj)))

def
_verify_file_size
(
self
,
obj
,
downloaded_file
)
:
file_size
=
os
.
path
.
getsize
(
downloaded_file
)
if
int
(
obj
[
'ContentLength'
]
)
!=
file_size
:
raise
RetryFailure
(
'Downloaded file size inconsistent: %s'
%
(
repr
(
obj
)
)
)

Verify the file size of the downloaded file .


def write_file_chunk(self, target, pos, chunk, body):
    '''Write local file chunk'''
    fd = os.open(target, os.O_CREAT | os.O_WRONLY)
    try:
      os.lseek(fd, pos, os.SEEK_SET)
      data = body.read(chunk)
      num_bytes_written = os.write(fd, data)
      if(num_bytes_written != len(data)):
        raise RetryFailure('Number of bytes written inconsistent: %s != %s' % (num_bytes_written, sys.getsizeof(data)))
    finally:
      os.close(fd)


def copy(self, source, target, mpi=None, pos=0, chunk=0, part=0, delete_source=False):
    '''Copy a single file from source to target using boto S3 library.'''

    if self.opt.dry_run:
      message('%s => %s' % (source, target))
      return

    source_url = S3URL(source)
    target_url = S3URL(target)

    if not mpi:
      obj = self.lookup(source_url)
      fsize = int(obj['ContentLength'])

      if fsize < self.opt.max_singlepart_copy_size:
        self.s3.copy_object(Bucket=target_url.bucket, Key=target_url.path,
                            CopySource={'Bucket': source_url.bucket, 'Key': source_url.path})

        message('%s => %s' % (source, target))
        if delete_source:
          self.delete(source)

        return

      response = self.s3.create_multipart_upload(Bucket=target_url.bucket,
                                                 Key=target_url.path,
                                                 Metadata=obj['Metadata'])
      upload_id = response['UploadId']

      for args in self.get_file_splits(upload_id, source, target, fsize, self.opt.multipart_split_size):
        self.pool.copy(*args, delete_source=delete_source)
      return

    response = self.s3.upload_part_copy(Bucket=target_url.bucket,
                                        Key=target_url.path,
                                        CopySource={'Bucket': source_url.bucket, 'Key': source_url.path},
                                        CopySourceRange='bytes=%d-%d' % (pos, pos + chunk - 1),
                                        UploadId=mpi.id,
                                        PartNumber=part)

    if mpi.complete({'ETag': response['CopyPartResult']['ETag'], 'PartNumber': part}):
      try:
        # Finalize copy operation.
        self.s3.complete_multipart_upload(Bucket=target_url.bucket, Key=target_url.path, UploadId=mpi.id, MultipartUpload={'Parts': mpi.sorted_parts()})

        if delete_source:
          self.delete(source)

        message('%s => %s' % (source, target))
      except Exception as e:
        message('Unable to complete upload: %s', str(e))
        self.s3.abort_multipart_upload(Bucket=source_url.bucket, Key=source_url.path, UploadId=mpi.id)
        raise RetryFailure('Copy failed: Unable to complete copy %s.' % source)

Copy a single file from source to target using boto S3 library .


def run(self, args):
    '''Main entry to handle commands. Dispatch to individual command handler.'''
    if len(args) == 0:
      raise InvalidArgument('No command provided')
    cmd = args[0]
    if cmd + '_handler' in CommandHandler.__dict__:
      CommandHandler.__dict__[cmd + '_handler'](self, args)
    else:
      raise InvalidArgument('Unknown command %s' % cmd)

def
run
(
self
,
args
)
:
if
len
(
args
)
==
0
:
raise
InvalidArgument
(
'No command provided'
)
cmd
=
args
[
0
]
if
cmd
+
'_handler'
in
CommandHandler
.
__dict__
:
CommandHandler
.
__dict__
[
cmd
+
'_handler'
]
(
self
,
args
)
else
:
raise
InvalidArgument
(
'Unknown command %s'
%
cmd
)

Main entry to handle commands . Dispatch to individual command handler .


def validate(self, format, args):
    '''Validate input parameters with given format.
       This function also checks for wildcards for recursive mode.
    '''
    fmtMap = {
      'cmd': 'Command',
      's3': 's3 path',
      'local': 'local path'
    }
    fmts = format.split('|')
    if len(fmts) != len(args):
      raise InvalidArgument('Invalid number of parameters')

    for i, fmt in enumerate(fmts):
      valid = False
      for f in fmt.split(','):
        if f == 'cmd' and args[i] + '_handler' in CommandHandler.__dict__:
          valid = True
        if f == 's3' and S3URL.is_valid(args[i]):
          valid = True
        if f == 'local' and not S3URL.is_valid(args[i]):
          valid = True
      if not valid:
        raise InvalidArgument('Invalid parameter: %s, %s expected' % (args[i], fmtMap[fmt.split(',')[0]]))

def
validate
(
self
,
format
,
args
)
:
fmtMap
=
{
'cmd'
:
'Command'
,
's3'
:
's3 path'
,
'local'
:
'local path'
}
fmts
=
format
.
split
(
'|'
)
if
len
(
fmts
)
!=
len
(
args
)
:
raise
InvalidArgument
(
'Invalid number of parameters'
)
for
i
,
fmt
in
enumerate
(
fmts
)
:
valid
=
False
for
f
in
fmt
.
split
(
','
)
:
if
f
==
'cmd'
and
args
[
i
]
+
'_handler'
in
CommandHandler
.
__dict__
:
valid
=
True
if
f
==
's3'
and
S3URL
.
is_valid
(
args
[
i
]
)
:
valid
=
True
if
f
==
'local'
and
not
S3URL
.
is_valid
(
args
[
i
]
)
:
valid
=
True
if
not
valid
:
raise
InvalidArgument
(
'Invalid parameter: %s, %s expected'
%
(
args
[
i
]
,
fmtMap
[
fmt
.
split
(
','
)
[
0
]
]
)
)

Validate input parameters with given format . This function also checks for wildcards for recursive mode .


def pretty_print(self, objlist):
    '''Pretty print the result of s3walk. Here we calculate the maximum width
       of each column and align them.
    '''

    def normalize_time(timestamp):
      '''Normalize the timestamp format for pretty print.'''
      if timestamp is None:
        return ' ' * 16

      return TIMESTAMP_FORMAT % (timestamp.year, timestamp.month, timestamp.day, timestamp.hour, timestamp.minute)

    cwidth = [0, 0, 0]
    format = '%%%ds %%%ds %%-%ds'

    # Calculate maximum width for each column.
    result = []
    for obj in objlist:
      last_modified = normalize_time(obj['last_modified'])
      size = str(obj['size']) if not obj['is_dir'] else 'DIR'
      name = obj['name']
      item = (last_modified, size, name)
      for i, value in enumerate(item):
        if cwidth[i] < len(value):
          cwidth[i] = len(value)
      result.append(item)

    # Format output.
    for item in result:
      text = (format % tuple(cwidth)) % item
      message('%s', text.rstrip())

def
pretty_print
(
self
,
objlist
)
:
def
normalize_time
(
timestamp
)
:
if
timestamp
is
:
return
' '
*
16
return
TIMESTAMP_FORMAT
%
(
timestamp
.
year
,
timestamp
.
month
,
timestamp
.
day
,
timestamp
.
hour
,
timestamp
.
minute
)
cwidth
=
[
0
,
0
,
0
]
format
=
'%%%ds %%%ds %%-%ds'
result
=
[
]
for
obj
in
objlist
:
last_modified
=
normalize_time
(
obj
[
'last_modified'
]
)
size
=
str
(
obj
[
'size'
]
)
if
not
obj
[
'is_dir'
]
else
'DIR'
name
=
obj
[
'name'
]
item
=
(
last_modified
,
size
,
name
)
for
i
,
value
in
enumerate
(
item
)
:
if
cwidth
[
i
]
<
len
(
value
)
:
cwidth
[
i
]
=
len
(
value
)
result
.
append
(
item
)
for
item
in
result
:
text
=
(
format
%
tuple
(
cwidth
)
)
%
item
message
(
'%s'
,
text
.
rstrip
(
)
)

Pretty print the result of s3walk . Here we calculate the maximum width of each column and align them .


def ls_handler(self, args):
    '''Handler for ls command'''
    if len(args) == 1:
      self.pretty_print(self.s3handler().list_buckets())
      return

    self.validate('cmd|s3', args)
    self.pretty_print(self.s3handler().s3walk(args[1]))

def ls_handler ( self , args ) : if len ( args ) == 1 : self . pretty_print ( self . s3handler ( ) . list_buckets ( ) ) return self . validate ( 'cmd|s3' , args ) self . pretty_print ( self . s3handler ( ) . s3walk ( args [ 1 ] ) )


def mb_handler(self, args):
    '''Handler for mb command'''
    if len(args) == 1:
      raise InvalidArgument('No s3 bucketname provided')

    self.validate('cmd|s3', args)
    self.s3handler().create_bucket(args[1])

def
mb_handler
(
self
,
args
)
:
if
len
(
args
)
==
1
:
raise
InvalidArgument
(
'No s3 bucketname provided'
)
self
.
validate
(
'cmd|s3'
,
args
)
self
.
s3handler
(
)
.
create_bucket
(
args
[
1
]
)


def put_handler(self, args):
    '''Handler for put command'''

    # Special check for shell expansion
    if len(args) < 3:
      raise InvalidArgument('Invalid number of parameters')
    self.validate('|'.join(['cmd'] + ['local'] * (len(args) - 2) + ['s3']), args)

    source = args[1:-1] # shell expansion
    target = args[-1]

    self.s3handler().put_files(source, target)

def
put_handler
(
self
,
args
)
:
if
len
(
args
)
<
3
:
raise
InvalidArgument
(
'Invalid number of parameters'
)
self
.
validate
(
'|'
.
join
(
[
'cmd'
]
+
[
'local'
]
*
(
len
(
args
)
2
)
+
[
's3'
]
)
,
args
)
source
=
args
[
1
:
1
]
target
=
args
[
1
]
self
.
s3handler
(
)
.
put_files
(
source
,
target
)


def get_handler(self, args):
    '''Handler for get command'''

    # Special case when we don't have target directory.
    if len(args) == 2:
      args += ['.']

    self.validate('cmd|s3|local', args)
    source = args[1]
    target = args[2]
    self.s3handler().get_files(source, target)

def get_handler ( self , args ) : if len ( args ) == 2 : args += [ '.' ] self . validate ( 'cmd|s3|local' , args ) source = args [ 1 ] target = args [ 2 ] self . s3handler ( ) . get_files ( source , target )


def cat_handler(self, args):
    '''Handler for cat command'''

    self.validate('cmd|s3', args)
    source = args[1]

    self.s3handler().print_files(source)

def cat_handler ( self , args ) : self . validate ( 'cmd|s3' , args ) source = args [ 1 ] self . s3handler ( ) . print_files ( source )


def dsync_handler(self, args):
    '''Handler for dsync command.'''
    self.opt.recursive = True
    self.opt.sync_check = True
    self.opt.force = True

    self.validate('cmd|s3,local|s3,local', args)
    source = args[1]
    target = args[2]

    self.s3handler().dsync_files(source, target)

def dsync_handler ( self , args ) : self . opt . recursive = True self . opt . sync_check = True self . opt . force = True self . validate ( 'cmd|s3,local|s3,local' , args ) source = args [ 1 ] target = args [ 2 ] self . s3handler ( ) . dsync_files ( source , target )


def cp_handler(self, args):
    '''Handler for cp command'''

    self.validate('cmd|s3|s3', args)
    source = args[1]
    target = args[2]
    self.s3handler().cp_files(source, target)

def cp_handler ( self , args ) : self . validate ( 'cmd|s3|s3' , args ) source = args [ 1 ] target = args [ 2 ] self . s3handler ( ) . cp_files ( source , target )


def mv_handler(self, args):
    '''Handler for mv command'''

    self.validate('cmd|s3|s3', args)
    source = args[1]
    target = args[2]
    self.s3handler().cp_files(source, target, delete_source=True)

def mv_handler ( self , args ) : self . validate ( 'cmd|s3|s3' , args ) source = args [ 1 ] target = args [ 2 ] self . s3handler ( ) . cp_files ( source , target , delete_source = True )


def del_handler(self, args):
    '''Handler for del command'''
    self.validate('cmd|s3', args)
    source = args[1]
    self.s3handler().del_files(source)

def del_handler ( self , args ) : self . validate ( 'cmd|s3' , args ) source = args [ 1 ] self . s3handler ( ) . del_files ( source )


def du_handler(self, args):
    '''Handler for size command'''
    for src, size in self.s3handler().size(args[1:]):
      message('%s\t%s' % (size, src))

def du_handler ( self , args ) : for src , size in self . s3handler ( ) . size ( args [ 1 : ] ) : message ( '%s\t%s' % ( size , src ) )


def _totalsize_handler(self, args):
    '''Handler of total_size command'''
    total_size = 0
    for src, size in self.s3handler().size(args[1:]):
      total_size += size
    message(str(total_size))

def _totalsize_handler ( self , args ) : total_size = 0 for src , size in self . s3handler ( ) . size ( args [ 1 : ] ) : total_size += size message ( str ( total_size ) )


def match_date(self, value):
    '''Search for date information in the string'''
    m = self.REGEX_DATE.search(value)
    date = datetime.datetime.utcnow().date()
    if m:
      date = datetime.date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
      value = self.REGEX_DATE.sub('', value)
    return (date, value)

def match_date ( self , value ) : m = self . REGEX_DATE . search ( value ) date = datetime . datetime . utcnow ( ) . date ( ) if m : date = datetime . date ( int ( m . group ( 1 ) ) , int ( m . group ( 2 ) ) , int ( m . group ( 3 ) ) ) value = self . REGEX_DATE . sub ( '' , value ) return ( date , value )


def match_time(self, value):
    '''Search for time information in the string'''
    m = self.REGEX_TIME.search(value)
    time = datetime.datetime.utcnow().time()
    if m:
      time = datetime.time(int(m.group(1)), int(m.group(2)))
      value = self.REGEX_TIME.sub('', value)
    return (time, value)

def match_time ( self , value ) : m = self . REGEX_TIME . search ( value ) time = datetime . datetime . utcnow ( ) . time ( ) if m : time = datetime . time ( int ( m . group ( 1 ) ) , int ( m . group ( 2 ) ) ) value = self . REGEX_TIME . sub ( '' , value ) return ( time , value )


def match_delta(self, value):
    '''Search for timedelta information in the string'''
    m = self.REGEX_DELTA.search(value)
    delta = datetime.timedelta(days=0)
    if m:
      d = int(m.group(1))
      if m.group(3) == 'ago' or m.group(3) == 'before':
        d = -d

      if m.group(2) == 'minute':
        delta = datetime.timedelta(minutes=d)
      elif m.group(2) == 'hour':
        delta = datetime.timedelta(hours=d)
      elif m.group(2) == 'day':
        delta = datetime.timedelta(days=d)
      elif m.group(2) == 'week':
        delta = datetime.timedelta(weeks=d)
      value = self.REGEX_DELTA.sub('', value)
    return (delta, value)

def match_delta ( self , value ) : m = self . REGEX_DELTA . search ( value ) delta = datetime . timedelta ( days = 0 ) if m : d = int ( m . group ( 1 ) ) if m . group ( 3 ) == 'ago' or m . group ( 3 ) == 'before' : d = - d if m . group ( 2 ) == 'minute' : delta = datetime . timedelta ( minutes = d ) elif m . group ( 2 ) == 'hour' : delta = datetime . timedelta ( hours = d ) elif m . group ( 2 ) == 'day' : delta = datetime . timedelta ( days = d ) elif m . group ( 2 ) == 'week' : delta = datetime . timedelta ( weeks = d ) value = self . REGEX_DELTA . sub ( '' , value ) return ( delta , value )


def check_dict(self, opt, value):
    '''Take json as dictionary parameter'''
    try:
      return json.loads(value)
    except:
      raise optparse.OptionValueError("Option %s: invalid dict value: %r" % (opt, value))

def
check_dict
(
self
,
opt
,
value
)
:
try
:
return
json
.
loads
(
value
)
except
:
raise
optparse
.
OptionValueError
(
"Option %s: invalid dict value: %r"
%
(
opt
,
value
)
)


def discover_gateways(self):
        """Discover gateways using multicast"""

        _socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        _socket.settimeout(5.0)
        if self._interface != 'any':
            _socket.bind((self._interface, 0))

        for gateway in self._gateways_config:
            host = gateway.get('host')
            port = gateway.get('port')
            sid = gateway.get('sid')

            if not (host and port and sid):
                continue
            try:
                ip_address = socket.gethostbyname(host)
                if gateway.get('disable'):
                    _LOGGER.info(
                        'Xiaomi Gateway %s is disabled by configuration', sid)
                    self.disabled_gateways.append(ip_address)
                    continue
                _LOGGER.info(
                    'Xiaomi Gateway %s configured at IP %s:%s',
                    sid, ip_address, port)

                self.gateways[ip_address] = XiaomiGateway(
                    ip_address, port, sid,
                    gateway.get('key'), self._device_discovery_retries,
                    self._interface, gateway.get('proto'))
            except OSError as error:
                _LOGGER.error(
                    "Could not resolve %s: %s", host, error)

        try:
            _socket.sendto('{"cmd":"whois"}'.encode(),
                           (self.MULTICAST_ADDRESS, self.GATEWAY_DISCOVERY_PORT))

            while True:
                data, (ip_add, _) = _socket.recvfrom(1024)
                if len(data) is None or ip_add in self.gateways:
                    continue

                if ip_add in self.gateways.keys() or ip_add in self.disabled_gateways:
                    continue

                resp = json.loads(data.decode())
                if resp["cmd"] != 'iam':
                    _LOGGER.error("Response does not match return cmd")
                    continue

                if resp["model"] not in GATEWAY_MODELS:
                    _LOGGER.error("Response must be gateway model")
                    continue

                disabled = False
                gateway_key = None
                for gateway in self._gateways_config:
                    sid = gateway.get('sid')
                    if sid is None or sid == resp["sid"]:
                        gateway_key = gateway.get('key')
                    if sid and sid == resp['sid'] and gateway.get('disable'):
                        disabled = True

                sid = resp["sid"]
                if disabled:
                    _LOGGER.info("Xiaomi Gateway %s is disabled by configuration",
                                 sid)
                    self.disabled_gateways.append(ip_add)
                else:
                    _LOGGER.info('Xiaomi Gateway %s found at IP %s', sid, ip_add)
                    self.gateways[ip_add] = XiaomiGateway(
                        ip_add, resp["port"], sid, gateway_key,
                        self._device_discovery_retries, self._interface,
                        resp["proto_version"] if "proto_version" in resp else None)

        except socket.timeout:
            _LOGGER.info("Gateway discovery finished in 5 seconds")
            _socket.close()


def listen(self):
        """Start listening."""

        _LOGGER.info('Creating Multicast Socket')
        self._mcastsocket = self._create_mcast_socket()
        self._listening = True
        thread = Thread(target=self._listen_to_msg, args=())
        self._threads.append(thread)
        thread.daemon = True
        thread.start()

def
listen
(
self
)
:
_LOGGER
.
info
(
'Creating Multicast Socket'
)
self
.
_mcastsocket
=
self
.
_create_mcast_socket
(
)
self
.
_listening
=
True
thread
=
Thread
(
target
=
self
.
_listen_to_msg
,
args
=
(
)
)
self
.
_threads
.
append
(
thread
)
thread
.
daemon
=
True
thread
.
start
(
)


def get_from_hub(self, sid):
        """Get data from gateway"""
        cmd = '{ "cmd":"read","sid":"' + sid + '"}'
        resp = self._send_cmd(cmd, "read_ack") if int(self.proto[0:1]) == 1 else self._send_cmd(cmd, "read_rsp")
        _LOGGER.debug("read_ack << %s", resp)
        return self.push_data(resp)

def
get_from_hub
(
self
,
sid
)
:
cmd
=
'{ "cmd":"read","sid":"'
+
sid
+
'"}'
resp
=
self
.
_send_cmd
(
cmd
,
"read_ack"
)
if
int
(
self
.
proto
[
0
:
1
]
)
==
1
else
self
.
_send_cmd
(
cmd
,
"read_rsp"
)
_LOGGER
.
debug
(
"read_ack << %s"
,
resp
)
return
self
.
push_data
(
resp
)


def push_data(self, data):
        """Push data broadcasted from gateway to device"""
        if not _validate_data(data):
            return False
        jdata = json.loads(data['data']) if int(self.proto[0:1]) == 1 else _list2map(data['params'])
        if jdata is None:
            return False
        sid = data['sid']
        for func in self.callbacks[sid]:
            func(jdata, data)
        return True

def push_data ( self , data ) : if not _validate_data ( data ) : return False jdata = json . loads ( data [ 'data' ] ) if int ( self . proto [ 0 : 1 ] ) == 1 else _list2map ( data [ 'params' ] ) if jdata is None : return False sid = data [ 'sid' ] for func in self . callbacks [ sid ] : func ( jdata , data ) return True


def _get_key(self):
        """Get key using token from gateway"""
        init_vector = bytes(bytearray.fromhex('17996d093d28ddb3ba695a2e6f58562e'))
        encryptor = Cipher(algorithms.AES(self.key.encode()), modes.CBC(init_vector),
                           backend=default_backend()).encryptor()
        ciphertext = encryptor.update(self.token.encode()) + encryptor.finalize()
        if isinstance(ciphertext, str):  # For Python 2 compatibility
            return ''.join('{:02x}'.format(ord(x)) for x in ciphertext)
        return ''.join('{:02x}'.format(x) for x in ciphertext)

def _get_key ( self ) : init_vector = bytes ( bytearray . fromhex ( '17996d093d28ddb3ba695a2e6f58562e' ) ) encryptor = Cipher ( algorithms . AES ( self . key . encode ( ) ) , modes . CBC ( init_vector ) , backend = default_backend ( ) ) . encryptor ( ) ciphertext = encryptor . update ( self . token . encode ( ) ) + encryptor . finalize ( ) if isinstance ( ciphertext , str ) : return '' . join ( '{:02x}' . format ( ord ( x ) ) for x in ciphertext ) return '' . join ( '{:02x}' . format ( x ) for x in ciphertext )


def exception_handler(job, *exc_info):
    """
    Called by RQ when there is a failure in a worker.

    NOTE: Make sure that in your RQ worker process, rollbar.init() has been called with
    handler='blocking'. The default handler, 'thread', does not work from inside an RQ worker.
    """
    # Report data about the job with the exception.
    job_info = job.to_dict()
    # job_info['data'] is the pickled representation of the job, and doesn't json-serialize well.
    # repr() works nicely.
    job_info['data'] = repr(job_info['data'])

    extra_data = {'job': job_info}
    payload_data = {'framework': 'rq'}

    rollbar.report_exc_info(exc_info, extra_data=extra_data, payload_data=payload_data)

    # continue to the next handler
    return True

def exception_handler ( job , * exc_info ) : job_info = job . to_dict ( ) job_info [ 'data' ] = repr ( job_info [ 'data' ] ) extra_data = { 'job' : job_info } payload_data = { 'framework' : 'rq' } rollbar . report_exc_info ( exc_info , extra_data = extra_data , payload_data = payload_data ) return True

Called by RQ when there is a failure in a worker .


def includeme(config):
    """
    Pyramid entry point
    """
    settings = config.registry.settings

    config.add_tween('rollbar.contrib.pyramid.rollbar_tween_factory', over=EXCVIEW)

    # run patch_debugtoolbar, unless they disabled it
    if asbool(settings.get('rollbar.patch_debugtoolbar', True)):
        patch_debugtoolbar(settings)

    def hook(request, data):
        data['framework'] = 'pyramid'

        if request:
            request.environ['rollbar.uuid'] = data['uuid']

            if request.matched_route:
                data['context'] = request.matched_route.name

    rollbar.BASE_DATA_HOOK = hook

    kw = parse_settings(settings)

    access_token = kw.pop('access_token')
    environment = kw.pop('environment', 'production')

    if kw.get('scrub_fields'):
        kw['scrub_fields'] = set([str.strip(x) for x in kw.get('scrub_fields').split('\n') if x])

    if kw.get('exception_level_filters'):
        r = DottedNameResolver()
        exception_level_filters = []
        for line in kw.get('exception_level_filters').split('\n'):
            if line:
                dotted_path, level = line.split()

                try:
                    cls = r.resolve(dotted_path)
                    exception_level_filters.append((cls, level))
                except ImportError:
                    log.error('Could not import %r' % dotted_path)

        kw['exception_level_filters'] = exception_level_filters

    kw['enabled'] = asbool(kw.get('enabled', True))

    rollbar.init(access_token, environment, **kw)


def _ensure_log_handler(self):
        """
        If there's no log configuration, set up a default handler.
        """
        if log.handlers:
            return
        handler = logging.StreamHandler()
        formatter = logging.Formatter(
            '%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s')
        handler.setFormatter(formatter)
        log.addHandler(handler)

If there s no log configuration set up a default handler .


def get_request():
    """
    Get the current request object. Implementation varies on
    library support. Modified below when we know which framework
    is being used.
    """

    # TODO(cory): add in a generic _get_locals_request() which
    # will iterate up through the call stack and look for a variable
    # that appears to be valid request object.
    for fn in (_get_bottle_request,
               _get_flask_request,
               _get_pyramid_request,
               _get_pylons_request):
        try:
            req = fn()
            if req is not None:
                return req
        except:
            pass

    return None

def get_request ( ) : for fn in ( _get_bottle_request , _get_flask_request , _get_pyramid_request , _get_pylons_request ) : try : req = fn ( ) if req is not None : return req except : pass return None

Get the current request object . Implementation varies on library support . Modified below when we know which framework is being used .


def init(access_token, environment='production', scrub_fields=None, url_fields=None, **kw):
    """
    Saves configuration variables in this module's SETTINGS.

    access_token: project access token. Get this from the Rollbar UI:
                  - click "Settings" in the top nav
                  - click "Projects" in the left nav
                  - copy-paste the appropriate token.
    environment: environment name. Can be any string; suggestions: 'production', 'development',
                 'staging', 'yourname'
    **kw: provided keyword arguments will override keys in SETTINGS.
    """
    global SETTINGS, agent_log, _initialized, _transforms, _serialize_transform, _threads

    if scrub_fields is not None:
       SETTINGS['scrub_fields'] = list(scrub_fields)
    if url_fields is not None:
       SETTINGS['url_fields'] = list(url_fields)

    # Merge the extra config settings into SETTINGS
    SETTINGS = dict_merge(SETTINGS, kw)
    if _initialized:
        # NOTE: Temp solution to not being able to re-init.
        # New versions of pyrollbar will support re-initialization
        # via the (not-yet-implemented) configure() method.
        if not SETTINGS.get('suppress_reinit_warning'):
            log.warning('Rollbar already initialized. Ignoring re-init.')
        return

    SETTINGS['access_token'] = access_token
    SETTINGS['environment'] = environment

    if SETTINGS.get('allow_logging_basic_config'):
        logging.basicConfig()

    if SETTINGS.get('handler') == 'agent':
        agent_log = _create_agent_log()

    # We will perform these transforms in order:
    # 1. Serialize the payload to be all python built-in objects
    # 2. Scrub the payloads based on the key suffixes in SETTINGS['scrub_fields']
    # 3. Scrub URLs in the payload for keys that end with 'url'
    # 4. Optional - If local variable gathering is enabled, transform the
    #       trace frame values using the ShortReprTransform.
    _serialize_transform = SerializableTransform(safe_repr=SETTINGS['locals']['safe_repr'],
                                                 whitelist_types=SETTINGS['locals']['whitelisted_types'])
    _transforms = [
        ScrubRedactTransform(),
        _serialize_transform,
        ScrubTransform(suffixes=[(field,) for field in SETTINGS['scrub_fields']], redact_char='*'),
        ScrubUrlTransform(suffixes=[(field,) for field in SETTINGS['url_fields']], params_to_scrub=SETTINGS['scrub_fields'])
    ]

    # A list of key prefixes to apply our shortener transform to. The request
    # being included in the body key is old behavior and is being retained for
    # backwards compatibility.
    shortener_keys = [
        ('request', 'POST'),
        ('request', 'json'),
        ('body', 'request', 'POST'),
        ('body', 'request', 'json'),
    ]

    if SETTINGS['locals']['enabled']:
        shortener_keys.append(('body', 'trace', 'frames', '*', 'code'))
        shortener_keys.append(('body', 'trace', 'frames', '*', 'args', '*'))
        shortener_keys.append(('body', 'trace', 'frames', '*', 'kwargs', '*'))
        shortener_keys.append(('body', 'trace', 'frames', '*', 'locals', '*'))

    shortener_keys.extend(SETTINGS['shortener_keys'])

    shortener = ShortenerTransform(safe_repr=SETTINGS['locals']['safe_repr'],
                                   keys=shortener_keys,
                                   **SETTINGS['locals']['sizes'])
    _transforms.append(shortener)
    _threads = queue.Queue()
    events.reset()
    filters.add_builtin_filters(SETTINGS)

    _initialized = True

Saves configuration variables in this module s SETTINGS .


def lambda_function(f):
    """
    Decorator for making error handling on AWS Lambda easier
    """
    @functools.wraps(f)
    def wrapper(event, context):
        global _CURRENT_LAMBDA_CONTEXT
        _CURRENT_LAMBDA_CONTEXT = context
        try:
            result = f(event, context)
            return wait(lambda: result)
        except:
            cls, exc, trace = sys.exc_info()
            report_exc_info((cls, exc, trace.tb_next))
            wait()
            raise
    return wrapper

def lambda_function ( f ) : @ functools . wraps ( f ) def wrapper ( event , context ) : global _CURRENT_LAMBDA_CONTEXT _CURRENT_LAMBDA_CONTEXT = context try : result = f ( event , context ) return wait ( lambda : result ) except : cls , exc , trace = sys . exc_info ( ) report_exc_info ( ( cls , exc , trace . tb_next ) ) wait ( ) raise return wrapper


def report_message(message, level='error', request=None, extra_data=None, payload_data=None):
    """
    Reports an arbitrary string message to Rollbar.

    message: the string body of the message
    level: level to report at. One of: 'critical', 'error', 'warning', 'info', 'debug'
    request: the request object for the context of the message
    extra_data: dictionary of params to include with the message. 'body' is reserved.
    payload_data: param names to pass in the 'data' level of the payload; overrides defaults.
    """
    try:
        return _report_message(message, level, request, extra_data, payload_data)
    except Exception as e:
        log.exception("Exception while reporting message to Rollbar. %r", e)

Reports an arbitrary string message to Rollbar .


def search_items(title, return_fields=None, access_token=None, endpoint=None, **search_fields):
    """
    Searches a project for items that match the input criteria.

    title: all or part of the item's title to search for.
    return_fields: the fields that should be returned for each item.
            e.g. ['id', 'project_id', 'status'] will return a dict containing
                 only those fields for each item.
    access_token: a project access token. If this is not provided,
                  the one provided to init() will be used instead.
    search_fields: additional fields to include in the search.
            currently supported: status, level, environment
    """
    if not title:
        return []

    if return_fields is not None:
        return_fields = ','.join(return_fields)

    return _get_api('search/',
                    title=title,
                    fields=return_fields,
                    access_token=access_token,
                    endpoint=endpoint,
                    **search_fields)

def search_items ( title , return_fields = None , access_token = None , endpoint = None , ** search_fields ) : if not title : return [ ] if return_fields is not None : return_fields = ',' . join ( return_fields ) return _get_api ( 'search/' , title = title , fields = return_fields , access_token = access_token , endpoint = endpoint , ** search_fields )

Searches a project for items that match the input criteria .


def _create_agent_log():
    """
    Creates .rollbar log file for use with rollbar-agent
    """
    log_file = SETTINGS['agent.log_file']
    if not log_file.endswith('.rollbar'):
        log.error("Provided agent log file does not end with .rollbar, which it must. "
                  "Using default instead.")
        log_file = DEFAULTS['agent.log_file']

    retval = logging.getLogger('rollbar_agent')
    handler = logging.FileHandler(log_file, 'a', 'utf-8')
    formatter = logging.Formatter('%(message)s')
    handler.setFormatter(formatter)
    retval.addHandler(handler)
    retval.setLevel(logging.WARNING)
    return retval

Creates . rollbar log file for use with rollbar - agent


def _build_person_data(request):
    """
    Returns a dictionary describing the logged-in user using data from `request.

    Try request.rollbar_person first, then 'user', then 'user_id'
    """
    if hasattr(request, 'rollbar_person'):
        rollbar_person_prop = request.rollbar_person
        try:
            person = rollbar_person_prop()
        except TypeError:
            person = rollbar_person_prop

        if person and isinstance(person, dict):
            return person
        else:
            return None

    if hasattr(request, 'user'):
        user_prop = request.user
        try:
            user = user_prop()
        except TypeError:
            user = user_prop

        if not user:
            return None
        elif isinstance(user, dict):
            return user
        else:
            retval = {}
            if getattr(user, 'id', None):
                retval['id'] = text(user.id)
            elif getattr(user, 'user_id', None):
                retval['id'] = text(user.user_id)

            # id is required, so only include username/email if we have an id
            if retval.get('id'):
                username = getattr(user, 'username', None)
                email = getattr(user, 'email', None)
                retval.update({
                    'username': username,
                    'email': email
                })
            return retval

    if hasattr(request, 'user_id'):
        user_id_prop = request.user_id
        try:
            user_id = user_id_prop()
        except TypeError:
            user_id = user_id_prop

        if not user_id:
            return None
        return {'id': text(user_id)}

def _build_person_data ( request ) : if hasattr ( request , 'rollbar_person' ) : rollbar_person_prop = request . rollbar_person try : person = rollbar_person_prop ( ) except TypeError : person = rollbar_person_prop if person and isinstance ( person , dict ) : return person else : return None if hasattr ( request , 'user' ) : user_prop = request . user try : user = user_prop ( ) except TypeError : user = user_prop if not user : return None elif isinstance ( user , dict ) : return user else : retval = { } if getattr ( user , 'id' , None ) : retval [ 'id' ] = text ( user . id ) elif getattr ( user , 'user_id' , None ) : retval [ 'id' ] = text ( user . user_id ) if retval . get ( 'id' ) : username = getattr ( user , 'username' , None ) email = getattr ( user , 'email' , None ) retval . update ( { 'username' : username , 'email' : email } ) return retval if hasattr ( request , 'user_id' ) : user_id_prop = request . user_id try : user_id = user_id_prop ( ) except TypeError : user_id = user_id_prop if not user_id : return None return { 'id' : text ( user_id ) }

Returns a dictionary describing the logged - in user using data from request .


def _add_lambda_context_data(data):
    """
    Attempts to add information from the lambda context if it exists
    """
    global _CURRENT_LAMBDA_CONTEXT
    context = _CURRENT_LAMBDA_CONTEXT
    if context is None:
        return
    try:
        lambda_data = {
            'lambda': {
                'remaining_time_in_millis': context.get_remaining_time_in_millis(),
                'function_name': context.function_name,
                'function_version': context.function_version,
                'arn': context.invoked_function_arn,
                'request_id': context.aws_request_id,
            }
        }
        if 'custom' in data:
            data['custom'] = dict_merge(data['custom'], lambda_data)
        else:
            data['custom'] = lambda_data
    except Exception as e:
        log.exception("Exception while adding lambda context data: %r", e)
    finally:
        _CURRENT_LAMBDA_CONTEXT = None


def _add_request_data(data, request):
    """
    Attempts to build request data; if successful, sets the 'request' key on `data`.
    """
    try:
        request_data = _build_request_data(request)
    except Exception as e:
        log.exception("Exception while building request_data for Rollbar payload: %r", e)
    else:
        if request_data:
            _filter_ip(request_data, SETTINGS['capture_ip'])
            data['request'] = request_data

Attempts to build request data ; if successful sets the request key on data .


def _check_add_locals(frame, frame_num, total_frames):
    """
    Returns True if we should record local variables for the given frame.
    """
    # Include the last frames locals
    # Include any frame locals that came from a file in the project's root
    return any(((frame_num == total_frames - 1),
                ('root' in SETTINGS and (frame.get('filename') or '').lower().startswith((SETTINGS['root'] or '').lower()))))

def _check_add_locals ( frame , frame_num , total_frames ) : return any ( ( ( frame_num == total_frames - 1 ) , ( 'root' in SETTINGS and ( frame . get ( 'filename' ) or '' ) . lower ( ) . startswith ( ( SETTINGS [ 'root' ] or '' ) . lower ( ) ) ) ) )

Returns True if we should record local variables for the given frame .


def _build_request_data(request):
    """
    Returns a dictionary containing data from the request.
    Can handle webob or werkzeug-based request objects.
    """

    # webob (pyramid)
    if WebobBaseRequest and isinstance(request, WebobBaseRequest):
        return _build_webob_request_data(request)

    # django
    if DjangoHttpRequest and isinstance(request, DjangoHttpRequest):
        return _build_django_request_data(request)

    # django rest framework
    if RestFrameworkRequest and isinstance(request, RestFrameworkRequest):
        return _build_django_request_data(request)

    # werkzeug (flask)
    if WerkzeugRequest and isinstance(request, WerkzeugRequest):
        return _build_werkzeug_request_data(request)

    # tornado
    if TornadoRequest and isinstance(request, TornadoRequest):
        return _build_tornado_request_data(request)

    # bottle
    if BottleRequest and isinstance(request, BottleRequest):
        return _build_bottle_request_data(request)

    # Sanic
    if SanicRequest and isinstance(request, SanicRequest):
        return _build_sanic_request_data(request)

    # falcon
    if FalconRequest and isinstance(request, FalconRequest):
        return _build_falcon_request_data(request)

    # Plain wsgi (should be last)
    if isinstance(request, dict) and 'wsgi.version' in request:
        return _build_wsgi_request_data(request)

    return None

def _build_request_data ( request ) : if WebobBaseRequest and isinstance ( request , WebobBaseRequest ) : return _build_webob_request_data ( request ) if DjangoHttpRequest and isinstance ( request , DjangoHttpRequest ) : return _build_django_request_data ( request ) if RestFrameworkRequest and isinstance ( request , RestFrameworkRequest ) : return _build_django_request_data ( request ) if WerkzeugRequest and isinstance ( request , WerkzeugRequest ) : return _build_werkzeug_request_data ( request ) if TornadoRequest and isinstance ( request , TornadoRequest ) : return _build_tornado_request_data ( request ) if BottleRequest and isinstance ( request , BottleRequest ) : return _build_bottle_request_data ( request ) if SanicRequest and isinstance ( request , SanicRequest ) : return _build_sanic_request_data ( request ) if FalconRequest and isinstance ( request , FalconRequest ) : return _build_falcon_request_data ( request ) if isinstance ( request , dict ) and 'wsgi.version' in request : return _build_wsgi_request_data ( request ) return None

Returns a dictionary containing data from the request . Can handle webob or werkzeug - based request objects .


def _build_server_data():
    """
    Returns a dictionary containing information about the server environment.
    """
    # server environment
    server_data = {
        'host': socket.gethostname(),
        'pid': os.getpid()
    }

    # argv does not always exist in embedded python environments
    argv = getattr(sys, 'argv', None)
    if argv:
         server_data['argv'] = argv

    for key in ['branch', 'root']:
        if SETTINGS.get(key):
            server_data[key] = SETTINGS[key]

    return server_data

def _build_server_data ( ) : server_data = { 'host' : socket . gethostname ( ) , 'pid' : os . getpid ( ) } argv = getattr ( sys , 'argv' , None ) if argv : server_data [ 'argv' ] = argv for key in [ 'branch' , 'root' ] : if SETTINGS . get ( key ) : server_data [ key ] = SETTINGS [ key ] return server_data

Returns a dictionary containing information about the server environment .


def _build_payload(data):
    """
    Returns the full payload as a string.
    """

    for k, v in iteritems(data):
        data[k] = _transform(v, key=(k,))

    payload = {
        'access_token': SETTINGS['access_token'],
        'data': data
    }

    return payload

def _build_payload ( data ) : for k , v in iteritems ( data ) : data [ k ] = _transform ( v , key = ( k , ) ) payload = { 'access_token' : SETTINGS [ 'access_token' ] , 'data' : data } return payload


def main():
    rollbar.init('ACCESS_TOKEN', environment='test', handler='twisted')

    """This runs the protocol on port 8000"""
    factory = protocol.ServerFactory()
    factory.protocol = Echo
    reactor.listenTCP(8000, factory)
    reactor.run()

def main ( ) : rollbar . init ( 'ACCESS_TOKEN' , environment = 'test' , handler = 'twisted' ) factory = protocol . ServerFactory ( ) factory . protocol = Echo reactor . listenTCP ( 8000 , factory ) reactor . run ( )


def compose(chosung, joongsung, jongsung=u''):
    """This function returns a Hangul letter by composing the specified chosung, joongsung, and jongsung.
    @param chosung
    @param joongsung
    @param jongsung the terminal Hangul letter. This is optional if you do not need a jongsung."""

    if jongsung is None: jongsung = u''

    try:
        chosung_index = CHO.index(chosung)
        joongsung_index = JOONG.index(joongsung)
        jongsung_index = JONG.index(jongsung)
    except Exception:
        raise NotHangulException('No valid Hangul character index')

    return unichr(0xAC00 + chosung_index * NUM_JOONG * NUM_JONG + joongsung_index * NUM_JONG + jongsung_index)

def
compose
(
chosung
,
joongsung
,
jongsung
=
u''
)
:
if
jongsung
is
:
jongsung
=
u''
try
:
chosung_index
=
CHO
.
index
(
chosung
)
joongsung_index
=
JOONG
.
index
(
joongsung
)
jongsung_index
=
JONG
.
index
(
jongsung
)
except
Exception
:
raise
NotHangulException
(
'No valid Hangul character index'
)
return
unichr
(
0xAC00
+
chosung_index
*
NUM_JOONG
*
NUM_JONG
+
joongsung_index
*
NUM_JONG
+
jongsung_index
)

This function returns a Hangul letter by composing the specified chosung joongsung and jongsung .


def decompose(hangul_letter):
    """This function returns letters by decomposing the specified Hangul letter."""

    from . import checker

    if len(hangul_letter) < 1:
        raise NotLetterException('')
    elif not checker.is_hangul(hangul_letter):
        raise NotHangulException('')

    if hangul_letter in CHO:
        return hangul_letter, '', ''

    if hangul_letter in JOONG:
        return '', hangul_letter, ''

    if hangul_letter in JONG:
        return '', '', hangul_letter

    code = hangul_index(hangul_letter)
    cho, joong, jong = decompose_index(code)

    if cho < 0:
        cho = 0

    try:
        return CHO[cho], JOONG[joong], JONG[jong]
    except:
        print("%d / %d  / %d"%(cho, joong, jong))
        print("%s / %s " %( JOONG[joong].encode("utf8"), JONG[jong].encode('utf8')))
        raise Exception()

def
decompose
(
hangul_letter
)
:
from
.
import
checker
if
len
(
hangul_letter
)
<
1
:
raise
NotLetterException
(
''
)
elif
not
checker
.
is_hangul
(
hangul_letter
)
:
raise
NotHangulException
(
''
)
if
hangul_letter
in
CHO
:
return
hangul_letter
,
''
,
''
if
hangul_letter
in
JOONG
:
return
''
,
hangul_letter
,
''
if
hangul_letter
in
JONG
:
return
''
,
''
,
hangul_letter
code
=
hangul_index
(
hangul_letter
)
cho
,
joong
,
jong
=
decompose_index
(
code
)
if
cho
<
0
:
cho
=
0
try
:
return
CHO
[
cho
]
,
JOONG
[
joong
]
,
JONG
[
jong
]
except
:
print
(
"%d / %d  / %d"
%
(
cho
,
joong
,
jong
)
)
print
(
"%s / %s "
%
(
JOONG
[
joong
]
.
encode
(
"utf8"
)
,
JONG
[
jong
]
.
encode
(
'utf8'
)
)
)
raise
Exception
(
)

This function returns letters by decomposing the specified Hangul letter .


def has_jongsung(letter):
    """Check whether this letter contains Jongsung"""
    if len(letter) != 1:
        raise Exception('The target string must be one letter.')
    if not is_hangul(letter):
        raise NotHangulException('The target string must be Hangul')

    code = lt.hangul_index(letter)
    return code % NUM_JONG > 0

def
has_jongsung
(
letter
)
:
if
len
(
letter
)
!=
1
:
raise
Exception
(
'The target string must be one letter.'
)
if
not
is_hangul
(
letter
)
:
raise
NotHangulException
(
'The target string must be Hangul'
)
code
=
lt
.
hangul_index
(
letter
)
return
code
%
NUM_JONG
>
0


def attach(word, josa=EUN_NEUN):
    """add josa at the end of this word"""
    last_letter = word.strip()[-1]
    try:
        _, _, letter_jong = letter.decompose(last_letter)
    except NotHangulException:
        letter_jong = letter.get_substituent_of(last_letter)

    if letter_jong in ('', josa['except']):
        return word + josa['has']

    return word + josa['not']

def attach ( word , josa = EUN_NEUN ) : last_letter = word . strip ( ) [ - 1 ] try : _ , _ , letter_jong = letter . decompose ( last_letter ) except NotHangulException : letter_jong = letter . get_substituent_of ( last_letter ) if letter_jong in ( '' , josa [ 'except' ] ) : return word + josa [ 'has' ] return word + josa [ 'not' ]


def is_inside_except(node):
    """Returns true if node is inside the name of an except handler."""
    current = node
    while current and not isinstance(current.parent, astroid.ExceptHandler):
        current = current.parent

    return current and current is current.parent.name

def is_inside_except ( node ) : current = node while current and not isinstance ( current . parent , astroid . ExceptHandler ) : current = current . parent return current and current is current . parent . name

Returns true if node is inside the name of an except handler .


def is_inside_lambda(node: astroid.node_classes.NodeNG) -> bool:
    """Return true if given node is inside lambda"""
    parent = node.parent
    while parent is not None:
        if isinstance(parent, astroid.Lambda):
            return True
        parent = parent.parent
    return False

def is_inside_lambda ( node : astroid . node_classes . NodeNG ) -> bool : parent = node . parent while parent is not None : if isinstance ( parent , astroid . Lambda ) : return True parent = parent . parent return False


def get_all_elements(
    node: astroid.node_classes.NodeNG
) -> Iterable[astroid.node_classes.NodeNG]:
    """Recursively returns all atoms in nested lists and tuples."""
    if isinstance(node, (astroid.Tuple, astroid.List)):
        for child in node.elts:
            for e in get_all_elements(child):
                yield e
    else:
        yield node

def get_all_elements ( node : astroid . node_classes . NodeNG ) -> Iterable [ astroid . node_classes . NodeNG ] : if isinstance ( node , ( astroid . Tuple , astroid . List ) ) : for child in node . elts : for e in get_all_elements ( child ) : yield e else : yield node

Recursively returns all atoms in nested lists and tuples .


def clobber_in_except(
    node: astroid.node_classes.NodeNG
) -> Tuple[bool, Tuple[str, str]]:
    """Checks if an assignment node in an except handler clobbers an existing
    variable.

    Returns (True, args for W0623) if assignment clobbers an existing variable,
    (False, None) otherwise.
    """
    if isinstance(node, astroid.AssignAttr):
        return True, (node.attrname, "object %r" % (node.expr.as_string(),))
    if isinstance(node, astroid.AssignName):
        name = node.name
        if is_builtin(name):
            return (True, (name, "builtins"))

        stmts = node.lookup(name)[1]
        if stmts and not isinstance(
            stmts[0].assign_type(),
            (astroid.Assign, astroid.AugAssign, astroid.ExceptHandler),
        ):
            return True, (name, "outer scope (line %s)" % stmts[0].fromlineno)
    return False, None

def
clobber_in_except
(
node
:
astroid
.
node_classes
.
NodeNG
)
->
Tuple
[
bool
,
Tuple
[
str
,
str
]
]
:
if
isinstance
(
node
,
astroid
.
AssignAttr
)
:
return
True
,
(
node
.
attrname
,
"object %r"
%
(
node
.
expr
.
as_string
(
)
,
)
)
if
isinstance
(
node
,
astroid
.
AssignName
)
:
name
=
node
.
name
if
is_builtin
(
name
)
:
return
(
True
,
(
name
,
"builtins"
)
)
stmts
=
node
.
lookup
(
name
)
[
1
]
if
stmts
and
not
isinstance
(
stmts
[
0
]
.
assign_type
(
)
,
(
astroid
.
Assign
,
astroid
.
AugAssign
,
astroid
.
ExceptHandler
)
,
)
:
return
True
,
(
name
,
"outer scope (line %s)"
%
stmts
[
0
]
.
fromlineno
)
return
False
,

Checks if an assignment node in an except handler clobbers an existing variable .


def is_super(node: astroid.node_classes.NodeNG) -> bool:
    """return True if the node is referencing the "super" builtin function
    """
    if getattr(node, "name", None) == "super" and node.root().name == BUILTINS_NAME:
        return True
    return False

def is_super ( node : astroid . node_classes . NodeNG ) -> bool : if getattr ( node , "name" , None ) == "super" and node . root ( ) . name == BUILTINS_NAME : return True return False

return True if the node is referencing the super builtin function


def is_error(node: astroid.node_classes.NodeNG) -> bool:
    """return true if the function does nothing but raising an exception"""
    for child_node in node.get_children():
        if isinstance(child_node, astroid.Raise):
            return True
    return False

def is_error ( node : astroid . node_classes . NodeNG ) -> bool : for child_node in node . get_children ( ) : if isinstance ( child_node , astroid . Raise ) : return True return False


def is_default_argument(node: astroid.node_classes.NodeNG) -> bool:
    """return true if the given Name node is used in function or lambda
    default argument's value
    """
    parent = node.scope()
    if isinstance(parent, (astroid.FunctionDef, astroid.Lambda)):
        for default_node in parent.args.defaults:
            for default_name_node in default_node.nodes_of_class(astroid.Name):
                if default_name_node is node:
                    return True
    return False

def is_default_argument ( node : astroid . node_classes . NodeNG ) -> bool : parent = node . scope ( ) if isinstance ( parent , ( astroid . FunctionDef , astroid . Lambda ) ) : for default_node in parent . args . defaults : for default_name_node in default_node . nodes_of_class ( astroid . Name ) : if default_name_node is node : return True return False

return true if the given Name node is used in function or lambda default argument s value


def is_func_decorator(node: astroid.node_classes.NodeNG) -> bool:
    """return true if the name is used in function decorator"""
    parent = node.parent
    while parent is not None:
        if isinstance(parent, astroid.Decorators):
            return True
        if parent.is_statement or isinstance(
            parent,
            (astroid.Lambda, scoped_nodes.ComprehensionScope, scoped_nodes.ListComp),
        ):
            break
        parent = parent.parent
    return False

def is_func_decorator ( node : astroid . node_classes . NodeNG ) -> bool : parent = node . parent while parent is not None : if isinstance ( parent , astroid . Decorators ) : return True if parent . is_statement or isinstance ( parent , ( astroid . Lambda , scoped_nodes . ComprehensionScope , scoped_nodes . ListComp ) , ) : break parent = parent . parent return False


def is_ancestor_name(
    frame: astroid.node_classes.NodeNG, node: astroid.node_classes.NodeNG
) -> bool:
    """return True if `frame` is an astroid.Class node with `node` in the
    subtree of its bases attribute
    """
    try:
        bases = frame.bases
    except AttributeError:
        return False
    for base in bases:
        if node in base.nodes_of_class(astroid.Name):
            return True
    return False

def is_ancestor_name ( frame : astroid . node_classes . NodeNG , node : astroid . node_classes . NodeNG ) -> bool : try : bases = frame . bases except AttributeError : return False for base in bases : if node in base . nodes_of_class ( astroid . Name ) : return True return False

return True if frame is an astroid . Class node with node in the subtree of its bases attribute


def assign_parent(node: astroid.node_classes.NodeNG) -> astroid.node_classes.NodeNG:
    """return the higher parent which is not an AssignName, Tuple or List node
    """
    while node and isinstance(node, (astroid.AssignName, astroid.Tuple, astroid.List)):
        node = node.parent
    return node

def assign_parent ( node : astroid . node_classes . NodeNG ) -> astroid . node_classes . NodeNG : while node and isinstance ( node , ( astroid . AssignName , astroid . Tuple , astroid . List ) ) : node = node . parent return node

return the higher parent which is not an AssignName Tuple or List node


def check_messages(*messages: str) -> Callable:
    """decorator to store messages that are handled by a checker method"""

    def store_messages(func):
        func.checks_msgs = messages
        return func

    return store_messages

def check_messages ( * messages : str ) -> Callable : def store_messages ( func ) : func . checks_msgs = messages return func return store_messages


def collect_string_fields(format_string) -> Iterable[Optional[str]]:
    """ Given a format string, return an iterator
    of all the valid format fields. It handles nested fields
    as well.
    """
    formatter = string.Formatter()
    try:
        parseiterator = formatter.parse(format_string)
        for result in parseiterator:
            if all(item is None for item in result[1:]):
                # not a replacement format
                continue
            name = result[1]
            nested = result[2]
            yield name
            if nested:
                for field in collect_string_fields(nested):
                    yield field
    except ValueError as exc:
        # Probably the format string is invalid.
        if exc.args[0].startswith("cannot switch from manual"):
            # On Jython, parsing a string with both manual
            # and automatic positions will fail with a ValueError,
            # while on CPython it will simply return the fields,
            # the validation being done in the interpreter (?).
            # We're just returning two mixed fields in order
            # to trigger the format-combined-specification check.
            yield ""
            yield "1"
            return
        raise IncompleteFormatString(format_string)

def
collect_string_fields
(
format_string
)
->
Iterable
[
Optional
[
str
]
]
:
formatter
=
string
.
Formatter
(
)
try
:
parseiterator
=
formatter
.
parse
(
format_string
)
for
result
in
parseiterator
:
if
all
(
item
is
for
item
in
result
[
1
:
]
)
:
continue
name
=
result
[
1
]
nested
=
result
[
2
]
yield
name
if
nested
:
for
field
in
collect_string_fields
(
nested
)
:
yield
field
except
ValueError
as
exc
:
if
exc
.
args
[
0
]
.
startswith
(
"cannot switch from manual"
)
:
yield
""
yield
"1"
return
raise
IncompleteFormatString
(
format_string
)

Given a format string return an iterator of all the valid format fields . It handles nested fields as well .


def get_argument_from_call(
    call_node: astroid.Call, position: int = None, keyword: str = None
) -> astroid.Name:
    """Returns the specified argument from a function call.

    :param astroid.Call call_node: Node representing a function call to check.
    :param int position: position of the argument.
    :param str keyword: the keyword of the argument.

    :returns: The node representing the argument, None if the argument is not found.
    :rtype: astroid.Name
    :raises ValueError: if both position and keyword are None.
    :raises NoSuchArgumentError: if no argument at the provided position or with
    the provided keyword.
    """
    if position is None and keyword is None:
        raise ValueError("Must specify at least one of: position or keyword.")
    if position is not None:
        try:
            return call_node.args[position]
        except IndexError:
            pass
    if keyword and call_node.keywords:
        for arg in call_node.keywords:
            if arg.arg == keyword:
                return arg.value

    raise NoSuchArgumentError

Returns the specified argument from a function call .


def inherit_from_std_ex(node: astroid.node_classes.NodeNG) -> bool:
    """
    Return true if the given class node is subclass of
    exceptions.Exception.
    """
    ancestors = node.ancestors() if hasattr(node, "ancestors") else []
    for ancestor in itertools.chain([node], ancestors):
        if (
            ancestor.name in ("Exception", "BaseException")
            and ancestor.root().name == EXCEPTIONS_MODULE
        ):
            return True
    return False

def inherit_from_std_ex ( node : astroid . node_classes . NodeNG ) -> bool : ancestors = node . ancestors ( ) if hasattr ( node , "ancestors" ) else [ ] for ancestor in itertools . chain ( [ node ] , ancestors ) : if ( ancestor . name in ( "Exception" , "BaseException" ) and ancestor . root ( ) . name == EXCEPTIONS_MODULE ) : return True return False

Return true if the given class node is subclass of exceptions . Exception .


def error_of_type(handler: astroid.ExceptHandler, error_type) -> bool:
    """
    Check if the given exception handler catches
    the given error_type.

    The *handler* parameter is a node, representing an ExceptHandler node.
    The *error_type* can be an exception, such as AttributeError,
    the name of an exception, or it can be a tuple of errors.
    The function will return True if the handler catches any of the
    given errors.
    """

    def stringify_error(error):
        if not isinstance(error, str):
            return error.__name__
        return error

    if not isinstance(error_type, tuple):
        error_type = (error_type,)  # type: ignore
    expected_errors = {stringify_error(error) for error in error_type}  # type: ignore
    if not handler.type:
        return True
    return handler.catch(expected_errors)

def error_of_type ( handler : astroid . ExceptHandler , error_type ) -> bool : def stringify_error ( error ) : if not isinstance ( error , str ) : return error . __name__ return error if not isinstance ( error_type , tuple ) : error_type = ( error_type , ) expected_errors = { stringify_error ( error ) for error in error_type } if not handler . type : return True return handler . catch ( expected_errors )

Check if the given exception handler catches the given error_type .


def decorated_with_property(node: astroid.FunctionDef) -> bool:
    """ Detect if the given function node is decorated with a property. """
    if not node.decorators:
        return False
    for decorator in node.decorators.nodes:
        if not isinstance(decorator, astroid.Name):
            continue
        try:
            if _is_property_decorator(decorator):
                return True
        except astroid.InferenceError:
            pass
    return False

def decorated_with_property ( node : astroid . FunctionDef ) -> bool : if not node . decorators : return False for decorator in node . decorators . nodes : if not isinstance ( decorator , astroid . Name ) : continue try : if _is_property_decorator ( decorator ) : return True except astroid . InferenceError : pass return False

Detect if the given function node is decorated with a property .


def decorated_with(func: astroid.FunctionDef, qnames: Iterable[str]) -> bool:
    """Determine if the `func` node has a decorator with the qualified name `qname`."""
    decorators = func.decorators.nodes if func.decorators else []
    for decorator_node in decorators:
        try:
            if any(
                i is not None and i.qname() in qnames for i in decorator_node.infer()
            ):
                return True
        except astroid.InferenceError:
            continue
    return False

def decorated_with ( func : astroid . FunctionDef , qnames : Iterable [ str ] ) -> bool : decorators = func . decorators . nodes if func . decorators else [ ] for decorator_node in decorators : try : if any ( i is not None and i . qname ( ) in qnames for i in decorator_node . infer ( ) ) : return True except astroid . InferenceError : continue return False

Determine if the func node has a decorator with the qualified name qname .


def find_try_except_wrapper_node(
    node: astroid.node_classes.NodeNG
) -> Union[astroid.ExceptHandler, astroid.TryExcept]:
    """Return the ExceptHandler or the TryExcept node in which the node is."""
    current = node
    ignores = (astroid.ExceptHandler, astroid.TryExcept)
    while current and not isinstance(current.parent, ignores):
        current = current.parent

    if current and isinstance(current.parent, ignores):
        return current.parent
    return None

def find_try_except_wrapper_node ( node : astroid . node_classes . NodeNG ) -> Union [ astroid . ExceptHandler , astroid . TryExcept ] : current = node ignores = ( astroid . ExceptHandler , astroid . TryExcept ) while current and not isinstance ( current . parent , ignores ) : current = current . parent if current and isinstance ( current . parent , ignores ) : return current . parent return None

Return the ExceptHandler or the TryExcept node in which the node is .


def is_from_fallback_block(node: astroid.node_classes.NodeNG) -> bool:
    """Check if the given node is from a fallback import block."""
    context = find_try_except_wrapper_node(node)
    if not context:
        return False

    if isinstance(context, astroid.ExceptHandler):
        other_body = context.parent.body
        handlers = context.parent.handlers
    else:
        other_body = itertools.chain.from_iterable(
            handler.body for handler in context.handlers
        )
        handlers = context.handlers

    has_fallback_imports = any(
        isinstance(import_node, (astroid.ImportFrom, astroid.Import))
        for import_node in other_body
    )
    ignores_import_error = _except_handlers_ignores_exception(handlers, ImportError)
    return ignores_import_error or has_fallback_imports

def is_from_fallback_block ( node : astroid . node_classes . NodeNG ) -> bool : context = find_try_except_wrapper_node ( node ) if not context : return False if isinstance ( context , astroid . ExceptHandler ) : other_body = context . parent . body handlers = context . parent . handlers else : other_body = itertools . chain . from_iterable ( handler . body for handler in context . handlers ) handlers = context . handlers has_fallback_imports = any ( isinstance ( import_node , ( astroid . ImportFrom , astroid . Import ) ) for import_node in other_body ) ignores_import_error = _except_handlers_ignores_exception ( handlers , ImportError ) return ignores_import_error or has_fallback_imports

Check if the given node is from a fallback import block .


def get_exception_handlers(
    node: astroid.node_classes.NodeNG, exception=Exception
) -> List[astroid.ExceptHandler]:
    """Return the collections of handlers handling the exception in arguments.

    Args:
        node (astroid.NodeNG): A node that is potentially wrapped in a try except.
        exception (builtin.Exception or str): exception or name of the exception.

    Returns:
        list: the collection of handlers that are handling the exception or None.

    """
    context = find_try_except_wrapper_node(node)
    if isinstance(context, astroid.TryExcept):
        return [
            handler for handler in context.handlers if error_of_type(handler, exception)
        ]
    return None

def get_exception_handlers ( node : astroid . node_classes . NodeNG , exception = Exception ) -> List [ astroid . ExceptHandler ] : context = find_try_except_wrapper_node ( node ) if isinstance ( context , astroid . TryExcept ) : return [ handler for handler in context . handlers if error_of_type ( handler , exception ) ] return None

Return the collections of handlers handling the exception in arguments .


def node_ignores_exception(
    node: astroid.node_classes.NodeNG, exception=Exception
) -> bool:
    """Check if the node is in a TryExcept which handles the given exception.

    If the exception is not given, the function is going to look for bare
    excepts.
    """
    managing_handlers = get_exception_handlers(node, exception)
    if not managing_handlers:
        return False
    return any(managing_handlers)

def node_ignores_exception ( node : astroid . node_classes . NodeNG , exception = Exception ) -> bool : managing_handlers = get_exception_handlers ( node , exception ) if not managing_handlers : return False return any ( managing_handlers )

Check if the node is in a TryExcept which handles the given exception .


def class_is_abstract(node: astroid.ClassDef) -> bool:
    """return true if the given class node should be considered as an abstract
    class
    """
    for method in node.methods():
        if method.parent.frame() is node:
            if method.is_abstract(pass_is_abstract=False):
                return True
    return False

def class_is_abstract ( node : astroid . ClassDef ) -> bool : for method in node . methods ( ) : if method . parent . frame ( ) is node : if method . is_abstract ( pass_is_abstract = False ) : return True return False


def safe_infer(
    node: astroid.node_classes.NodeNG, context=None
) -> Optional[astroid.node_classes.NodeNG]:
    """Return the inferred value for the given node.

    Return None if inference failed or if there is some ambiguity (more than
    one node has been inferred).
    """
    try:
        inferit = node.infer(context=context)
        value = next(inferit)
    except astroid.InferenceError:
        return None
    try:
        next(inferit)
        return None  # None if there is ambiguity on the inferred node
    except astroid.InferenceError:
        return None  # there is some kind of ambiguity
    except StopIteration:
        return value

def safe_infer ( node : astroid . node_classes . NodeNG , context = None ) -> Optional [ astroid . node_classes . NodeNG ] : try : inferit = node . infer ( context = context ) value = next ( inferit ) except astroid . InferenceError : return None try : next ( inferit ) return None except astroid . InferenceError : return None except StopIteration : return value

Return the inferred value for the given node .


def node_type(node: astroid.node_classes.NodeNG) -> Optional[type]:
    """Return the inferred type for `node`

    If there is more than one possible type, or if inferred type is Uninferable or None,
    return None
    """
    # check there is only one possible type for the assign node. Else we
    # don't handle it for now
    types = set()
    try:
        for var_type in node.infer():
            if var_type == astroid.Uninferable or is_none(var_type):
                continue
            types.add(var_type)
            if len(types) > 1:
                return None
    except astroid.InferenceError:
        return None
    return types.pop() if types else None

def node_type ( node : astroid . node_classes . NodeNG ) -> Optional [ type ] : types = set ( ) try : for var_type in node . infer ( ) : if var_type == astroid . Uninferable or is_none ( var_type ) : continue types . add ( var_type ) if len ( types ) > 1 : return None except astroid . InferenceError : return None return types . pop ( ) if types else None


def is_registered_in_singledispatch_function(node: astroid.FunctionDef) -> bool:
    """Check if the given function node is a singledispatch function."""

    singledispatch_qnames = (
        "functools.singledispatch",
        "singledispatch.singledispatch",
    )

    if not isinstance(node, astroid.FunctionDef):
        return False

    decorators = node.decorators.nodes if node.decorators else []
    for decorator in decorators:
        # func.register are function calls
        if not isinstance(decorator, astroid.Call):
            continue

        func = decorator.func
        if not isinstance(func, astroid.Attribute) or func.attrname != "register":
            continue

        try:
            func_def = next(func.expr.infer())
        except astroid.InferenceError:
            continue

        if isinstance(func_def, astroid.FunctionDef):
            # pylint: disable=redundant-keyword-arg; some flow inference goes wrong here
            return decorated_with(func_def, singledispatch_qnames)

    return False

Check if the given function node is a singledispatch function .


def is_postponed_evaluation_enabled(node: astroid.node_classes.NodeNG) -> bool:
    """Check if the postponed evaluation of annotations is enabled"""
    name = "annotations"
    module = node.root()
    stmt = module.locals.get(name)
    return (
        stmt
        and isinstance(stmt[0], astroid.ImportFrom)
        and stmt[0].modname == "__future__"
    )

def is_postponed_evaluation_enabled ( node : astroid . node_classes . NodeNG ) -> bool : name = "annotations" module = node . root ( ) stmt = module . locals . get ( name ) return ( stmt and isinstance ( stmt [ 0 ] , astroid . ImportFrom ) and stmt [ 0 ] . modname == "__future__" )


def _qualified_names(modname):
    """Split the names of the given module into subparts

    For example,
        _qualified_names('pylint.checkers.ImportsChecker')
    returns
        ['pylint', 'pylint.checkers', 'pylint.checkers.ImportsChecker']
    """
    names = modname.split(".")
    return [".".join(names[0 : i + 1]) for i in range(len(names))]

def _qualified_names ( modname ) : names = modname . split ( "." ) return [ "." . join ( names [ 0 : i + 1 ] ) for i in range ( len ( names ) ) ]


def _get_import_name(importnode, modname):
    """Get a prepared module name from the given import node

    In the case of relative imports, this will return the
    absolute qualified module name, which might be useful
    for debugging. Otherwise, the initial module name
    is returned unchanged.
    """
    if isinstance(importnode, astroid.ImportFrom):
        if importnode.level:
            root = importnode.root()
            if isinstance(root, astroid.Module):
                modname = root.relative_to_absolute_name(
                    modname, level=importnode.level
                )
    return modname

def _get_import_name ( importnode , modname ) : if isinstance ( importnode , astroid . ImportFrom ) : if importnode . level : root = importnode . root ( ) if isinstance ( root , astroid . Module ) : modname = root . relative_to_absolute_name ( modname , level = importnode . level ) return modname


def _repr_tree_defs(data, indent_str=None):
    """return a string which represents imports as a tree"""
    lines = []
    nodes = data.items()
    for i, (mod, (sub, files)) in enumerate(sorted(nodes, key=lambda x: x[0])):
        if not files:
            files = ""
        else:
            files = "(%s)" % ",".join(sorted(files))
        if indent_str is None:
            lines.append("%s %s" % (mod, files))
            sub_indent_str = "  "
        else:
            lines.append(r"%s\-%s %s" % (indent_str, mod, files))
            if i == len(nodes) - 1:
                sub_indent_str = "%s  " % indent_str
            else:
                sub_indent_str = "%s| " % indent_str
        if sub:
            lines.append(_repr_tree_defs(sub, sub_indent_str))
    return "\n".join(lines)

def
_repr_tree_defs
(
data
,
indent_str
=
)
:
lines
=
[
]
nodes
=
data
.
items
(
)
for
i
,
(
mod
,
(
sub
,
files
)
)
in
enumerate
(
sorted
(
nodes
,
key
=
lambda
x
:
x
[
0
]
)
)
:
if
not
files
:
files
=
""
else
:
files
=
"(%s)"
%
","
.
join
(
sorted
(
files
)
)
if
indent_str
is
:
lines
.
append
(
"%s %s"
%
(
mod
,
files
)
)
sub_indent_str
=
"  "
else
:
lines
.
append
(
r"%s\-%s %s"
%
(
indent_str
,
mod
,
files
)
)
if
i
==
len
(
nodes
)
1
:
sub_indent_str
=
"%s  "
%
indent_str
else
:
sub_indent_str
=
"%s| "
%
indent_str
if
sub
:
lines
.
append
(
_repr_tree_defs
(
sub
,
sub_indent_str
)
)
return
"\n"
.
join
(
lines
)


def _make_graph(filename, dep_info, sect, gtype):
    """generate a dependencies graph and add some information about it in the
    report's section
    """
    _dependencies_graph(filename, dep_info)
    sect.append(Paragraph("%simports graph has been written to %s" % (gtype, filename)))

generate a dependencies graph and add some information about it in the report s section


def visit_import(self, node):
        """triggered when an import statement is seen"""
        self._check_reimport(node)
        self._check_import_as_rename(node)

        modnode = node.root()
        names = [name for name, _ in node.names]
        if len(names) >= 2:
            self.add_message("multiple-imports", args=", ".join(names), node=node)

        for name in names:
            self._check_deprecated_module(node, name)
            self._check_preferred_module(node, name)
            imported_module = self._get_imported_module(node, name)
            if isinstance(node.parent, astroid.Module):
                # Allow imports nested
                self._check_position(node)
            if isinstance(node.scope(), astroid.Module):
                self._record_import(node, imported_module)

            if imported_module is None:
                continue

            self._check_relative_import(modnode, node, imported_module, name)
            self._add_imported_module(node, imported_module.name)

def
visit_import
(
self
,
node
)
:
self
.
_check_reimport
(
node
)
self
.
_check_import_as_rename
(
node
)
modnode
=
node
.
root
(
)
names
=
[
name
for
name
,
_
in
node
.
names
]
if
len
(
names
)
>=
2
:
self
.
add_message
(
"multiple-imports"
,
args
=
", "
.
join
(
names
)
,
node
=
node
)
for
name
in
names
:
self
.
_check_deprecated_module
(
node
,
name
)
self
.
_check_preferred_module
(
node
,
name
)
imported_module
=
self
.
_get_imported_module
(
node
,
name
)
if
isinstance
(
node
.
parent
,
astroid
.
Module
)
:
self
.
_check_position
(
node
)
if
isinstance
(
node
.
scope
(
)
,
astroid
.
Module
)
:
self
.
_record_import
(
node
,
imported_module
)
if
imported_module
is
:
continue
self
.
_check_relative_import
(
modnode
,
node
,
imported_module
,
name
)
self
.
_add_imported_module
(
node
,
imported_module
.
name
)


def visit_importfrom(self, node):
        """triggered when a from statement is seen"""
        basename = node.modname
        imported_module = self._get_imported_module(node, basename)

        self._check_import_as_rename(node)
        self._check_misplaced_future(node)
        self._check_deprecated_module(node, basename)
        self._check_preferred_module(node, basename)
        self._check_wildcard_imports(node, imported_module)
        self._check_same_line_imports(node)
        self._check_reimport(node, basename=basename, level=node.level)

        if isinstance(node.parent, astroid.Module):
            # Allow imports nested
            self._check_position(node)
        if isinstance(node.scope(), astroid.Module):
            self._record_import(node, imported_module)
        if imported_module is None:
            return
        modnode = node.root()
        self._check_relative_import(modnode, node, imported_module, basename)

        for name, _ in node.names:
            if name != "*":
                self._add_imported_module(node, "%s.%s" % (imported_module.name, name))
            else:
                self._add_imported_module(node, imported_module.name)

def visit_importfrom ( self , node ) : basename = node . modname imported_module = self . _get_imported_module ( node , basename ) self . _check_import_as_rename ( node ) self . _check_misplaced_future ( node ) self . _check_deprecated_module ( node , basename ) self . _check_preferred_module ( node , basename ) self . _check_wildcard_imports ( node , imported_module ) self . _check_same_line_imports ( node ) self . _check_reimport ( node , basename = basename , level = node . level ) if isinstance ( node . parent , astroid . Module ) : self . _check_position ( node ) if isinstance ( node . scope ( ) , astroid . Module ) : self . _record_import ( node , imported_module ) if imported_module is None : return modnode = node . root ( ) self . _check_relative_import ( modnode , node , imported_module , basename ) for name , _ in node . names : if name != "*" : self . _add_imported_module ( node , "%s.%s" % ( imported_module . name , name ) ) else : self . _add_imported_module ( node , imported_module . name )


def _check_position(self, node):
        """Check `node` import or importfrom node position is correct

        Send a message  if `node` comes before another instruction
        """
        # if a first non-import instruction has already been encountered,
        # it means the import comes after it and therefore is not well placed
        if self._first_non_import_node:
            self.add_message("wrong-import-position", node=node, args=node.as_string())

def _check_position ( self , node ) : if self . _first_non_import_node : self . add_message ( "wrong-import-position" , node = node , args = node . as_string ( ) )

Check node import or importfrom node position is correct


def _record_import(self, node, importedmodnode):
        """Record the package `node` imports from"""
        if isinstance(node, astroid.ImportFrom):
            importedname = node.modname
        else:
            importedname = importedmodnode.name if importedmodnode else None
        if not importedname:
            importedname = node.names[0][0].split(".")[0]

        if isinstance(node, astroid.ImportFrom) and (node.level or 0) >= 1:
            # We need the importedname with first point to detect local package
            # Example of node:
            #  'from .my_package1 import MyClass1'
            #  the output should be '.my_package1' instead of 'my_package1'
            # Example of node:
            #  'from . import my_package2'
            #  the output should be '.my_package2' instead of '{pyfile}'
            importedname = "." + importedname

        self._imports_stack.append((node, importedname))

def _record_import ( self , node , importedmodnode ) : if isinstance ( node , astroid . ImportFrom ) : importedname = node . modname else : importedname = importedmodnode . name if importedmodnode else None if not importedname : importedname = node . names [ 0 ] [ 0 ] . split ( "." ) [ 0 ] if isinstance ( node , astroid . ImportFrom ) and ( node . level or 0 ) >= 1 : importedname = "." + importedname self . _imports_stack . append ( ( node , importedname ) )