def hits(self, keywords):
        """Fetch information about a list of keywords."""

        if len(keywords) == 1:
            query_str = keywords[0]
        else:
            query_str = ' '.join([k for k in keywords])

        logger.info("Fetching hits for '%s'", query_str)
        params = {'q': query_str}

        # Make the request
        req = self.fetch(GOOGLE_SEARCH_URL, payload=params)

        return req.text

def
hits
(
self
,
keywords
)
:
if
len
(
keywords
)
==
1
:
query_str
=
keywords
[
0
]
else
:
query_str
=
' '
.
join
(
[
k
for
k
in
keywords
]
)
logger
.
info
(
"Fetching hits for '%s'"
,
query_str
)
params
=
{
'q'
:
query_str
}
req
=
self
.
fetch
(
GOOGLE_SEARCH_URL
,
payload
=
params
)
return
req
.
text

Fetch information about a list of keywords .


def metadata_updated_on(item):
        """Extracts the update time from a GitHub item.

        The timestamp used is extracted from 'updated_at' field.
        This date is converted to UNIX timestamp format. As GitHub
        dates are in UTC the conversion is straightforward.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        if "forks_count" in item:
            return item['fetched_on']
        else:
            ts = item['updated_at']
            ts = str_to_datetime(ts)

            return ts.timestamp()

def metadata_updated_on ( item ) : if "forks_count" in item : return item [ 'fetched_on' ] else : ts = item [ 'updated_at' ] ts = str_to_datetime ( ts ) return ts . timestamp ( )

Extracts the update time from a GitHub item .


def metadata_category(item):
        """Extracts the category from a GitHub item.

        This backend generates two types of item which are
        'issue' and 'pull_request'.
        """

        if "base" in item:
            category = CATEGORY_PULL_REQUEST
        elif "forks_count" in item:
            category = CATEGORY_REPO
        else:
            category = CATEGORY_ISSUE

        return category

def metadata_category ( item ) : if "base" in item : category = CATEGORY_PULL_REQUEST elif "forks_count" in item : category = CATEGORY_REPO else : category = CATEGORY_ISSUE return category

Extracts the category from a GitHub item .


def __fetch_pull_requests(self, from_date, to_date):
        """Fetch the pull requests"""

        raw_pulls = self.client.pulls(from_date=from_date)
        for raw_pull in raw_pulls:
            pull = json.loads(raw_pull)

            if str_to_datetime(pull['updated_at']) > to_date:
                return

            self.__init_extra_pull_fields(pull)
            for field in TARGET_PULL_FIELDS:

                if not pull[field]:
                    continue

                if field == 'user':
                    pull[field + '_data'] = self.__get_user(pull[field]['login'])
                elif field == 'merged_by':
                    pull[field + '_data'] = self.__get_user(pull[field]['login'])
                elif field == 'review_comments':
                    pull[field + '_data'] = self.__get_pull_review_comments(pull['number'])
                elif field == 'requested_reviewers':
                    pull[field + '_data'] = self.__get_pull_requested_reviewers(pull['number'])
                elif field == 'commits':
                    pull[field + '_data'] = self.__get_pull_commits(pull['number'])

            yield pull

def __fetch_pull_requests ( self , from_date , to_date ) : raw_pulls = self . client . pulls ( from_date = from_date ) for raw_pull in raw_pulls : pull = json . loads ( raw_pull ) if str_to_datetime ( pull [ 'updated_at' ] ) > to_date : return self . __init_extra_pull_fields ( pull ) for field in TARGET_PULL_FIELDS : if not pull [ field ] : continue if field == 'user' : pull [ field + '_data' ] = self . __get_user ( pull [ field ] [ 'login' ] ) elif field == 'merged_by' : pull [ field + '_data' ] = self . __get_user ( pull [ field ] [ 'login' ] ) elif field == 'review_comments' : pull [ field + '_data' ] = self . __get_pull_review_comments ( pull [ 'number' ] ) elif field == 'requested_reviewers' : pull [ field + '_data' ] = self . __get_pull_requested_reviewers ( pull [ 'number' ] ) elif field == 'commits' : pull [ field + '_data' ] = self . __get_pull_commits ( pull [ 'number' ] ) yield pull


def __fetch_repo_info(self):
        """Get repo info about stars, watchers and forks"""

        raw_repo = self.client.repo()
        repo = json.loads(raw_repo)

        fetched_on = datetime_utcnow()
        repo['fetched_on'] = fetched_on.timestamp()

        yield repo

def __fetch_repo_info ( self ) : raw_repo = self . client . repo ( ) repo = json . loads ( raw_repo ) fetched_on = datetime_utcnow ( ) repo [ 'fetched_on' ] = fetched_on . timestamp ( ) yield repo

Get repo info about stars watchers and forks


def __get_issue_reactions(self, issue_number, total_count):
        """Get issue reactions"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.issue_reactions(issue_number)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

def __get_issue_reactions ( self , issue_number , total_count ) : reactions = [ ] if total_count == 0 : return reactions group_reactions = self . client . issue_reactions ( issue_number ) for raw_reactions in group_reactions : for reaction in json . loads ( raw_reactions ) : reaction [ 'user_data' ] = self . __get_user ( reaction [ 'user' ] [ 'login' ] ) reactions . append ( reaction ) return reactions


def __get_issue_comment_reactions(self, comment_id, total_count):
        """Get reactions on issue comments"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.issue_comment_reactions(comment_id)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

def __get_issue_comment_reactions ( self , comment_id , total_count ) : reactions = [ ] if total_count == 0 : return reactions group_reactions = self . client . issue_comment_reactions ( comment_id ) for raw_reactions in group_reactions : for reaction in json . loads ( raw_reactions ) : reaction [ 'user_data' ] = self . __get_user ( reaction [ 'user' ] [ 'login' ] ) reactions . append ( reaction ) return reactions


def __get_issue_assignees(self, raw_assignees):
        """Get issue assignees"""

        assignees = []
        for ra in raw_assignees:
            assignees.append(self.__get_user(ra['login']))

        return assignees

def __get_issue_assignees ( self , raw_assignees ) : assignees = [ ] for ra in raw_assignees : assignees . append ( self . __get_user ( ra [ 'login' ] ) ) return assignees


def __get_pull_requested_reviewers(self, pr_number):
        """Get pull request requested reviewers"""

        requested_reviewers = []
        group_requested_reviewers = self.client.pull_requested_reviewers(pr_number)

        for raw_requested_reviewers in group_requested_reviewers:
            group_requested_reviewers = json.loads(raw_requested_reviewers)

            for requested_reviewer in group_requested_reviewers['users']:
                user_data = self.__get_user(requested_reviewer['login'])
                requested_reviewers.append(user_data)

        return requested_reviewers

def __get_pull_requested_reviewers ( self , pr_number ) : requested_reviewers = [ ] group_requested_reviewers = self . client . pull_requested_reviewers ( pr_number ) for raw_requested_reviewers in group_requested_reviewers : group_requested_reviewers = json . loads ( raw_requested_reviewers ) for requested_reviewer in group_requested_reviewers [ 'users' ] : user_data = self . __get_user ( requested_reviewer [ 'login' ] ) requested_reviewers . append ( user_data ) return requested_reviewers


def __get_pull_commits(self, pr_number):
        """Get pull request commit hashes"""

        hashes = []
        group_pull_commits = self.client.pull_commits(pr_number)

        for raw_pull_commits in group_pull_commits:

            for commit in json.loads(raw_pull_commits):
                commit_hash = commit['sha']
                hashes.append(commit_hash)

        return hashes

def __get_pull_commits ( self , pr_number ) : hashes = [ ] group_pull_commits = self . client . pull_commits ( pr_number ) for raw_pull_commits in group_pull_commits : for commit in json . loads ( raw_pull_commits ) : commit_hash = commit [ 'sha' ] hashes . append ( commit_hash ) return hashes


def __get_pull_review_comment_reactions(self, comment_id, total_count):
        """Get pull review comment reactions"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.pull_review_comment_reactions(comment_id)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

def __get_pull_review_comment_reactions ( self , comment_id , total_count ) : reactions = [ ] if total_count == 0 : return reactions group_reactions = self . client . pull_review_comment_reactions ( comment_id ) for raw_reactions in group_reactions : for reaction in json . loads ( raw_reactions ) : reaction [ 'user_data' ] = self . __get_user ( reaction [ 'user' ] [ 'login' ] ) reactions . append ( reaction ) return reactions


def __get_user(self, login):
        """Get user and org data for the login"""

        user = {}

        if not login:
            return user

        user_raw = self.client.user(login)
        user = json.loads(user_raw)
        user_orgs_raw = \
            self.client.user_orgs(login)
        user['organizations'] = json.loads(user_orgs_raw)

        return user

def __get_user ( self , login ) : user = { } if not login : return user user_raw = self . client . user ( login ) user = json . loads ( user_raw ) user_orgs_raw = self . client . user_orgs ( login ) user [ 'organizations' ] = json . loads ( user_orgs_raw ) return user


def issue_reactions(self, issue_number):
        """Get reactions of an issue"""

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("issues", str(issue_number), "reactions")
        return self.fetch_items(path, payload)

def issue_reactions ( self , issue_number ) : payload = { 'per_page' : PER_PAGE , 'direction' : 'asc' , 'sort' : 'updated' } path = urijoin ( "issues" , str ( issue_number ) , "reactions" ) return self . fetch_items ( path , payload )


def issues(self, from_date=None):
        """Fetch the issues from the repository.

        The method retrieves, from a GitHub repository, the issues
        updated since the given date.

        :param from_date: obtain issues updated since this date

        :returns: a generator of issues
        """
        payload = {
            'state': 'all',
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'}

        if from_date:
            payload['since'] = from_date.isoformat()

        path = urijoin("issues")
        return self.fetch_items(path, payload)

def issues ( self , from_date = None ) : payload = { 'state' : 'all' , 'per_page' : PER_PAGE , 'direction' : 'asc' , 'sort' : 'updated' } if from_date : payload [ 'since' ] = from_date . isoformat ( ) path = urijoin ( "issues" ) return self . fetch_items ( path , payload )


def pulls(self, from_date=None):
        """Fetch the pull requests from the repository.

        The method retrieves, from a GitHub repository, the pull requests
        updated since the given date.

        :param from_date: obtain pull requests updated since this date

        :returns: a generator of pull requests
        """
        issues_groups = self.issues(from_date=from_date)

        for raw_issues in issues_groups:
            issues = json.loads(raw_issues)
            for issue in issues:

                if "pull_request" not in issue:
                    continue

                pull_number = issue["number"]
                path = urijoin(self.base_url, 'repos', self.owner, self.repository, "pulls", pull_number)

                r = self.fetch(path)
                pull = r.text

                yield pull

def pulls ( self , from_date = None ) : issues_groups = self . issues ( from_date = from_date ) for raw_issues in issues_groups : issues = json . loads ( raw_issues ) for issue in issues : if "pull_request" not in issue : continue pull_number = issue [ "number" ] path = urijoin ( self . base_url , 'repos' , self . owner , self . repository , "pulls" , pull_number ) r = self . fetch ( path ) pull = r . text yield pull

Fetch the pull requests from the repository .


def repo(self):
        """Get repository data"""

        path = urijoin(self.base_url, 'repos', self.owner, self.repository)

        r = self.fetch(path)
        repo = r.text

        return repo

def repo ( self ) : path = urijoin ( self . base_url , 'repos' , self . owner , self . repository ) r = self . fetch ( path ) repo = r . text return repo


def pull_requested_reviewers(self, pr_number):
        """Get pull requested reviewers"""

        requested_reviewers_url = urijoin("pulls", str(pr_number), "requested_reviewers")
        return self.fetch_items(requested_reviewers_url, {})

def pull_requested_reviewers ( self , pr_number ) : requested_reviewers_url = urijoin ( "pulls" , str ( pr_number ) , "requested_reviewers" ) return self . fetch_items ( requested_reviewers_url , { } )


def pull_commits(self, pr_number):
        """Get pull request commits"""

        payload = {
            'per_page': PER_PAGE,
        }

        commit_url = urijoin("pulls", str(pr_number), "commits")
        return self.fetch_items(commit_url, payload)

def pull_commits ( self , pr_number ) : payload = { 'per_page' : PER_PAGE , } commit_url = urijoin ( "pulls" , str ( pr_number ) , "commits" ) return self . fetch_items ( commit_url , payload )


def pull_review_comment_reactions(self, comment_id):
        """Get reactions of a review comment"""

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("pulls", "comments", str(comment_id), "reactions")
        return self.fetch_items(path, payload)

def pull_review_comment_reactions ( self , comment_id ) : payload = { 'per_page' : PER_PAGE , 'direction' : 'asc' , 'sort' : 'updated' } path = urijoin ( "pulls" , "comments" , str ( comment_id ) , "reactions" ) return self . fetch_items ( path , payload )


def user(self, login):
        """Get the user information and update the user cache"""
        user = None

        if login in self._users:
            return self._users[login]

        url_user = urijoin(self.base_url, 'users', login)

        logging.info("Getting info for %s" % (url_user))

        r = self.fetch(url_user)
        user = r.text
        self._users[login] = user

        return user

def
user
(
self
,
login
)
:
user
=
if
login
in
self
.
_users
:
return
self
.
_users
[
login
]
url_user
=
urijoin
(
self
.
base_url
,
'users'
,
login
)
logging
.
info
(
"Getting info for %s"
%
(
url_user
)
)
r
=
self
.
fetch
(
url_user
)
user
=
r
.
text
self
.
_users
[
login
]
=
user
return
user


def user_orgs(self, login):
        """Get the user public organizations"""
        if login in self._users_orgs:
            return self._users_orgs[login]

        url = urijoin(self.base_url, 'users', login, 'orgs')
        try:
            r = self.fetch(url)
            orgs = r.text
        except requests.exceptions.HTTPError as error:
            # 404 not found is wrongly received sometimes
            if error.response.status_code == 404:
                logger.error("Can't get github login orgs: %s", error)
                orgs = '[]'
            else:
                raise error

        self._users_orgs[login] = orgs

        return orgs

def
user_orgs
(
self
,
login
)
:
if
login
in
self
.
_users_orgs
:
return
self
.
_users_orgs
[
login
]
url
=
urijoin
(
self
.
base_url
,
'users'
,
login
,
'orgs'
)
try
:
r
=
self
.
fetch
(
url
)
orgs
=
r
.
text
except
requests
.
exceptions
.
HTTPError
as
error
:
if
error
.
response
.
status_code
==
404
:
logger
.
error
(
"Can't get github login orgs: %s"
,
error
)
orgs
=
'[]'
else
:
raise
error
self
.
_users_orgs
[
login
]
=
orgs
return
orgs


def _get_token_rate_limit(self, token):
        """Return token's remaining API points"""

        rate_url = urijoin(self.base_url, "rate_limit")
        self.session.headers.update({'Authorization': 'token ' + token})
        remaining = 0
        try:
            headers = super().fetch(rate_url).headers
            if self.rate_limit_header in headers:
                remaining = int(headers[self.rate_limit_header])
        except requests.exceptions.HTTPError as error:
            logger.warning("Rate limit not initialized: %s", error)
        return remaining

def
_get_token_rate_limit
(
self
,
token
)
:
rate_url
=
urijoin
(
self
.
base_url
,
"rate_limit"
)
self
.
session
.
headers
.
update
(
{
'Authorization'
:
'token '
+
token
}
)
remaining
=
0
try
:
headers
=
super
(
)
.
fetch
(
rate_url
)
.
headers
if
self
.
rate_limit_header
in
headers
:
remaining
=
int
(
headers
[
self
.
rate_limit_header
]
)
except
requests
.
exceptions
.
HTTPError
as
error
:
logger
.
warning
(
"Rate limit not initialized: %s"
,
error
)
return
remaining


def _get_tokens_rate_limits(self):
        """Return array of all tokens remaining API points"""

        remainings = [0] * self.n_tokens
        # Turn off archiving when checking rates, because that would cause
        # archive key conflict (the same URLs giving different responses)
        arch = self.archive
        self.archive = None
        for idx, token in enumerate(self.tokens):
            # Pass flag to skip disabling archiving because this function doies it
            remainings[idx] = self._get_token_rate_limit(token)
        # Restore archiving to whatever state it was
        self.archive = arch
        logger.debug("Remaining API points: {}".format(remainings))
        return remainings

def
_get_tokens_rate_limits
(
self
)
:
remainings
=
[
0
]
*
self
.
n_tokens
arch
=
self
.
archive
self
.
archive
=
for
idx
,
token
in
enumerate
(
self
.
tokens
)
:
remainings
[
idx
]
=
self
.
_get_token_rate_limit
(
token
)
self
.
archive
=
arch
logger
.
debug
(
"Remaining API points: {}"
.
format
(
remainings
)
)
return
remainings


def _choose_best_api_token(self):
        """Check all API tokens defined and choose one with most remaining API points"""

        # Return if no tokens given
        if self.n_tokens == 0:
            return

        # If multiple tokens given, choose best
        token_idx = 0
        if self.n_tokens > 1:
            remainings = self._get_tokens_rate_limits()
            token_idx = remainings.index(max(remainings))
            logger.debug("Remaining API points: {}, choosen index: {}".format(remainings, token_idx))

        # If we have any tokens - use best of them
        self.current_token = self.tokens[token_idx]
        self.session.headers.update({'Authorization': 'token ' + self.current_token})
        # Update rate limit data for the current token
        self._update_current_rate_limit()


def _need_check_tokens(self):
        """Check if we need to switch GitHub API tokens"""

        if self.n_tokens <= 1 or self.rate_limit is None:
            return False
        elif self.last_rate_limit_checked is None:
            self.last_rate_limit_checked = self.rate_limit
            return True

        # If approaching minimum rate limit for sleep
        approaching_limit = float(self.min_rate_to_sleep) * (1.0 + TOKEN_USAGE_BEFORE_SWITCH) + 1
        if self.rate_limit <= approaching_limit:
            self.last_rate_limit_checked = self.rate_limit
            return True

        # Only switch token when used predefined factor of the current token's remaining API points
        ratio = float(self.rate_limit) / float(self.last_rate_limit_checked)
        if ratio < 1.0 - TOKEN_USAGE_BEFORE_SWITCH:
            self.last_rate_limit_checked = self.rate_limit
            return True
        elif ratio > 1.0:
            self.last_rate_limit_checked = self.rate_limit
            return False
        else:
            return False

def _need_check_tokens ( self ) : if self . n_tokens <= 1 or self . rate_limit is None : return False elif self . last_rate_limit_checked is None : self . last_rate_limit_checked = self . rate_limit return True approaching_limit = float ( self . min_rate_to_sleep ) * ( 1.0 + TOKEN_USAGE_BEFORE_SWITCH ) + 1 if self . rate_limit <= approaching_limit : self . last_rate_limit_checked = self . rate_limit return True ratio = float ( self . rate_limit ) / float ( self . last_rate_limit_checked ) if ratio < 1.0 - TOKEN_USAGE_BEFORE_SWITCH : self . last_rate_limit_checked = self . rate_limit return True elif ratio > 1.0 : self . last_rate_limit_checked = self . rate_limit return False else : return False


def _update_current_rate_limit(self):
        """Update rate limits data for the current token"""

        url = urijoin(self.base_url, "rate_limit")
        try:
            # Turn off archiving when checking rates, because that would cause
            # archive key conflict (the same URLs giving different responses)
            arch = self.archive
            self.archive = None
            response = super().fetch(url)
            self.archive = arch
            self.update_rate_limit(response)
            self.last_rate_limit_checked = self.rate_limit
        except requests.exceptions.HTTPError as error:
            if error.response.status_code == 404:
                logger.warning("Rate limit not initialized: %s", error)
            else:
                raise error

def
_update_current_rate_limit
(
self
)
:
url
=
urijoin
(
self
.
base_url
,
"rate_limit"
)
try
:
arch
=
self
.
archive
self
.
archive
=
response
=
super
(
)
.
fetch
(
url
)
self
.
archive
=
arch
self
.
update_rate_limit
(
response
)
self
.
last_rate_limit_checked
=
self
.
rate_limit
except
requests
.
exceptions
.
HTTPError
as
error
:
if
error
.
response
.
status_code
==
404
:
logger
.
warning
(
"Rate limit not initialized: %s"
,
error
)
else
:
raise
error


def init_metadata(self, origin, backend_name, backend_version,
                      category, backend_params):
        """Init metadata information.

        Metatada is composed by basic information needed to identify
        where archived data came from and how it can be retrieved
        and built into Perceval items.

        :param: origin: identifier of the repository
        :param: backend_name: name of the backend
        :param: backend_version: version of the backend
        :param: category: category of the items fetched
        :param: backend_params: dict representation of the fetch parameters

        raises ArchiveError: when an error occurs initializing the metadata
        """
        created_on = datetime_to_utc(datetime_utcnow())
        created_on_dumped = created_on.isoformat()
        backend_params_dumped = pickle.dumps(backend_params, 0)

        metadata = (origin, backend_name, backend_version, category,
                    backend_params_dumped, created_on_dumped,)

        try:
            cursor = self._db.cursor()
            insert_stmt = "INSERT INTO " + self.METADATA_TABLE + " "\
                          "(origin, backend_name, backend_version, " \
                          "category, backend_params, created_on) " \
                          "VALUES (?, ?, ?, ?, ?, ?)"
            cursor.execute(insert_stmt, metadata)

            self._db.commit()
            cursor.close()
        except sqlite3.DatabaseError as e:
            msg = "metadata initialization error; cause: %s" % str(e)
            raise ArchiveError(cause=msg)

        self.origin = origin
        self.backend_name = backend_name
        self.backend_version = backend_version
        self.category = category
        self.backend_params = backend_params
        self.created_on = created_on

        logger.debug("Metadata of archive %s initialized to %s",
                     self.archive_path, metadata)


def store(self, uri, payload, headers, data):
        """Store a raw item in this archive.

        The method will store `data` content in this archive. The unique
        identifier for that item will be generated using the rest of the
        parameters.

        :param uri: request URI
        :param payload: request payload
        :param headers: request headers
        :param data: data to store in this archive

        :raises ArchiveError: when an error occurs storing the given data
        """
        hashcode = self.make_hashcode(uri, payload, headers)
        payload_dump = pickle.dumps(payload, 0)
        headers_dump = pickle.dumps(headers, 0)
        data_dump = pickle.dumps(data, 0)

        logger.debug("Archiving %s with %s %s %s in %s",
                     hashcode, uri, payload, headers, self.archive_path)

        try:
            cursor = self._db.cursor()
            insert_stmt = "INSERT INTO " + self.ARCHIVE_TABLE + " (" \
                          "id, hashcode, uri, payload, headers, data) " \
                          "VALUES(?,?,?,?,?,?)"
            cursor.execute(insert_stmt, (None, hashcode, uri,
                                         payload_dump, headers_dump, data_dump))
            self._db.commit()
            cursor.close()
        except sqlite3.IntegrityError as e:
            msg = "data storage error; cause: duplicated entry %s" % hashcode
            raise ArchiveError(cause=msg)
        except sqlite3.DatabaseError as e:
            msg = "data storage error; cause: %s" % str(e)
            raise ArchiveError(cause=msg)

        logger.debug("%s data archived in %s", hashcode, self.archive_path)


def retrieve(self, uri, payload, headers):
        """Retrieve a raw item from the archive.

        The method will return the `data` content corresponding to the
        hascode derived from the given parameters.

        :param uri: request URI
        :param payload: request payload
        :param headers: request headers

        :returns: the archived data

        :raises ArchiveError: when an error occurs retrieving data
        """
        hashcode = self.make_hashcode(uri, payload, headers)

        logger.debug("Retrieving entry %s with %s %s %s in %s",
                     hashcode, uri, payload, headers, self.archive_path)

        self._db.row_factory = sqlite3.Row

        try:
            cursor = self._db.cursor()
            select_stmt = "SELECT data " \
                          "FROM " + self.ARCHIVE_TABLE + " " \
                          "WHERE hashcode = ?"
            cursor.execute(select_stmt, (hashcode,))
            row = cursor.fetchone()
            cursor.close()
        except sqlite3.DatabaseError as e:
            msg = "data retrieval error; cause: %s" % str(e)
            raise ArchiveError(cause=msg)

        if row:
            found = pickle.loads(row['data'])
        else:
            msg = "entry %s not found in archive %s" % (hashcode, self.archive_path)
            raise ArchiveError(cause=msg)

        return found


def create(cls, archive_path):
        """Create a brand new archive.

         Call this method to create a new and empty archive. It will initialize
         the storage file in the path defined by `archive_path`.

        :param archive_path: absolute path where the archive file will be created

        :raises ArchiveError: when the archive file already exists
        """
        if os.path.exists(archive_path):
            msg = "archive %s already exists; remove it before creating a new one"
            raise ArchiveError(cause=msg % (archive_path))

        conn = sqlite3.connect(archive_path)

        cursor = conn.cursor()
        cursor.execute(cls.METADATA_CREATE_STMT)
        cursor.execute(cls.ARCHIVE_CREATE_STMT)
        conn.commit()

        cursor.close()
        conn.close()

        logger.debug("Creating archive %s", archive_path)
        archive = cls(archive_path)
        logger.debug("Achive %s was created", archive_path)

        return archive


def make_hashcode(uri, payload, headers):
        """Generate a SHA1 based on the given arguments.

        Hashcodes created by this method will used as unique identifiers
        for the raw items or resources stored by this archive.

        :param uri: URI to the resource
        :param payload: payload of the request needed to fetch the resource
        :param headers: headers of the request needed to fetch the resource

        :returns: a SHA1 hash code
        """
        def dict_to_json_str(data):
            return json.dumps(data, sort_keys=True)

        content = ':'.join([uri, dict_to_json_str(payload), dict_to_json_str(headers)])
        hashcode = hashlib.sha1(content.encode('utf-8'))
        return hashcode.hexdigest()

def make_hashcode ( uri , payload , headers ) : def dict_to_json_str ( data ) : return json . dumps ( data , sort_keys = True ) content = ':' . join ( [ uri , dict_to_json_str ( payload ) , dict_to_json_str ( headers ) ] ) hashcode = hashlib . sha1 ( content . encode ( 'utf-8' ) ) return hashcode . hexdigest ( )

Generate a SHA1 based on the given arguments .


def _verify_archive(self):
        """Check whether the archive is valid or not.

        This method will check if tables were created and if they
        contain valid data.
        """
        nentries = self._count_table_rows(self.ARCHIVE_TABLE)
        nmetadata = self._count_table_rows(self.METADATA_TABLE)

        if nmetadata > 1:
            msg = "archive %s metadata corrupted; multiple metadata entries" % (self.archive_path)
            raise ArchiveError(cause=msg)
        if nmetadata == 0 and nentries > 0:
            msg = "archive %s metadata is empty but %s entries were achived" % (self.archive_path)
            raise ArchiveError(cause=msg)

        logger.debug("Integrity of archive %s OK; entries: %s rows, metadata: %s rows",
                     self.archive_path, nentries, nmetadata)

Check whether the archive is valid or not .


def _load_metadata(self):
        """Load metadata from the archive file"""

        logger.debug("Loading metadata infomation of archive %s", self.archive_path)

        cursor = self._db.cursor()
        select_stmt = "SELECT origin, backend_name, backend_version, " \
                      "category, backend_params, created_on " \
                      "FROM " + self.METADATA_TABLE + " " \
                      "LIMIT 1"
        cursor.execute(select_stmt)
        row = cursor.fetchone()
        cursor.close()

        if row:
            self.origin = row[0]
            self.backend_name = row[1]
            self.backend_version = row[2]
            self.category = row[3]
            self.backend_params = pickle.loads(row[4])
            self.created_on = str_to_datetime(row[5])
        else:
            logger.debug("Metadata of archive %s was empty", self.archive_path)

        logger.debug("Metadata of archive %s loaded", self.archive_path)


def _count_table_rows(self, table_name):
        """Fetch the number of rows in a table"""

        cursor = self._db.cursor()
        select_stmt = "SELECT COUNT(*) FROM " + table_name

        try:
            cursor.execute(select_stmt)
            row = cursor.fetchone()
        except sqlite3.DatabaseError as e:
            msg = "invalid archive file; cause: %s" % str(e)
            raise ArchiveError(cause=msg)
        finally:
            cursor.close()

        return row[0]

def
_count_table_rows
(
self
,
table_name
)
:
cursor
=
self
.
_db
.
cursor
(
)
select_stmt
=
"SELECT COUNT(*) FROM "
+
table_name
try
:
cursor
.
execute
(
select_stmt
)
row
=
cursor
.
fetchone
(
)
except
sqlite3
.
DatabaseError
as
e
:
msg
=
"invalid archive file; cause: %s"
%
str
(
e
)
raise
ArchiveError
(
cause
=
msg
)
finally
:
cursor
.
close
(
)
return
row
[
0
]


def create_archive(self):
        """Create a new archive.

        The method creates in the filesystem a brand new archive with
        a random SHA1 as its name. The first byte of the hashcode will
        be the name of the subdirectory; the remaining bytes, the
        archive name.

        :returns: a new `Archive` object

        :raises ArchiveManagerError: when an error occurs creating the
            new archive
        """
        hashcode = uuid.uuid4().hex
        archive_dir = os.path.join(self.dirpath, hashcode[0:2])
        archive_name = hashcode[2:] + self.STORAGE_EXT
        archive_path = os.path.join(archive_dir, archive_name)

        if not os.path.exists(archive_dir):
            os.makedirs(archive_dir)

        try:
            archive = Archive.create(archive_path)
        except ArchiveError as e:
            raise ArchiveManagerError(cause=str(e))

        return archive

def create_archive ( self ) : hashcode = uuid . uuid4 ( ) . hex archive_dir = os . path . join ( self . dirpath , hashcode [ 0 : 2 ] ) archive_name = hashcode [ 2 : ] + self . STORAGE_EXT archive_path = os . path . join ( archive_dir , archive_name ) if not os . path . exists ( archive_dir ) : os . makedirs ( archive_dir ) try : archive = Archive . create ( archive_path ) except ArchiveError as e : raise ArchiveManagerError ( cause = str ( e ) ) return archive


def remove_archive(self, archive_path):
        """Remove an archive.

        This method deletes from the filesystem the archive stored
        in `archive_path`.

        :param archive_path: path to the archive

        :raises ArchiveManangerError: when an error occurs removing the
            archive
        """
        try:
            Archive(archive_path)
        except ArchiveError as e:
            raise ArchiveManagerError(cause=str(e))

        os.remove(archive_path)

def remove_archive ( self , archive_path ) : try : Archive ( archive_path ) except ArchiveError as e : raise ArchiveManagerError ( cause = str ( e ) ) os . remove ( archive_path )


def search(self, origin, backend_name, category, archived_after):
        """Search archives.

        Get the archives which store data based on the given parameters.
        These parameters define which the origin was (`origin`), how data
        was fetched (`backend_name`) and data type ('category').
        Only those archives created on or after `archived_after` will be
        returned.

        The method returns a list with the file paths to those archives.
        The list is sorted by the date of creation of each archive.

        :param origin: data origin
        :param backend_name: backed used to fetch data
        :param category: type of the items fetched by the backend
        :param archived_after: get archives created on or after this date

        :returns: a list with archive names which match the search criteria
        """
        archives = self._search_archives(origin, backend_name,
                                         category, archived_after)
        archives = [(fp, date) for fp, date in archives]
        archives = [fp for fp, _ in sorted(archives, key=lambda x: x[1])]

        return archives

def search ( self , origin , backend_name , category , archived_after ) : archives = self . _search_archives ( origin , backend_name , category , archived_after ) archives = [ ( fp , date ) for fp , date in archives ] archives = [ fp for fp , _ in sorted ( archives , key = lambda x : x [ 1 ] ) ] return archives


def _search_archives(self, origin, backend_name, category, archived_after):
        """Search archives using filters."""

        for archive_path in self._search_files():
            try:
                archive = Archive(archive_path)
            except ArchiveError:
                continue

            match = archive.origin == origin and \
                archive.backend_name == backend_name and \
                archive.category == category and \
                archive.created_on >= archived_after

            if not match:
                continue

            yield archive_path, archive.created_on

def _search_archives ( self , origin , backend_name , category , archived_after ) : for archive_path in self . _search_files ( ) : try : archive = Archive ( archive_path ) except ArchiveError : continue match = archive . origin == origin and archive . backend_name == backend_name and archive . category == category and archive . created_on >= archived_after if not match : continue yield archive_path , archive . created_on


def _search_files(self):
        """Retrieve the file paths stored under the base path."""

        for root, _, files in os.walk(self.dirpath):
            for filename in files:
                location = os.path.join(root, filename)
                yield location

def _search_files ( self ) : for root , _ , files in os . walk ( self . dirpath ) : for filename in files : location = os . path . join ( root , filename ) yield location

Retrieve the file paths stored under the base path .


def check_compressed_file_type(filepath):
    """Check if filename is a compressed file supported by the tool.

    This function uses magic numbers (first four bytes) to determine
    the type of the file. Supported types are 'gz' and 'bz2'. When
    the filetype is not supported, the function returns `None`.

    :param filepath: path to the file

    :returns: 'gz' or 'bz2'; `None` if the type is not supported
    """
    def compressed_file_type(content):
        magic_dict = {
            b'\x1f\x8b\x08': 'gz',
            b'\x42\x5a\x68': 'bz2',
            b'PK\x03\x04': 'zip'
        }

        for magic, filetype in magic_dict.items():
            if content.startswith(magic):
                return filetype

        return None

    with open(filepath, mode='rb') as f:
        magic_number = f.read(4)
    return compressed_file_type(magic_number)

def check_compressed_file_type ( filepath ) : def compressed_file_type ( content ) : magic_dict = { b'\x1f\x8b\x08' : 'gz' , b'\x42\x5a\x68' : 'bz2' , b'PK\x03\x04' : 'zip' } for magic , filetype in magic_dict . items ( ) : if content . startswith ( magic ) : return filetype return None with open ( filepath , mode = 'rb' ) as f : magic_number = f . read ( 4 ) return compressed_file_type ( magic_number )

Check if filename is a compressed file supported by the tool .


def months_range(from_date, to_date):
    """Generate a months range.

    Generator of months starting on `from_date` util `to_date`. Each
    returned item is a tuple of two datatime objects like in (month, month+1).
    Thus, the result will follow the sequence:
        ((fd, fd+1), (fd+1, fd+2), ..., (td-2, td-1), (td-1, td))

    :param from_date: generate dates starting on this month
    :param to_date: generate dates until this month

    :result: a generator of months range
    """
    start = datetime.datetime(from_date.year, from_date.month, 1)
    end = datetime.datetime(to_date.year, to_date.month, 1)

    month_gen = dateutil.rrule.rrule(freq=dateutil.rrule.MONTHLY,
                                     dtstart=start, until=end)
    months = [d for d in month_gen]

    pos = 0
    for x in range(1, len(months)):
        yield months[pos], months[x]
        pos = x

def months_range ( from_date , to_date ) : start = datetime . datetime ( from_date . year , from_date . month , 1 ) end = datetime . datetime ( to_date . year , to_date . month , 1 ) month_gen = dateutil . rrule . rrule ( freq = dateutil . rrule . MONTHLY , dtstart = start , until = end ) months = [ d for d in month_gen ] pos = 0 for x in range ( 1 , len ( months ) ) : yield months [ pos ] , months [ x ] pos = x


def message_to_dict(msg):
    """Convert an email message into a dictionary.

    This function transforms an `email.message.Message` object
    into a dictionary. Headers are stored as key:value pairs
    while the body of the message is stored inside `body` key.
    Body may have two other keys inside, 'plain', for plain body
    messages and 'html', for HTML encoded messages.

    The returned dictionary has the type `requests.structures.CaseInsensitiveDict`
    due to same headers with different case formats can appear in
    the same message.

    :param msg: email message of type `email.message.Message`

    :returns : dictionary of type `requests.structures.CaseInsensitiveDict`

    :raises ParseError: when an error occurs transforming the message
        to a dictionary
    """
    def parse_headers(msg):
        headers = {}

        for header, value in msg.items():
            hv = []

            for text, charset in email.header.decode_header(value):
                if type(text) == bytes:
                    charset = charset if charset else 'utf-8'
                    try:
                        text = text.decode(charset, errors='surrogateescape')
                    except (UnicodeError, LookupError):
                        # Try again with a 7bit encoding
                        text = text.decode('ascii', errors='surrogateescape')
                hv.append(text)

            v = ' '.join(hv)
            headers[header] = v if v else None

        return headers

    def parse_payload(msg):
        body = {}

        if not msg.is_multipart():
            payload = decode_payload(msg)
            subtype = msg.get_content_subtype()
            body[subtype] = [payload]
        else:
            # Include all the attached texts if it is multipart
            # Ignores binary parts by default
            for part in email.iterators.typed_subpart_iterator(msg):
                payload = decode_payload(part)
                subtype = part.get_content_subtype()
                body.setdefault(subtype, []).append(payload)

        return {k: '\n'.join(v) for k, v in body.items()}

    def decode_payload(msg_or_part):
        charset = msg_or_part.get_content_charset('utf-8')
        payload = msg_or_part.get_payload(decode=True)

        try:
            payload = payload.decode(charset, errors='surrogateescape')
        except (UnicodeError, LookupError):
            # Try again with a 7bit encoding
            payload = payload.decode('ascii', errors='surrogateescape')
        return payload

    # The function starts here
    message = requests.structures.CaseInsensitiveDict()

    if isinstance(msg, mailbox.mboxMessage):
        message['unixfrom'] = msg.get_from()
    else:
        message['unixfrom'] = None

    try:
        for k, v in parse_headers(msg).items():
            message[k] = v
        message['body'] = parse_payload(msg)
    except UnicodeError as e:
        raise ParseError(cause=str(e))

    return message

def
message_to_dict
(
msg
)
:
def
parse_headers
(
msg
)
:
headers
=
{
}
for
header
,
value
in
msg
.
items
(
)
:
hv
=
[
]
for
text
,
charset
in
email
.
header
.
decode_header
(
value
)
:
if
type
(
text
)
==
bytes
:
charset
=
charset
if
charset
else
'utf-8'
try
:
text
=
text
.
decode
(
charset
,
errors
=
'surrogateescape'
)
except
(
UnicodeError
,
LookupError
)
:
text
=
text
.
decode
(
'ascii'
,
errors
=
'surrogateescape'
)
hv
.
append
(
text
)
v
=
' '
.
join
(
hv
)
headers
[
header
]
=
v
if
v
else
return
headers
def
parse_payload
(
msg
)
:
body
=
{
}
if
not
msg
.
is_multipart
(
)
:
payload
=
decode_payload
(
msg
)
subtype
=
msg
.
get_content_subtype
(
)
body
[
subtype
]
=
[
payload
]
else
:
for
part
in
email
.
iterators
.
typed_subpart_iterator
(
msg
)
:
payload
=
decode_payload
(
part
)
subtype
=
part
.
get_content_subtype
(
)
body
.
setdefault
(
subtype
,
[
]
)
.
append
(
payload
)
return
{
k
:
'\n'
.
join
(
v
)
for
k
,
v
in
body
.
items
(
)
}
def
decode_payload
(
msg_or_part
)
:
charset
=
msg_or_part
.
get_content_charset
(
'utf-8'
)
payload
=
msg_or_part
.
get_payload
(
decode
=
True
)
try
:
payload
=
payload
.
decode
(
charset
,
errors
=
'surrogateescape'
)
except
(
UnicodeError
,
LookupError
)
:
payload
=
payload
.
decode
(
'ascii'
,
errors
=
'surrogateescape'
)
return
payload
message
=
requests
.
structures
.
CaseInsensitiveDict
(
)
if
isinstance
(
msg
,
mailbox
.
mboxMessage
)
:
message
[
'unixfrom'
]
=
msg
.
get_from
(
)
else
:
message
[
'unixfrom'
]
=
try
:
for
k
,
v
in
parse_headers
(
msg
)
.
items
(
)
:
message
[
k
]
=
v
message
[
'body'
]
=
parse_payload
(
msg
)
except
UnicodeError
as
e
:
raise
ParseError
(
cause
=
str
(
e
)
)
return
message

Convert an email message into a dictionary .


def remove_invalid_xml_chars(raw_xml):
    """Remove control and invalid characters from an xml stream.

    Looks for invalid characters and subtitutes them with whitespaces.
    This solution is based on these two posts: Olemis Lang's reponse
    on StackOverflow (http://stackoverflow.com/questions/1707890) and
    lawlesst's on GitHub Gist (https://gist.github.com/lawlesst/4110923),
    that is based on the previous answer.

    :param xml: XML stream

    :returns: a purged XML stream
    """
    illegal_unichrs = [(0x00, 0x08), (0x0B, 0x1F),
                       (0x7F, 0x84), (0x86, 0x9F)]

    illegal_ranges = ['%s-%s' % (chr(low), chr(high))
                      for (low, high) in illegal_unichrs
                      if low < sys.maxunicode]

    illegal_xml_re = re.compile('[%s]' % ''.join(illegal_ranges))

    purged_xml = ''

    for c in raw_xml:
        if illegal_xml_re.search(c) is not None:
            c = ' '
        purged_xml += c

    return purged_xml

def
remove_invalid_xml_chars
(
raw_xml
)
:
illegal_unichrs
=
[
(
0x00
,
0x08
)
,
(
0x0B
,
0x1F
)
,
(
0x7F
,
0x84
)
,
(
0x86
,
0x9F
)
]
illegal_ranges
=
[
'%s-%s'
%
(
chr
(
low
)
,
chr
(
high
)
)
for
(
low
,
high
)
in
illegal_unichrs
if
low
<
sys
.
maxunicode
]
illegal_xml_re
=
re
.
compile
(
'[%s]'
%
''
.
join
(
illegal_ranges
)
)
purged_xml
=
''
for
c
in
raw_xml
:
if
illegal_xml_re
.
search
(
c
)
is
not
:
c
=
' '
purged_xml
+=
c
return
purged_xml

Remove control and invalid characters from an xml stream .


def xml_to_dict(raw_xml):
    """Convert a XML stream into a dictionary.

    This function transforms a xml stream into a dictionary. The
    attributes are stored as single elements while child nodes are
    stored into lists. The text node is stored using the special
    key '__text__'.

    This code is based on Winston Ewert's solution to this problem.
    See http://codereview.stackexchange.com/questions/10400/convert-elementtree-to-dict
    for more info. The code was licensed as cc by-sa 3.0.

    :param raw_xml: XML stream

    :returns: a dict with the XML data

    :raises ParseError: raised when an error occurs parsing the given
        XML stream
    """
    def node_to_dict(node):
        d = {}
        d.update(node.items())

        text = getattr(node, 'text', None)

        if text is not None:
            d['__text__'] = text

        childs = {}
        for child in node:
            childs.setdefault(child.tag, []).append(node_to_dict(child))

        d.update(childs.items())

        return d

    purged_xml = remove_invalid_xml_chars(raw_xml)

    try:
        tree = xml.etree.ElementTree.fromstring(purged_xml)
    except xml.etree.ElementTree.ParseError as e:
        cause = "XML stream %s" % (str(e))
        raise ParseError(cause=cause)

    d = node_to_dict(tree)

    return d

def
xml_to_dict
(
raw_xml
)
:
def
node_to_dict
(
node
)
:
d
=
{
}
d
.
update
(
node
.
items
(
)
)
text
=
getattr
(
node
,
'text'
,
)
if
text
is
not
:
d
[
'__text__'
]
=
text
childs
=
{
}
for
child
in
node
:
childs
.
setdefault
(
child
.
tag
,
[
]
)
.
append
(
node_to_dict
(
child
)
)
d
.
update
(
childs
.
items
(
)
)
return
d
purged_xml
=
remove_invalid_xml_chars
(
raw_xml
)
try
:
tree
=
xml
.
etree
.
ElementTree
.
fromstring
(
purged_xml
)
except
xml
.
etree
.
ElementTree
.
ParseError
as
e
:
cause
=
"XML stream %s"
%
(
str
(
e
)
)
raise
ParseError
(
cause
=
cause
)
d
=
node_to_dict
(
tree
)
return
d

Convert a XML stream into a dictionary .


def parse_issues(raw_json):
        """Parse a Redmine issues JSON stream.

        The method parses a JSON stream and returns a list iterator.
        Each item is a dictionary that contains the issue parsed data.

        :param raw_json: JSON string to parse

        :returns: a generator of parsed issues
        """
        results = json.loads(raw_json)

        issues = results['issues']
        for issue in issues:
            yield issue

def parse_issues ( raw_json ) : results = json . loads ( raw_json ) issues = results [ 'issues' ] for issue in issues : yield issue


def issues(self, from_date=DEFAULT_DATETIME,
               offset=None, max_issues=MAX_ISSUES):
        """Get the information of a list of issues.

        :param from_date: retrieve issues that where updated from that date;
            dates are converted to UTC
        :param offset: starting position for the search
        :param max_issues: maximum number of issues to reteurn per query
        """
        resource = self.RISSUES + self.CJSON

        ts = datetime_to_utc(from_date)
        ts = ts.strftime("%Y-%m-%dT%H:%M:%SZ")

        # By default, Redmine returns open issues only.
        # Parameter 'status_id' is set to get all the statuses.
        params = {
            self.PSTATUS_ID: '*',
            self.PSORT: self.PUPDATED_ON,
            self.PUPDATED_ON: '>=' + ts,
            self.PLIMIT: max_issues
        }

        if offset is not None:
            params[self.POFFSET] = offset

        response = self._call(resource, params)

        return response

def issues ( self , from_date = DEFAULT_DATETIME , offset = None , max_issues = MAX_ISSUES ) : resource = self . RISSUES + self . CJSON ts = datetime_to_utc ( from_date ) ts = ts . strftime ( "%Y-%m-%dT%H:%M:%SZ" ) params = { self . PSTATUS_ID : '*' , self . PSORT : self . PUPDATED_ON , self . PUPDATED_ON : '>=' + ts , self . PLIMIT : max_issues } if offset is not None : params [ self . POFFSET ] = offset response = self . _call ( resource , params ) return response

Get the information of a list of issues .


def issue(self, issue_id):
        """Get the information of the given issue.

        :param issue_id: issue identifier
        """
        resource = urijoin(self.RISSUES, str(issue_id) + self.CJSON)

        params = {
            self.PINCLUDE: ','.join([self.CATTACHMENTS, self.CCHANGESETS,
                                     self.CCHILDREN, self.CJOURNALS,
                                     self.CRELATIONS, self.CWATCHERS])
        }

        response = self._call(resource, params)

        return response

def issue ( self , issue_id ) : resource = urijoin ( self . RISSUES , str ( issue_id ) + self . CJSON ) params = { self . PINCLUDE : ',' . join ( [ self . CATTACHMENTS , self . CCHANGESETS , self . CCHILDREN , self . CJOURNALS , self . CRELATIONS , self . CWATCHERS ] ) } response = self . _call ( resource , params ) return response

Get the information of the given issue .


def user(self, user_id):
        """Get the information of the given user.

        :param user_id: user identifier
        """
        resource = urijoin(self.RUSERS, str(user_id) + self.CJSON)

        params = {}

        response = self._call(resource, params)

        return response

def user ( self , user_id ) : resource = urijoin ( self . RUSERS , str ( user_id ) + self . CJSON ) params = { } response = self . _call ( resource , params ) return response


def _call(self, resource, params):
        """Call to get a resource.

        :param method: resource to get
        :param params: dict with the HTTP parameters needed to get
            the given resource
        """
        url = self.URL % {'base': self.base_url, 'resource': resource}

        if self.api_token:
            params[self.PKEY] = self.api_token

        logger.debug("Redmine client requests: %s params: %s",
                     resource, str(params))

        r = self.fetch(url, payload=params, verify=False)

        return r.text


def fetch(self, category=CATEGORY_DOCKERHUB_DATA):
        """Fetch data from a Docker Hub repository.

        The method retrieves, from a repository stored in Docker Hub,
        its data which includes number of pulls, stars, description,
        among other data.

        :param category: the category of items to fetch

        :returns: a generator of data
        """
        kwargs = {}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_DOCKERHUB_DATA ) : kwargs = { } items = super ( ) . fetch ( category , ** kwargs ) return items

Fetch data from a Docker Hub repository .


def fetch_items(self, category, **kwargs):
        """Fetch the Dockher Hub items

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        logger.info("Fetching data from '%s' repository of '%s' owner",
                    self.repository, self.owner)

        raw_data = self.client.repository(self.owner, self.repository)
        fetched_on = datetime_utcnow().timestamp()

        data = self.parse_json(raw_data)
        data['fetched_on'] = fetched_on
        yield data

        logger.info("Fetch process completed")


def repository(self, owner, repository):
        """Fetch information about a repository."""

        url = urijoin(self.base_url, self.RREPOSITORY, owner, repository)

        logger.debug("DockerHub client requests: %s", url)

        response = self.fetch(url)

        return response.text

def
repository
(
self
,
owner
,
repository
)
:
url
=
urijoin
(
self
.
base_url
,
self
.
RREPOSITORY
,
owner
,
repository
)
logger
.
debug
(
"DockerHub client requests: %s"
,
url
)
response
=
self
.
fetch
(
url
)
return
response
.
text


def map_custom_field(custom_fields, fields):
    """Add extra information for custom fields.

    :param custom_fields: set of custom fields with the extra information
    :param fields: fields of the issue where to add the extra information

    :returns: an set of items with the extra information mapped
    """
    def build_cf(cf, v):
        return {'id': cf['id'], 'name': cf['name'], 'value': v}

    return {
        k: build_cf(custom_fields[k], v)
        for k, v in fields.items()
        if k in custom_fields
    }

def map_custom_field ( custom_fields , fields ) : def build_cf ( cf , v ) : return { 'id' : cf [ 'id' ] , 'name' : cf [ 'name' ] , 'value' : v } return { k : build_cf ( custom_fields [ k ] , v ) for k , v in fields . items ( ) if k in custom_fields }

Add extra information for custom fields .


def filter_custom_fields(fields):
    """Filter custom fields from a given set of fields.

    :param fields: set of fields

    :returns: an object with the filtered custom fields
    """

    custom_fields = {}

    sorted_fields = [field for field in fields if field['custom'] is True]

    for custom_field in sorted_fields:
        custom_fields[custom_field['id']] = custom_field

    return custom_fields

def filter_custom_fields ( fields ) : custom_fields = { } sorted_fields = [ field for field in fields if field [ 'custom' ] is True ] for custom_field in sorted_fields : custom_fields [ custom_field [ 'id' ] ] = custom_field return custom_fields

Filter custom fields from a given set of fields .


def parse_issues(raw_page):
        """Parse a JIRA API raw response.

        The method parses the API response retrieving the
        issues from the received items

        :param items: items from where to parse the issues

        :returns: a generator of issues
        """
        raw_issues = json.loads(raw_page)
        issues = raw_issues['issues']
        for issue in issues:
            yield issue

def parse_issues ( raw_page ) : raw_issues = json . loads ( raw_page ) issues = raw_issues [ 'issues' ] for issue in issues : yield issue


def get_items(self, from_date, url, expand_fields=True):
        """Retrieve all the items from a given date.

        :param url: endpoint API url
        :param from_date: obtain items updated since this date
        :param expand_fields: if True, it includes the expand fields in the payload
        """
        start_at = 0

        req = self.fetch(url, payload=self.__build_payload(start_at, from_date, expand_fields))
        issues = req.text

        data = req.json()
        titems = data['total']
        nitems = data['maxResults']

        start_at += min(nitems, titems)
        self.__log_status(start_at, titems, url)

        while issues:
            yield issues
            issues = None

            if data['startAt'] + nitems < titems:
                req = self.fetch(url, payload=self.__build_payload(start_at, from_date, expand_fields))

                data = req.json()
                start_at += nitems
                issues = req.text
                self.__log_status(start_at, titems, url)

def get_items ( self , from_date , url , expand_fields = True ) : start_at = 0 req = self . fetch ( url , payload = self . __build_payload ( start_at , from_date , expand_fields ) ) issues = req . text data = req . json ( ) titems = data [ 'total' ] nitems = data [ 'maxResults' ] start_at += min ( nitems , titems ) self . __log_status ( start_at , titems , url ) while issues : yield issues issues = None if data [ 'startAt' ] + nitems < titems : req = self . fetch ( url , payload = self . __build_payload ( start_at , from_date , expand_fields ) ) data = req . json ( ) start_at += nitems issues = req . text self . __log_status ( start_at , titems , url )

Retrieve all the items from a given date .


def get_issues(self, from_date):
        """Retrieve all the issues from a given date.

        :param from_date: obtain issues updated since this date
        """
        url = urijoin(self.base_url, self.RESOURCE, self.VERSION_API, 'search')
        issues = self.get_items(from_date, url)

        return issues

def get_issues ( self , from_date ) : url = urijoin ( self . base_url , self . RESOURCE , self . VERSION_API , 'search' ) issues = self . get_items ( from_date , url ) return issues

Retrieve all the issues from a given date .


def get_comments(self, issue_id):
        """Retrieve all the comments of a given issue.

        :param issue_id: ID of the issue
        """
        url = urijoin(self.base_url, self.RESOURCE, self.VERSION_API, self.ISSUE, issue_id, self.COMMENT)
        comments = self.get_items(DEFAULT_DATETIME, url, expand_fields=False)

        return comments

def get_comments ( self , issue_id ) : url = urijoin ( self . base_url , self . RESOURCE , self . VERSION_API , self . ISSUE , issue_id , self . COMMENT ) comments = self . get_items ( DEFAULT_DATETIME , url , expand_fields = False ) return comments

Retrieve all the comments of a given issue .


def get_fields(self):
        """Retrieve all the fields available."""

        url = urijoin(self.base_url, self.RESOURCE, self.VERSION_API, 'field')
        req = self.fetch(url)

        return req.text

def get_fields ( self ) : url = urijoin ( self . base_url , self . RESOURCE , self . VERSION_API , 'field' ) req = self . fetch ( url ) return req . text


def fetch(self, category=CATEGORY_BUILD):
        """Fetch the builds from the url.

        The method retrieves, from a Jenkins url, the
        builds updated since the given date.

        :param category: the category of items to fetch

        :returns: a generator of builds
        """

        kwargs = {}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_BUILD ) : kwargs = { } items = super ( ) . fetch ( category , ** kwargs ) return items


def get_jobs(self):
        """ Retrieve all jobs"""

        url_jenkins = urijoin(self.base_url, "api", "json")

        response = self.fetch(url_jenkins)
        return response.text

def get_jobs ( self ) : url_jenkins = urijoin ( self . base_url , "api" , "json" ) response = self . fetch ( url_jenkins ) return response . text


def get_builds(self, job_name):
        """ Retrieve all builds from a job"""

        if self.blacklist_jobs and job_name in self.blacklist_jobs:
            logger.warning("Not getting blacklisted job: %s", job_name)
            return

        payload = {'depth': self.detail_depth}
        url_build = urijoin(self.base_url, "job", job_name, "api", "json")

        response = self.fetch(url_build, payload=payload)
        return response.text

def
get_builds
(
self
,
job_name
)
:
if
self
.
blacklist_jobs
and
job_name
in
self
.
blacklist_jobs
:
logger
.
warning
(
"Not getting blacklisted job: %s"
,
job_name
)
return
payload
=
{
'depth'
:
self
.
detail_depth
}
url_build
=
urijoin
(
self
.
base_url
,
"job"
,
job_name
,
"api"
,
"json"
)
response
=
self
.
fetch
(
url_build
,
payload
=
payload
)
return
response
.
text


def parse_questions(raw_page):
        """Parse a StackExchange API raw response.

        The method parses the API response retrieving the
        questions from the received items

        :param items: items from where to parse the questions

        :returns: a generator of questions
        """
        raw_questions = json.loads(raw_page)
        questions = raw_questions['items']
        for question in questions:
            yield question

def parse_questions ( raw_page ) : raw_questions = json . loads ( raw_page ) questions = raw_questions [ 'items' ] for question in questions : yield question

Parse a StackExchange API raw response .


def get_questions(self, from_date):
        """Retrieve all the questions from a given date.

        :param from_date: obtain questions updated since this date
        """

        page = 1
        url = urijoin(self.base_url, self.VERSION_API, "questions")

        req = self.fetch(url, payload=self.__build_payload(page, from_date))
        questions = req.text

        data = req.json()
        tquestions = data['total']
        nquestions = data['page_size']

        self.__log_status(data['quota_remaining'],
                          data['quota_max'],
                          nquestions,
                          tquestions)

        while questions:
            yield questions
            questions = None

            if data['has_more']:
                page += 1

                backoff = data.get('backoff', None)
                if backoff:
                    logger.debug("Expensive query. Wait %s secs to send a new request",
                                 backoff)
                    time.sleep(float(backoff))

                req = self.fetch(url, payload=self.__build_payload(page, from_date))
                data = req.json()
                questions = req.text
                nquestions += data['page_size']
                self.__log_status(data['quota_remaining'],
                                  data['quota_max'],
                                  nquestions,
                                  tquestions)

Retrieve all the questions from a given date .


def setup_cmd_parser(cls):
        """Returns the StackExchange argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              token_auth=True,
                                              archive=True)

        # StackExchange options
        group = parser.parser.add_argument_group('StackExchange arguments')
        group.add_argument('--site', dest='site',
                           required=True,
                           help="StackExchange site")
        group.add_argument('--tagged', dest='tagged',
                           help="filter items by question Tag")
        group.add_argument('--max-questions', dest='max_questions',
                           type=int, default=MAX_QUESTIONS,
                           help="Maximum number of questions requested in the same query")

        return parser

Returns the StackExchange argument parser .


def fetch_items(self, category, **kwargs):
        """Fetch the pages

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']
        reviews_api = kwargs['reviews_api']

        mediawiki_version = self.client.get_version()
        logger.info("MediaWiki version: %s", mediawiki_version)

        if reviews_api:
            if ((mediawiki_version[0] == 1 and mediawiki_version[1] >= 27) or mediawiki_version[0] > 1):
                fetcher = self.__fetch_1_27(from_date)
            else:
                logger.warning("Reviews API only available in MediaWiki >= 1.27")
                logger.warning("Using the Pages API instead")
                fetcher = self.__fetch_pre1_27(from_date)
        else:
            fetcher = self.__fetch_pre1_27(from_date)

        for page_reviews in fetcher:
            yield page_reviews


def __get_max_date(self, reviews):
        """"Get the max date in unixtime format from reviews."""
        max_ts = 0
        for review in reviews:
            ts = str_to_datetime(review['timestamp'])
            ts = datetime_to_utc(ts)
            if ts.timestamp() > max_ts:
                max_ts = ts.timestamp()
        return max_ts

def __get_max_date ( self , reviews ) : max_ts = 0 for review in reviews : ts = str_to_datetime ( review [ 'timestamp' ] ) ts = datetime_to_utc ( ts ) if ts . timestamp ( ) > max_ts : max_ts = ts . timestamp ( ) return max_ts

Get the max date in unixtime format from reviews .


def __fetch_1_27(self, from_date=None):
        """Fetch the pages from the backend url for MediaWiki >=1.27

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
        """

        logger.info("Looking for pages at url '%s'", self.url)

        npages = 0  # number of pages processed
        tpages = 0  # number of total pages
        pages_done = []  # pages already retrieved in reviews API

        namespaces_contents = self.__get_namespaces_contents()

        arvcontinue = ''  # pagination for getting revisions and their pages
        while arvcontinue is not None:
            raw_pages = self.client.get_pages_from_allrevisions(namespaces_contents, from_date, arvcontinue)
            data_json = json.loads(raw_pages)
            arvcontinue = data_json['continue']['arvcontinue'] if 'continue' in data_json else None
            pages_json = data_json['query']['allrevisions']
            for page in pages_json:

                if page['pageid'] in pages_done:
                    logger.debug("Page %s already processed; skipped", page['pageid'])
                    continue

                tpages += 1
                pages_done.append(page['pageid'])
                page_reviews = self.__get_page_reviews(page)

                if not page_reviews:
                    logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                   page['title'], page['pageid'])
                    continue

                yield page_reviews
                npages += 1

        logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)

Fetch the pages from the backend url for MediaWiki > = 1 . 27


def get_pages(self, namespace, apcontinue=''):
        """Retrieve all pages from a namespace starting from apcontinue."""
        params = {
            "action": "query",
            "list": "allpages",
            "aplimit": self.limit,
            "apnamespace": namespace,
            "format": "json"
        }
        if apcontinue:
            params['apcontinue'] = apcontinue

        return self.call(params)

def get_pages ( self , namespace , apcontinue = '' ) : params = { "action" : "query" , "list" : "allpages" , "aplimit" : self . limit , "apnamespace" : namespace , "format" : "json" } if apcontinue : params [ 'apcontinue' ] = apcontinue return self . call ( params )

Retrieve all pages from a namespace starting from apcontinue .


def get_recent_pages(self, namespaces, rccontinue=''):
        """Retrieve recent pages from all namespaces starting from rccontinue."""

        namespaces.sort()
        params = {
            "action": "query",
            "list": "recentchanges",
            "rclimit": self.limit,
            "rcnamespace": "|".join(namespaces),
            "rcprop": "title|timestamp|ids",
            "format": "json"
        }
        if rccontinue:
            params['rccontinue'] = rccontinue

        return self.call(params)

def get_recent_pages ( self , namespaces , rccontinue = '' ) : namespaces . sort ( ) params = { "action" : "query" , "list" : "recentchanges" , "rclimit" : self . limit , "rcnamespace" : "|" . join ( namespaces ) , "rcprop" : "title|timestamp|ids" , "format" : "json" } if rccontinue : params [ 'rccontinue' ] = rccontinue return self . call ( params )

Retrieve recent pages from all namespaces starting from rccontinue .


def fetch(self, category=CATEGORY_MESSAGE, offset=DEFAULT_OFFSET, chats=None):
        """Fetch the messages the bot can read from the server.

        The method retrieves, from the Telegram server, the messages
        sent with an offset equal or greater than the given.

        A list of chats, groups and channels identifiers can be set
        using the parameter `chats`. When it is set, only those
        messages sent to any of these will be returned. An empty list
        will return no messages.

        :param category: the category of items to fetch
        :param offset: obtain messages from this offset
        :param chats: list of chat names used to filter messages

        :returns: a generator of messages

        :raises ValueError: when `chats` is an empty list
        """
        if not offset:
            offset = DEFAULT_OFFSET

        kwargs = {"offset": offset, "chats": chats}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_MESSAGE , offset = DEFAULT_OFFSET , chats = None ) : if not offset : offset = DEFAULT_OFFSET kwargs = { "offset" : offset , "chats" : chats } items = super ( ) . fetch ( category , ** kwargs ) return items

Fetch the messages the bot can read from the server .


def parse_messages(raw_json):
        """Parse a Telegram JSON messages list.

        The method parses the JSON stream and returns an iterator of
        dictionaries. Each one of this, contains a Telegram message.

        :param raw_json: JSON string to parse

        :returns: a generator of parsed messages
        """
        result = json.loads(raw_json)

        messages = result['result']
        for msg in messages:
            yield msg

def parse_messages ( raw_json ) : result = json . loads ( raw_json ) messages = result [ 'result' ] for msg in messages : yield msg


def _filter_message_by_chats(self, message, chats):
        """Check if a message can be filtered based in a list of chats.

        This method returns `True` when the message was sent to a chat
        of the given list. It also returns `True` when chats is `None`.

        :param message: Telegram message
        :param chats: list of chat, groups and channels identifiers

        :returns: `True` when the message can be filtered; otherwise,
            it returns `False`
        """
        if chats is None:
            return True

        chat_id = message['message']['chat']['id']

        return chat_id in chats

def _filter_message_by_chats ( self , message , chats ) : if chats is None : return True chat_id = message [ 'message' ] [ 'chat' ] [ 'id' ] return chat_id in chats

Check if a message can be filtered based in a list of chats .


def updates(self, offset=None):
        """Fetch the messages that a bot can read.

        When the `offset` is given it will retrieve all the messages
        that are greater or equal to that offset. Take into account
        that, due to how the API works, all previous messages will
        be removed from the server.

        :param offset: fetch the messages starting on this offset
        """
        params = {}

        if offset:
            params[self.OFFSET] = offset

        response = self._call(self.UPDATES_METHOD, params)

        return response

def updates ( self , offset = None ) : params = { } if offset : params [ self . OFFSET ] = offset response = self . _call ( self . UPDATES_METHOD , params ) return response

Fetch the messages that a bot can read .


def fetch_items(self, category, **kwargs):
        """Fetch the articles

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        offset = kwargs['offset']

        logger.info("Fetching articles of '%s' group on '%s' offset %s",
                    self.group, self.host, str(offset))

        narts, iarts, tarts = (0, 0, 0)

        _, _, first, last, _ = self.client.group(self.group)

        if offset <= last:
            first = max(first, offset)
            _, overview = self.client.over((first, last))
        else:
            overview = []

        tarts = len(overview)

        logger.debug("Total number of articles to fetch: %s", tarts)

        for article_id, _ in overview:
            try:
                article_raw = self.client.article(article_id)
                article = self.__parse_article(article_raw)
            except ParseError:
                logger.warning("Error parsing %s article; skipping",
                               article_id)
                iarts += 1
                continue
            except nntplib.NNTPTemporaryError as e:
                logger.warning("Error '%s' fetching article %s; skipping",
                               e.response, article_id)
                iarts += 1
                continue

            yield article
            narts += 1


def metadata(self, item, filter_classified=False):
        """NNTP metadata.

        This method takes items, overriding `metadata` decorator,
        to add extra information related to NNTP.

        :param item: an item fetched by a backend
        :param filter_classified: sets if classified fields were filtered
        """
        item = super().metadata(item, filter_classified=filter_classified)
        item['offset'] = item['data']['offset']

        return item

def metadata ( self , item , filter_classified = False ) : item = super ( ) . metadata ( item , filter_classified = filter_classified ) item [ 'offset' ] = item [ 'data' ] [ 'offset' ] return item


def parse_article(raw_article):
        """Parse a NNTP article.

        This method parses a NNTP article stored in a string object
        and returns an dictionary.

        :param raw_article: NNTP article string

        :returns: a dictionary of type `requests.structures.CaseInsensitiveDict`

        :raises ParseError: when an error is found parsing the article
        """
        try:
            message = email.message_from_string(raw_article)
            article = message_to_dict(message)
        except UnicodeEncodeError as e:
            raise ParseError(cause=str(e))
        return article

def parse_article ( raw_article ) : try : message = email . message_from_string ( raw_article ) article = message_to_dict ( message ) except UnicodeEncodeError as e : raise ParseError ( cause = str ( e ) ) return article


def _fetch(self, method, args):
        """Fetch NNTP data from the server or from the archive

        :param method: the name of the command to execute
        :param args: the arguments required by the command
        """
        if self.from_archive:
            data = self._fetch_from_archive(method, args)
        else:
            data = self._fetch_from_remote(method, args)

        return data

def _fetch ( self , method , args ) : if self . from_archive : data = self . _fetch_from_archive ( method , args ) else : data = self . _fetch_from_remote ( method , args ) return data


def _fetch_article(self, article_id):
        """Fetch article data

        :param article_id: id of the article to fetch
        """
        fetched_data = self.handler.article(article_id)
        data = {
            'number': fetched_data[1].number,
            'message_id': fetched_data[1].message_id,
            'lines': fetched_data[1].lines
        }

        return data

def _fetch_article ( self , article_id ) : fetched_data = self . handler . article ( article_id ) data = { 'number' : fetched_data [ 1 ] . number , 'message_id' : fetched_data [ 1 ] . message_id , 'lines' : fetched_data [ 1 ] . lines } return data


def _fetch_from_remote(self, method, args):
        """Fetch data from NNTP

        :param method: the name of the command to execute
        :param args: the arguments required by the command
        """
        try:
            if method == NNTTPClient.GROUP:
                data = self.handler.group(args)
            elif method == NNTTPClient.OVER:
                data = self.handler.over(args)
            elif method == NNTTPClient.ARTICLE:
                data = self._fetch_article(args)
        except nntplib.NNTPTemporaryError as e:
            data = e
            raise e
        finally:
            if self.archive:
                self.archive.store(method, args, None, data)

        return data

def _fetch_from_remote ( self , method , args ) : try : if method == NNTTPClient . GROUP : data = self . handler . group ( args ) elif method == NNTTPClient . OVER : data = self . handler . over ( args ) elif method == NNTTPClient . ARTICLE : data = self . _fetch_article ( args ) except nntplib . NNTPTemporaryError as e : data = e raise e finally : if self . archive : self . archive . store ( method , args , None , data ) return data


def _fetch_from_archive(self, method, args):
        """Fetch data from the archive

        :param method: the name of the command to execute
        :param args: the arguments required by the command
        """
        if not self.archive:
            raise ArchiveError(cause="Archive not provided")

        data = self.archive.retrieve(method, args, None)

        if isinstance(data, nntplib.NNTPTemporaryError):
            raise data

        return data

def
_fetch_from_archive
(
self
,
method
,
args
)
:
if
not
self
.
archive
:
raise
ArchiveError
(
cause
=
"Archive not provided"
)
data
=
self
.
archive
.
retrieve
(
method
,
args
,
)
if
isinstance
(
data
,
nntplib
.
NNTPTemporaryError
)
:
raise
data
return
data


def _create_http_session(self):
        """Create a http session and initialize the retry object."""

        self.session = requests.Session()

        if self.headers:
            self.session.headers.update(self.headers)

        retries = urllib3.util.Retry(total=self.max_retries,
                                     connect=self.max_retries_on_connect,
                                     read=self.max_retries_on_read,
                                     redirect=self.max_retries_on_redirect,
                                     status=self.max_retries_on_status,
                                     method_whitelist=self.method_whitelist,
                                     status_forcelist=self.status_forcelist,
                                     backoff_factor=self.sleep_time,
                                     raise_on_redirect=self.raise_on_redirect,
                                     raise_on_status=self.raise_on_status,
                                     respect_retry_after_header=self.respect_retry_after_header)

        self.session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
        self.session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))

def _create_http_session ( self ) : self . session = requests . Session ( ) if self . headers : self . session . headers . update ( self . headers ) retries = urllib3 . util . Retry ( total = self . max_retries , connect = self . max_retries_on_connect , read = self . max_retries_on_read , redirect = self . max_retries_on_redirect , status = self . max_retries_on_status , method_whitelist = self . method_whitelist , status_forcelist = self . status_forcelist , backoff_factor = self . sleep_time , raise_on_redirect = self . raise_on_redirect , raise_on_status = self . raise_on_status , respect_retry_after_header = self . respect_retry_after_header ) self . session . mount ( 'http://' , requests . adapters . HTTPAdapter ( max_retries = retries ) ) self . session . mount ( 'https://' , requests . adapters . HTTPAdapter ( max_retries = retries ) )

Create a http session and initialize the retry object .


def setup_rate_limit_handler(self, sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT,
                                 rate_limit_header=RATE_LIMIT_HEADER,
                                 rate_limit_reset_header=RATE_LIMIT_RESET_HEADER):
        """Setup the rate limit handler.

        :param sleep_for_rate: sleep until rate limit is reset
        :param min_rate_to_sleep: minimun rate needed to make the fecthing process sleep
        :param rate_limit_header: header from where extract the rate limit data
        :param rate_limit_reset_header: header from where extract the rate limit reset data
        """
        self.rate_limit = None
        self.rate_limit_reset_ts = None
        self.sleep_for_rate = sleep_for_rate
        self.rate_limit_header = rate_limit_header
        self.rate_limit_reset_header = rate_limit_reset_header

        if min_rate_to_sleep > self.MAX_RATE_LIMIT:
            msg = "Minimum rate to sleep value exceeded (%d)."
            msg += "High values might cause the client to sleep forever."
            msg += "Reset to %d."
            self.min_rate_to_sleep = self.MAX_RATE_LIMIT
            logger.warning(msg, min_rate_to_sleep, self.MAX_RATE_LIMIT)
        else:
            self.min_rate_to_sleep = min_rate_to_sleep


def sleep_for_rate_limit(self):
        """The fetching process sleeps until the rate limit is restored or
           raises a RateLimitError exception if sleep_for_rate flag is disabled.
        """
        if self.rate_limit is not None and self.rate_limit <= self.min_rate_to_sleep:
            seconds_to_reset = self.calculate_time_to_reset()

            if seconds_to_reset < 0:
                logger.warning("Value of sleep for rate limit is negative, reset it to 0")
                seconds_to_reset = 0

            cause = "Rate limit exhausted."
            if self.sleep_for_rate:
                logger.info("%s Waiting %i secs for rate limit reset.", cause, seconds_to_reset)
                time.sleep(seconds_to_reset)
            else:
                raise RateLimitError(cause=cause, seconds_to_reset=seconds_to_reset)

The fetching process sleeps until the rate limit is restored or raises a RateLimitError exception if sleep_for_rate flag is disabled .


def update_rate_limit(self, response):
        """Update the rate limit and the time to reset
        from the response headers.

        :param: response: the response object
        """
        if self.rate_limit_header in response.headers:
            self.rate_limit = int(response.headers[self.rate_limit_header])
            logger.debug("Rate limit: %s", self.rate_limit)
        else:
            self.rate_limit = None

        if self.rate_limit_reset_header in response.headers:
            self.rate_limit_reset_ts = int(response.headers[self.rate_limit_reset_header])
            logger.debug("Rate limit reset: %s", self.calculate_time_to_reset())
        else:
            self.rate_limit_reset_ts = None

def
update_rate_limit
(
self
,
response
)
:
if
self
.
rate_limit_header
in
response
.
headers
:
self
.
rate_limit
=
int
(
response
.
headers
[
self
.
rate_limit_header
]
)
logger
.
debug
(
"Rate limit: %s"
,
self
.
rate_limit
)
else
:
self
.
rate_limit
=
if
self
.
rate_limit_reset_header
in
response
.
headers
:
self
.
rate_limit_reset_ts
=
int
(
response
.
headers
[
self
.
rate_limit_reset_header
]
)
logger
.
debug
(
"Rate limit reset: %s"
,
self
.
calculate_time_to_reset
(
)
)
else
:
self
.
rate_limit_reset_ts
=

Update the rate limit and the time to reset from the response headers .


def parse_supybot_log(filepath):
        """Parse a Supybot IRC log file.

        The method parses the Supybot IRC log file and returns an iterator of
        dictionaries. Each one of this, contains a message from the file.

        :param filepath: path to the IRC log file

        :returns: a generator of parsed messages

        :raises ParseError: raised when the format of the Supybot log file
            is invalid
        :raises OSError: raised when an error occurs reading the
            given file
        """
        with open(filepath, 'r', errors='surrogateescape',
                  newline=os.linesep) as f:
            parser = SupybotParser(f)

            try:
                for message in parser.parse():
                    yield message
            except ParseError as e:
                cause = "file: %s; reason: %s" % (filepath, str(e))
                raise ParseError(cause=cause)

def
parse_supybot_log
(
filepath
)
:
with
open
(
filepath
,
'r'
,
errors
=
'surrogateescape'
,
newline
=
os
.
linesep
)
as
f
:
parser
=
SupybotParser
(
f
)
try
:
for
message
in
parser
.
parse
(
)
:
yield
message
except
ParseError
as
e
:
cause
=
"file: %s; reason: %s"
%
(
filepath
,
str
(
e
)
)
raise
ParseError
(
cause
=
cause
)


def __retrieve_archives(self, from_date):
        """Retrieve the Supybot archives after the given date"""

        archives = []

        candidates = self.__list_supybot_archives()

        for candidate in candidates:
            dt = self.__parse_date_from_filepath(candidate)

            if dt.date() >= from_date.date():
                archives.append((dt, candidate))
            else:
                logger.debug("Archive %s stored before %s; skipped",
                             candidate, str(from_date))

        archives.sort(key=lambda x: x[0])

        return [archive[1] for archive in archives]

def
__retrieve_archives
(
self
,
from_date
)
:
archives
=
[
]
candidates
=
self
.
__list_supybot_archives
(
)
for
candidate
in
candidates
:
dt
=
self
.
__parse_date_from_filepath
(
candidate
)
if
dt
.
date
(
)
>=
from_date
.
date
(
)
:
archives
.
append
(
(
dt
,
candidate
)
)
else
:
logger
.
debug
(
"Archive %s stored before %s; skipped"
,
candidate
,
str
(
from_date
)
)
archives
.
sort
(
key
=
lambda
x
:
x
[
0
]
)
return
[
archive
[
1
]
for
archive
in
archives
]


def __list_supybot_archives(self):
        """List the filepath of the archives stored in dirpath"""

        archives = []

        for root, _, files in os.walk(self.dirpath):
            for filename in files:
                location = os.path.join(root, filename)
                archives.append(location)

        return archives

def __list_supybot_archives ( self ) : archives = [ ] for root , _ , files in os . walk ( self . dirpath ) : for filename in files : location = os . path . join ( root , filename ) archives . append ( location ) return archives


def parse(self):
        """Parse a Supybot IRC stream.

        Returns an iterator of dicts. Each dicts contains information
        about the date, type, nick and body of a single log entry.

        :returns: iterator of parsed lines

        :raises ParseError: when an invalid line is found parsing the given
            stream
        """
        for line in self.stream:
            line = line.rstrip('\n')
            self.nline += 1

            if self.SUPYBOT_EMPTY_REGEX.match(line):
                continue

            ts, msg = self._parse_supybot_timestamp(line)

            if self.SUPYBOT_EMPTY_COMMENT_REGEX.match(msg):
                continue
            elif self.SUPYBOT_EMPTY_COMMENT_ACTION_REGEX.match(msg):
                continue
            elif self.SUPYBOT_EMPTY_BOT_REGEX.match(msg):
                continue

            itype, nick, body = self._parse_supybot_msg(msg)
            item = self._build_item(ts, itype, nick, body)

            yield item

def parse ( self ) : for line in self . stream : line = line . rstrip ( '\n' ) self . nline += 1 if self . SUPYBOT_EMPTY_REGEX . match ( line ) : continue ts , msg = self . _parse_supybot_timestamp ( line ) if self . SUPYBOT_EMPTY_COMMENT_REGEX . match ( msg ) : continue elif self . SUPYBOT_EMPTY_COMMENT_ACTION_REGEX . match ( msg ) : continue elif self . SUPYBOT_EMPTY_BOT_REGEX . match ( msg ) : continue itype , nick , body = self . _parse_supybot_msg ( msg ) item = self . _build_item ( ts , itype , nick , body ) yield item


def _parse_supybot_timestamp(self, line):
        """Parse timestamp section"""

        m = self.SUPYBOT_TIMESTAMP_REGEX.match(line)

        if not m:
            msg = "date expected on line %s" % (str(self.nline))
            raise ParseError(cause=msg)

        ts = m.group('ts')
        msg = m.group('msg')

        return ts, msg

def
_parse_supybot_timestamp
(
self
,
line
)
:
m
=
self
.
SUPYBOT_TIMESTAMP_REGEX
.
match
(
line
)
if
not
m
:
msg
=
"date expected on line %s"
%
(
str
(
self
.
nline
)
)
raise
ParseError
(
cause
=
msg
)
ts
=
m
.
group
(
'ts'
)
msg
=
m
.
group
(
'msg'
)
return
ts
,
msg


def _parse_supybot_msg(self, line):
        """Parse message section"""

        patterns = [(self.SUPYBOT_COMMENT_REGEX, self.TCOMMENT),
                    (self.SUPYBOT_COMMENT_ACTION_REGEX, self.TCOMMENT),
                    (self.SUPYBOT_SERVER_REGEX, self.TSERVER),
                    (self.SUPYBOT_BOT_REGEX, self.TCOMMENT)]

        for p in patterns:
            m = p[0].match(line)
            if not m:
                continue
            return p[1], m.group('nick'), m.group('body').strip()

        msg = "invalid message on line %s" % (str(self.nline))
        raise ParseError(cause=msg)

def
_parse_supybot_msg
(
self
,
line
)
:
patterns
=
[
(
self
.
SUPYBOT_COMMENT_REGEX
,
self
.
TCOMMENT
)
,
(
self
.
SUPYBOT_COMMENT_ACTION_REGEX
,
self
.
TCOMMENT
)
,
(
self
.
SUPYBOT_SERVER_REGEX
,
self
.
TSERVER
)
,
(
self
.
SUPYBOT_BOT_REGEX
,
self
.
TCOMMENT
)
]
for
p
in
patterns
:
m
=
p
[
0
]
.
match
(
line
)
if
not
m
:
continue
return
p
[
1
]
,
m
.
group
(
'nick'
)
,
m
.
group
(
'body'
)
.
strip
(
)
msg
=
"invalid message on line %s"
%
(
str
(
self
.
nline
)
)
raise
ParseError
(
cause
=
msg
)


def fetch_items(self, category, **kwargs):
        """Fetch the topics

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """

        from_date = kwargs['from_date']

        logger.info("Looking for topics at '%s', updated from '%s'",
                    self.url, str(from_date))

        ntopics = 0

        topics_ids = self.__fetch_and_parse_topics_ids(from_date)

        for topic_id in topics_ids:
            topic = self.__fetch_and_parse_topic(topic_id)
            ntopics += 1
            yield topic

        logger.info("Fetch process completed: %s topics fetched",
                    ntopics)


def __parse_topics_page(self, raw_json):
        """Parse a topics page stream.

        The result of parsing process is a generator of tuples. Each
        tuple contains de identifier of the topic, the last date
        when it was updated and whether is pinned or not.

        :param raw_json: JSON stream to parse

        :returns: a generator of parsed bugs
        """
        topics_page = json.loads(raw_json)

        topics_ids = []

        for topic in topics_page['topic_list']['topics']:
            topic_id = topic['id']
            if topic['last_posted_at'] is None:
                logger.warning("Topic %s with last_posted_at null. Ignoring it.", topic['title'])
                continue
            updated_at = str_to_datetime(topic['last_posted_at'])
            pinned = topic['pinned']
            topics_ids.append((topic_id, updated_at, pinned))

        return topics_ids


def topic(self, topic_id):
        """Retrive the topic with `topic_id` identifier.

        :param topic_id: identifier of the topic to retrieve
        """
        params = {
            self.PKEY: self.api_key
        }

        # http://example.com/t/8.json
        response = self._call(self.TOPIC, topic_id,
                              params=params)

        return response

def topic ( self , topic_id ) : params = { self . PKEY : self . api_key } response = self . _call ( self . TOPIC , topic_id , params = params ) return response

Retrive the topic with topic_id identifier .


def post(self, post_id):
        """Retrieve the post whit `post_id` identifier.

        :param post_id: identifier of the post to retrieve
        """
        params = {
            self.PKEY: self.api_key
        }

        # http://example.com/posts/10.json
        response = self._call(self.POSTS, post_id,
                              params=params)

        return response

def post ( self , post_id ) : params = { self . PKEY : self . api_key } response = self . _call ( self . POSTS , post_id , params = params ) return response

Retrieve the post whit post_id identifier .


def fetch_items(self, category, **kwargs):
        """Fetch the tasks

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']

        logger.info("Fetching tasks of '%s' from %s", self.url, str(from_date))

        ntasks = 0

        for task in self.__fetch_tasks(from_date):
            yield task
            ntasks += 1

        logger.info("Fetch process completed: %s tasks fetched", ntasks)


def parse_tasks(raw_json):
        """Parse a Phabricator tasks JSON stream.

        The method parses a JSON stream and returns a list iterator.
        Each item is a dictionary that contains the task parsed data.

        :param raw_json: JSON string to parse

        :returns: a generator of parsed tasks
        """
        results = json.loads(raw_json)

        tasks = results['result']['data']
        for t in tasks:
            yield t

def parse_tasks ( raw_json ) : results = json . loads ( raw_json ) tasks = results [ 'result' ] [ 'data' ] for t in tasks : yield t


def parse_users(raw_json):
        """Parse a Phabricator users JSON stream.

        The method parses a JSON stream and returns a list iterator.
        Each item is a dictionary that contais the user parsed data.

        :param raw_json: JSON string to parse

        :returns: a generator of parsed users
        """
        results = json.loads(raw_json)

        users = results['result']
        for u in users:
            yield u

def parse_users ( raw_json ) : results = json . loads ( raw_json ) users = results [ 'result' ] for u in users : yield u


def tasks(self, from_date=DEFAULT_DATETIME):
        """Retrieve tasks.

        :param from_date: retrieve tasks that where updated from that date;
            dates are converted epoch time.
        """
        # Convert 'from_date' to epoch timestamp.
        # Zero value (1970-01-01 00:00:00) is not allowed for
        # 'modifiedStart' so it will be set to 1, by default.
        ts = int(datetime_to_utc(from_date).timestamp()) or 1

        consts = {
            self.PMODIFIED_START: ts
        }

        attachments = {
            self. PPROJECTS: True
        }

        params = {
            self.PCONSTRAINTS: consts,
            self.PATTACHMENTS: attachments,
            self.PORDER: self.VOUTDATED,
        }

        while True:
            r = self._call(self.MANIPHEST_TASKS, params)
            yield r
            j = json.loads(r)
            after = j['result']['cursor']['after']
            if not after:
                break
            params[self.PAFTER] = after

def tasks ( self , from_date = DEFAULT_DATETIME ) : ts = int ( datetime_to_utc ( from_date ) . timestamp ( ) ) or 1 consts = { self . PMODIFIED_START : ts } attachments = { self . PPROJECTS : True } params = { self . PCONSTRAINTS : consts , self . PATTACHMENTS : attachments , self . PORDER : self . VOUTDATED , } while True : r = self . _call ( self . MANIPHEST_TASKS , params ) yield r j = json . loads ( r ) after = j [ 'result' ] [ 'cursor' ] [ 'after' ] if not after : break params [ self . PAFTER ] = after


def transactions(self, *phids):
        """Retrieve tasks transactions.

        :param phids: list of tasks identifiers
        """
        params = {
            self.PIDS: phids
        }

        response = self._call(self.MANIPHEST_TRANSACTIONS, params)

        return response

def transactions ( self , * phids ) : params = { self . PIDS : phids } response = self . _call ( self . MANIPHEST_TRANSACTIONS , params ) return response


def users(self, *phids):
        """Retrieve users.

        :params phids: list of users identifiers
        """
        params = {
            self.PHIDS: phids
        }

        response = self._call(self.PHAB_USERS, params)

        return response

def users ( self , * phids ) : params = { self . PHIDS : phids } response = self . _call ( self . PHAB_USERS , params ) return response


def phids(self, *phids):
        """Retrieve data about PHIDs.

        :params phids: list of PHIDs
        """
        params = {
            self.PHIDS: phids
        }

        response = self._call(self.PHAB_PHIDS, params)

        return response

def phids ( self , * phids ) : params = { self . PHIDS : phids } response = self . _call ( self . PHAB_PHIDS , params ) return response


def _call(self, method, params):
        """Call a method.

        :param method: method to call
        :param params: dict with the HTTP parameters needed to call
            the given method

        :raises ConduitError: when an error is returned by the server
        """
        url = self.URL % {'base': self.base_url, 'method': method}

        # Conduit and POST parameters
        params['__conduit__'] = {'token': self.api_token}

        data = {
            'params': json.dumps(params, sort_keys=True),
            'output': 'json',
            '__conduit__': True
        }

        logger.debug("Phabricator Conduit client requests: %s params: %s",
                     method, str(data))

        r = self.fetch(url, payload=data, method=HttpClient.POST, verify=False)

        # Check for possible Conduit API errors
        result = r.json()

        if result['error_code']:
            raise ConduitError(error=result['error_info'],
                               code=result['error_code'])

        return r.text